Merge pull request #7747 from penpot/niwinz-develop-storage-changes

 Make the binfile exportation process more reliable
This commit is contained in:
Alejandro Alonso
2025-11-20 07:58:57 +01:00
committed by GitHub
25 changed files with 321 additions and 409 deletions

View File

@@ -255,6 +255,8 @@
(write-entry! output path params)
(events/tap :progress {:section :storage-object :id id})
(with-open [input (sto/get-object-data storage sobject)]
(.putNextEntry ^ZipOutputStream output (ZipEntry. (str "objects/" id ext)))
(io/copy input output :size (:size sobject))
@@ -279,6 +281,8 @@
thumbnails (bfc/get-file-object-thumbnails cfg file-id)]
(events/tap :progress {:section :file :id file-id})
(vswap! bfc/*state* update :files assoc file-id
{:id file-id
:name (:name file)

View File

@@ -11,9 +11,9 @@
[app.binfile.v1 :as bf.v1]
[app.binfile.v3 :as bf.v3]
[app.common.features :as cfeat]
[app.common.logging :as l]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.uri :as u]
[app.config :as cf]
[app.db :as db]
[app.http.sse :as sse]
@@ -25,10 +25,12 @@
[app.rpc.commands.projects :as projects]
[app.rpc.commands.teams :as teams]
[app.rpc.doc :as-alias doc]
[app.rpc.helpers :as rph]
[app.storage :as sto]
[app.storage.tmp :as tmp]
[app.tasks.file-gc]
[app.util.services :as sv]
[app.worker :as-alias wrk]))
[app.worker :as-alias wrk]
[datoteka.fs :as fs]))
(set! *warn-on-reflection* true)
@@ -38,52 +40,42 @@
schema:export-binfile
[:map {:title "export-binfile"}
[:file-id ::sm/uuid]
[:version {:optional true} ::sm/int]
[:include-libraries ::sm/boolean]
[:embed-assets ::sm/boolean]])
(defn stream-export-v1
[cfg {:keys [file-id include-libraries embed-assets] :as params}]
(rph/stream
(fn [_ output-stream]
(try
(-> cfg
(assoc ::bfc/ids #{file-id})
(assoc ::bfc/embed-assets embed-assets)
(assoc ::bfc/include-libraries include-libraries)
(bf.v1/export-files! output-stream))
(catch Throwable cause
(l/err :hint "exception on exporting file"
:file-id (str file-id)
:cause cause))))))
(defn- export-binfile
[{:keys [::sto/storage] :as cfg} {:keys [file-id include-libraries embed-assets]}]
(let [output (tmp/tempfile*)]
(try
(-> cfg
(assoc ::bfc/ids #{file-id})
(assoc ::bfc/embed-assets embed-assets)
(assoc ::bfc/include-libraries include-libraries)
(bf.v3/export-files! output))
(defn stream-export-v3
[cfg {:keys [file-id include-libraries embed-assets] :as params}]
(rph/stream
(fn [_ output-stream]
(try
(-> cfg
(assoc ::bfc/ids #{file-id})
(assoc ::bfc/embed-assets embed-assets)
(assoc ::bfc/include-libraries include-libraries)
(bf.v3/export-files! output-stream))
(catch Throwable cause
(l/err :hint "exception on exporting file"
:file-id (str file-id)
:cause cause))))))
(let [data (sto/content output)
object (sto/put-object! storage
{::sto/content data
::sto/touched-at (ct/in-future {:minutes 60})
:content-type "application/zip"
:bucket "tempfile"})]
(-> (cf/get :public-uri)
(u/join "/assets/by-id/")
(u/join (str (:id object)))))
(finally
(fs/delete output)))))
(sv/defmethod ::export-binfile
"Export a penpot file in a binary format."
{::doc/added "1.15"
::doc/changes [["2.12" "Remove version parameter, only one version is supported"]]
::webhooks/event? true
::sm/params schema:export-binfile}
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id version file-id] :as params}]
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
(files/check-read-permissions! pool profile-id file-id)
(let [version (or version 1)]
(case (int version)
1 (stream-export-v1 cfg params)
2 (throw (ex-info "not-implemented" {}))
3 (stream-export-v3 cfg params))))
(sse/response (partial export-binfile cfg params)))
;; --- Command: import-binfile

View File

@@ -41,6 +41,7 @@
"file-object-thumbnail"
"file-thumbnail"
"profile"
"tempfile"
"file-data"
"file-data-fragment"
"file-change"})
@@ -163,9 +164,6 @@
backend
(:metadata result))))
(def ^:private sql:retrieve-storage-object
"select * from storage_object where id = ? and (deleted_at is null or deleted_at > now())")
(defn row->storage-object [res]
(let [mdata (or (some-> (:metadata res) (db/decode-transit-pgobject)) {})]
(impl/storage-object
@@ -177,9 +175,15 @@
(keyword (:backend res))
mdata)))
(defn- retrieve-database-object
(def ^:private sql:get-storage-object
"SELECT *
FROM storage_object
WHERE id = ?
AND (deleted_at IS NULL)")
(defn- get-database-object
[conn id]
(some-> (db/exec-one! conn [sql:retrieve-storage-object id])
(some-> (db/exec-one! conn [sql:get-storage-object id])
(row->storage-object)))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -202,7 +206,7 @@
(defn get-object
[{:keys [::db/connectable] :as storage} id]
(assert (valid-storage? storage))
(retrieve-database-object connectable id))
(get-database-object connectable id))
(defn put-object!
"Creates a new object with the provided content."

View File

@@ -37,7 +37,6 @@
(into #{} (map :id))
(not-empty))))
(def ^:private sql:delete-sobjects
"DELETE FROM storage_object
WHERE id = ANY(?::uuid[])")
@@ -77,47 +76,37 @@
(d/group-by (comp keyword :backend) :id #{} items))
(def ^:private sql:get-deleted-sobjects
"SELECT s.* FROM storage_object AS s
"SELECT s.*
FROM storage_object AS s
WHERE s.deleted_at IS NOT NULL
AND s.deleted_at < now() - ?::interval
AND s.deleted_at <= ?
ORDER BY s.deleted_at ASC")
(defn- get-buckets
[conn min-age]
(let [age (db/interval min-age)]
[conn]
(let [now (ct/now)]
(sequence
(comp (partition-all 25)
(mapcat group-by-backend))
(db/cursor conn [sql:get-deleted-sobjects age]))))
(db/cursor conn [sql:get-deleted-sobjects now]))))
(defn- clean-deleted!
[{:keys [::db/conn ::min-age] :as cfg}]
[{:keys [::db/conn] :as cfg}]
(reduce (fn [total [backend-id ids]]
(let [deleted (delete-in-bulk! cfg backend-id ids)]
(+ total (or deleted 0))))
0
(get-buckets conn min-age)))
(get-buckets conn)))
(defmethod ig/assert-key ::handler
[_ params]
(assert (sto/valid-storage? (::sto/storage params)) "expect valid storage")
(assert (db/pool? (::db/pool params)) "expect valid storage"))
(defmethod ig/expand-key ::handler
[k v]
{k (assoc v ::min-age (ct/duration {:hours 2}))})
(defmethod ig/init-key ::handler
[_ {:keys [::min-age] :as cfg}]
(fn [{:keys [props] :as task}]
(let [min-age (ct/duration (or (:min-age props) min-age))]
(db/tx-run! cfg (fn [cfg]
(let [cfg (assoc cfg ::min-age min-age)
total (clean-deleted! cfg)]
(l/inf :hint "task finished"
:min-age (ct/format-duration min-age)
:total total)
{:deleted total}))))))
[_ cfg]
(fn [_]
(db/tx-run! cfg (fn [cfg]
(let [total (clean-deleted! cfg)]
(l/inf :hint "task finished" :total total)
{:deleted total})))))

View File

@@ -22,6 +22,8 @@
[app.common.data.macros :as dm]
[app.common.exceptions :as ex]
[app.common.logging :as l]
[app.common.time :as ct]
[app.config :as cf]
[app.db :as db]
[app.storage :as-alias sto]
[app.storage.impl :as impl]
@@ -101,14 +103,15 @@
(def ^:private sql:mark-delete-in-bulk
"UPDATE storage_object
SET deleted_at = now(),
SET deleted_at = ?,
touched_at = NULL
WHERE id = ANY(?::uuid[])")
(defn- mark-delete-in-bulk!
[conn ids]
(let [ids (db/create-array conn "uuid" ids)]
(db/exec-one! conn [sql:mark-delete-in-bulk ids])))
[conn deletion-delay ids]
(let [ids (db/create-array conn "uuid" ids)
now (ct/plus (ct/now) deletion-delay)]
(db/exec-one! conn [sql:mark-delete-in-bulk now ids])))
;; NOTE: A getter that retrieves the key which will be used for group
;; ids; previously we have no value, then we introduced the
@@ -137,18 +140,20 @@
(if-let [{:keys [id] :as object} (first objects)]
(if (has-refs? conn object)
(do
(l/debug :id (str id)
:status "freeze"
:bucket bucket)
(l/dbg :id (str id)
:status "freeze"
:bucket bucket)
(recur (conj to-freeze id) to-delete (rest objects)))
(do
(l/debug :id (str id)
:status "delete"
:bucket bucket)
(l/dbg :id (str id)
:status "delete"
:bucket bucket)
(recur to-freeze (conj to-delete id) (rest objects))))
(do
(let [deletion-delay (if (= bucket "tempfile")
(ct/duration {:hours 2})
(cf/get-deletion-delay))]
(some->> (seq to-freeze) (mark-freeze-in-bulk! conn))
(some->> (seq to-delete) (mark-delete-in-bulk! conn))
(some->> (seq to-delete) (mark-delete-in-bulk! conn deletion-delay))
[(count to-freeze) (count to-delete)]))))
(defn- process-bucket!
@@ -160,6 +165,7 @@
"file-thumbnail" (process-objects! conn has-file-thumbnails-refs? bucket objects)
"profile" (process-objects! conn has-profile-refs? bucket objects)
"file-data" (process-objects! conn has-file-data-refs? bucket objects)
"tempfile" (process-objects! conn (constantly false) bucket objects)
(ex/raise :type :internal
:code :unexpected-unknown-reference
:hint (dm/fmt "unknown reference '%'" bucket))))
@@ -173,27 +179,27 @@
[0 0]
(d/group-by lookup-bucket identity #{} chunk)))
(def ^:private
sql:get-touched-storage-objects
(def ^:private sql:get-touched-storage-objects
"SELECT so.*
FROM storage_object AS so
WHERE so.touched_at IS NOT NULL
AND so.touched_at <= ?
ORDER BY touched_at ASC
FOR UPDATE
SKIP LOCKED
LIMIT 10")
(defn get-chunk
[conn]
(->> (db/exec! conn [sql:get-touched-storage-objects])
[conn timestamp]
(->> (db/exec! conn [sql:get-touched-storage-objects timestamp])
(map impl/decode-row)
(not-empty)))
(defn- process-touched!
[{:keys [::db/pool] :as cfg}]
[{:keys [::db/pool ::timestamp] :as cfg}]
(loop [freezed 0
deleted 0]
(if-let [chunk (get-chunk pool)]
(if-let [chunk (get-chunk pool timestamp)]
(let [[nfo ndo] (db/tx-run! cfg process-chunk! chunk)]
(recur (long (+ freezed nfo))
(long (+ deleted ndo))))
@@ -209,5 +215,6 @@
(defmethod ig/init-key ::handler
[_ cfg]
(fn [_] (process-touched! cfg)))
(fn [_]
(process-touched! (assoc cfg ::timestamp (ct/now)))))

View File

@@ -79,14 +79,17 @@
;; API
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn tempfile
[& {:keys [suffix prefix min-age]
(defn tempfile*
[& {:keys [suffix prefix]
:or {prefix "penpot."
suffix ".tmp"}}]
(let [attrs (fs/make-permissions "rw-r--r--")
path (fs/join default-tmp-dir (str prefix (uuid/next) suffix))
path (Files/createFile path attrs)]
(fs/delete-on-exit! path)
path (fs/join default-tmp-dir (str prefix (uuid/next) suffix))]
(Files/createFile path attrs)))
(defn tempfile
[& {:keys [min-age] :as opts}]
(let [path (tempfile* opts)]
(sp/offer! queue [path (some-> min-age ct/duration)])
path))

View File

@@ -18,15 +18,15 @@
(def ^:private sql:get-profiles
"SELECT id, photo_id FROM profile
WHERE deleted_at IS NOT NULL
AND deleted_at < now() + ?::interval
AND deleted_at <= ?
ORDER BY deleted_at ASC
LIMIT ?
FOR UPDATE
SKIP LOCKED")
(defn- delete-profiles!
[{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-profiles deletion-threshold chunk-size] {:fetch-size 5})
[{:keys [::db/conn ::timestamp ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-profiles timestamp chunk-size] {:fetch-size 5})
(reduce (fn [total {:keys [id photo-id]}]
(l/trc :obj "profile" :id (str id))
@@ -41,15 +41,15 @@
(def ^:private sql:get-teams
"SELECT deleted_at, id, photo_id FROM team
WHERE deleted_at IS NOT NULL
AND deleted_at < now() + ?::interval
AND deleted_at <= ?
ORDER BY deleted_at ASC
LIMIT ?
FOR UPDATE
SKIP LOCKED")
(defn- delete-teams!
[{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-teams deletion-threshold chunk-size] {:fetch-size 5})
[{:keys [::db/conn ::timestamp ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-teams timestamp chunk-size] {:fetch-size 5})
(reduce (fn [total {:keys [id photo-id deleted-at]}]
(l/trc :obj "team"
:id (str id)
@@ -68,15 +68,15 @@
"SELECT id, team_id, deleted_at, woff1_file_id, woff2_file_id, otf_file_id, ttf_file_id
FROM team_font_variant
WHERE deleted_at IS NOT NULL
AND deleted_at < now() + ?::interval
AND deleted_at <= ?
ORDER BY deleted_at ASC
LIMIT ?
FOR UPDATE
SKIP LOCKED")
(defn- delete-fonts!
[{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-fonts deletion-threshold chunk-size] {:fetch-size 5})
[{:keys [::db/conn ::timestamp ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-fonts timestamp chunk-size] {:fetch-size 5})
(reduce (fn [total {:keys [id team-id deleted-at] :as font}]
(l/trc :obj "font-variant"
:id (str id)
@@ -98,15 +98,15 @@
"SELECT id, deleted_at, team_id
FROM project
WHERE deleted_at IS NOT NULL
AND deleted_at < now() + ?::interval
AND deleted_at <= ?
ORDER BY deleted_at ASC
LIMIT ?
FOR UPDATE
SKIP LOCKED")
(defn- delete-projects!
[{:keys [::db/conn ::deletion-threshold ::chunk-size] :as cfg}]
(->> (db/plan conn [sql:get-projects deletion-threshold chunk-size] {:fetch-size 5})
[{:keys [::db/conn ::timestamp ::chunk-size] :as cfg}]
(->> (db/plan conn [sql:get-projects timestamp chunk-size] {:fetch-size 5})
(reduce (fn [total {:keys [id team-id deleted-at]}]
(l/trc :obj "project"
:id (str id)
@@ -124,15 +124,15 @@
f.project_id
FROM file AS f
WHERE f.deleted_at IS NOT NULL
AND f.deleted_at < now() + ?::interval
AND f.deleted_at <= ?
ORDER BY f.deleted_at ASC
LIMIT ?
FOR UPDATE
SKIP LOCKED")
(defn- delete-files!
[{:keys [::db/conn ::deletion-threshold ::chunk-size] :as cfg}]
(->> (db/plan conn [sql:get-files deletion-threshold chunk-size] {:fetch-size 5})
[{:keys [::db/conn ::timestamp ::chunk-size] :as cfg}]
(->> (db/plan conn [sql:get-files timestamp chunk-size] {:fetch-size 5})
(reduce (fn [total {:keys [id deleted-at project-id] :as file}]
(l/trc :obj "file"
:id (str id)
@@ -148,15 +148,15 @@
"SELECT file_id, revn, media_id, deleted_at
FROM file_thumbnail
WHERE deleted_at IS NOT NULL
AND deleted_at < now() + ?::interval
AND deleted_at <= ?
ORDER BY deleted_at ASC
LIMIT ?
FOR UPDATE
SKIP LOCKED")
(defn delete-file-thumbnails!
[{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-file-thumbnails deletion-threshold chunk-size] {:fetch-size 5})
[{:keys [::db/conn ::timestamp ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-file-thumbnails timestamp chunk-size] {:fetch-size 5})
(reduce (fn [total {:keys [file-id revn media-id deleted-at]}]
(l/trc :obj "file-thumbnail"
:file-id (str file-id)
@@ -175,15 +175,15 @@
"SELECT file_id, object_id, media_id, deleted_at
FROM file_tagged_object_thumbnail
WHERE deleted_at IS NOT NULL
AND deleted_at < now() + ?::interval
AND deleted_at <= ?
ORDER BY deleted_at ASC
LIMIT ?
FOR UPDATE
SKIP LOCKED")
(defn delete-file-object-thumbnails!
[{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-file-object-thumbnails deletion-threshold chunk-size] {:fetch-size 5})
[{:keys [::db/conn ::timestamp ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-file-object-thumbnails timestamp chunk-size] {:fetch-size 5})
(reduce (fn [total {:keys [file-id object-id media-id deleted-at]}]
(l/trc :obj "file-object-thumbnail"
:file-id (str file-id)
@@ -203,15 +203,15 @@
"SELECT id, file_id, media_id, thumbnail_id, deleted_at
FROM file_media_object
WHERE deleted_at IS NOT NULL
AND deleted_at < now() + ?::interval
AND deleted_at <= ?
ORDER BY deleted_at ASC
LIMIT ?
FOR UPDATE
SKIP LOCKED")
(defn- delete-file-media-objects!
[{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-file-media-objects deletion-threshold chunk-size] {:fetch-size 5})
[{:keys [::db/conn ::timestamp ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-file-media-objects timestamp chunk-size] {:fetch-size 5})
(reduce (fn [total {:keys [id file-id deleted-at] :as fmo}]
(l/trc :obj "file-media-object"
:id (str id)
@@ -231,16 +231,15 @@
"SELECT file_id, id, type, deleted_at, metadata, backend
FROM file_data
WHERE deleted_at IS NOT NULL
AND deleted_at < now() + ?::interval
AND deleted_at <= ?
ORDER BY deleted_at ASC
LIMIT ?
FOR UPDATE
SKIP LOCKED")
(defn- delete-file-data!
[{:keys [::db/conn ::deletion-threshold ::chunk-size] :as cfg}]
(->> (db/plan conn [sql:get-file-data deletion-threshold chunk-size] {:fetch-size 5})
[{:keys [::db/conn ::timestamp ::chunk-size] :as cfg}]
(->> (db/plan conn [sql:get-file-data timestamp chunk-size] {:fetch-size 5})
(reduce (fn [total {:keys [file-id id type deleted-at metadata backend]}]
(some->> metadata
@@ -266,15 +265,15 @@
"SELECT id, file_id, deleted_at
FROM file_change
WHERE deleted_at IS NOT NULL
AND deleted_at < now() + ?::interval
AND deleted_at <= ?
ORDER BY deleted_at ASC
LIMIT ?
FOR UPDATE
SKIP LOCKED")
(defn- delete-file-changes!
[{:keys [::db/conn ::deletion-threshold ::chunk-size] :as cfg}]
(->> (db/plan conn [sql:get-file-change deletion-threshold chunk-size] {:fetch-size 5})
[{:keys [::db/conn ::timestamp ::chunk-size] :as cfg}]
(->> (db/plan conn [sql:get-file-change timestamp chunk-size] {:fetch-size 5})
(reduce (fn [total {:keys [id file-id deleted-at] :as xlog}]
(l/trc :obj "file-change"
:id (str id)
@@ -322,9 +321,8 @@
(defmethod ig/init-key ::handler
[_ cfg]
(fn [{:keys [props] :as task}]
(let [threshold (ct/duration (get props :deletion-threshold 0))
cfg (assoc cfg ::deletion-threshold (db/interval threshold))]
(fn [_]
(let [cfg (assoc cfg ::timestamp (ct/now))]
(loop [procs (map deref deletion-proc-vars)
total 0]
(if-let [proc-fn (first procs)]

View File

@@ -27,7 +27,7 @@
(throw (IllegalArgumentException. "Missing arguments on `defmethod` macro.")))
(let [mdata (assoc mdata
::docstring (some-> docs str/<<-)
::docstring (some-> docs str/unindent)
::spec sname
::name (name sname))

View File

@@ -9,6 +9,7 @@
[app.common.features :as cfeat]
[app.common.pprint :as pp]
[app.common.thumbnails :as thc]
[app.common.time :as ct]
[app.common.types.shape :as cts]
[app.common.uuid :as uuid]
[app.config :as cf]
@@ -16,6 +17,7 @@
[app.db.sql :as sql]
[app.http :as http]
[app.rpc :as-alias rpc]
[app.setup.clock :as clock]
[app.storage :as sto]
[backend-tests.helpers :as th]
[clojure.test :as t]
@@ -132,9 +134,10 @@
;; this will run pending task triggered by deleting user snapshot
(th/run-pending-tasks!)
(let [res (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
;; delete 2 snapshots and 2 file data entries
(t/is (= 4 (:processed res))))))))
(binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
(let [res (th/run-task! :objects-gc {})]
;; delete 2 snapshots and 2 file data entries
(t/is (= 4 (:processed res)))))))))
(t/deftest snapshots-locking
(let [profile-1 (th/create-profile* 1 {:is-active true})

View File

@@ -313,7 +313,7 @@
;; freeze because of the deduplication (we have uploaded 2 times
;; the same files).
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 2 (:freeze res)))
(t/is (= 0 (:delete res))))
@@ -372,14 +372,14 @@
(th/db-exec! ["update file_change set deleted_at = now() where file_id = ? and label is not null" (:id file)])
(th/db-exec! ["update file set has_media_trimmed = false where id = ?" (:id file)])
(let [res (th/run-task! :objects-gc {:deletion-threshold 0})]
(let [res (th/run-task! :objects-gc {})]
;; this will remove the file change and file data entries for two snapshots
(t/is (= 4 (:processed res))))
;; Rerun the file-gc and objects-gc
(t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
(let [res (th/run-task! :objects-gc {:deletion-threshold 0})]
(let [res (th/run-task! :objects-gc {})]
;; this will remove the file media objects marked as deleted
;; on prev file-gc
(t/is (= 2 (:processed res))))
@@ -387,7 +387,7 @@
;; Now that file-gc have deleted the file-media-object usage,
;; lets execute the touched-gc task, we should see that two of
;; them are marked to be deleted
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 0 (:freeze res)))
(t/is (= 2 (:delete res))))
@@ -572,7 +572,7 @@
;; Now that file-gc have deleted the file-media-object usage,
;; lets execute the touched-gc task, we should see that two of
;; them are marked to be deleted.
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 0 (:freeze res)))
(t/is (= 2 (:delete res))))
@@ -665,7 +665,7 @@
;; because of the deduplication (we have uploaded 2 times the
;; same files).
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 1 (:freeze res)))
(t/is (= 0 (:delete res))))
@@ -715,7 +715,7 @@
;; Now that objects-gc have deleted the object thumbnail lets
;; execute the touched-gc task
(let [res (th/run-task! "storage-gc-touched" {:min-age 0})]
(let [res (th/run-task! "storage-gc-touched" {})]
(t/is (= 1 (:freeze res))))
;; check file media objects
@@ -750,7 +750,7 @@
;; Now that file-gc have deleted the object thumbnail lets
;; execute the touched-gc task
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 1 (:delete res))))
;; check file media objects
@@ -922,8 +922,9 @@
(t/is (= 0 (:processed result))))
;; run permanent deletion
(let [result (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
(t/is (= 3 (:processed result))))
(binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
(let [result (th/run-task! :objects-gc {})]
(t/is (= 3 (:processed result)))))
;; query the list of file libraries of a after hard deletion
(let [data {::th/type :get-file-libraries
@@ -1134,7 +1135,7 @@
(th/sleep 300)
;; run the task
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
(t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
;; check that object thumbnails are still here
(let [rows (th/db-query :file-tagged-object-thumbnail {:file-id (:id file)})]
@@ -1163,7 +1164,7 @@
(t/is (= 2 (count rows))))
;; run the task again
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
(t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
;; check that we have all object thumbnails
(let [rows (th/db-query :file-tagged-object-thumbnail {:file-id (:id file)})]
@@ -1226,7 +1227,7 @@
(t/is (= 2 (count rows)))))
(t/testing "gc task"
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
(t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
(let [rows (th/db-query :file-thumbnail {:file-id (:id file)})]
(t/is (= 2 (count rows)))
@@ -1273,7 +1274,7 @@
;; The FileGC task will schedule an inner taskq
(th/run-pending-tasks!)
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 2 (:freeze res)))
(t/is (= 0 (:delete res))))
@@ -1367,7 +1368,7 @@
;; we ensure that once object-gc is passed and marked two storage
;; objects to delete
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 0 (:freeze res)))
(t/is (= 2 (:delete res))))
@@ -1489,7 +1490,7 @@
(t/is (some? (not-empty (:objects component))))))
;; Re-run the file-gc task
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
(t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
(let [row (th/db-get :file {:id (:id file)})]
(t/is (true? (:has-media-trimmed row))))
@@ -1519,7 +1520,7 @@
;; Now, we have deleted the usage of component if we pass file-gc,
;; that component should be deleted
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
(t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
;; Check that component is properly removed
(let [data {::th/type :get-file
@@ -1610,8 +1611,8 @@
:component-id c-id})}])
;; Run the file-gc on file and library
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file-1)})))
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file-2)})))
(t/is (true? (th/run-task! :file-gc {:file-id (:id file-1)})))
(t/is (true? (th/run-task! :file-gc {:file-id (:id file-2)})))
;; Check that component exists
(let [data {::th/type :get-file
@@ -1684,7 +1685,7 @@
;; Now, we have deleted the usage of component if we pass file-gc,
;; that component should be deleted
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file-1)})))
(t/is (true? (th/run-task! :file-gc {:file-id (:id file-1)})))
;; Check that component is properly removed
(let [data {::th/type :get-file
@@ -1833,8 +1834,8 @@
(t/is (not= (:id fill) (:id fmedia)))))
;; Run the file-gc on file and library
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file-1)})))
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file-2)})))
(t/is (true? (th/run-task! :file-gc {:file-id (:id file-1)})))
(t/is (true? (th/run-task! :file-gc {:file-id (:id file-2)})))
;; Now proceed to delete file and absorb it
(let [data {::th/type :delete-file

View File

@@ -8,12 +8,14 @@
(:require
[app.common.pprint :as pp]
[app.common.thumbnails :as thc]
[app.common.time :as ct]
[app.common.types.shape :as cts]
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
[app.rpc :as-alias rpc]
[app.rpc.commands.auth :as cauth]
[app.setup.clock :as clock]
[app.storage :as sto]
[app.tokens :as tokens]
[backend-tests.helpers :as th]
@@ -83,7 +85,8 @@
(t/is (map? (:result out))))
;; run the task again
(let [res (th/run-task! "storage-gc-touched" {:min-age 0})]
(let [res (binding [ct/*clock* (clock/fixed (ct/in-future {:minutes 31}))]
(th/run-task! "storage-gc-touched" {}))]
(t/is (= 2 (:freeze res))))
(let [[row1 row2 :as rows] (th/db-query :file-tagged-object-thumbnail
@@ -114,9 +117,9 @@
;; Run the File GC task that should remove unused file object
;; thumbnails
(th/run-task! :file-gc {:min-age 0 :file-id (:id file)})
(th/run-task! :file-gc {:file-id (:id file)})
(let [result (th/run-task! :objects-gc {:min-age 0})]
(let [result (th/run-task! :objects-gc {})]
(t/is (= 3 (:processed result))))
;; check if row2 related thumbnail row still exists
@@ -133,7 +136,8 @@
(t/is (some? (sto/get-object storage (:media-id row2))))
;; run the task again
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(let [res (binding [ct/*clock* (clock/fixed (ct/in-future {:minutes 31}))]
(th/run-task! :storage-gc-touched {}))]
(t/is (= 1 (:delete res)))
(t/is (= 0 (:freeze res))))
@@ -143,8 +147,9 @@
;; Run the storage gc deleted task, it should permanently delete
;; all storage objects related to the deleted thumbnails
(let [result (th/run-task! :storage-gc-deleted {:min-age 0})]
(t/is (= 1 (:deleted result))))
(binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
(let [res (th/run-task! :storage-gc-deleted {})]
(t/is (= 1 (:deleted res)))))
(t/is (nil? (sto/get-object storage (:media-id row1))))
(t/is (some? (sto/get-object storage (:media-id row2))))
@@ -216,9 +221,9 @@
;; Run the File GC task that should remove unused file object
;; thumbnails
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
(t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
(let [result (th/run-task! :objects-gc {:min-age 0})]
(let [result (th/run-task! :objects-gc {})]
(t/is (= 2 (:processed result))))
;; check if row1 related thumbnail row still exists
@@ -230,7 +235,7 @@
(t/is (= (:object-id data1) (:object-id row)))
(t/is (uuid? (:media-id row1))))
(let [result (th/run-task! :storage-gc-touched {:min-age 0})]
(let [result (th/run-task! :storage-gc-touched {})]
(t/is (= 1 (:delete result))))
;; Check if storage objects still exists after file-gc
@@ -242,8 +247,9 @@
;; Run the storage gc deleted task, it should permanently delete
;; all storage objects related to the deleted thumbnails
(let [result (th/run-task! :storage-gc-deleted {:min-age 0})]
(t/is (= 1 (:deleted result))))
(binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
(let [result (th/run-task! :storage-gc-deleted {})]
(t/is (= 1 (:deleted result)))))
(t/is (some? (sto/get-object storage (:media-id row2)))))))

View File

@@ -6,11 +6,13 @@
(ns backend-tests.rpc-font-test
(:require
[app.common.time :as ct]
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
[app.http :as http]
[app.rpc :as-alias rpc]
[app.setup.clock :as clock]
[app.storage :as sto]
[backend-tests.helpers :as th]
[clojure.test :as t]
@@ -129,7 +131,7 @@
;; (th/print-result! out)
(t/is (nil? (:error out))))
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 6 (:freeze res))))
(let [params {::th/type :delete-font
@@ -141,16 +143,17 @@
(t/is (nil? (:error out)))
(t/is (nil? (:result out))))
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 0 (:freeze res)))
(t/is (= 0 (:delete res))))
(let [res (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
(t/is (= 2 (:processed res))))
(binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
(let [res (th/run-task! :objects-gc {})]
(t/is (= 2 (:processed res))))
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(t/is (= 0 (:freeze res)))
(t/is (= 6 (:delete res))))))
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 0 (:freeze res)))
(t/is (= 6 (:delete res)))))))
(t/deftest font-deletion-2
(let [prof (th/create-profile* 1 {:is-active true})
@@ -189,7 +192,7 @@
;; (th/print-result! out)
(t/is (nil? (:error out))))
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 6 (:freeze res))))
(let [params {::th/type :delete-font
@@ -201,16 +204,17 @@
(t/is (nil? (:error out)))
(t/is (nil? (:result out))))
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 0 (:freeze res)))
(t/is (= 0 (:delete res))))
(let [res (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
(t/is (= 1 (:processed res))))
(binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
(let [res (th/run-task! :objects-gc {})]
(t/is (= 1 (:processed res))))
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(t/is (= 0 (:freeze res)))
(t/is (= 3 (:delete res))))))
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 0 (:freeze res)))
(t/is (= 3 (:delete res)))))))
(t/deftest font-deletion-3
(let [prof (th/create-profile* 1 {:is-active true})
@@ -248,7 +252,7 @@
(t/is (nil? (:error out1)))
(t/is (nil? (:error out2)))
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 6 (:freeze res))))
(let [params {::th/type :delete-font-variant
@@ -260,13 +264,14 @@
(t/is (nil? (:error out)))
(t/is (nil? (:result out))))
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 0 (:freeze res)))
(t/is (= 0 (:delete res))))
(let [res (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
(t/is (= 1 (:processed res))))
(binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
(let [res (th/run-task! :objects-gc {})]
(t/is (= 1 (:processed res))))
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(t/is (= 0 (:freeze res)))
(t/is (= 3 (:delete res))))))
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 0 (:freeze res)))
(t/is (= 3 (:delete res)))))))

View File

@@ -6,11 +6,13 @@
(ns backend-tests.rpc-project-test
(:require
[app.common.time :as ct]
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
[app.http :as http]
[app.rpc :as-alias rpc]
[app.setup.clock :as clock]
[backend-tests.helpers :as th]
[clojure.test :as t]))
@@ -226,8 +228,9 @@
(t/is (= 0 (count result)))))
;; run permanent deletion
(let [result (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
(t/is (= 1 (:processed result))))
(binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
(let [result (th/run-task! :objects-gc {})]
(t/is (= 1 (:processed result)))))
;; query the list of files of a after hard deletion
(let [data {::th/type :get-project-files

View File

@@ -13,6 +13,7 @@
[app.db :as db]
[app.http :as http]
[app.rpc :as-alias rpc]
[app.setup.clock :as clock]
[app.storage :as sto]
[app.tokens :as tokens]
[backend-tests.helpers :as th]
@@ -525,8 +526,9 @@
(t/is (= :not-found (:type edata)))))
;; run permanent deletion
(let [result (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
(t/is (= 2 (:processed result))))
(binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
(let [result (th/run-task! :objects-gc {})]
(t/is (= 2 (:processed result)))))
;; query the list of projects of a after hard deletion
(let [data {::th/type :get-projects
@@ -581,8 +583,9 @@
(t/is (= 1 (count rows)))
(t/is (ct/inst? (:deleted-at (first rows)))))
(let [result (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
(t/is (= 7 (:processed result))))))
(binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
(let [result (th/run-task! :objects-gc {})]
(t/is (= 7 (:processed result)))))))
(t/deftest create-team-access-request
(with-mocks [mock {:target 'app.email/send! :return nil}]

View File

@@ -11,6 +11,7 @@
[app.common.uuid :as uuid]
[app.db :as db]
[app.rpc :as-alias rpc]
[app.setup.clock :as clock]
[app.storage :as sto]
[backend-tests.helpers :as th]
[clojure.test :as t]
@@ -53,19 +54,13 @@
(configure-storage-backend))
content (sto/content "content")
object (sto/put-object! storage {::sto/content content
::sto/expired-at (ct/in-future {:seconds 1})
::sto/expired-at (ct/in-future {:hours 1})
:content-type "text/plain"})]
(t/is (sto/object? object))
(t/is (ct/inst? (:expired-at object)))
(t/is (ct/is-after? (:expired-at object) (ct/now)))
(t/is (= object (sto/get-object storage (:id object))))
(th/sleep 1000)
(t/is (nil? (sto/get-object storage (:id object))))
(t/is (nil? (sto/get-object-data storage object)))
(t/is (nil? (sto/get-object-url storage object)))
(t/is (nil? (sto/get-object-path storage object)))))
(t/is (nil? (sto/get-object storage (:id object))))))
(t/deftest put-and-delete-object
(let [storage (-> (:app.storage/storage th/*system*)
@@ -98,20 +93,25 @@
::sto/expired-at (ct/now)
:content-type "text/plain"})
object2 (sto/put-object! storage {::sto/content content2
::sto/expired-at (ct/in-past {:hours 2})
::sto/expired-at (ct/in-future {:hours 2})
:content-type "text/plain"})
object3 (sto/put-object! storage {::sto/content content3
::sto/expired-at (ct/in-past {:hours 1})
::sto/expired-at (ct/in-future {:hours 1})
:content-type "text/plain"})]
(th/sleep 200)
(let [res (th/run-task! :storage-gc-deleted {})]
(t/is (= 1 (:deleted res))))
(binding [ct/*clock* (clock/fixed (ct/in-future {:minutes 0}))]
(let [res (th/run-task! :storage-gc-deleted {})]
(t/is (= 1 (:deleted res)))))
(let [res (th/db-exec-one! ["select count(*) from storage_object;"])]
(t/is (= 2 (:count res))))))
(t/is (= 2 (:count res))))
(binding [ct/*clock* (clock/fixed (ct/in-future {:minutes 61}))]
(let [res (th/run-task! :storage-gc-deleted {})]
(t/is (= 1 (:deleted res)))))
(let [res (th/db-exec-one! ["select count(*) from storage_object;"])]
(t/is (= 1 (:count res))))))
(t/deftest touched-gc-task-1
(let [storage (-> (:app.storage/storage th/*system*)
@@ -158,7 +158,7 @@
{:id (:id result-1)})
;; run the objects gc task for permanent deletion
(let [res (th/run-task! :objects-gc {:min-age 0})]
(let [res (th/run-task! :objects-gc {})]
(t/is (= 1 (:processed res))))
;; check that we still have all the storage objects
@@ -182,7 +182,6 @@
(let [res (th/db-exec-one! ["select count(*) from storage_object where deleted_at is not null"])]
(t/is (= 0 (:count res)))))))
(t/deftest touched-gc-task-2
(let [storage (-> (:app.storage/storage th/*system*)
(configure-storage-backend))
@@ -243,11 +242,12 @@
{:id (:id result-2)})
;; run the objects gc task for permanent deletion
(let [res (th/run-task! :objects-gc {:min-age 0})]
(let [res (th/run-task! :objects-gc {})]
(t/is (= 1 (:processed res))))
;; revert touched state to all storage objects
(th/db-exec-one! ["update storage_object set touched_at=now()"])
(th/db-exec-one! ["update storage_object set touched_at=?" (ct/now)])
;; Run the task again
(let [res (th/run-task! :storage-gc-touched {})]
@@ -293,10 +293,10 @@
result-2 (:result out2)]
;; now we proceed to manually mark all storage objects touched
(th/db-exec! ["update storage_object set touched_at=now()"])
(th/db-exec! ["update storage_object set touched_at=?" (ct/now)])
;; run the touched gc task
(let [res (th/run-task! "storage-gc-touched" {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 2 (:freeze res)))
(t/is (= 0 (:delete res))))
@@ -305,13 +305,13 @@
(t/is (= 2 (count rows)))))
;; now we proceed to manually delete all file_media_object
(th/db-exec! ["update file_media_object set deleted_at = now()"])
(th/db-exec! ["update file_media_object set deleted_at = ?" (ct/now)])
(let [res (th/run-task! "objects-gc" {:min-age 0})]
(let [res (th/run-task! :objects-gc {})]
(t/is (= 2 (:processed res))))
;; run the touched gc task
(let [res (th/run-task! "storage-gc-touched" {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 0 (:freeze res)))
(t/is (= 2 (:delete res))))