mirror of
https://github.com/penpot/penpot.git
synced 2025-12-11 22:14:05 +01:00
Merge pull request #7747 from penpot/niwinz-develop-storage-changes
✨ Make the binfile exportation process more reliable
This commit is contained in:
@@ -255,6 +255,8 @@
|
||||
|
||||
(write-entry! output path params)
|
||||
|
||||
(events/tap :progress {:section :storage-object :id id})
|
||||
|
||||
(with-open [input (sto/get-object-data storage sobject)]
|
||||
(.putNextEntry ^ZipOutputStream output (ZipEntry. (str "objects/" id ext)))
|
||||
(io/copy input output :size (:size sobject))
|
||||
@@ -279,6 +281,8 @@
|
||||
|
||||
thumbnails (bfc/get-file-object-thumbnails cfg file-id)]
|
||||
|
||||
(events/tap :progress {:section :file :id file-id})
|
||||
|
||||
(vswap! bfc/*state* update :files assoc file-id
|
||||
{:id file-id
|
||||
:name (:name file)
|
||||
|
||||
@@ -11,9 +11,9 @@
|
||||
[app.binfile.v1 :as bf.v1]
|
||||
[app.binfile.v3 :as bf.v3]
|
||||
[app.common.features :as cfeat]
|
||||
[app.common.logging :as l]
|
||||
[app.common.schema :as sm]
|
||||
[app.common.time :as ct]
|
||||
[app.common.uri :as u]
|
||||
[app.config :as cf]
|
||||
[app.db :as db]
|
||||
[app.http.sse :as sse]
|
||||
@@ -25,10 +25,12 @@
|
||||
[app.rpc.commands.projects :as projects]
|
||||
[app.rpc.commands.teams :as teams]
|
||||
[app.rpc.doc :as-alias doc]
|
||||
[app.rpc.helpers :as rph]
|
||||
[app.storage :as sto]
|
||||
[app.storage.tmp :as tmp]
|
||||
[app.tasks.file-gc]
|
||||
[app.util.services :as sv]
|
||||
[app.worker :as-alias wrk]))
|
||||
[app.worker :as-alias wrk]
|
||||
[datoteka.fs :as fs]))
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
@@ -38,52 +40,42 @@
|
||||
schema:export-binfile
|
||||
[:map {:title "export-binfile"}
|
||||
[:file-id ::sm/uuid]
|
||||
[:version {:optional true} ::sm/int]
|
||||
[:include-libraries ::sm/boolean]
|
||||
[:embed-assets ::sm/boolean]])
|
||||
|
||||
(defn stream-export-v1
|
||||
[cfg {:keys [file-id include-libraries embed-assets] :as params}]
|
||||
(rph/stream
|
||||
(fn [_ output-stream]
|
||||
(try
|
||||
(-> cfg
|
||||
(assoc ::bfc/ids #{file-id})
|
||||
(assoc ::bfc/embed-assets embed-assets)
|
||||
(assoc ::bfc/include-libraries include-libraries)
|
||||
(bf.v1/export-files! output-stream))
|
||||
(catch Throwable cause
|
||||
(l/err :hint "exception on exporting file"
|
||||
:file-id (str file-id)
|
||||
:cause cause))))))
|
||||
(defn- export-binfile
|
||||
[{:keys [::sto/storage] :as cfg} {:keys [file-id include-libraries embed-assets]}]
|
||||
(let [output (tmp/tempfile*)]
|
||||
(try
|
||||
(-> cfg
|
||||
(assoc ::bfc/ids #{file-id})
|
||||
(assoc ::bfc/embed-assets embed-assets)
|
||||
(assoc ::bfc/include-libraries include-libraries)
|
||||
(bf.v3/export-files! output))
|
||||
|
||||
(defn stream-export-v3
|
||||
[cfg {:keys [file-id include-libraries embed-assets] :as params}]
|
||||
(rph/stream
|
||||
(fn [_ output-stream]
|
||||
(try
|
||||
(-> cfg
|
||||
(assoc ::bfc/ids #{file-id})
|
||||
(assoc ::bfc/embed-assets embed-assets)
|
||||
(assoc ::bfc/include-libraries include-libraries)
|
||||
(bf.v3/export-files! output-stream))
|
||||
(catch Throwable cause
|
||||
(l/err :hint "exception on exporting file"
|
||||
:file-id (str file-id)
|
||||
:cause cause))))))
|
||||
(let [data (sto/content output)
|
||||
object (sto/put-object! storage
|
||||
{::sto/content data
|
||||
::sto/touched-at (ct/in-future {:minutes 60})
|
||||
:content-type "application/zip"
|
||||
:bucket "tempfile"})]
|
||||
|
||||
(-> (cf/get :public-uri)
|
||||
(u/join "/assets/by-id/")
|
||||
(u/join (str (:id object)))))
|
||||
|
||||
(finally
|
||||
(fs/delete output)))))
|
||||
|
||||
(sv/defmethod ::export-binfile
|
||||
"Export a penpot file in a binary format."
|
||||
{::doc/added "1.15"
|
||||
::doc/changes [["2.12" "Remove version parameter, only one version is supported"]]
|
||||
::webhooks/event? true
|
||||
::sm/params schema:export-binfile}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id version file-id] :as params}]
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
|
||||
(files/check-read-permissions! pool profile-id file-id)
|
||||
(let [version (or version 1)]
|
||||
(case (int version)
|
||||
1 (stream-export-v1 cfg params)
|
||||
2 (throw (ex-info "not-implemented" {}))
|
||||
3 (stream-export-v3 cfg params))))
|
||||
(sse/response (partial export-binfile cfg params)))
|
||||
|
||||
;; --- Command: import-binfile
|
||||
|
||||
|
||||
@@ -41,6 +41,7 @@
|
||||
"file-object-thumbnail"
|
||||
"file-thumbnail"
|
||||
"profile"
|
||||
"tempfile"
|
||||
"file-data"
|
||||
"file-data-fragment"
|
||||
"file-change"})
|
||||
@@ -163,9 +164,6 @@
|
||||
backend
|
||||
(:metadata result))))
|
||||
|
||||
(def ^:private sql:retrieve-storage-object
|
||||
"select * from storage_object where id = ? and (deleted_at is null or deleted_at > now())")
|
||||
|
||||
(defn row->storage-object [res]
|
||||
(let [mdata (or (some-> (:metadata res) (db/decode-transit-pgobject)) {})]
|
||||
(impl/storage-object
|
||||
@@ -177,9 +175,15 @@
|
||||
(keyword (:backend res))
|
||||
mdata)))
|
||||
|
||||
(defn- retrieve-database-object
|
||||
(def ^:private sql:get-storage-object
|
||||
"SELECT *
|
||||
FROM storage_object
|
||||
WHERE id = ?
|
||||
AND (deleted_at IS NULL)")
|
||||
|
||||
(defn- get-database-object
|
||||
[conn id]
|
||||
(some-> (db/exec-one! conn [sql:retrieve-storage-object id])
|
||||
(some-> (db/exec-one! conn [sql:get-storage-object id])
|
||||
(row->storage-object)))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
@@ -202,7 +206,7 @@
|
||||
(defn get-object
|
||||
[{:keys [::db/connectable] :as storage} id]
|
||||
(assert (valid-storage? storage))
|
||||
(retrieve-database-object connectable id))
|
||||
(get-database-object connectable id))
|
||||
|
||||
(defn put-object!
|
||||
"Creates a new object with the provided content."
|
||||
|
||||
@@ -37,7 +37,6 @@
|
||||
(into #{} (map :id))
|
||||
(not-empty))))
|
||||
|
||||
|
||||
(def ^:private sql:delete-sobjects
|
||||
"DELETE FROM storage_object
|
||||
WHERE id = ANY(?::uuid[])")
|
||||
@@ -77,47 +76,37 @@
|
||||
(d/group-by (comp keyword :backend) :id #{} items))
|
||||
|
||||
(def ^:private sql:get-deleted-sobjects
|
||||
"SELECT s.* FROM storage_object AS s
|
||||
"SELECT s.*
|
||||
FROM storage_object AS s
|
||||
WHERE s.deleted_at IS NOT NULL
|
||||
AND s.deleted_at < now() - ?::interval
|
||||
AND s.deleted_at <= ?
|
||||
ORDER BY s.deleted_at ASC")
|
||||
|
||||
(defn- get-buckets
|
||||
[conn min-age]
|
||||
(let [age (db/interval min-age)]
|
||||
[conn]
|
||||
(let [now (ct/now)]
|
||||
(sequence
|
||||
(comp (partition-all 25)
|
||||
(mapcat group-by-backend))
|
||||
(db/cursor conn [sql:get-deleted-sobjects age]))))
|
||||
|
||||
(db/cursor conn [sql:get-deleted-sobjects now]))))
|
||||
|
||||
(defn- clean-deleted!
|
||||
[{:keys [::db/conn ::min-age] :as cfg}]
|
||||
[{:keys [::db/conn] :as cfg}]
|
||||
(reduce (fn [total [backend-id ids]]
|
||||
(let [deleted (delete-in-bulk! cfg backend-id ids)]
|
||||
(+ total (or deleted 0))))
|
||||
0
|
||||
(get-buckets conn min-age)))
|
||||
(get-buckets conn)))
|
||||
|
||||
(defmethod ig/assert-key ::handler
|
||||
[_ params]
|
||||
(assert (sto/valid-storage? (::sto/storage params)) "expect valid storage")
|
||||
(assert (db/pool? (::db/pool params)) "expect valid storage"))
|
||||
|
||||
(defmethod ig/expand-key ::handler
|
||||
[k v]
|
||||
{k (assoc v ::min-age (ct/duration {:hours 2}))})
|
||||
|
||||
(defmethod ig/init-key ::handler
|
||||
[_ {:keys [::min-age] :as cfg}]
|
||||
(fn [{:keys [props] :as task}]
|
||||
(let [min-age (ct/duration (or (:min-age props) min-age))]
|
||||
(db/tx-run! cfg (fn [cfg]
|
||||
(let [cfg (assoc cfg ::min-age min-age)
|
||||
total (clean-deleted! cfg)]
|
||||
|
||||
(l/inf :hint "task finished"
|
||||
:min-age (ct/format-duration min-age)
|
||||
:total total)
|
||||
|
||||
{:deleted total}))))))
|
||||
[_ cfg]
|
||||
(fn [_]
|
||||
(db/tx-run! cfg (fn [cfg]
|
||||
(let [total (clean-deleted! cfg)]
|
||||
(l/inf :hint "task finished" :total total)
|
||||
{:deleted total})))))
|
||||
|
||||
@@ -22,6 +22,8 @@
|
||||
[app.common.data.macros :as dm]
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.logging :as l]
|
||||
[app.common.time :as ct]
|
||||
[app.config :as cf]
|
||||
[app.db :as db]
|
||||
[app.storage :as-alias sto]
|
||||
[app.storage.impl :as impl]
|
||||
@@ -101,14 +103,15 @@
|
||||
|
||||
(def ^:private sql:mark-delete-in-bulk
|
||||
"UPDATE storage_object
|
||||
SET deleted_at = now(),
|
||||
SET deleted_at = ?,
|
||||
touched_at = NULL
|
||||
WHERE id = ANY(?::uuid[])")
|
||||
|
||||
(defn- mark-delete-in-bulk!
|
||||
[conn ids]
|
||||
(let [ids (db/create-array conn "uuid" ids)]
|
||||
(db/exec-one! conn [sql:mark-delete-in-bulk ids])))
|
||||
[conn deletion-delay ids]
|
||||
(let [ids (db/create-array conn "uuid" ids)
|
||||
now (ct/plus (ct/now) deletion-delay)]
|
||||
(db/exec-one! conn [sql:mark-delete-in-bulk now ids])))
|
||||
|
||||
;; NOTE: A getter that retrieves the key which will be used for group
|
||||
;; ids; previously we have no value, then we introduced the
|
||||
@@ -137,18 +140,20 @@
|
||||
(if-let [{:keys [id] :as object} (first objects)]
|
||||
(if (has-refs? conn object)
|
||||
(do
|
||||
(l/debug :id (str id)
|
||||
:status "freeze"
|
||||
:bucket bucket)
|
||||
(l/dbg :id (str id)
|
||||
:status "freeze"
|
||||
:bucket bucket)
|
||||
(recur (conj to-freeze id) to-delete (rest objects)))
|
||||
(do
|
||||
(l/debug :id (str id)
|
||||
:status "delete"
|
||||
:bucket bucket)
|
||||
(l/dbg :id (str id)
|
||||
:status "delete"
|
||||
:bucket bucket)
|
||||
(recur to-freeze (conj to-delete id) (rest objects))))
|
||||
(do
|
||||
(let [deletion-delay (if (= bucket "tempfile")
|
||||
(ct/duration {:hours 2})
|
||||
(cf/get-deletion-delay))]
|
||||
(some->> (seq to-freeze) (mark-freeze-in-bulk! conn))
|
||||
(some->> (seq to-delete) (mark-delete-in-bulk! conn))
|
||||
(some->> (seq to-delete) (mark-delete-in-bulk! conn deletion-delay))
|
||||
[(count to-freeze) (count to-delete)]))))
|
||||
|
||||
(defn- process-bucket!
|
||||
@@ -160,6 +165,7 @@
|
||||
"file-thumbnail" (process-objects! conn has-file-thumbnails-refs? bucket objects)
|
||||
"profile" (process-objects! conn has-profile-refs? bucket objects)
|
||||
"file-data" (process-objects! conn has-file-data-refs? bucket objects)
|
||||
"tempfile" (process-objects! conn (constantly false) bucket objects)
|
||||
(ex/raise :type :internal
|
||||
:code :unexpected-unknown-reference
|
||||
:hint (dm/fmt "unknown reference '%'" bucket))))
|
||||
@@ -173,27 +179,27 @@
|
||||
[0 0]
|
||||
(d/group-by lookup-bucket identity #{} chunk)))
|
||||
|
||||
(def ^:private
|
||||
sql:get-touched-storage-objects
|
||||
(def ^:private sql:get-touched-storage-objects
|
||||
"SELECT so.*
|
||||
FROM storage_object AS so
|
||||
WHERE so.touched_at IS NOT NULL
|
||||
AND so.touched_at <= ?
|
||||
ORDER BY touched_at ASC
|
||||
FOR UPDATE
|
||||
SKIP LOCKED
|
||||
LIMIT 10")
|
||||
|
||||
(defn get-chunk
|
||||
[conn]
|
||||
(->> (db/exec! conn [sql:get-touched-storage-objects])
|
||||
[conn timestamp]
|
||||
(->> (db/exec! conn [sql:get-touched-storage-objects timestamp])
|
||||
(map impl/decode-row)
|
||||
(not-empty)))
|
||||
|
||||
(defn- process-touched!
|
||||
[{:keys [::db/pool] :as cfg}]
|
||||
[{:keys [::db/pool ::timestamp] :as cfg}]
|
||||
(loop [freezed 0
|
||||
deleted 0]
|
||||
(if-let [chunk (get-chunk pool)]
|
||||
(if-let [chunk (get-chunk pool timestamp)]
|
||||
(let [[nfo ndo] (db/tx-run! cfg process-chunk! chunk)]
|
||||
(recur (long (+ freezed nfo))
|
||||
(long (+ deleted ndo))))
|
||||
@@ -209,5 +215,6 @@
|
||||
|
||||
(defmethod ig/init-key ::handler
|
||||
[_ cfg]
|
||||
(fn [_] (process-touched! cfg)))
|
||||
(fn [_]
|
||||
(process-touched! (assoc cfg ::timestamp (ct/now)))))
|
||||
|
||||
|
||||
@@ -79,14 +79,17 @@
|
||||
;; API
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
(defn tempfile
|
||||
[& {:keys [suffix prefix min-age]
|
||||
(defn tempfile*
|
||||
[& {:keys [suffix prefix]
|
||||
:or {prefix "penpot."
|
||||
suffix ".tmp"}}]
|
||||
(let [attrs (fs/make-permissions "rw-r--r--")
|
||||
path (fs/join default-tmp-dir (str prefix (uuid/next) suffix))
|
||||
path (Files/createFile path attrs)]
|
||||
(fs/delete-on-exit! path)
|
||||
path (fs/join default-tmp-dir (str prefix (uuid/next) suffix))]
|
||||
(Files/createFile path attrs)))
|
||||
|
||||
(defn tempfile
|
||||
[& {:keys [min-age] :as opts}]
|
||||
(let [path (tempfile* opts)]
|
||||
(sp/offer! queue [path (some-> min-age ct/duration)])
|
||||
path))
|
||||
|
||||
|
||||
@@ -18,15 +18,15 @@
|
||||
(def ^:private sql:get-profiles
|
||||
"SELECT id, photo_id FROM profile
|
||||
WHERE deleted_at IS NOT NULL
|
||||
AND deleted_at < now() + ?::interval
|
||||
AND deleted_at <= ?
|
||||
ORDER BY deleted_at ASC
|
||||
LIMIT ?
|
||||
FOR UPDATE
|
||||
SKIP LOCKED")
|
||||
|
||||
(defn- delete-profiles!
|
||||
[{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
|
||||
(->> (db/plan conn [sql:get-profiles deletion-threshold chunk-size] {:fetch-size 5})
|
||||
[{:keys [::db/conn ::timestamp ::chunk-size ::sto/storage] :as cfg}]
|
||||
(->> (db/plan conn [sql:get-profiles timestamp chunk-size] {:fetch-size 5})
|
||||
(reduce (fn [total {:keys [id photo-id]}]
|
||||
(l/trc :obj "profile" :id (str id))
|
||||
|
||||
@@ -41,15 +41,15 @@
|
||||
(def ^:private sql:get-teams
|
||||
"SELECT deleted_at, id, photo_id FROM team
|
||||
WHERE deleted_at IS NOT NULL
|
||||
AND deleted_at < now() + ?::interval
|
||||
AND deleted_at <= ?
|
||||
ORDER BY deleted_at ASC
|
||||
LIMIT ?
|
||||
FOR UPDATE
|
||||
SKIP LOCKED")
|
||||
|
||||
(defn- delete-teams!
|
||||
[{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
|
||||
(->> (db/plan conn [sql:get-teams deletion-threshold chunk-size] {:fetch-size 5})
|
||||
[{:keys [::db/conn ::timestamp ::chunk-size ::sto/storage] :as cfg}]
|
||||
(->> (db/plan conn [sql:get-teams timestamp chunk-size] {:fetch-size 5})
|
||||
(reduce (fn [total {:keys [id photo-id deleted-at]}]
|
||||
(l/trc :obj "team"
|
||||
:id (str id)
|
||||
@@ -68,15 +68,15 @@
|
||||
"SELECT id, team_id, deleted_at, woff1_file_id, woff2_file_id, otf_file_id, ttf_file_id
|
||||
FROM team_font_variant
|
||||
WHERE deleted_at IS NOT NULL
|
||||
AND deleted_at < now() + ?::interval
|
||||
AND deleted_at <= ?
|
||||
ORDER BY deleted_at ASC
|
||||
LIMIT ?
|
||||
FOR UPDATE
|
||||
SKIP LOCKED")
|
||||
|
||||
(defn- delete-fonts!
|
||||
[{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
|
||||
(->> (db/plan conn [sql:get-fonts deletion-threshold chunk-size] {:fetch-size 5})
|
||||
[{:keys [::db/conn ::timestamp ::chunk-size ::sto/storage] :as cfg}]
|
||||
(->> (db/plan conn [sql:get-fonts timestamp chunk-size] {:fetch-size 5})
|
||||
(reduce (fn [total {:keys [id team-id deleted-at] :as font}]
|
||||
(l/trc :obj "font-variant"
|
||||
:id (str id)
|
||||
@@ -98,15 +98,15 @@
|
||||
"SELECT id, deleted_at, team_id
|
||||
FROM project
|
||||
WHERE deleted_at IS NOT NULL
|
||||
AND deleted_at < now() + ?::interval
|
||||
AND deleted_at <= ?
|
||||
ORDER BY deleted_at ASC
|
||||
LIMIT ?
|
||||
FOR UPDATE
|
||||
SKIP LOCKED")
|
||||
|
||||
(defn- delete-projects!
|
||||
[{:keys [::db/conn ::deletion-threshold ::chunk-size] :as cfg}]
|
||||
(->> (db/plan conn [sql:get-projects deletion-threshold chunk-size] {:fetch-size 5})
|
||||
[{:keys [::db/conn ::timestamp ::chunk-size] :as cfg}]
|
||||
(->> (db/plan conn [sql:get-projects timestamp chunk-size] {:fetch-size 5})
|
||||
(reduce (fn [total {:keys [id team-id deleted-at]}]
|
||||
(l/trc :obj "project"
|
||||
:id (str id)
|
||||
@@ -124,15 +124,15 @@
|
||||
f.project_id
|
||||
FROM file AS f
|
||||
WHERE f.deleted_at IS NOT NULL
|
||||
AND f.deleted_at < now() + ?::interval
|
||||
AND f.deleted_at <= ?
|
||||
ORDER BY f.deleted_at ASC
|
||||
LIMIT ?
|
||||
FOR UPDATE
|
||||
SKIP LOCKED")
|
||||
|
||||
(defn- delete-files!
|
||||
[{:keys [::db/conn ::deletion-threshold ::chunk-size] :as cfg}]
|
||||
(->> (db/plan conn [sql:get-files deletion-threshold chunk-size] {:fetch-size 5})
|
||||
[{:keys [::db/conn ::timestamp ::chunk-size] :as cfg}]
|
||||
(->> (db/plan conn [sql:get-files timestamp chunk-size] {:fetch-size 5})
|
||||
(reduce (fn [total {:keys [id deleted-at project-id] :as file}]
|
||||
(l/trc :obj "file"
|
||||
:id (str id)
|
||||
@@ -148,15 +148,15 @@
|
||||
"SELECT file_id, revn, media_id, deleted_at
|
||||
FROM file_thumbnail
|
||||
WHERE deleted_at IS NOT NULL
|
||||
AND deleted_at < now() + ?::interval
|
||||
AND deleted_at <= ?
|
||||
ORDER BY deleted_at ASC
|
||||
LIMIT ?
|
||||
FOR UPDATE
|
||||
SKIP LOCKED")
|
||||
|
||||
(defn delete-file-thumbnails!
|
||||
[{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
|
||||
(->> (db/plan conn [sql:get-file-thumbnails deletion-threshold chunk-size] {:fetch-size 5})
|
||||
[{:keys [::db/conn ::timestamp ::chunk-size ::sto/storage] :as cfg}]
|
||||
(->> (db/plan conn [sql:get-file-thumbnails timestamp chunk-size] {:fetch-size 5})
|
||||
(reduce (fn [total {:keys [file-id revn media-id deleted-at]}]
|
||||
(l/trc :obj "file-thumbnail"
|
||||
:file-id (str file-id)
|
||||
@@ -175,15 +175,15 @@
|
||||
"SELECT file_id, object_id, media_id, deleted_at
|
||||
FROM file_tagged_object_thumbnail
|
||||
WHERE deleted_at IS NOT NULL
|
||||
AND deleted_at < now() + ?::interval
|
||||
AND deleted_at <= ?
|
||||
ORDER BY deleted_at ASC
|
||||
LIMIT ?
|
||||
FOR UPDATE
|
||||
SKIP LOCKED")
|
||||
|
||||
(defn delete-file-object-thumbnails!
|
||||
[{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
|
||||
(->> (db/plan conn [sql:get-file-object-thumbnails deletion-threshold chunk-size] {:fetch-size 5})
|
||||
[{:keys [::db/conn ::timestamp ::chunk-size ::sto/storage] :as cfg}]
|
||||
(->> (db/plan conn [sql:get-file-object-thumbnails timestamp chunk-size] {:fetch-size 5})
|
||||
(reduce (fn [total {:keys [file-id object-id media-id deleted-at]}]
|
||||
(l/trc :obj "file-object-thumbnail"
|
||||
:file-id (str file-id)
|
||||
@@ -203,15 +203,15 @@
|
||||
"SELECT id, file_id, media_id, thumbnail_id, deleted_at
|
||||
FROM file_media_object
|
||||
WHERE deleted_at IS NOT NULL
|
||||
AND deleted_at < now() + ?::interval
|
||||
AND deleted_at <= ?
|
||||
ORDER BY deleted_at ASC
|
||||
LIMIT ?
|
||||
FOR UPDATE
|
||||
SKIP LOCKED")
|
||||
|
||||
(defn- delete-file-media-objects!
|
||||
[{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
|
||||
(->> (db/plan conn [sql:get-file-media-objects deletion-threshold chunk-size] {:fetch-size 5})
|
||||
[{:keys [::db/conn ::timestamp ::chunk-size ::sto/storage] :as cfg}]
|
||||
(->> (db/plan conn [sql:get-file-media-objects timestamp chunk-size] {:fetch-size 5})
|
||||
(reduce (fn [total {:keys [id file-id deleted-at] :as fmo}]
|
||||
(l/trc :obj "file-media-object"
|
||||
:id (str id)
|
||||
@@ -231,16 +231,15 @@
|
||||
"SELECT file_id, id, type, deleted_at, metadata, backend
|
||||
FROM file_data
|
||||
WHERE deleted_at IS NOT NULL
|
||||
AND deleted_at < now() + ?::interval
|
||||
AND deleted_at <= ?
|
||||
ORDER BY deleted_at ASC
|
||||
LIMIT ?
|
||||
FOR UPDATE
|
||||
SKIP LOCKED")
|
||||
|
||||
(defn- delete-file-data!
|
||||
[{:keys [::db/conn ::deletion-threshold ::chunk-size] :as cfg}]
|
||||
|
||||
(->> (db/plan conn [sql:get-file-data deletion-threshold chunk-size] {:fetch-size 5})
|
||||
[{:keys [::db/conn ::timestamp ::chunk-size] :as cfg}]
|
||||
(->> (db/plan conn [sql:get-file-data timestamp chunk-size] {:fetch-size 5})
|
||||
(reduce (fn [total {:keys [file-id id type deleted-at metadata backend]}]
|
||||
|
||||
(some->> metadata
|
||||
@@ -266,15 +265,15 @@
|
||||
"SELECT id, file_id, deleted_at
|
||||
FROM file_change
|
||||
WHERE deleted_at IS NOT NULL
|
||||
AND deleted_at < now() + ?::interval
|
||||
AND deleted_at <= ?
|
||||
ORDER BY deleted_at ASC
|
||||
LIMIT ?
|
||||
FOR UPDATE
|
||||
SKIP LOCKED")
|
||||
|
||||
(defn- delete-file-changes!
|
||||
[{:keys [::db/conn ::deletion-threshold ::chunk-size] :as cfg}]
|
||||
(->> (db/plan conn [sql:get-file-change deletion-threshold chunk-size] {:fetch-size 5})
|
||||
[{:keys [::db/conn ::timestamp ::chunk-size] :as cfg}]
|
||||
(->> (db/plan conn [sql:get-file-change timestamp chunk-size] {:fetch-size 5})
|
||||
(reduce (fn [total {:keys [id file-id deleted-at] :as xlog}]
|
||||
(l/trc :obj "file-change"
|
||||
:id (str id)
|
||||
@@ -322,9 +321,8 @@
|
||||
|
||||
(defmethod ig/init-key ::handler
|
||||
[_ cfg]
|
||||
(fn [{:keys [props] :as task}]
|
||||
(let [threshold (ct/duration (get props :deletion-threshold 0))
|
||||
cfg (assoc cfg ::deletion-threshold (db/interval threshold))]
|
||||
(fn [_]
|
||||
(let [cfg (assoc cfg ::timestamp (ct/now))]
|
||||
(loop [procs (map deref deletion-proc-vars)
|
||||
total 0]
|
||||
(if-let [proc-fn (first procs)]
|
||||
|
||||
@@ -27,7 +27,7 @@
|
||||
(throw (IllegalArgumentException. "Missing arguments on `defmethod` macro.")))
|
||||
|
||||
(let [mdata (assoc mdata
|
||||
::docstring (some-> docs str/<<-)
|
||||
::docstring (some-> docs str/unindent)
|
||||
::spec sname
|
||||
::name (name sname))
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
[app.common.features :as cfeat]
|
||||
[app.common.pprint :as pp]
|
||||
[app.common.thumbnails :as thc]
|
||||
[app.common.time :as ct]
|
||||
[app.common.types.shape :as cts]
|
||||
[app.common.uuid :as uuid]
|
||||
[app.config :as cf]
|
||||
@@ -16,6 +17,7 @@
|
||||
[app.db.sql :as sql]
|
||||
[app.http :as http]
|
||||
[app.rpc :as-alias rpc]
|
||||
[app.setup.clock :as clock]
|
||||
[app.storage :as sto]
|
||||
[backend-tests.helpers :as th]
|
||||
[clojure.test :as t]
|
||||
@@ -132,9 +134,10 @@
|
||||
;; this will run pending task triggered by deleting user snapshot
|
||||
(th/run-pending-tasks!)
|
||||
|
||||
(let [res (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
|
||||
;; delete 2 snapshots and 2 file data entries
|
||||
(t/is (= 4 (:processed res))))))))
|
||||
(binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
|
||||
(let [res (th/run-task! :objects-gc {})]
|
||||
;; delete 2 snapshots and 2 file data entries
|
||||
(t/is (= 4 (:processed res)))))))))
|
||||
|
||||
(t/deftest snapshots-locking
|
||||
(let [profile-1 (th/create-profile* 1 {:is-active true})
|
||||
|
||||
@@ -313,7 +313,7 @@
|
||||
;; freeze because of the deduplication (we have uploaded 2 times
|
||||
;; the same files).
|
||||
|
||||
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
|
||||
(let [res (th/run-task! :storage-gc-touched {})]
|
||||
(t/is (= 2 (:freeze res)))
|
||||
(t/is (= 0 (:delete res))))
|
||||
|
||||
@@ -372,14 +372,14 @@
|
||||
(th/db-exec! ["update file_change set deleted_at = now() where file_id = ? and label is not null" (:id file)])
|
||||
(th/db-exec! ["update file set has_media_trimmed = false where id = ?" (:id file)])
|
||||
|
||||
(let [res (th/run-task! :objects-gc {:deletion-threshold 0})]
|
||||
(let [res (th/run-task! :objects-gc {})]
|
||||
;; this will remove the file change and file data entries for two snapshots
|
||||
(t/is (= 4 (:processed res))))
|
||||
|
||||
;; Rerun the file-gc and objects-gc
|
||||
(t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
|
||||
|
||||
(let [res (th/run-task! :objects-gc {:deletion-threshold 0})]
|
||||
(let [res (th/run-task! :objects-gc {})]
|
||||
;; this will remove the file media objects marked as deleted
|
||||
;; on prev file-gc
|
||||
(t/is (= 2 (:processed res))))
|
||||
@@ -387,7 +387,7 @@
|
||||
;; Now that file-gc have deleted the file-media-object usage,
|
||||
;; lets execute the touched-gc task, we should see that two of
|
||||
;; them are marked to be deleted
|
||||
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
|
||||
(let [res (th/run-task! :storage-gc-touched {})]
|
||||
(t/is (= 0 (:freeze res)))
|
||||
(t/is (= 2 (:delete res))))
|
||||
|
||||
@@ -572,7 +572,7 @@
|
||||
;; Now that file-gc have deleted the file-media-object usage,
|
||||
;; lets execute the touched-gc task, we should see that two of
|
||||
;; them are marked to be deleted.
|
||||
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
|
||||
(let [res (th/run-task! :storage-gc-touched {})]
|
||||
(t/is (= 0 (:freeze res)))
|
||||
(t/is (= 2 (:delete res))))
|
||||
|
||||
@@ -665,7 +665,7 @@
|
||||
;; because of the deduplication (we have uploaded 2 times the
|
||||
;; same files).
|
||||
|
||||
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
|
||||
(let [res (th/run-task! :storage-gc-touched {})]
|
||||
(t/is (= 1 (:freeze res)))
|
||||
(t/is (= 0 (:delete res))))
|
||||
|
||||
@@ -715,7 +715,7 @@
|
||||
|
||||
;; Now that objects-gc have deleted the object thumbnail lets
|
||||
;; execute the touched-gc task
|
||||
(let [res (th/run-task! "storage-gc-touched" {:min-age 0})]
|
||||
(let [res (th/run-task! "storage-gc-touched" {})]
|
||||
(t/is (= 1 (:freeze res))))
|
||||
|
||||
;; check file media objects
|
||||
@@ -750,7 +750,7 @@
|
||||
|
||||
;; Now that file-gc have deleted the object thumbnail lets
|
||||
;; execute the touched-gc task
|
||||
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
|
||||
(let [res (th/run-task! :storage-gc-touched {})]
|
||||
(t/is (= 1 (:delete res))))
|
||||
|
||||
;; check file media objects
|
||||
@@ -922,8 +922,9 @@
|
||||
(t/is (= 0 (:processed result))))
|
||||
|
||||
;; run permanent deletion
|
||||
(let [result (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
|
||||
(t/is (= 3 (:processed result))))
|
||||
(binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
|
||||
(let [result (th/run-task! :objects-gc {})]
|
||||
(t/is (= 3 (:processed result)))))
|
||||
|
||||
;; query the list of file libraries of a after hard deletion
|
||||
(let [data {::th/type :get-file-libraries
|
||||
@@ -1134,7 +1135,7 @@
|
||||
(th/sleep 300)
|
||||
|
||||
;; run the task
|
||||
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
|
||||
(t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
|
||||
|
||||
;; check that object thumbnails are still here
|
||||
(let [rows (th/db-query :file-tagged-object-thumbnail {:file-id (:id file)})]
|
||||
@@ -1163,7 +1164,7 @@
|
||||
(t/is (= 2 (count rows))))
|
||||
|
||||
;; run the task again
|
||||
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
|
||||
(t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
|
||||
|
||||
;; check that we have all object thumbnails
|
||||
(let [rows (th/db-query :file-tagged-object-thumbnail {:file-id (:id file)})]
|
||||
@@ -1226,7 +1227,7 @@
|
||||
(t/is (= 2 (count rows)))))
|
||||
|
||||
(t/testing "gc task"
|
||||
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
|
||||
(t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
|
||||
|
||||
(let [rows (th/db-query :file-thumbnail {:file-id (:id file)})]
|
||||
(t/is (= 2 (count rows)))
|
||||
@@ -1273,7 +1274,7 @@
|
||||
;; The FileGC task will schedule an inner taskq
|
||||
(th/run-pending-tasks!)
|
||||
|
||||
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
|
||||
(let [res (th/run-task! :storage-gc-touched {})]
|
||||
(t/is (= 2 (:freeze res)))
|
||||
(t/is (= 0 (:delete res))))
|
||||
|
||||
@@ -1367,7 +1368,7 @@
|
||||
|
||||
;; we ensure that once object-gc is passed and marked two storage
|
||||
;; objects to delete
|
||||
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
|
||||
(let [res (th/run-task! :storage-gc-touched {})]
|
||||
(t/is (= 0 (:freeze res)))
|
||||
(t/is (= 2 (:delete res))))
|
||||
|
||||
@@ -1489,7 +1490,7 @@
|
||||
(t/is (some? (not-empty (:objects component))))))
|
||||
|
||||
;; Re-run the file-gc task
|
||||
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
|
||||
(t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
|
||||
(let [row (th/db-get :file {:id (:id file)})]
|
||||
(t/is (true? (:has-media-trimmed row))))
|
||||
|
||||
@@ -1519,7 +1520,7 @@
|
||||
|
||||
;; Now, we have deleted the usage of component if we pass file-gc,
|
||||
;; that component should be deleted
|
||||
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
|
||||
(t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
|
||||
|
||||
;; Check that component is properly removed
|
||||
(let [data {::th/type :get-file
|
||||
@@ -1610,8 +1611,8 @@
|
||||
:component-id c-id})}])
|
||||
|
||||
;; Run the file-gc on file and library
|
||||
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file-1)})))
|
||||
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file-2)})))
|
||||
(t/is (true? (th/run-task! :file-gc {:file-id (:id file-1)})))
|
||||
(t/is (true? (th/run-task! :file-gc {:file-id (:id file-2)})))
|
||||
|
||||
;; Check that component exists
|
||||
(let [data {::th/type :get-file
|
||||
@@ -1684,7 +1685,7 @@
|
||||
|
||||
;; Now, we have deleted the usage of component if we pass file-gc,
|
||||
;; that component should be deleted
|
||||
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file-1)})))
|
||||
(t/is (true? (th/run-task! :file-gc {:file-id (:id file-1)})))
|
||||
|
||||
;; Check that component is properly removed
|
||||
(let [data {::th/type :get-file
|
||||
@@ -1833,8 +1834,8 @@
|
||||
(t/is (not= (:id fill) (:id fmedia)))))
|
||||
|
||||
;; Run the file-gc on file and library
|
||||
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file-1)})))
|
||||
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file-2)})))
|
||||
(t/is (true? (th/run-task! :file-gc {:file-id (:id file-1)})))
|
||||
(t/is (true? (th/run-task! :file-gc {:file-id (:id file-2)})))
|
||||
|
||||
;; Now proceed to delete file and absorb it
|
||||
(let [data {::th/type :delete-file
|
||||
|
||||
@@ -8,12 +8,14 @@
|
||||
(:require
|
||||
[app.common.pprint :as pp]
|
||||
[app.common.thumbnails :as thc]
|
||||
[app.common.time :as ct]
|
||||
[app.common.types.shape :as cts]
|
||||
[app.common.uuid :as uuid]
|
||||
[app.config :as cf]
|
||||
[app.db :as db]
|
||||
[app.rpc :as-alias rpc]
|
||||
[app.rpc.commands.auth :as cauth]
|
||||
[app.setup.clock :as clock]
|
||||
[app.storage :as sto]
|
||||
[app.tokens :as tokens]
|
||||
[backend-tests.helpers :as th]
|
||||
@@ -83,7 +85,8 @@
|
||||
(t/is (map? (:result out))))
|
||||
|
||||
;; run the task again
|
||||
(let [res (th/run-task! "storage-gc-touched" {:min-age 0})]
|
||||
(let [res (binding [ct/*clock* (clock/fixed (ct/in-future {:minutes 31}))]
|
||||
(th/run-task! "storage-gc-touched" {}))]
|
||||
(t/is (= 2 (:freeze res))))
|
||||
|
||||
(let [[row1 row2 :as rows] (th/db-query :file-tagged-object-thumbnail
|
||||
@@ -114,9 +117,9 @@
|
||||
|
||||
;; Run the File GC task that should remove unused file object
|
||||
;; thumbnails
|
||||
(th/run-task! :file-gc {:min-age 0 :file-id (:id file)})
|
||||
(th/run-task! :file-gc {:file-id (:id file)})
|
||||
|
||||
(let [result (th/run-task! :objects-gc {:min-age 0})]
|
||||
(let [result (th/run-task! :objects-gc {})]
|
||||
(t/is (= 3 (:processed result))))
|
||||
|
||||
;; check if row2 related thumbnail row still exists
|
||||
@@ -133,7 +136,8 @@
|
||||
(t/is (some? (sto/get-object storage (:media-id row2))))
|
||||
|
||||
;; run the task again
|
||||
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
|
||||
(let [res (binding [ct/*clock* (clock/fixed (ct/in-future {:minutes 31}))]
|
||||
(th/run-task! :storage-gc-touched {}))]
|
||||
(t/is (= 1 (:delete res)))
|
||||
(t/is (= 0 (:freeze res))))
|
||||
|
||||
@@ -143,8 +147,9 @@
|
||||
|
||||
;; Run the storage gc deleted task, it should permanently delete
|
||||
;; all storage objects related to the deleted thumbnails
|
||||
(let [result (th/run-task! :storage-gc-deleted {:min-age 0})]
|
||||
(t/is (= 1 (:deleted result))))
|
||||
(binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
|
||||
(let [res (th/run-task! :storage-gc-deleted {})]
|
||||
(t/is (= 1 (:deleted res)))))
|
||||
|
||||
(t/is (nil? (sto/get-object storage (:media-id row1))))
|
||||
(t/is (some? (sto/get-object storage (:media-id row2))))
|
||||
@@ -216,9 +221,9 @@
|
||||
|
||||
;; Run the File GC task that should remove unused file object
|
||||
;; thumbnails
|
||||
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
|
||||
(t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
|
||||
|
||||
(let [result (th/run-task! :objects-gc {:min-age 0})]
|
||||
(let [result (th/run-task! :objects-gc {})]
|
||||
(t/is (= 2 (:processed result))))
|
||||
|
||||
;; check if row1 related thumbnail row still exists
|
||||
@@ -230,7 +235,7 @@
|
||||
(t/is (= (:object-id data1) (:object-id row)))
|
||||
(t/is (uuid? (:media-id row1))))
|
||||
|
||||
(let [result (th/run-task! :storage-gc-touched {:min-age 0})]
|
||||
(let [result (th/run-task! :storage-gc-touched {})]
|
||||
(t/is (= 1 (:delete result))))
|
||||
|
||||
;; Check if storage objects still exists after file-gc
|
||||
@@ -242,8 +247,9 @@
|
||||
|
||||
;; Run the storage gc deleted task, it should permanently delete
|
||||
;; all storage objects related to the deleted thumbnails
|
||||
(let [result (th/run-task! :storage-gc-deleted {:min-age 0})]
|
||||
(t/is (= 1 (:deleted result))))
|
||||
(binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
|
||||
(let [result (th/run-task! :storage-gc-deleted {})]
|
||||
(t/is (= 1 (:deleted result)))))
|
||||
|
||||
(t/is (some? (sto/get-object storage (:media-id row2)))))))
|
||||
|
||||
|
||||
@@ -6,11 +6,13 @@
|
||||
|
||||
(ns backend-tests.rpc-font-test
|
||||
(:require
|
||||
[app.common.time :as ct]
|
||||
[app.common.uuid :as uuid]
|
||||
[app.config :as cf]
|
||||
[app.db :as db]
|
||||
[app.http :as http]
|
||||
[app.rpc :as-alias rpc]
|
||||
[app.setup.clock :as clock]
|
||||
[app.storage :as sto]
|
||||
[backend-tests.helpers :as th]
|
||||
[clojure.test :as t]
|
||||
@@ -129,7 +131,7 @@
|
||||
;; (th/print-result! out)
|
||||
(t/is (nil? (:error out))))
|
||||
|
||||
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
|
||||
(let [res (th/run-task! :storage-gc-touched {})]
|
||||
(t/is (= 6 (:freeze res))))
|
||||
|
||||
(let [params {::th/type :delete-font
|
||||
@@ -141,16 +143,17 @@
|
||||
(t/is (nil? (:error out)))
|
||||
(t/is (nil? (:result out))))
|
||||
|
||||
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
|
||||
(let [res (th/run-task! :storage-gc-touched {})]
|
||||
(t/is (= 0 (:freeze res)))
|
||||
(t/is (= 0 (:delete res))))
|
||||
|
||||
(let [res (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
|
||||
(t/is (= 2 (:processed res))))
|
||||
(binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
|
||||
(let [res (th/run-task! :objects-gc {})]
|
||||
(t/is (= 2 (:processed res))))
|
||||
|
||||
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
|
||||
(t/is (= 0 (:freeze res)))
|
||||
(t/is (= 6 (:delete res))))))
|
||||
(let [res (th/run-task! :storage-gc-touched {})]
|
||||
(t/is (= 0 (:freeze res)))
|
||||
(t/is (= 6 (:delete res)))))))
|
||||
|
||||
(t/deftest font-deletion-2
|
||||
(let [prof (th/create-profile* 1 {:is-active true})
|
||||
@@ -189,7 +192,7 @@
|
||||
;; (th/print-result! out)
|
||||
(t/is (nil? (:error out))))
|
||||
|
||||
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
|
||||
(let [res (th/run-task! :storage-gc-touched {})]
|
||||
(t/is (= 6 (:freeze res))))
|
||||
|
||||
(let [params {::th/type :delete-font
|
||||
@@ -201,16 +204,17 @@
|
||||
(t/is (nil? (:error out)))
|
||||
(t/is (nil? (:result out))))
|
||||
|
||||
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
|
||||
(let [res (th/run-task! :storage-gc-touched {})]
|
||||
(t/is (= 0 (:freeze res)))
|
||||
(t/is (= 0 (:delete res))))
|
||||
|
||||
(let [res (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
|
||||
(t/is (= 1 (:processed res))))
|
||||
(binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
|
||||
(let [res (th/run-task! :objects-gc {})]
|
||||
(t/is (= 1 (:processed res))))
|
||||
|
||||
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
|
||||
(t/is (= 0 (:freeze res)))
|
||||
(t/is (= 3 (:delete res))))))
|
||||
(let [res (th/run-task! :storage-gc-touched {})]
|
||||
(t/is (= 0 (:freeze res)))
|
||||
(t/is (= 3 (:delete res)))))))
|
||||
|
||||
(t/deftest font-deletion-3
|
||||
(let [prof (th/create-profile* 1 {:is-active true})
|
||||
@@ -248,7 +252,7 @@
|
||||
(t/is (nil? (:error out1)))
|
||||
(t/is (nil? (:error out2)))
|
||||
|
||||
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
|
||||
(let [res (th/run-task! :storage-gc-touched {})]
|
||||
(t/is (= 6 (:freeze res))))
|
||||
|
||||
(let [params {::th/type :delete-font-variant
|
||||
@@ -260,13 +264,14 @@
|
||||
(t/is (nil? (:error out)))
|
||||
(t/is (nil? (:result out))))
|
||||
|
||||
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
|
||||
(let [res (th/run-task! :storage-gc-touched {})]
|
||||
(t/is (= 0 (:freeze res)))
|
||||
(t/is (= 0 (:delete res))))
|
||||
|
||||
(let [res (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
|
||||
(t/is (= 1 (:processed res))))
|
||||
(binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
|
||||
(let [res (th/run-task! :objects-gc {})]
|
||||
(t/is (= 1 (:processed res))))
|
||||
|
||||
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
|
||||
(t/is (= 0 (:freeze res)))
|
||||
(t/is (= 3 (:delete res))))))
|
||||
(let [res (th/run-task! :storage-gc-touched {})]
|
||||
(t/is (= 0 (:freeze res)))
|
||||
(t/is (= 3 (:delete res)))))))
|
||||
|
||||
@@ -6,11 +6,13 @@
|
||||
|
||||
(ns backend-tests.rpc-project-test
|
||||
(:require
|
||||
[app.common.time :as ct]
|
||||
[app.common.uuid :as uuid]
|
||||
[app.config :as cf]
|
||||
[app.db :as db]
|
||||
[app.http :as http]
|
||||
[app.rpc :as-alias rpc]
|
||||
[app.setup.clock :as clock]
|
||||
[backend-tests.helpers :as th]
|
||||
[clojure.test :as t]))
|
||||
|
||||
@@ -226,8 +228,9 @@
|
||||
(t/is (= 0 (count result)))))
|
||||
|
||||
;; run permanent deletion
|
||||
(let [result (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
|
||||
(t/is (= 1 (:processed result))))
|
||||
(binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
|
||||
(let [result (th/run-task! :objects-gc {})]
|
||||
(t/is (= 1 (:processed result)))))
|
||||
|
||||
;; query the list of files of a after hard deletion
|
||||
(let [data {::th/type :get-project-files
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
[app.db :as db]
|
||||
[app.http :as http]
|
||||
[app.rpc :as-alias rpc]
|
||||
[app.setup.clock :as clock]
|
||||
[app.storage :as sto]
|
||||
[app.tokens :as tokens]
|
||||
[backend-tests.helpers :as th]
|
||||
@@ -525,8 +526,9 @@
|
||||
(t/is (= :not-found (:type edata)))))
|
||||
|
||||
;; run permanent deletion
|
||||
(let [result (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
|
||||
(t/is (= 2 (:processed result))))
|
||||
(binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
|
||||
(let [result (th/run-task! :objects-gc {})]
|
||||
(t/is (= 2 (:processed result)))))
|
||||
|
||||
;; query the list of projects of a after hard deletion
|
||||
(let [data {::th/type :get-projects
|
||||
@@ -581,8 +583,9 @@
|
||||
(t/is (= 1 (count rows)))
|
||||
(t/is (ct/inst? (:deleted-at (first rows)))))
|
||||
|
||||
(let [result (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
|
||||
(t/is (= 7 (:processed result))))))
|
||||
(binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
|
||||
(let [result (th/run-task! :objects-gc {})]
|
||||
(t/is (= 7 (:processed result)))))))
|
||||
|
||||
(t/deftest create-team-access-request
|
||||
(with-mocks [mock {:target 'app.email/send! :return nil}]
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
[app.common.uuid :as uuid]
|
||||
[app.db :as db]
|
||||
[app.rpc :as-alias rpc]
|
||||
[app.setup.clock :as clock]
|
||||
[app.storage :as sto]
|
||||
[backend-tests.helpers :as th]
|
||||
[clojure.test :as t]
|
||||
@@ -53,19 +54,13 @@
|
||||
(configure-storage-backend))
|
||||
content (sto/content "content")
|
||||
object (sto/put-object! storage {::sto/content content
|
||||
::sto/expired-at (ct/in-future {:seconds 1})
|
||||
::sto/expired-at (ct/in-future {:hours 1})
|
||||
:content-type "text/plain"})]
|
||||
|
||||
(t/is (sto/object? object))
|
||||
(t/is (ct/inst? (:expired-at object)))
|
||||
(t/is (ct/is-after? (:expired-at object) (ct/now)))
|
||||
(t/is (= object (sto/get-object storage (:id object))))
|
||||
|
||||
(th/sleep 1000)
|
||||
(t/is (nil? (sto/get-object storage (:id object))))
|
||||
(t/is (nil? (sto/get-object-data storage object)))
|
||||
(t/is (nil? (sto/get-object-url storage object)))
|
||||
(t/is (nil? (sto/get-object-path storage object)))))
|
||||
(t/is (nil? (sto/get-object storage (:id object))))))
|
||||
|
||||
(t/deftest put-and-delete-object
|
||||
(let [storage (-> (:app.storage/storage th/*system*)
|
||||
@@ -98,20 +93,25 @@
|
||||
::sto/expired-at (ct/now)
|
||||
:content-type "text/plain"})
|
||||
object2 (sto/put-object! storage {::sto/content content2
|
||||
::sto/expired-at (ct/in-past {:hours 2})
|
||||
::sto/expired-at (ct/in-future {:hours 2})
|
||||
:content-type "text/plain"})
|
||||
object3 (sto/put-object! storage {::sto/content content3
|
||||
::sto/expired-at (ct/in-past {:hours 1})
|
||||
::sto/expired-at (ct/in-future {:hours 1})
|
||||
:content-type "text/plain"})]
|
||||
|
||||
|
||||
(th/sleep 200)
|
||||
|
||||
(let [res (th/run-task! :storage-gc-deleted {})]
|
||||
(t/is (= 1 (:deleted res))))
|
||||
(binding [ct/*clock* (clock/fixed (ct/in-future {:minutes 0}))]
|
||||
(let [res (th/run-task! :storage-gc-deleted {})]
|
||||
(t/is (= 1 (:deleted res)))))
|
||||
|
||||
(let [res (th/db-exec-one! ["select count(*) from storage_object;"])]
|
||||
(t/is (= 2 (:count res))))))
|
||||
(t/is (= 2 (:count res))))
|
||||
|
||||
(binding [ct/*clock* (clock/fixed (ct/in-future {:minutes 61}))]
|
||||
(let [res (th/run-task! :storage-gc-deleted {})]
|
||||
(t/is (= 1 (:deleted res)))))
|
||||
|
||||
(let [res (th/db-exec-one! ["select count(*) from storage_object;"])]
|
||||
(t/is (= 1 (:count res))))))
|
||||
|
||||
(t/deftest touched-gc-task-1
|
||||
(let [storage (-> (:app.storage/storage th/*system*)
|
||||
@@ -158,7 +158,7 @@
|
||||
{:id (:id result-1)})
|
||||
|
||||
;; run the objects gc task for permanent deletion
|
||||
(let [res (th/run-task! :objects-gc {:min-age 0})]
|
||||
(let [res (th/run-task! :objects-gc {})]
|
||||
(t/is (= 1 (:processed res))))
|
||||
|
||||
;; check that we still have all the storage objects
|
||||
@@ -182,7 +182,6 @@
|
||||
(let [res (th/db-exec-one! ["select count(*) from storage_object where deleted_at is not null"])]
|
||||
(t/is (= 0 (:count res)))))))
|
||||
|
||||
|
||||
(t/deftest touched-gc-task-2
|
||||
(let [storage (-> (:app.storage/storage th/*system*)
|
||||
(configure-storage-backend))
|
||||
@@ -243,11 +242,12 @@
|
||||
{:id (:id result-2)})
|
||||
|
||||
;; run the objects gc task for permanent deletion
|
||||
(let [res (th/run-task! :objects-gc {:min-age 0})]
|
||||
(let [res (th/run-task! :objects-gc {})]
|
||||
(t/is (= 1 (:processed res))))
|
||||
|
||||
;; revert touched state to all storage objects
|
||||
(th/db-exec-one! ["update storage_object set touched_at=now()"])
|
||||
|
||||
(th/db-exec-one! ["update storage_object set touched_at=?" (ct/now)])
|
||||
|
||||
;; Run the task again
|
||||
(let [res (th/run-task! :storage-gc-touched {})]
|
||||
@@ -293,10 +293,10 @@
|
||||
result-2 (:result out2)]
|
||||
|
||||
;; now we proceed to manually mark all storage objects touched
|
||||
(th/db-exec! ["update storage_object set touched_at=now()"])
|
||||
(th/db-exec! ["update storage_object set touched_at=?" (ct/now)])
|
||||
|
||||
;; run the touched gc task
|
||||
(let [res (th/run-task! "storage-gc-touched" {:min-age 0})]
|
||||
(let [res (th/run-task! :storage-gc-touched {})]
|
||||
(t/is (= 2 (:freeze res)))
|
||||
(t/is (= 0 (:delete res))))
|
||||
|
||||
@@ -305,13 +305,13 @@
|
||||
(t/is (= 2 (count rows)))))
|
||||
|
||||
;; now we proceed to manually delete all file_media_object
|
||||
(th/db-exec! ["update file_media_object set deleted_at = now()"])
|
||||
(th/db-exec! ["update file_media_object set deleted_at = ?" (ct/now)])
|
||||
|
||||
(let [res (th/run-task! "objects-gc" {:min-age 0})]
|
||||
(let [res (th/run-task! :objects-gc {})]
|
||||
(t/is (= 2 (:processed res))))
|
||||
|
||||
;; run the touched gc task
|
||||
(let [res (th/run-task! "storage-gc-touched" {:min-age 0})]
|
||||
(let [res (th/run-task! :storage-gc-touched {})]
|
||||
(t/is (= 0 (:freeze res)))
|
||||
(t/is (= 2 (:delete res))))
|
||||
|
||||
|
||||
Reference in New Issue
Block a user