♻️ Make storage and other objects deletion task vclock aware

This simplifes the mental model on how it works and simplifies testing
of the related code.

This also normalizes storage object deletion in the same way as the
rest of objects in penpot (now future deletion date on storage object
also means storage object to be deleted).
This commit is contained in:
Andrey Antukh
2025-11-13 14:59:14 +01:00
parent 9532dea2c6
commit 94719eebf8
11 changed files with 189 additions and 172 deletions

View File

@@ -9,6 +9,7 @@
[app.common.features :as cfeat]
[app.common.pprint :as pp]
[app.common.thumbnails :as thc]
[app.common.time :as ct]
[app.common.types.shape :as cts]
[app.common.uuid :as uuid]
[app.config :as cf]
@@ -16,6 +17,7 @@
[app.db.sql :as sql]
[app.http :as http]
[app.rpc :as-alias rpc]
[app.setup.clock :as clock]
[app.storage :as sto]
[backend-tests.helpers :as th]
[clojure.test :as t]
@@ -132,9 +134,10 @@
;; this will run pending task triggered by deleting user snapshot
(th/run-pending-tasks!)
(let [res (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
;; delete 2 snapshots and 2 file data entries
(t/is (= 4 (:processed res))))))))
(binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
(let [res (th/run-task! :objects-gc {})]
;; delete 2 snapshots and 2 file data entries
(t/is (= 4 (:processed res)))))))))
(t/deftest snapshots-locking
(let [profile-1 (th/create-profile* 1 {:is-active true})

View File

@@ -313,7 +313,7 @@
;; freeze because of the deduplication (we have uploaded 2 times
;; the same files).
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 2 (:freeze res)))
(t/is (= 0 (:delete res))))
@@ -372,14 +372,14 @@
(th/db-exec! ["update file_change set deleted_at = now() where file_id = ? and label is not null" (:id file)])
(th/db-exec! ["update file set has_media_trimmed = false where id = ?" (:id file)])
(let [res (th/run-task! :objects-gc {:deletion-threshold 0})]
(let [res (th/run-task! :objects-gc {})]
;; this will remove the file change and file data entries for two snapshots
(t/is (= 4 (:processed res))))
;; Rerun the file-gc and objects-gc
(t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
(let [res (th/run-task! :objects-gc {:deletion-threshold 0})]
(let [res (th/run-task! :objects-gc {})]
;; this will remove the file media objects marked as deleted
;; on prev file-gc
(t/is (= 2 (:processed res))))
@@ -387,7 +387,7 @@
;; Now that file-gc have deleted the file-media-object usage,
;; lets execute the touched-gc task, we should see that two of
;; them are marked to be deleted
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 0 (:freeze res)))
(t/is (= 2 (:delete res))))
@@ -572,7 +572,7 @@
;; Now that file-gc have deleted the file-media-object usage,
;; lets execute the touched-gc task, we should see that two of
;; them are marked to be deleted.
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 0 (:freeze res)))
(t/is (= 2 (:delete res))))
@@ -665,7 +665,7 @@
;; because of the deduplication (we have uploaded 2 times the
;; same files).
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 1 (:freeze res)))
(t/is (= 0 (:delete res))))
@@ -715,7 +715,7 @@
;; Now that objects-gc have deleted the object thumbnail lets
;; execute the touched-gc task
(let [res (th/run-task! "storage-gc-touched" {:min-age 0})]
(let [res (th/run-task! "storage-gc-touched" {})]
(t/is (= 1 (:freeze res))))
;; check file media objects
@@ -750,7 +750,7 @@
;; Now that file-gc have deleted the object thumbnail lets
;; execute the touched-gc task
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 1 (:delete res))))
;; check file media objects
@@ -922,8 +922,9 @@
(t/is (= 0 (:processed result))))
;; run permanent deletion
(let [result (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
(t/is (= 3 (:processed result))))
(binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
(let [result (th/run-task! :objects-gc {})]
(t/is (= 3 (:processed result)))))
;; query the list of file libraries of a after hard deletion
(let [data {::th/type :get-file-libraries
@@ -1134,7 +1135,7 @@
(th/sleep 300)
;; run the task
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
(t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
;; check that object thumbnails are still here
(let [rows (th/db-query :file-tagged-object-thumbnail {:file-id (:id file)})]
@@ -1163,7 +1164,7 @@
(t/is (= 2 (count rows))))
;; run the task again
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
(t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
;; check that we have all object thumbnails
(let [rows (th/db-query :file-tagged-object-thumbnail {:file-id (:id file)})]
@@ -1226,7 +1227,7 @@
(t/is (= 2 (count rows)))))
(t/testing "gc task"
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
(t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
(let [rows (th/db-query :file-thumbnail {:file-id (:id file)})]
(t/is (= 2 (count rows)))
@@ -1273,7 +1274,7 @@
;; The FileGC task will schedule an inner taskq
(th/run-pending-tasks!)
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 2 (:freeze res)))
(t/is (= 0 (:delete res))))
@@ -1367,7 +1368,7 @@
;; we ensure that once object-gc is passed and marked two storage
;; objects to delete
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 0 (:freeze res)))
(t/is (= 2 (:delete res))))
@@ -1489,7 +1490,7 @@
(t/is (some? (not-empty (:objects component))))))
;; Re-run the file-gc task
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
(t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
(let [row (th/db-get :file {:id (:id file)})]
(t/is (true? (:has-media-trimmed row))))
@@ -1519,7 +1520,7 @@
;; Now, we have deleted the usage of component if we pass file-gc,
;; that component should be deleted
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
(t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
;; Check that component is properly removed
(let [data {::th/type :get-file
@@ -1610,8 +1611,8 @@
:component-id c-id})}])
;; Run the file-gc on file and library
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file-1)})))
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file-2)})))
(t/is (true? (th/run-task! :file-gc {:file-id (:id file-1)})))
(t/is (true? (th/run-task! :file-gc {:file-id (:id file-2)})))
;; Check that component exists
(let [data {::th/type :get-file
@@ -1684,7 +1685,7 @@
;; Now, we have deleted the usage of component if we pass file-gc,
;; that component should be deleted
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file-1)})))
(t/is (true? (th/run-task! :file-gc {:file-id (:id file-1)})))
;; Check that component is properly removed
(let [data {::th/type :get-file
@@ -1833,8 +1834,8 @@
(t/is (not= (:id fill) (:id fmedia)))))
;; Run the file-gc on file and library
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file-1)})))
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file-2)})))
(t/is (true? (th/run-task! :file-gc {:file-id (:id file-1)})))
(t/is (true? (th/run-task! :file-gc {:file-id (:id file-2)})))
;; Now proceed to delete file and absorb it
(let [data {::th/type :delete-file

View File

@@ -8,12 +8,14 @@
(:require
[app.common.pprint :as pp]
[app.common.thumbnails :as thc]
[app.common.time :as ct]
[app.common.types.shape :as cts]
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
[app.rpc :as-alias rpc]
[app.rpc.commands.auth :as cauth]
[app.setup.clock :as clock]
[app.storage :as sto]
[app.tokens :as tokens]
[backend-tests.helpers :as th]
@@ -83,7 +85,8 @@
(t/is (map? (:result out))))
;; run the task again
(let [res (th/run-task! "storage-gc-touched" {:min-age 0})]
(let [res (binding [ct/*clock* (clock/fixed (ct/in-future {:minutes 31}))]
(th/run-task! "storage-gc-touched" {}))]
(t/is (= 2 (:freeze res))))
(let [[row1 row2 :as rows] (th/db-query :file-tagged-object-thumbnail
@@ -114,9 +117,9 @@
;; Run the File GC task that should remove unused file object
;; thumbnails
(th/run-task! :file-gc {:min-age 0 :file-id (:id file)})
(th/run-task! :file-gc {:file-id (:id file)})
(let [result (th/run-task! :objects-gc {:min-age 0})]
(let [result (th/run-task! :objects-gc {})]
(t/is (= 3 (:processed result))))
;; check if row2 related thumbnail row still exists
@@ -133,7 +136,8 @@
(t/is (some? (sto/get-object storage (:media-id row2))))
;; run the task again
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(let [res (binding [ct/*clock* (clock/fixed (ct/in-future {:minutes 31}))]
(th/run-task! :storage-gc-touched {}))]
(t/is (= 1 (:delete res)))
(t/is (= 0 (:freeze res))))
@@ -143,8 +147,9 @@
;; Run the storage gc deleted task, it should permanently delete
;; all storage objects related to the deleted thumbnails
(let [result (th/run-task! :storage-gc-deleted {:min-age 0})]
(t/is (= 1 (:deleted result))))
(binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
(let [res (th/run-task! :storage-gc-deleted {})]
(t/is (= 1 (:deleted res)))))
(t/is (nil? (sto/get-object storage (:media-id row1))))
(t/is (some? (sto/get-object storage (:media-id row2))))
@@ -216,9 +221,9 @@
;; Run the File GC task that should remove unused file object
;; thumbnails
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
(t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
(let [result (th/run-task! :objects-gc {:min-age 0})]
(let [result (th/run-task! :objects-gc {})]
(t/is (= 2 (:processed result))))
;; check if row1 related thumbnail row still exists
@@ -230,7 +235,7 @@
(t/is (= (:object-id data1) (:object-id row)))
(t/is (uuid? (:media-id row1))))
(let [result (th/run-task! :storage-gc-touched {:min-age 0})]
(let [result (th/run-task! :storage-gc-touched {})]
(t/is (= 1 (:delete result))))
;; Check if storage objects still exists after file-gc
@@ -242,8 +247,9 @@
;; Run the storage gc deleted task, it should permanently delete
;; all storage objects related to the deleted thumbnails
(let [result (th/run-task! :storage-gc-deleted {:min-age 0})]
(t/is (= 1 (:deleted result))))
(binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
(let [result (th/run-task! :storage-gc-deleted {})]
(t/is (= 1 (:deleted result)))))
(t/is (some? (sto/get-object storage (:media-id row2)))))))

View File

@@ -6,11 +6,13 @@
(ns backend-tests.rpc-font-test
(:require
[app.common.time :as ct]
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
[app.http :as http]
[app.rpc :as-alias rpc]
[app.setup.clock :as clock]
[app.storage :as sto]
[backend-tests.helpers :as th]
[clojure.test :as t]
@@ -129,7 +131,7 @@
;; (th/print-result! out)
(t/is (nil? (:error out))))
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 6 (:freeze res))))
(let [params {::th/type :delete-font
@@ -141,16 +143,17 @@
(t/is (nil? (:error out)))
(t/is (nil? (:result out))))
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 0 (:freeze res)))
(t/is (= 0 (:delete res))))
(let [res (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
(t/is (= 2 (:processed res))))
(binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
(let [res (th/run-task! :objects-gc {})]
(t/is (= 2 (:processed res))))
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(t/is (= 0 (:freeze res)))
(t/is (= 6 (:delete res))))))
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 0 (:freeze res)))
(t/is (= 6 (:delete res)))))))
(t/deftest font-deletion-2
(let [prof (th/create-profile* 1 {:is-active true})
@@ -189,7 +192,7 @@
;; (th/print-result! out)
(t/is (nil? (:error out))))
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 6 (:freeze res))))
(let [params {::th/type :delete-font
@@ -201,16 +204,17 @@
(t/is (nil? (:error out)))
(t/is (nil? (:result out))))
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 0 (:freeze res)))
(t/is (= 0 (:delete res))))
(let [res (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
(t/is (= 1 (:processed res))))
(binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
(let [res (th/run-task! :objects-gc {})]
(t/is (= 1 (:processed res))))
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(t/is (= 0 (:freeze res)))
(t/is (= 3 (:delete res))))))
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 0 (:freeze res)))
(t/is (= 3 (:delete res)))))))
(t/deftest font-deletion-3
(let [prof (th/create-profile* 1 {:is-active true})
@@ -248,7 +252,7 @@
(t/is (nil? (:error out1)))
(t/is (nil? (:error out2)))
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 6 (:freeze res))))
(let [params {::th/type :delete-font-variant
@@ -260,13 +264,14 @@
(t/is (nil? (:error out)))
(t/is (nil? (:result out))))
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 0 (:freeze res)))
(t/is (= 0 (:delete res))))
(let [res (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
(t/is (= 1 (:processed res))))
(binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
(let [res (th/run-task! :objects-gc {})]
(t/is (= 1 (:processed res))))
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(t/is (= 0 (:freeze res)))
(t/is (= 3 (:delete res))))))
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 0 (:freeze res)))
(t/is (= 3 (:delete res)))))))

View File

@@ -6,11 +6,13 @@
(ns backend-tests.rpc-project-test
(:require
[app.common.time :as ct]
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
[app.http :as http]
[app.rpc :as-alias rpc]
[app.setup.clock :as clock]
[backend-tests.helpers :as th]
[clojure.test :as t]))
@@ -226,8 +228,9 @@
(t/is (= 0 (count result)))))
;; run permanent deletion
(let [result (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
(t/is (= 1 (:processed result))))
(binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
(let [result (th/run-task! :objects-gc {})]
(t/is (= 1 (:processed result)))))
;; query the list of files of a after hard deletion
(let [data {::th/type :get-project-files

View File

@@ -13,6 +13,7 @@
[app.db :as db]
[app.http :as http]
[app.rpc :as-alias rpc]
[app.setup.clock :as clock]
[app.storage :as sto]
[app.tokens :as tokens]
[backend-tests.helpers :as th]
@@ -525,8 +526,9 @@
(t/is (= :not-found (:type edata)))))
;; run permanent deletion
(let [result (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
(t/is (= 2 (:processed result))))
(binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
(let [result (th/run-task! :objects-gc {})]
(t/is (= 2 (:processed result)))))
;; query the list of projects of a after hard deletion
(let [data {::th/type :get-projects
@@ -581,8 +583,9 @@
(t/is (= 1 (count rows)))
(t/is (ct/inst? (:deleted-at (first rows)))))
(let [result (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
(t/is (= 7 (:processed result))))))
(binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
(let [result (th/run-task! :objects-gc {})]
(t/is (= 7 (:processed result)))))))
(t/deftest create-team-access-request
(with-mocks [mock {:target 'app.email/send! :return nil}]

View File

@@ -11,6 +11,7 @@
[app.common.uuid :as uuid]
[app.db :as db]
[app.rpc :as-alias rpc]
[app.setup.clock :as clock]
[app.storage :as sto]
[backend-tests.helpers :as th]
[clojure.test :as t]
@@ -53,19 +54,13 @@
(configure-storage-backend))
content (sto/content "content")
object (sto/put-object! storage {::sto/content content
::sto/expired-at (ct/in-future {:seconds 1})
::sto/expired-at (ct/in-future {:hours 1})
:content-type "text/plain"})]
(t/is (sto/object? object))
(t/is (ct/inst? (:expired-at object)))
(t/is (ct/is-after? (:expired-at object) (ct/now)))
(t/is (= object (sto/get-object storage (:id object))))
(th/sleep 1000)
(t/is (nil? (sto/get-object storage (:id object))))
(t/is (nil? (sto/get-object-data storage object)))
(t/is (nil? (sto/get-object-url storage object)))
(t/is (nil? (sto/get-object-path storage object)))))
(t/is (nil? (sto/get-object storage (:id object))))))
(t/deftest put-and-delete-object
(let [storage (-> (:app.storage/storage th/*system*)
@@ -98,20 +93,25 @@
::sto/expired-at (ct/now)
:content-type "text/plain"})
object2 (sto/put-object! storage {::sto/content content2
::sto/expired-at (ct/in-past {:hours 2})
::sto/expired-at (ct/in-future {:hours 2})
:content-type "text/plain"})
object3 (sto/put-object! storage {::sto/content content3
::sto/expired-at (ct/in-past {:hours 1})
::sto/expired-at (ct/in-future {:hours 1})
:content-type "text/plain"})]
(th/sleep 200)
(let [res (th/run-task! :storage-gc-deleted {})]
(t/is (= 1 (:deleted res))))
(binding [ct/*clock* (clock/fixed (ct/in-future {:minutes 0}))]
(let [res (th/run-task! :storage-gc-deleted {})]
(t/is (= 1 (:deleted res)))))
(let [res (th/db-exec-one! ["select count(*) from storage_object;"])]
(t/is (= 2 (:count res))))))
(t/is (= 2 (:count res))))
(binding [ct/*clock* (clock/fixed (ct/in-future {:minutes 61}))]
(let [res (th/run-task! :storage-gc-deleted {})]
(t/is (= 1 (:deleted res)))))
(let [res (th/db-exec-one! ["select count(*) from storage_object;"])]
(t/is (= 1 (:count res))))))
(t/deftest touched-gc-task-1
(let [storage (-> (:app.storage/storage th/*system*)
@@ -158,7 +158,7 @@
{:id (:id result-1)})
;; run the objects gc task for permanent deletion
(let [res (th/run-task! :objects-gc {:min-age 0})]
(let [res (th/run-task! :objects-gc {})]
(t/is (= 1 (:processed res))))
;; check that we still have all the storage objects
@@ -182,7 +182,6 @@
(let [res (th/db-exec-one! ["select count(*) from storage_object where deleted_at is not null"])]
(t/is (= 0 (:count res)))))))
(t/deftest touched-gc-task-2
(let [storage (-> (:app.storage/storage th/*system*)
(configure-storage-backend))
@@ -243,11 +242,12 @@
{:id (:id result-2)})
;; run the objects gc task for permanent deletion
(let [res (th/run-task! :objects-gc {:min-age 0})]
(let [res (th/run-task! :objects-gc {})]
(t/is (= 1 (:processed res))))
;; revert touched state to all storage objects
(th/db-exec-one! ["update storage_object set touched_at=now()"])
(th/db-exec-one! ["update storage_object set touched_at=?" (ct/now)])
;; Run the task again
(let [res (th/run-task! :storage-gc-touched {})]
@@ -293,10 +293,10 @@
result-2 (:result out2)]
;; now we proceed to manually mark all storage objects touched
(th/db-exec! ["update storage_object set touched_at=now()"])
(th/db-exec! ["update storage_object set touched_at=?" (ct/now)])
;; run the touched gc task
(let [res (th/run-task! "storage-gc-touched" {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 2 (:freeze res)))
(t/is (= 0 (:delete res))))
@@ -305,13 +305,13 @@
(t/is (= 2 (count rows)))))
;; now we proceed to manually delete all file_media_object
(th/db-exec! ["update file_media_object set deleted_at = now()"])
(th/db-exec! ["update file_media_object set deleted_at = ?" (ct/now)])
(let [res (th/run-task! "objects-gc" {:min-age 0})]
(let [res (th/run-task! :objects-gc {})]
(t/is (= 2 (:processed res))))
;; run the touched gc task
(let [res (th/run-task! "storage-gc-touched" {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 0 (:freeze res)))
(t/is (= 2 (:delete res))))