From 83763b46ce2bc920d67bc73c509df6ff7278527b Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 22 Oct 2025 14:51:33 +0200 Subject: [PATCH] :sparkles: Add RPC methods for manage deleted files This includes: get already deletedf files, restore deleted files and permanently delete files marked for deletion. --- backend/src/app/rpc/commands/files.clj | 166 ++++++++++++++++++- backend/src/app/setup/clock.clj | 10 +- backend/src/app/srepl/main.clj | 44 +---- backend/test/backend_tests/rpc_file_test.clj | 123 ++++++++++++++ 4 files changed, 302 insertions(+), 41 deletions(-) diff --git a/backend/src/app/rpc/commands/files.clj b/backend/src/app/rpc/commands/files.clj index eb5ecf9c0a..f6b262eecc 100644 --- a/backend/src/app/rpc/commands/files.clj +++ b/backend/src/app/rpc/commands/files.clj @@ -26,6 +26,7 @@ [app.db.sql :as-alias sql] [app.features.fdata :as feat.fdata] [app.features.logical-deletion :as ldel] + [app.http.sse :as sse] [app.loggers.audit :as-alias audit] [app.loggers.webhooks :as-alias webhooks] [app.msgbus :as mbus] @@ -38,6 +39,7 @@ [app.rpc.helpers :as rph] [app.rpc.permissions :as perms] [app.util.blob :as blob] + [app.util.events :as events] [app.util.pointer-map :as pmap] [app.util.services :as sv] [app.worker :as wrk] @@ -423,7 +425,6 @@ ;; --- QUERY COMMAND: get-page - (defn- prune-objects "Given the page data and the object-id returns the page data with all other not needed objects removed from the `:objects` data @@ -764,6 +765,54 @@ (teams/check-read-permissions! conn profile-id team-id) (get-team-recent-files conn team-id))) + +;; --- COMMAND QUERY: get-team-deleted-files + +(def sql:team-deleted-files + "WITH deleted_files AS ( + SELECT f.id, + f.revn, + f.vern, + f.project_id, + f.created_at, + f.modified_at, + f.name, + f.is_shared, + f.deleted_at AS will_be_deleted_at, + ft.media_id AS thumbnail_id, + row_number() OVER w AS row_num, + p.team_id + FROM file AS f + INNER JOIN project AS p ON (p.id = f.project_id) + LEFT JOIN file_thumbnail AS ft on (ft.file_id = f.id + AND ft.revn = f.revn + AND ft.deleted_at is null) + WHERE p.team_id = ? + AND (p.deleted_at > ?::timestamptz OR + f.deleted_at > ?::timestamptz) + WINDOW w AS (PARTITION BY f.project_id + ORDER BY f.modified_at DESC) + ORDER BY f.modified_at DESC + ) + SELECT * FROM deleted_files") + +(defn get-team-deleted-files + [conn team-id] + (let [now (ct/now)] + (db/exec! conn [sql:team-deleted-files team-id now now]))) + +(def ^:private schema:get-team-deleted-files + [:map {:title "get-team-deleted-files"} + [:team-id ::sm/uuid]]) + +(sv/defmethod ::get-team-deleted-files + {::doc/added "2.12" + ::sm/params schema:get-team-deleted-files} + [cfg {:keys [::rpc/profile-id team-id]}] + (db/run! cfg (fn [{:keys [::db/conn]}] + (teams/check-read-permissions! conn profile-id team-id) + (get-team-deleted-files conn team-id)))) + ;; --- COMMAND QUERY: get-file-info @@ -1112,3 +1161,118 @@ (check-edition-permissions! conn profile-id file-id) (-> (ignore-sync conn params) (update :features db/decode-pgarray #{}))) + +;; --- MUTATION COMMAND: delete-files-immediatelly + +(def ^:private sql:delete-team-files + "UPDATE file AS uf SET deleted_at = ?::timestamptz + FROM ( + SELECT f.id + FROM file AS f + JOIN project AS p ON (p.id = f.project_id) + JOIN team AS t ON (t.id = p.team_id) + WHERE t.deleted_at IS NULL + AND t.id = ? + AND f.id = ANY(?::uuid[]) + ) AS subquery + WHERE uf.id = subquery.id + RETURNING uf.id, uf.deleted_at;") + +(def ^:private schema:permanently-delete-team-files + [:map {:title "permanently-delete-team-files"} + [:team-id ::sm/uuid] + [:ids [::sm/set ::sm/uuid]]]) + +(sv/defmethod ::permanently-delete-team-files + "Mark the specified files to be deleted immediatelly on the + specified team. The team-id on params will be used to filter and + check writable permissons on team." + + {::doc/added "2.12" + ::sm/params schema:permanently-delete-team-files + ::db/transaction true} + + [{:keys [::db/conn]} {:keys [::rpc/profile-id ::rpc/request-at team-id ids]}] + (teams/check-edition-permissions! conn profile-id team-id) + + (reduce (fn [acc {:keys [id deleted-at]}] + (wrk/submit! {::db/conn conn + ::wrk/task :delete-object + ::wrk/params {:object :file + :deleted-at deleted-at + :id id}}) + (conj acc id)) + #{} + (db/plan conn [sql:delete-team-files request-at team-id + (db/create-array conn "uuid" ids)]))) + +;; --- MUTATION COMMAND: restore-files-immediatelly + +(def ^:private sql:resolve-editable-files + "SELECT f.id + FROM file AS f + JOIN project AS p ON (p.id = f.project_id) + JOIN team AS t ON (t.id = p.team_id) + WHERE t.deleted_at IS NULL + AND t.id = ? + AND f.id = ANY(?::uuid[])") + +(defn- restore-file + [conn file-id] + (db/update! conn :file + {:deleted-at nil + :has-media-trimmed false} + {:id file-id} + {::db/return-keys false}) + + (db/update! conn :file-media-object + {:deleted-at nil} + {:file-id file-id} + {::db/return-keys false}) + + (db/update! conn :file-change + {:deleted-at nil} + {:file-id file-id} + {::db/return-keys false}) + + (db/update! conn :file-data + {:deleted-at nil} + {:file-id file-id} + {::db/return-keys false}) + + (db/update! conn :file-thumbnail + {:deleted-at nil} + {:file-id file-id} + {::db/return-keys false}) + + (db/update! conn :file-tagged-object-thumbnail + {:deleted-at nil} + {:file-id file-id} + {::db/return-keys false})) + +(defn- restore-deleted-team-files + [{:keys [::db/conn]} {:keys [::rpc/profile-id team-id ids]}] + (teams/check-edition-permissions! conn profile-id team-id) + + (reduce (fn [affected {:keys [id]}] + (let [index (inc (count affected))] + (events/tap :progress {:file-id id :index index :total (count ids)}) + (restore-file conn id) + (conj affected id))) + #{} + (db/plan conn [sql:resolve-editable-files team-id + (db/create-array conn "uuid" ids)]))) + +(def ^:private schema:restore-deleted-team-files + [:map {:title "restore-deleted-team-files"} + [:team-id ::sm/uuid] + [:ids [::sm/set ::sm/uuid]]]) + +(sv/defmethod ::restore-deleted-team-files + "Removes the deletion mark from the specified files (and respective projects)." + + {::doc/added "2.12" + ::sse/stream? true + ::sm/params schema:restore-deleted-team-files} + [cfg params] + (sse/response #(db/tx-run! cfg restore-deleted-team-files params))) diff --git a/backend/src/app/setup/clock.clj b/backend/src/app/setup/clock.clj index 09ab991856..f73f6f8501 100644 --- a/backend/src/app/setup/clock.clj +++ b/backend/src/app/setup/clock.clj @@ -14,7 +14,9 @@ [integrant.core :as ig]) (:import java.time.Clock - java.time.Duration)) + java.time.Duration + java.time.Instant + java.time.ZoneId)) (defonce current (atom {:clock (Clock/systemDefaultZone) @@ -36,6 +38,12 @@ [_ _] (remove-watch current ::common)) +(defn fixed + "Get fixed clock, mainly used in tests" + [instant] + (Clock/fixed ^Instant (ct/inst instant) + ^ZoneId (ZoneId/of "Z"))) + (defn set-offset! [duration] (swap! current assoc :offset (some-> duration ct/duration))) diff --git a/backend/src/app/srepl/main.clj b/backend/src/app/srepl/main.clj index f2286013aa..51832cefac 100644 --- a/backend/src/app/srepl/main.clj +++ b/backend/src/app/srepl/main.clj @@ -567,48 +567,12 @@ :id file-id}))) :deleted)) -(defn- restore-file* - [{:keys [::db/conn]} file-id] - (db/update! conn :file - {:deleted-at nil - :has-media-trimmed false} - {:id file-id} - {::db/return-keys false}) - - (db/update! conn :file-media-object - {:deleted-at nil} - {:file-id file-id} - {::db/return-keys false}) - - (db/update! conn :file-change - {:deleted-at nil} - {:file-id file-id} - {::db/return-keys false}) - - (db/update! conn :file-data - {:deleted-at nil} - {:file-id file-id} - {::db/return-keys false}) - - ;; Mark thumbnails to be deleted - (db/update! conn :file-thumbnail - {:deleted-at nil} - {:file-id file-id} - {::db/return-keys false}) - - (db/update! conn :file-tagged-object-thumbnail - {:deleted-at nil} - {:file-id file-id} - {::db/return-keys false}) - - :restored) - (defn restore-file! "Mark a file and all related objects as not deleted" [file-id] (let [file-id (h/parse-uuid file-id)] (db/tx-run! main/system - (fn [system] + (fn [{:keys [::db/conn] :as system}] (when-let [file (db/get* system :file {:id file-id} {::db/remove-deleted false @@ -622,7 +586,9 @@ :cause "explicit call to restore-file!"} ::audit/tracked-at (ct/now)}) - (restore-file* system file-id)))))) + + (#'files/restore-file conn file-id)) + :restored)))) (defn delete-project! "Mark a project for deletion" @@ -655,7 +621,7 @@ (doseq [{:keys [id]} (db/query conn :file {:project-id project-id} {::sql/columns [:id]})] - (restore-file* cfg id)) + (#'files/restore-file conn id)) :restored) diff --git a/backend/test/backend_tests/rpc_file_test.clj b/backend/test/backend_tests/rpc_file_test.clj index 3bbc5e0aa6..d34077bc2d 100644 --- a/backend/test/backend_tests/rpc_file_test.clj +++ b/backend/test/backend_tests/rpc_file_test.clj @@ -19,6 +19,7 @@ [app.http :as http] [app.rpc :as-alias rpc] [app.rpc.commands.files :as files] + [app.setup.clock :as clock] [app.storage :as sto] [backend-tests.helpers :as th] [clojure.test :as t] @@ -1865,3 +1866,125 @@ (t/is (= (:id file-2) (:file-id (get rows 0)))) (t/is (nil? (:deleted-at (get rows 0))))))) + +(t/deftest deleted-files-permanently-delete + (let [prof (th/create-profile* 1 {:is-active true}) + team-id (:default-team-id prof) + proj-id (:default-project-id prof) + file-id (uuid/next) + now (ct/inst "2025-10-31T00:00:00Z")] + + (binding [ct/*clock* (clock/fixed now)] + (let [data {::th/type :create-file + ::rpc/profile-id (:id prof) + :project-id proj-id + :id file-id + :name "foobar" + :is-shared false + :components-v2 true} + out (th/command! data)] + + ;; (th/print-result! out) + (t/is (nil? (:error out))) + + (let [result (:result out)] + (t/is (= (:name data) (:name result))) + (t/is (= proj-id (:project-id result))))) + + (let [data {::th/type :delete-file + :id file-id + ::rpc/profile-id (:id prof)} + out (th/command! data)] + ;; (th/print-result! out) + (t/is (nil? (:error out))) + (t/is (nil? (:result out)))) + + ;; get deleted files + (let [data {::th/type :get-team-deleted-files + ::rpc/profile-id (:id prof) + :team-id team-id} + out (th/command! data)] + ;; (th/print-result! out) + (t/is (nil? (:error out))) + (let [[row1 :as result] (:result out)] + (t/is (= 1 (count result))) + (t/is (= (:will-be-deleted-at row1) #penpot/inst "2025-11-07T00:00:00Z")) + (t/is (= (:created-at row1) #penpot/inst "2025-10-31T00:00:00Z")) + (t/is (= (:modified-at row1) #penpot/inst "2025-10-31T00:00:00Z")))) + + (let [data {::th/type :permanently-delete-team-files + ::rpc/profile-id (:id prof) + :team-id team-id + :ids #{file-id}} + out (th/command! data)] + ;; (th/print-result! out) + (t/is (nil? (:error out))) + (let [result (:result out)] + (t/is (= (:ids data) result))) + + (let [row (th/db-exec-one! ["select * from file where id = ?" file-id])] + (t/is (= (:deleted-at row) now))))))) + +(t/deftest deleted-files-restore + (let [prof (th/create-profile* 1 {:is-active true}) + team-id (:default-team-id prof) + proj-id (:default-project-id prof) + file-id (uuid/next) + now (ct/inst "2025-10-31T00:00:00Z")] + + (binding [ct/*clock* (clock/fixed now)] + (let [data {::th/type :create-file + ::rpc/profile-id (:id prof) + :project-id proj-id + :id file-id + :name "foobar" + :is-shared false + :components-v2 true} + out (th/command! data)] + + ;; (th/print-result! out) + (t/is (nil? (:error out))) + + (let [result (:result out)] + (t/is (= (:name data) (:name result))) + (t/is (= proj-id (:project-id result))))) + + (let [data {::th/type :delete-file + :id file-id + ::rpc/profile-id (:id prof)} + out (th/command! data)] + ;; (th/print-result! out) + (t/is (nil? (:error out))) + (t/is (nil? (:result out)))) + + ;; get deleted files + (let [data {::th/type :get-team-deleted-files + ::rpc/profile-id (:id prof) + :team-id team-id} + out (th/command! data)] + ;; (th/print-result! out) + (t/is (nil? (:error out))) + (let [[row1 :as result] (:result out)] + (t/is (= 1 (count result))) + (t/is (= (:will-be-deleted-at row1) #penpot/inst "2025-11-07T00:00:00Z")) + (t/is (= (:created-at row1) #penpot/inst "2025-10-31T00:00:00Z")) + (t/is (= (:modified-at row1) #penpot/inst "2025-10-31T00:00:00Z")))) + + (let [data {::th/type :restore-deleted-team-files + ::rpc/profile-id (:id prof) + :team-id team-id + :ids #{file-id}} + out (th/command! data)] + ;; (th/print-result! out) + (t/is (nil? (:error out))) + (let [result (:result out)] + (t/is (fn? result)) + + (let [events (th/consume-sse result)] + ;; (pp/pprint events) + (t/is (= 2 (count events))) + (t/is (= :end (first (last events)))) + (t/is (= (:ids data) (last (last events))))))) + + (let [row (th/db-exec-one! ["select * from file where id = ?" file-id])] + (t/is (nil? (:deleted-at row)))))))