mirror of
https://github.com/penpot/penpot.git
synced 2025-12-11 22:14:05 +01:00
88fb5e7ab5
This upgrade also includes complete elimination of use spec from the backend codebase, completing the long running migration to fully use malli for validation and decoding.
126 lines
4.7 KiB
Clojure
126 lines
4.7 KiB
Clojure
;; This Source Code Form is subject to the terms of the Mozilla Public
|
|
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
;;
|
|
;; Copyright (c) KALEIDOS INC
|
|
|
|
(ns app.tasks.offload-file-data
|
|
"A maintenance task responsible of moving file data from hot
|
|
storage (the database row) to a cold storage (fs or s3)."
|
|
(:require
|
|
[app.common.exceptions :as ex]
|
|
[app.common.logging :as l]
|
|
[app.db :as db]
|
|
[app.db.sql :as-alias sql]
|
|
[app.storage :as sto]
|
|
[integrant.core :as ig]))
|
|
|
|
(defn- offload-file-data!
|
|
[{:keys [::db/conn ::sto/storage ::file-id] :as cfg}]
|
|
(let [file (db/get conn :file {:id file-id}
|
|
{::sql/for-update true})]
|
|
(when (nil? (:data file))
|
|
(ex/raise :hint "file already offloaded"
|
|
:type :internal
|
|
:code :file-already-offloaded
|
|
:file-id file-id))
|
|
|
|
(let [data (sto/content (:data file))
|
|
sobj (sto/put-object! storage
|
|
{::sto/content data
|
|
::sto/touch true
|
|
:bucket "file-data"
|
|
:content-type "application/octet-stream"
|
|
:file-id file-id})]
|
|
|
|
(l/trc :hint "offload file data"
|
|
:file-id (str file-id)
|
|
:storage-id (str (:id sobj)))
|
|
|
|
(db/update! conn :file
|
|
{:data-backend "objects-storage"
|
|
:data-ref-id (:id sobj)
|
|
:data nil}
|
|
{:id file-id}
|
|
{::db/return-keys false}))))
|
|
|
|
(defn- offload-file-data-fragments!
|
|
[{:keys [::db/conn ::sto/storage ::file-id] :as cfg}]
|
|
(doseq [fragment (db/query conn :file-data-fragment
|
|
{:file-id file-id
|
|
:deleted-at nil
|
|
:data-backend nil}
|
|
{::db/for-update true})]
|
|
(let [data (sto/content (:data fragment))
|
|
sobj (sto/put-object! storage
|
|
{::sto/content data
|
|
::sto/touch true
|
|
:bucket "file-data-fragment"
|
|
:content-type "application/octet-stream"
|
|
:file-id file-id
|
|
:file-fragment-id (:id fragment)})]
|
|
|
|
(l/trc :hint "offload file data fragment"
|
|
:file-id (str file-id)
|
|
:file-fragment-id (str (:id fragment))
|
|
:storage-id (str (:id sobj)))
|
|
|
|
(db/update! conn :file-data-fragment
|
|
{:data-backend "objects-storage"
|
|
:data-ref-id (:id sobj)
|
|
:data nil}
|
|
{:id (:id fragment)}
|
|
{::db/return-keys false}))))
|
|
|
|
(def sql:get-snapshots
|
|
"SELECT fc.*
|
|
FROM file_change AS fc
|
|
WHERE fc.file_id = ?
|
|
AND fc.label IS NOT NULL
|
|
AND fc.data IS NOT NULL
|
|
AND fc.data_backend IS NULL")
|
|
|
|
(defn- offload-file-snapshots!
|
|
[{:keys [::db/conn ::sto/storage ::file-id] :as cfg}]
|
|
(doseq [snapshot (db/exec! conn [sql:get-snapshots file-id])]
|
|
(let [data (sto/content (:data snapshot))
|
|
sobj (sto/put-object! storage
|
|
{::sto/content data
|
|
::sto/touch true
|
|
:bucket "file-change"
|
|
:content-type "application/octet-stream"
|
|
:file-id file-id
|
|
:file-change-id (:id snapshot)})]
|
|
|
|
(l/trc :hint "offload file change"
|
|
:file-id (str file-id)
|
|
:file-change-id (str (:id snapshot))
|
|
:storage-id (str (:id sobj)))
|
|
|
|
(db/update! conn :file-change
|
|
{:data-backend "objects-storage"
|
|
:data-ref-id (:id sobj)
|
|
:data nil}
|
|
{:id (:id snapshot)}
|
|
{::db/return-keys false}))))
|
|
|
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
;; HANDLER
|
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
|
|
(defmethod ig/assert-key ::handler
|
|
[_ params]
|
|
(assert (db/pool? (::db/pool params)) "expected a valid database pool")
|
|
(assert (sto/valid-storage? (::sto/storage params)) "expected valid storage to be provided"))
|
|
|
|
(defmethod ig/init-key ::handler
|
|
[_ cfg]
|
|
(fn [{:keys [props] :as task}]
|
|
(-> cfg
|
|
(assoc ::db/rollback (:rollback? props))
|
|
(assoc ::file-id (:file-id props))
|
|
(db/tx-run! (fn [cfg]
|
|
(offload-file-data! cfg)
|
|
(offload-file-data-fragments! cfg)
|
|
(offload-file-snapshots! cfg))))))
|