johanels/penpot
1
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-12-12 06:24:17 +01:00
Code Issues Actions 9 Packages Projects Releases Wiki Activity

✨ Make the binfile import process more resilient (#7464)

Browse Source 
on small inconsistencies on file media object references
This commit is contained in:
Andrey Antukh
2025-10-13 11:13:10 +02:00
committed by GitHub
parent 71fd6640af
commit 1b9deecefc
1 changed files with 73 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

175
backend/src/app/binfile/v3.clj
View File

@@ -713,7 +713,7 @@
:plugin-data plugin-data})) :plugin-data plugin-data}))
(defn- import-file (defn- import-file
[{:keys [::bfc/project-id] :as cfg} {file-id :id file-name :name}] [{:keys [::db/conn ::bfc/project-id] :as cfg} {file-id :id file-name :name}]
(let [file-id' (bfc/lookup-index file-id) (let [file-id' (bfc/lookup-index file-id)
file (read-file cfg file-id) file (read-file cfg file-id)
media (read-file-media cfg file-id) media (read-file-media cfg file-id)
@@ -726,26 +726,48 @@
:version (:version file) :version (:version file)
::l/sync? true) ::l/sync? true)
(events/tap :progress {:section :file :name file-name}) (vswap! bfc/*state* update :index bfc/update-index media :id)
(when media (events/tap :progress {:section :media :file-id file-id})
;; Update index with media
(l/dbg :hint "update media index"
:file-id (str file-id')
:total (count media)
::l/sync? true)
(vswap! bfc/*state* update :index bfc/update-index (map :id media)) (doseq [item media]
(vswap! bfc/*state* update :media into media)) (let [params (-> item
(update :id bfc/lookup-index)
(assoc :file-id file-id')
(d/update-when :media-id bfc/lookup-index)
(d/update-when :thumbnail-id bfc/lookup-index))]
(when thumbnails (l/dbg :hint "inserting media object"
(l/dbg :hint "update thumbnails index" :file-id (str file-id')
:file-id (str file-id') :id (str (:id params))
:total (count thumbnails) :media-id (str (:media-id params))
::l/sync? true) :thumbnail-id (str (:thumbnail-id params))
:old-id (str (:id item))
::l/sync? true)
(vswap! bfc/*state* update :index bfc/update-index (map :media-id thumbnails)) (db/insert! conn :file-media-object params
(vswap! bfc/*state* update :thumbnails into thumbnails)) ::db/on-conflict-do-nothing? (::bfc/overwrite cfg))))
(events/tap :progress {:section :thumbnails :file-id file-id})
(doseq [item thumbnails]
(let [media-id (bfc/lookup-index (:media-id item))
object-id (-> (assoc item :file-id file-id')
(cth/fmt-object-id))
params {:file-id file-id'
:object-id object-id
:tag (:tag item)
:media-id media-id}]
(l/dbg :hint "inserting object thumbnail"
:file-id (str file-id')
:media-id (str media-id)
::l/sync? true)
(db/insert! conn :file-tagged-object-thumbnail params
::db/on-conflict-do-nothing? true)))
(events/tap :progress {:section :file :file-id file-id})
(let [data (-> (read-file-data cfg file-id) (let [data (-> (read-file-data cfg file-id)
(d/without-nils) (d/without-nils)
@@ -794,95 +816,47 @@
entries (keep (match-storage-entry-fn) entries)] entries (keep (match-storage-entry-fn) entries)]
(doseq [{:keys [id entry]} entries] (doseq [{:keys [id entry]} entries]
(let [object (->> (read-entry input entry) (let [object (->> (read-entry input entry)
(decode-storage-object) (decode-storage-object)
(validate-storage-object))] (validate-storage-object))
(when (not= id (:id object)) ext (cmedia/mtype->extension (:content-type object))
path (str "objects/" id ext)
content (->> path
(get-zip-entry input)
(zip-entry-storage-content input))]
(when (not= (:size object) (sto/get-size content))
(ex/raise :type :validation (ex/raise :type :validation
:code :inconsistent-penpot-file :code :inconsistent-penpot-file
:hint "the penpot file seems corrupt, found unexpected uuid (storage-object-id)" :hint "found corrupted storage object: size does not match"
:expected-id (str id) :path path
:found-id (str (:id object)))) :expected-size (:size object)
:found-size (sto/get-size content)))
(let [ext (cmedia/mtype->extension (:content-type object)) (when-let [hash (get object :hash)]
path (str "objects/" id ext) (when (not= hash (sto/get-hash content))
content (->> path
(get-zip-entry input)
(zip-entry-storage-content input))]
(when (not= (:size object) (sto/get-size content))
(ex/raise :type :validation (ex/raise :type :validation
:code :inconsistent-penpot-file :code :inconsistent-penpot-file
:hint "found corrupted storage object: size does not match" :hint "found corrupted storage object: hash does not match"
:path path :path path
:expected-size (:size object) :expected-hash (:hash object)
:found-size (sto/get-size content))) :found-hash (sto/get-hash content))))
(when-let [hash (get object :hash)] (let [params (-> object
(when (not= hash (sto/get-hash content)) (dissoc :id :size)
(ex/raise :type :validation (assoc ::sto/content content)
:code :inconsistent-penpot-file (assoc ::sto/deduplicate? true)
:hint "found corrupted storage object: hash does not match" (assoc ::sto/touched-at timestamp))
:path path sobject (sto/put-object! storage params)]
:expected-hash (:hash object)
:found-hash (sto/get-hash content))))
(let [params (-> object (l/dbg :hint "persisted storage object"
(dissoc :id :size) :id (str (:id sobject))
(assoc ::sto/content content) :prev-id (str id)
(assoc ::sto/deduplicate? true) :bucket (:bucket params)
(assoc ::sto/touched-at timestamp)) ::l/sync? true)
sobject (sto/put-object! storage params)]
(l/dbg :hint "persisted storage object" (vswap! bfc/*state* update :index assoc id (:id sobject)))))))
:id (str (:id sobject))
:prev-id (str id)
:bucket (:bucket params)
::l/sync? true)
(vswap! bfc/*state* update :index assoc id (:id sobject))))))))
(defn- import-file-media
[{:keys [::db/conn] :as cfg}]
(events/tap :progress {:section :media})
(doseq [item (:media @bfc/*state*)]
(let [params (-> item
(update :id bfc/lookup-index)
(update :file-id bfc/lookup-index)
(d/update-when :media-id bfc/lookup-index)
(d/update-when :thumbnail-id bfc/lookup-index))]
(l/dbg :hint "inserting file media object"
:old-id (str (:id item))
:id (str (:id params))
:file-id (str (:file-id params))
::l/sync? true)
(db/insert! conn :file-media-object params
::db/on-conflict-do-nothing? (::bfc/overwrite cfg)))))
(defn- import-file-thumbnails
[{:keys [::db/conn] :as cfg}]
(events/tap :progress {:section :thumbnails})
(doseq [item (:thumbnails @bfc/*state*)]
(let [file-id (bfc/lookup-index (:file-id item))
media-id (bfc/lookup-index (:media-id item))
object-id (-> (assoc item :file-id file-id)
(cth/fmt-object-id))
params {:file-id file-id
:object-id object-id
:tag (:tag item)
:media-id media-id}]
(l/dbg :hint "inserting file object thumbnail"
:file-id (str file-id)
:media-id (str media-id)
::l/sync? true)
(db/insert! conn :file-tagged-object-thumbnail params
{::db/on-conflict-do-nothing? true}))))
(defn- import-files* (defn- import-files*
[{:keys [::manifest] :as cfg}] [{:keys [::manifest] :as cfg}]
@@ -890,6 +864,8 @@
(vswap! bfc/*state* update :index bfc/update-index (:files manifest) :id) (vswap! bfc/*state* update :index bfc/update-index (:files manifest) :id)
(import-storage-objects cfg)
(let [files (get manifest :files) (let [files (get manifest :files)
result (reduce (fn [result {:keys [id] :as file}] result (reduce (fn [result {:keys [id] :as file}]
(let [name' (get file :name) (let [name' (get file :name)
@@ -902,10 +878,6 @@
files)] files)]
(import-file-relations cfg) (import-file-relations cfg)
(import-storage-objects cfg)
(import-file-media cfg)
(import-file-thumbnails cfg)
(bfm/apply-pending-migrations! cfg) (bfm/apply-pending-migrations! cfg)
result)) result))
@@ -930,9 +902,8 @@
(binding [bfc/*options* cfg (binding [bfc/*options* cfg
bfc/*reference-file* ref-file] bfc/*reference-file* ref-file]
(import-file cfg file)
(import-storage-objects cfg) (import-storage-objects cfg)
(import-file-media cfg) (import-file cfg file)
(bfc/invalidate-thumbnails cfg file-id) (bfc/invalidate-thumbnails cfg file-id)
(bfm/apply-pending-migrations! cfg) (bfm/apply-pending-migrations! cfg)