johanels/penpot
1
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-12-11 22:14:05 +01:00
Code Issues Actions 9 Packages Projects Releases Wiki Activity

🔥 Remove already broken and unused internal components-v2 migration code

Browse Source
This commit is contained in:
Andrey Antukh
2025-05-15 12:06:01 +02:00
parent 2da8747485
commit c8a211742a
7 changed files with 117 additions and 2237 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
backend/src/app/binfile/migrations.clj
View File

@@ -10,7 +10,6 @@
[app.binfile.common :as bfc]
[app.common.exceptions :as ex]
[app.common.features :as cfeat]
[app.features.components-v2 :as feat.compv2]
[clojure.set :as set]
[cuerdas.core :as str]))
@@ -28,13 +27,11 @@
(defn apply-pending-migrations!
"Apply alredy registered pending migrations to files"
[cfg]
(doseq [[feature file-id] (-> bfc/*state* deref :pending-to-migrate)]
[_cfg]
(doseq [[feature _file-id] (-> bfc/*state* deref :pending-to-migrate)]
(case feature
"components/v2"
(feat.compv2/migrate-file! cfg file-id
:validate? (::validate cfg true)
:skip-on-graphic-error? true)
nil
"fdata/shape-data-type"
nil

1845
backend/src/app/features/components_v2.clj
View File

File diff suppressed because it is too large Load Diff

104
backend/src/app/media.clj
View File

@@ -8,12 +8,11 @@
"Media & Font postprocessing."
(:require
[app.common.data :as d]
[app.common.data.macros :as dm]
[app.common.exceptions :as ex]
[app.common.media :as cm]
[app.common.schema :as sm]
[app.common.schema.openapi :as-alias oapi]
[app.common.spec :as us]
[app.common.svg :as csvg]
[app.config :as cf]
[app.db :as-alias db]
[app.storage :as-alias sto]
@@ -22,30 +21,21 @@
[buddy.core.bytes :as bb]
[buddy.core.codecs :as bc]
[clojure.java.shell :as sh]
[clojure.spec.alpha :as s]
[clojure.xml :as xml]
[cuerdas.core :as str]
[datoteka.fs :as fs]
[datoteka.io :as io])
(:import
clojure.lang.XMLHandler
java.io.InputStream
javax.xml.XMLConstants
javax.xml.parsers.SAXParserFactory
org.apache.commons.io.IOUtils
org.im4java.core.ConvertCmd
org.im4java.core.IMOperation
org.im4java.core.Info))
(s/def ::path fs/path?)
(s/def ::filename string?)
(s/def ::size integer?)
(s/def ::headers (s/map-of string? string?))
(s/def ::mtype string?)
(s/def ::upload
(s/keys :req-un [::filename ::size ::path]
:opt-un [::mtype ::headers]))
;; A subset of fields from the ::upload spec
(s/def ::input
(s/keys :req-un [::path]
:opt-un [::mtype]))
(def schema:upload
(sm/register!
^{::sm/type ::upload}
[:map {:title "Upload"}
@@ -54,7 +44,15 @@
[:path ::fs/path]
[:mtype {:optional true} :string]
[:headers {:optional true}
[:map-of :string :string]]])
[:map-of :string :string]]]))
(def ^:private schema:input
[:map {:title "Input"}
[:path ::fs/path]
[:mtype {:optional true} ::sm/text]])
(def ^:private check-input
(sm/check-fn schema:input))
(defn validate-media-type!
([upload] (validate-media-type! upload cm/valid-image-types))
@@ -97,17 +95,44 @@
(catch Throwable e
(process-error e))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; SVG PARSING
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn- secure-parser-factory
[^InputStream input ^XMLHandler handler]
(.. (doto (SAXParserFactory/newInstance)
(.setFeature XMLConstants/FEATURE_SECURE_PROCESSING true)
(.setFeature "http://apache.org/xml/features/disallow-doctype-decl" true))
(newSAXParser)
(parse input handler)))
(defn- strip-doctype
[data]
(cond-> data
(str/includes? data "<!DOCTYPE")
(str/replace #"<\!DOCTYPE[^>]*>" "")))
(defn- parse-svg
[text]
(let [text (strip-doctype text)]
(dm/with-open [istream (IOUtils/toInputStream text "UTF-8")]
(xml/parse istream secure-parser-factory))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; IMAGE THUMBNAILS
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(s/def ::width integer?)
(s/def ::height integer?)
(s/def ::format #{:jpeg :webp :png})
(s/def ::quality #(< 0 % 101))
(def ^:private schema:thumbnail-params
[:map {:title "ThumbnailParams"}
[:input schema:input]
[:format [:enum :jpeg :webp :png]]
[:quality [:int {:min 1 :max 100}]]
[:width :int]
[:height :int]])
(s/def ::thumbnail-params
(s/keys :req-un [::input ::format ::width ::height]))
(def ^:private check-thumbnail-params
(sm/check-fn schema:thumbnail-params))
;; Related info on how thumbnails generation
;; http://www.imagemagick.org/Usage/thumbnails/
@@ -129,21 +154,28 @@
:data tmp)))
(defmethod process :generic-thumbnail
[{:keys [quality width height] :as params}]
(us/assert ::thumbnail-params params)
(let [op (doto (IMOperation.)
[params]
(let [{:keys [quality width height] :as params}
(check-thumbnail-params params)
operation
(doto (IMOperation.)
(.addImage)
(.autoOrient)
(.strip)
(.thumbnail ^Integer (int width) ^Integer (int height) ">")
(.quality (double quality))
(.addImage))]
(generic-process (assoc params :operation op))))
(generic-process (assoc params :operation operation))))
(defmethod process :profile-thumbnail
[{:keys [quality width height] :as params}]
(us/assert ::thumbnail-params params)
(let [op (doto (IMOperation.)
[params]
(let [{:keys [quality width height] :as params}
(check-thumbnail-params params)
operation
(doto (IMOperation.)
(.addImage)
(.autoOrient)
(.strip)
@@ -152,7 +184,8 @@
(.extent (int width) (int height))
(.quality (double quality))
(.addImage))]
(generic-process (assoc params :operation op))))
(generic-process (assoc params :operation operation))))
(defn get-basic-info-from-svg
[{:keys [tag attrs] :as data}]
@@ -184,10 +217,9 @@
(defmethod process :info
[{:keys [input] :as params}]
(us/assert ::input input)
(let [{:keys [path mtype]} input]
(let [{:keys [path mtype] :as input} (check-input input)]
(if (= mtype "image/svg+xml")
(let [info (some-> path slurp csvg/parse get-basic-info-from-svg)]
(let [info (some-> path slurp parse-svg get-basic-info-from-svg)]
(when-not info
(ex/raise :type :validation
:code :invalid-svg-file

16
backend/src/app/rpc/commands/files_temp.clj
View File

@@ -14,7 +14,6 @@
[app.config :as cf]
[app.db :as db]
[app.db.sql :as sql]
[app.features.components-v2 :as feat.compv2]
[app.features.fdata :as fdata]
[app.loggers.audit :as audit]
[app.rpc :as-alias rpc]
@@ -110,7 +109,7 @@
;; --- MUTATION COMMAND: persist-temp-file
(defn persist-temp-file
[{:keys [::db/conn] :as cfg} {:keys [id ::rpc/profile-id] :as params}]
[{:keys [::db/conn] :as cfg} {:keys [id] :as params}]
(let [file (files/get-file cfg id
:migrate? false
:lock-for-update? true)]
@@ -147,19 +146,6 @@
:revn 1
:data (blob/encode (:data file))}
{:id id})
(let [team (teams/get-team conn :profile-id profile-id :project-id (:project-id file))
file-features (:features file)
team-features (cfeat/get-team-enabled-features cf/flags team)]
(when (and (contains? team-features "components/v2")
(not (contains? file-features "components/v2")))
;; Migrate components v2
(feat.compv2/migrate-file! cfg
(:id file)
:max-procs 2
:validate? true
:throw-on-validate? true)))
nil)))
(def ^:private schema:persist-temp-file

306
backend/src/app/srepl/components_v2.clj
View File

@@ -1,306 +0,0 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) KALEIDOS INC
(ns app.srepl.components-v2
(:require
[app.common.fressian :as fres]
[app.common.logging :as l]
[app.db :as db]
[app.features.components-v2 :as feat]
[app.main :as main]
[app.srepl.helpers :as h]
[app.util.events :as events]
[app.util.time :as dt]
[app.worker :as-alias wrk]
[datoteka.fs :as fs]
[datoteka.io :as io]
[promesa.exec :as px]
[promesa.exec.semaphore :as ps]
[promesa.util :as pu]))
(def ^:dynamic *scope* nil)
(def ^:dynamic *semaphore* nil)
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; PRIVATE HELPERS
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(def ^:private sql:get-files-by-created-at
"SELECT id, features,
row_number() OVER (ORDER BY created_at DESC) AS rown
FROM file
WHERE deleted_at IS NULL
ORDER BY created_at DESC")
(defn- get-files
[conn]
(->> (db/cursor conn [sql:get-files-by-created-at] {:chunk-size 500})
(map feat/decode-row)
(remove (fn [{:keys [features]}]
(contains? features "components/v2")))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; PUBLIC API
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn migrate-file!
[file-id & {:keys [rollback? validate? label cache skip-on-graphic-error?]
:or {rollback? true
validate? false
skip-on-graphic-error? true}}]
(l/dbg :hint "migrate:start" :rollback rollback?)
(let [tpoint (dt/tpoint)
file-id (h/parse-uuid file-id)]
(binding [feat/*stats* (atom {})
feat/*cache* cache]
(try
(-> (assoc main/system ::db/rollback rollback?)
(feat/migrate-file! file-id
:validate? validate?
:skip-on-graphic-error? skip-on-graphic-error?
:label label))
(-> (deref feat/*stats*)
(assoc :elapsed (dt/format-duration (tpoint))))
(catch Throwable cause
(l/wrn :hint "migrate:error" :cause cause))
(finally
(let [elapsed (dt/format-duration (tpoint))]
(l/dbg :hint "migrate:end" :rollback rollback? :elapsed elapsed)))))))
(defn migrate-team!
[team-id & {:keys [rollback? skip-on-graphic-error? validate? label cache]
:or {rollback? true
validate? true
skip-on-graphic-error? true}}]
(l/dbg :hint "migrate:start" :rollback rollback?)
(let [team-id (h/parse-uuid team-id)
stats (atom {})
tpoint (dt/tpoint)]
(binding [feat/*stats* stats
feat/*cache* cache]
(try
(-> (assoc main/system ::db/rollback rollback?)
(feat/migrate-team! team-id
:label label
:validate? validate?
:skip-on-graphics-error? skip-on-graphic-error?))
(-> (deref feat/*stats*)
(assoc :elapsed (dt/format-duration (tpoint))))
(catch Throwable cause
(l/dbg :hint "migrate:error" :cause cause))
(finally
(let [elapsed (dt/format-duration (tpoint))]
(l/dbg :hint "migrate:end" :rollback rollback? :elapsed elapsed)))))))
(defn migrate-files!
"A REPL helper for migrate all files.
This function starts multiple concurrent file migration processes
until thw maximum number of jobs is reached which by default has the
value of `1`. This is controled with the `:max-jobs` option.
If you want to run this on multiple machines you will need to specify
the total number of partitions and the current partition.
In order to get the report table populated, you will need to provide
a correct `:label`. That label is also used for persist a file
snaphot before continue with the migration."
[& {:keys [max-jobs max-items rollback? validate?
cache skip-on-graphic-error?
label partitions current-partition]
:or {validate? false
rollback? true
max-jobs 1
current-partition 1
skip-on-graphic-error? true
max-items Long/MAX_VALUE}}]
(when (int? partitions)
(when-not (int? current-partition)
(throw (IllegalArgumentException. "missing `current-partition` parameter")))
(when-not (<= 0 current-partition partitions)
(throw (IllegalArgumentException. "invalid value on `current-partition` parameter"))))
(let [stats (atom {})
tpoint (dt/tpoint)
factory (px/thread-factory :virtual false :prefix "penpot/migration/")
executor (px/cached-executor :factory factory)
sjobs (ps/create :permits max-jobs)
migrate-file
(fn [file-id rown]
(try
(db/tx-run! (assoc main/system ::db/rollback rollback?)
(fn [system]
(db/exec-one! system ["SET LOCAL idle_in_transaction_session_timeout = 0"])
(feat/migrate-file! system file-id
:rown rown
:label label
:validate? validate?
:skip-on-graphic-error? skip-on-graphic-error?)))
(catch Throwable cause
(l/wrn :hint "unexpected error on processing file (skiping)"
:file-id (str file-id))
(events/tap :error
(ex-info "unexpected error on processing file (skiping)"
{:file-id file-id}
cause))
(swap! stats update :errors (fnil inc 0)))
(finally
(ps/release! sjobs))))
process-file
(fn [{:keys [id rown]}]
(ps/acquire! sjobs)
(px/run! executor (partial migrate-file id rown)))]
(l/dbg :hint "migrate:start"
:label label
:rollback rollback?
:max-jobs max-jobs
:max-items max-items)
(binding [feat/*stats* stats
feat/*cache* cache]
(try
(db/tx-run! main/system
(fn [{:keys [::db/conn] :as system}]
(db/exec! conn ["SET LOCAL statement_timeout = 0"])
(db/exec! conn ["SET LOCAL idle_in_transaction_session_timeout = 0"])
(run! process-file
(->> (get-files conn)
(filter (fn [{:keys [rown] :as row}]
(if (int? partitions)
(= current-partition (inc (mod rown partitions)))
true)))
(take max-items)))
;; Close and await tasks
(pu/close! executor)))
(-> (deref stats)
(assoc :elapsed (dt/format-duration (tpoint))))
(catch Throwable cause
(l/dbg :hint "migrate:error" :cause cause)
(events/tap :error cause))
(finally
(let [elapsed (dt/format-duration (tpoint))]
(l/dbg :hint "migrate:end"
:rollback rollback?
:elapsed elapsed)))))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; CACHE POPULATE
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(def sql:sobjects-for-cache
"SELECT id,
row_number() OVER (ORDER BY created_at) AS index
FROM storage_object
WHERE (metadata->>'~:bucket' = 'file-media-object' OR
metadata->>'~:bucket' IS NULL)
AND metadata->>'~:content-type' = 'image/svg+xml'
AND deleted_at IS NULL
AND size < 1135899
ORDER BY created_at ASC")
(defn populate-cache!
"A REPL helper for migrate all files.
This function starts multiple concurrent file migration processes
until thw maximum number of jobs is reached which by default has the
value of `1`. This is controled with the `:max-jobs` option.
If you want to run this on multiple machines you will need to specify
the total number of partitions and the current partition.
In order to get the report table populated, you will need to provide
a correct `:label`. That label is also used for persist a file
snaphot before continue with the migration."
[& {:keys [max-jobs] :or {max-jobs 1}}]
(let [tpoint (dt/tpoint)
factory (px/thread-factory :virtual false :prefix "penpot/cache/")
executor (px/cached-executor :factory factory)
sjobs (ps/create :permits max-jobs)
retrieve-sobject
(fn [id index]
(let [path (feat/get-sobject-cache-path id)
parent (fs/parent path)]
(try
(when-not (fs/exists? parent)
(fs/create-dir parent))
(if (fs/exists? path)
(l/inf :hint "create cache entry" :status "exists" :index index :id (str id) :path (str path))
(let [svg-data (feat/get-optimized-svg id)]
(with-open [^java.lang.AutoCloseable stream (io/output-stream path)]
(let [writer (fres/writer stream)]
(fres/write! writer svg-data)))
(l/inf :hint "create cache entry" :status "created"
:index index
:id (str id)
:path (str path))))
(catch Throwable cause
(l/wrn :hint "create cache entry"
:status "error"
:index index
:id (str id)
:path (str path)
:cause cause))
(finally
(ps/release! sjobs)))))
process-sobject
(fn [{:keys [id index]}]
(ps/acquire! sjobs)
(px/run! executor (partial retrieve-sobject id index)))]
(l/dbg :hint "migrate:start"
:max-jobs max-jobs)
(try
(binding [feat/*system* main/system]
(run! process-sobject
(db/exec! main/system [sql:sobjects-for-cache]))
;; Close and await tasks
(pu/close! executor))
{:elapsed (dt/format-duration (tpoint))}
(catch Throwable cause
(l/dbg :hint "populate:error" :cause cause))
(finally
(let [elapsed (dt/format-duration (tpoint))]
(l/dbg :hint "populate:end"
:elapsed elapsed))))))

31
backend/src/app/srepl/helpers.clj
View File

@@ -13,7 +13,6 @@
[app.common.files.migrations :as fmg]
[app.common.files.validate :as cfv]
[app.db :as db]
[app.features.components-v2 :as feat.comp-v2]
[app.main :as main]
[app.rpc.commands.files :as files]
[app.rpc.commands.files-snapshot :as fsnap]
@@ -62,6 +61,27 @@
{:id id})
team))
(def ^:private sql:get-and-lock-team-files
"SELECT f.id
FROM file AS f
JOIN project AS p ON (p.id = f.project_id)
WHERE p.team_id = ?
AND p.deleted_at IS NULL
AND f.deleted_at IS NULL
FOR UPDATE")
(defn get-team
[conn team-id]
(-> (db/get conn :team {:id team-id}
{::db/remove-deleted false
::db/check-deleted false})
(update :features db/decode-pgarray #{})))
(defn get-and-lock-team-files
[conn team-id]
(transduce (map :id) conj []
(db/plan conn [sql:get-and-lock-team-files team-id])))
(defn reset-file-data!
"Hardcode replace of the data of one file."
[system id data]
@@ -96,7 +116,7 @@
(defn take-team-snapshot!
[system team-id label]
(let [conn (db/get-connection system)]
(->> (feat.comp-v2/get-and-lock-team-files conn team-id)
(->> (get-and-lock-team-files conn team-id)
(reduce (fn [result file-id]
(let [file (fsnap/get-file-snapshots system file-id)]
(fsnap/create-file-snapshot! system file
@@ -108,19 +128,16 @@
(defn restore-team-snapshot!
[system team-id label]
(let [conn (db/get-connection system)
ids (->> (feat.comp-v2/get-and-lock-team-files conn team-id)
ids (->> (get-and-lock-team-files conn team-id)
(into #{}))
snap (search-file-snapshots conn ids label)
ids' (into #{} (map :file-id) snap)
team (-> (feat.comp-v2/get-team conn team-id)
(update :features disj "components/v2"))]
ids' (into #{} (map :file-id) snap)]
(when (not= ids ids')
(throw (RuntimeException. "no uniform snapshot available")))
(feat.comp-v2/update-team! conn team)
(reduce (fn [result {:keys [file-id id]}]
(fsnap/restore-file-snapshot! system file-id id)
(inc result))

3
backend/src/app/srepl/main.clj
View File

@@ -22,7 +22,6 @@
[app.config :as cf]
[app.db :as db]
[app.db.sql :as-alias sql]
[app.features.components-v2 :as feat.comp-v2]
[app.features.fdata :as feat.fdata]
[app.loggers.audit :as audit]
[app.main :as main]
@@ -439,7 +438,7 @@
(binding [h/*system* system
db/*conn* (db/get-connection system)]
(->> (feat.comp-v2/get-and-lock-team-files conn team-id)
(->> (h/get-and-lock-team-files conn team-id)
(reduce (fn [result file-id]
(if (h/process-file! system file-id update-fn opts)
(inc result)