johanels/penpot
1
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-12-11 22:14:05 +01:00
Code Issues Actions 9 Packages Projects Releases Wiki Activity

♻️ Make exporter upload resources using backend management api

Browse Source 
Instead of custon shared fs approach. This commit fixes the main
scalability issue of exporter removing the need of shared-fs
for make it work with multiple instances.
This commit is contained in:
Andrey Antukh
2025-11-18 15:17:06 +01:00
parent 4fddf3d986
commit 81632a03dd
16 changed files with 265 additions and 218 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
backend/src/app/rpc.clj
View File

@@ -295,7 +295,8 @@
[cfg] [cfg]
(let [cfg (assoc cfg ::type "management" ::metrics-id :rpc-management-timing)] (let [cfg (assoc cfg ::type "management" ::metrics-id :rpc-management-timing)]
(->> (sv/scan-ns (->> (sv/scan-ns
'app.rpc.management.subscription) 'app.rpc.management.subscription
'app.rpc.management.exporter)
(map (partial process-method cfg "management" wrap-management)) (map (partial process-method cfg "management" wrap-management))
(into {})))) (into {}))))

49
backend/src/app/rpc/management/exporter.clj Normal file
View File

@@ -0,0 +1,49 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) KALEIDOS INC
(ns app.rpc.management.exporter
(:require
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.uri :as u]
[app.config :as cf]
[app.media :refer [schema:upload]]
[app.rpc :as-alias rpc]
[app.rpc.doc :as doc]
[app.storage :as sto]
[app.util.services :as sv]))
;; ---- RPC METHOD: UPLOAD-TEMPFILE
(def ^:private
schema:upload-tempfile-params
[:map {:title "upload-templfile-params"}
[:content schema:upload]])
(def ^:private
schema:upload-tempfile-result
[:map {:title "upload-templfile-result"}])
(sv/defmethod ::upload-tempfile
{::doc/added "2.12"
::sm/params schema:upload-tempfile-params
::sm/result schema:upload-tempfile-result}
[cfg {:keys [::rpc/profile-id content]}]
(let [storage (sto/resolve cfg)
hash (sto/calculate-hash (:path content))
data (-> (sto/content (:path content))
(sto/wrap-with-hash hash))
content {::sto/content data
::sto/deduplicate? true
::sto/touched-at (ct/in-future {:minutes 10})
:profile-id profile-id
:content-type (:mtype content)
:bucket "tempfile"}
object (sto/put-object! storage content)]
{:id (:id object)
:uri (-> (cf/get :public-uri)
(u/join "/assets/by-id/")
(u/join (str (:id object))))}))

6
exporter/scripts/run Executable file
View File

@@ -0,0 +1,6 @@
#!/usr/bin/env bash
SCRIPT_DIR=$(dirname $0);
source $SCRIPT_DIR/../../backend/scripts/_env;
exec node target/app.js

6
exporter/scripts/wait-and-start.sh
View File

@@ -1,6 +1,10 @@
#!/usr/bin/env bash #!/usr/bin/env bash
SCRIPT_DIR=$(dirname $0);
source $SCRIPT_DIR/../../backend/scripts/_env;
bb -i '(babashka.wait/wait-for-port "localhost" 9630)'; bb -i '(babashka.wait/wait-for-port "localhost" 9630)';
bb -i '(babashka.wait/wait-for-path "target/app.js")'; bb -i '(babashka.wait/wait-for-path "target/app.js")';
sleep 2; sleep 2;
node target/app.js
exec node target/app.js

18
exporter/src/app/config.cljs
View File

@@ -7,7 +7,9 @@
(ns app.config (ns app.config
(:refer-clojure :exclude [get]) (:refer-clojure :exclude [get])
(:require (:require
["process" :as process] ["node:buffer" :as buffer]
["node:crypto" :as crypto]
["node:process" :as process]
[app.common.data :as d] [app.common.data :as d]
[app.common.flags :as flags] [app.common.flags :as flags]
[app.common.schema :as sm] [app.common.schema :as sm]
@@ -21,13 +23,14 @@
:host "localhost" :host "localhost"
:http-server-port 6061 :http-server-port 6061
:http-server-host "0.0.0.0" :http-server-host "0.0.0.0"
:tempdir "/tmp/penpot-exporter" :tempdir "/tmp/penpot"
:redis-uri "redis://redis/0"}) :redis-uri "redis://redis/0"})
(def ^:private (def ^:private schema:config
schema:config
[:map {:title "config"} [:map {:title "config"}
[:secret-key :string]
[:public-uri {:optional true} ::sm/uri] [:public-uri {:optional true} ::sm/uri]
[:management-api-key {:optional true} :string]
[:host {:optional true} :string] [:host {:optional true} :string]
[:tenant {:optional true} :string] [:tenant {:optional true} :string]
[:flags {:optional true} [::sm/set :keyword]] [:flags {:optional true} [::sm/set :keyword]]
@@ -93,3 +96,10 @@
(c/get config key)) (c/get config key))
([key default] ([key default]
(c/get config key default))) (c/get config key default)))
(def management-key
(or (c/get config :management-api-key)
(let [secret-key (c/get config :secret-key)
derived-key (crypto/hkdfSync "blake2b512" secret-key, "management" "" 32)]
(-> (.from buffer/Buffer derived-key)
(.toString "base64url")))))

5
exporter/src/app/handlers.cljs
View File

@@ -12,7 +12,6 @@
[app.common.spec :as us] [app.common.spec :as us]
[app.handlers.export-frames :as export-frames] [app.handlers.export-frames :as export-frames]
[app.handlers.export-shapes :as export-shapes] [app.handlers.export-shapes :as export-shapes]
[app.handlers.resources :as resources]
[app.util.transit :as t] [app.util.transit :as t]
[clojure.spec.alpha :as s] [clojure.spec.alpha :as s]
[cuerdas.core :as str])) [cuerdas.core :as str]))
@@ -54,7 +53,7 @@
:else :else
(let [data {:type :server-error (let [data {:type :server-error
:code type :code code
:hint (ex-message error) :hint (ex-message error)
:data data}] :data data}]
(l/error :hint "unexpected internal error" :cause error) (l/error :hint "unexpected internal error" :cause error)
@@ -71,7 +70,6 @@
(defmethod command-spec :export-shapes [_] ::export-shapes/params) (defmethod command-spec :export-shapes [_] ::export-shapes/params)
(defmethod command-spec :export-frames [_] ::export-frames/params) (defmethod command-spec :export-frames [_] ::export-frames/params)
(defmethod command-spec :get-resource [_] (s/keys :req-un [::id]))
(s/def ::params (s/def ::params
(s/and (s/keys :req-un [::cmd] (s/and (s/keys :req-un [::cmd]
@@ -83,7 +81,6 @@
(let [{:keys [cmd] :as params} (us/conform ::params params)] (let [{:keys [cmd] :as params} (us/conform ::params params)]
(l/debug :hint "process-request" :cmd cmd) (l/debug :hint "process-request" :cmd cmd)
(case cmd (case cmd
:get-resource (resources/handler exchange)
:export-shapes (export-shapes/handler exchange params) :export-shapes (export-shapes/handler exchange params)
:export-frames (export-frames/handler exchange params) :export-frames (export-frames/handler exchange params)
(ex/raise :type :internal (ex/raise :type :internal

92
exporter/src/app/handlers/export_frames.cljs
View File

@@ -43,90 +43,78 @@
;; datastructure preparation uses it for creating the groups. ;; datastructure preparation uses it for creating the groups.
(let [exports (-> (map #(assoc % :type :pdf :scale 1 :suffix "") exports) (let [exports (-> (map #(assoc % :type :pdf :scale 1 :suffix "") exports)
(prepare-exports auth-token))] (prepare-exports auth-token))]
(handle-export exchange (assoc params :exports exports)))) (handle-export exchange (assoc params :exports exports))))
(defn handle-export (defn handle-export
[exchange {:keys [exports wait name profile-id] :as params}] [{:keys [:request/auth-token] :as exchange} {:keys [exports name profile-id] :as params}]
(let [total (count exports) (let [topic (str profile-id)
topic (str profile-id) file-id (-> exports first :file-id)
resource (rsc/create :pdf (or name (-> exports first :name)))
on-progress (fn [{:keys [done]}] resource
(when-not wait (rsc/create :pdf (or name (-> exports first :name)))
on-progress
(fn [done]
(let [data {:type :export-update (let [data {:type :export-update
:resource-id (:id resource) :resource-id (:id resource)
:name (:name resource)
:filename (:filename resource)
:status "running" :status "running"
:total total
:done done}] :done done}]
(redis/pub! topic data)))) (redis/pub! topic data)))
on-complete (fn [] on-complete
(when-not wait (fn [resource]
(let [data {:type :export-update (let [data {:type :export-update
:resource-id (:id resource) :resource-id (:id resource)
:resource-uri (:uri resource)
:name (:name resource) :name (:name resource)
:filename (:filename resource) :filename (:filename resource)
:mtype (:mtype resource)
:status "ended"}] :status "ended"}]
(redis/pub! topic data)))) (redis/pub! topic data)))
on-error (fn [cause] on-error
(fn [cause]
(l/error :hint "unexpected error on frames exportation" :cause cause) (l/error :hint "unexpected error on frames exportation" :cause cause)
(if wait
(p/rejected cause)
(let [data {:type :export-update (let [data {:type :export-update
:resource-id (:id resource) :resource-id (:id resource)
:name (:name resource) :name (:name resource)
:filename (:filename resource) :filename (:filename resource)
:status "error" :status "error"
:cause (ex-message cause)}] :cause (ex-message cause)}]
(redis/pub! topic data)))) (redis/pub! topic data)))
proc (create-pdf :resource resource result-cache
:exports exports (atom [])
:on-progress on-progress
:on-complete on-complete
:on-error on-error)]
(if wait
(p/then proc #(assoc exchange :response/body (dissoc % :path)))
(assoc exchange :response/body (dissoc resource :path)))))
(defn create-pdf
[& {:keys [resource exports on-progress on-complete on-error]
:or {on-progress (constantly nil)
on-complete (constantly nil)
on-error p/rejected}}]
(let [file-id (-> exports first :file-id)
result (atom [])
on-object on-object
(fn [{:keys [path] :as object}] (fn [{:keys [path] :as object}]
(let [res (swap! result conj path)] (let [res (swap! result-cache conj path)]
(on-progress {:done (count res)})))] (on-progress (count res))))
(-> (p/loop [exports (seq exports)] procs
(when-let [export (first exports)] (->> (seq exports)
(p/do (map #(rd/render % on-object)))]
(rd/render export on-object)
(p/recur (rest exports)))))
(p/then (fn [_] (deref result))) (->> (p/all procs)
(p/then (partial join-pdf file-id)) (p/fmap (fn [] @result-cache))
(p/then (partial move-file resource)) (p/mcat (partial join-pdf file-id))
(p/then (constantly resource)) (p/mcat (partial move-file resource))
(p/then (fn [resource] (p/fmap (constantly resource))
(-> (sh/stat (:path resource)) (p/mcat (partial rsc/upload-resource auth-token))
(p/then #(merge resource %))))) (p/mcat (fn [resource]
(p/catch on-error) (->> (sh/stat (:path resource))
(p/finally (fn [_ cause] (p/fmap #(merge resource %)))))
(p/merr on-error)
(p/fnly (fn [resource cause]
(when-not cause (when-not cause
(on-complete))))))) (on-complete resource)))))
(assoc exchange :response/body (dissoc resource :path))))
(defn- join-pdf (defn- join-pdf
[file-id paths] [file-id paths]
(p/let [prefix (str/concat "penpot.tmp.pdfunite." file-id ".") (p/let [prefix (str/concat "penpot.pdfunite." file-id ".")
path (sh/tempfile :prefix prefix :suffix ".pdf")] path (sh/tempfile :prefix prefix :suffix ".pdf")]
(sh/run-cmd! (str "pdfunite " (str/join " " paths) " " path)) (sh/run-cmd! (str "pdfunite " (str/join " " paths) " " path))
path)) path))

92
exporter/src/app/handlers/export_shapes.cljs
View File

@@ -60,46 +60,26 @@
(handle-multiple-export exchange (assoc params :exports exports))))) (handle-multiple-export exchange (assoc params :exports exports)))))
(defn- handle-single-export (defn- handle-single-export
[exchange {:keys [export wait profile-id name skip-children] :as params}] [{:keys [:request/auth-token] :as exchange} {:keys [export name skip-children] :as params}]
(let [topic (str profile-id) (let [resource (rsc/create (:type export) (or name (:name export)))
resource (rsc/create (:type export) (or name (:name export))) export (assoc export :skip-children skip-children)]
on-progress (fn [{:keys [path] :as object}] (->> (rd/render export
(p/do (fn [{:keys [path] :as object}]
;; Move the generated path to the resource (sh/move! path (:path resource))))
;; path destination. (p/fmap (constantly resource))
(sh/move! path (:path resource)) (p/mcat (partial rsc/upload-resource auth-token))
(p/fmap (fn [resource]
(when-not wait (dissoc resource :path)))
(redis/pub! topic {:type :export-update (p/fmap (fn [resource]
:resource-id (:id resource) (assoc exchange :response/body resource)))
:status "running" (p/merr (fn [cause]
:total 1
:done 1})
(redis/pub! topic {:type :export-update
:resource-id (:id resource)
:filename (:filename resource)
:name (:name resource)
:status "ended"}))))
on-error (fn [cause]
(l/error :hint "unexpected error on export multiple" (l/error :hint "unexpected error on export multiple"
:cause cause) :cause cause)
(if wait (p/rejected cause))))))
(p/rejected cause)
(redis/pub! topic {:type :export-update
:resource-id (:id resource)
:status "error"
:cause (ex-message cause)})))
export (assoc export :skip-children skip-children)
proc (-> (rd/render export on-progress)
(p/then (constantly resource))
(p/catch on-error))]
(if wait
(p/then proc #(assoc exchange :response/body (dissoc % :path)))
(assoc exchange :response/body (dissoc resource :path)))))
(defn- handle-multiple-export (defn- handle-multiple-export
[exchange {:keys [exports wait profile-id name skip-children] :as params}] [{:keys [:request/auth-token] :as exchange} {:keys [exports wait profile-id name] :as params}]
(let [resource (rsc/create :zip (or name (-> exports first :name))) (let [resource (rsc/create :zip (or name (-> exports first :name)))
total (count exports) total (count exports)
topic (str profile-id) topic (str profile-id)
@@ -113,15 +93,6 @@
:done done}] :done done}]
(redis/pub! topic data)))) (redis/pub! topic data))))
on-complete (fn []
(when-not wait
(let [data {:type :export-update
:name (:name resource)
:filename (:filename resource)
:resource-id (:id resource)
:status "ended"}]
(redis/pub! topic data))))
on-error (fn [cause] on-error (fn [cause]
(l/error :hint "unexpected error on multiple exportation" :cause cause) (l/error :hint "unexpected error on multiple exportation" :cause cause)
(if wait (if wait
@@ -132,30 +103,35 @@
:cause (ex-message cause)}))) :cause (ex-message cause)})))
zip (rsc/create-zip :resource resource zip (rsc/create-zip :resource resource
:on-complete on-complete
:on-error on-error :on-error on-error
:on-progress on-progress) :on-progress on-progress)
append (fn [{:keys [filename path] :as object}] append (fn [{:keys [filename path] :as resource}]
(rsc/add-to-zip! zip path (str/replace filename sanitize-file-regex "_"))) (rsc/add-to-zip! zip path (str/replace filename sanitize-file-regex "_")))
proc (-> (p/do proc (->> exports
(p/loop [exports (seq exports)] (map (fn [export] (rd/render export append)))
(when-let [export (some-> (first exports) (p/all)
(assoc :skip-children skip-children))] (p/fnly (fn [_] (.finalize zip)))
(p/do (p/fmap (constantly resource))
(rd/render export append) (p/mcat (partial rsc/upload-resource auth-token))
(p/recur (rest exports))))) (p/fmap (fn [resource]
(.finalize zip)) (let [data {:type :export-update
(p/then (constantly resource)) :name (:name resource)
(p/catch on-error))] :filename (:filename resource)
:resource-id (:id resource)
:resource-uri (:uri resource)
:mtype (:mtype resource)
:status "ended"}]
(p/do (redis/pub! topic data)
(assoc exchange :response/body resource)))))
(p/merr on-error))]
(if wait (if wait
(p/then proc #(assoc exchange :response/body (dissoc % :path))) (p/then proc #(assoc exchange :response/body (dissoc % :path)))
(assoc exchange :response/body (dissoc resource :path))))) (assoc exchange :response/body (dissoc resource :path)))))
(defn- assoc-file-name (defn- assoc-file-name
"A transducer that assocs a candidate filename and avoid duplicates." "A transducer that assocs a candidate filename and avoid duplicates"
[] []
(letfn [(find-candidate [params used] (letfn [(find-candidate [params used]
(loop [index 0] (loop [index 0]

66
exporter/src/app/handlers/resources.cljs
View File

@@ -9,9 +9,13 @@
(:require (:require
["archiver$default" :as arc] ["archiver$default" :as arc]
["node:fs" :as fs] ["node:fs" :as fs]
["node:fs/promises" :as fsp]
["node:path" :as path] ["node:path" :as path]
[app.common.exceptions :as ex] [app.common.exceptions :as ex]
[app.common.transit :as t]
[app.common.uri :as u]
[app.common.uuid :as uuid] [app.common.uuid :as uuid]
[app.config :as cf]
[app.util.mime :as mime] [app.util.mime :as mime]
[app.util.shell :as sh] [app.util.shell :as sh]
[cljs.core :as c] [cljs.core :as c]
@@ -25,44 +29,20 @@
(defn create (defn create
"Generates ephimeral resource object." "Generates ephimeral resource object."
[type name] [type name]
(let [task-id (uuid/next)] (let [task-id (uuid/next)
{:path (get-path type task-id) path (-> (get-path type task-id)
(sh/schedule-deletion))]
{:path path
:mtype (mime/get type) :mtype (mime/get type)
:name name :name name
:filename (str/concat name (mime/get-extension type)) :filename (str/concat name (mime/get-extension type))
:id (str/concat (c/name type) "." task-id)})) :id task-id}))
(defn- lookup
[id]
(p/let [[type task-id] (str/split id "." 2)
path (get-path type task-id)
mtype (mime/get (keyword type))
stat (sh/stat path)]
(when-not stat
(ex/raise :type :not-found))
{:stream (fs/createReadStream path)
:headers {"content-type" mtype
"content-length" (:size stat)}}))
(defn handler
[{:keys [:request/params] :as exchange}]
(when-not (contains? params :id)
(ex/raise :type :validation
:code :missing-id))
(-> (lookup (get params :id))
(p/then (fn [{:keys [stream headers] :as resource}]
(-> exchange
(assoc :response/status 200)
(assoc :response/body stream)
(assoc :response/headers headers))))))
(defn create-zip (defn create-zip
[& {:keys [resource on-complete on-progress on-error]}] [& {:keys [resource on-complete on-progress on-error]}]
(let [^js zip (arc/create "zip") (let [^js zip (arc/create "zip")
^js out (fs/createWriteStream (:path resource)) ^js out (fs/createWriteStream (:path resource))
on-complete (or on-complete (constantly nil))
progress (atom 0)] progress (atom 0)]
(.on zip "error" on-error) (.on zip "error" on-error)
(.on zip "end" on-complete) (.on zip "end" on-complete)
@@ -80,3 +60,29 @@
(defn close-zip! (defn close-zip!
[zip] [zip]
(.finalize ^js zip)) (.finalize ^js zip))
(defn upload-resource
[auth-token resource]
(->> (fsp/readFile (:path resource))
(p/fmap (fn [buffer]
(new js/Blob #js [buffer] #js {:type (:mtype resource)})))
(p/mcat (fn [blob]
(let [fdata (new js/FormData)
uri (-> (cf/get :public-uri)
(u/ensure-path-slash)
(u/join "api/management/methods/upload-tempfile")
(str))]
(.append fdata "content" blob (:filename resource))
(js/fetch uri #js {:headers #js {"X-Shared-Key" cf/management-key
"Authorization" (str "Bearer " auth-token)}
:method "POST"
:body fdata}))))
(p/mcat (fn [response]
(if (not= (.-status response) 200)
(ex/raise :type :internal
:code :unable-to-upload-resource
:response-status (.-status response))
(.text response))))
(p/fmap t/decode-str)
(p/fmap (fn [result]
(merge resource (dissoc result :id))))))

5
exporter/src/app/renderer/bitmap.cljs
View File

@@ -29,13 +29,13 @@
:userAgent bw/default-user-agent}) :userAgent bw/default-user-agent})
(render-object [page {:keys [id] :as object}] (render-object [page {:keys [id] :as object}]
(p/let [path (sh/tempfile :prefix "penpot.tmp.render.bitmap." :suffix (mime/get-extension type)) (p/let [path (sh/tempfile :prefix "penpot.tmp.bitmap." :suffix (mime/get-extension type))
node (bw/select page (str/concat "#screenshot-" id))] node (bw/select page (str/concat "#screenshot-" id))]
(bw/wait-for node) (bw/wait-for node)
(case type (case type
:png (bw/screenshot node {:omit-background? true :type type :path path}) :png (bw/screenshot node {:omit-background? true :type type :path path})
:jpeg (bw/screenshot node {:omit-background? false :type type :path path}) :jpeg (bw/screenshot node {:omit-background? false :type type :path path})
:webp (p/let [png-path (sh/tempfile :prefix "penpot.tmp.render.bitmap." :suffix ".png")] :webp (p/let [png-path (sh/tempfile :prefix "penpot.tmp.bitmap." :suffix ".png")]
;; playwright only supports jpg and png, we need to convert it afterwards ;; playwright only supports jpg and png, we need to convert it afterwards
(bw/screenshot node {:omit-background? true :type :png :path png-path}) (bw/screenshot node {:omit-background? true :type :png :path png-path})
(sh/run-cmd! (str "convert " png-path " -quality 100 WEBP:" path)))) (sh/run-cmd! (str "convert " png-path " -quality 100 WEBP:" path))))
@@ -52,6 +52,7 @@
;; take the screnshot of requested objects, one by one ;; take the screnshot of requested objects, one by one
(p/run (partial render-object page) objects) (p/run (partial render-object page) objects)
nil))] nil))]
(p/let [params {:file-id file-id (p/let [params {:file-id file-id
:page-id page-id :page-id page-id
:share-id share-id :share-id share-id

2
exporter/src/app/renderer/pdf.cljs
View File

@@ -40,7 +40,7 @@
(render-object [page base-uri {:keys [id] :as object}] (render-object [page base-uri {:keys [id] :as object}]
(p/let [uri (prepare-uri base-uri id) (p/let [uri (prepare-uri base-uri id)
path (sh/tempfile :prefix "penpot.tmp.render.pdf." :suffix (mime/get-extension type))] path (sh/tempfile :prefix "penpot.tmp.pdf." :suffix (mime/get-extension type))]
(l/info :uri uri) (l/info :uri uri)
(bw/nav! page uri) (bw/nav! page uri)
(p/let [dom (bw/select page (dm/str "#screenshot-" id))] (p/let [dom (bw/select page (dm/str "#screenshot-" id))]

49
exporter/src/app/util/shell.cljs
View File

@@ -7,11 +7,12 @@
(ns app.util.shell (ns app.util.shell
"Shell & FS utilities." "Shell & FS utilities."
(:require (:require
["child_process" :as proc] ["node:child_process" :as proc]
["fs" :as fs] ["node:fs" :as fs]
["path" :as path] ["node:path" :as path]
[app.common.exceptions :as ex] [app.common.exceptions :as ex]
[app.common.logging :as l] [app.common.logging :as l]
[app.common.time :as ct]
[app.common.uuid :as uuid] [app.common.uuid :as uuid]
[app.config :as cf] [app.config :as cf]
[cuerdas.core :as str] [cuerdas.core :as str]
@@ -19,7 +20,8 @@
(l/set-level! :trace) (l/set-level! :trace)
(def tempfile-minage (* 1000 60 60 1)) ;; 1h (def ^:const default-deletion-delay
(* 60 60 1)) ;; 1h
(def tmpdir (def tmpdir
(let [path (cf/get :tempdir)] (let [path (cf/get :tempdir)]
@@ -28,16 +30,28 @@
(fs/mkdirSync path #js {:recursive true})) (fs/mkdirSync path #js {:recursive true}))
path)) path))
(defn- schedule-deletion! (defn schedule-deletion
[path] ([path] (schedule-deletion path default-deletion-delay))
(letfn [(remote-tempfile [] ([path delay]
(let [remove-path
(fn []
(try
(when (fs/existsSync path) (when (fs/existsSync path)
(l/trace :hint "permanently remove tempfile" :path path) (fs/rmSync path #js {:recursive true})
(fs/rmSync path #js {:recursive true})))] (l/trc :hint "tempfile permanently deleted" :path path))
(l/trace :hint "schedule tempfile deletion" (catch :default cause
(l/err :hint "error on deleting temporal file"
:path path :path path
:scheduled-at (.. (js/Date. (+ (js/Date.now) tempfile-minage)) toString)) :cause cause))))
(js/setTimeout remote-tempfile tempfile-minage))) scheduled-at
(-> (ct/now) (ct/plus #js {:seconds delay}))]
(l/trc :hint "schedule tempfile deletion"
:path path
:scheduled-at (ct/format-inst scheduled-at))
(js/setTimeout remove-path (* delay 1000))
path)))
(defn tempfile (defn tempfile
[& {:keys [prefix suffix] [& {:keys [prefix suffix]
@@ -48,9 +62,7 @@
(let [path (path/join tmpdir (str/concat prefix (uuid/next) "-" i suffix))] (let [path (path/join tmpdir (str/concat prefix (uuid/next) "-" i suffix))]
(if (fs/existsSync path) (if (fs/existsSync path)
(recur (inc i)) (recur (inc i))
(do (schedule-deletion path)))
(schedule-deletion! path)
path)))
(ex/raise :type :internal (ex/raise :type :internal
:code :unable-to-locate-temporal-file :code :unable-to-locate-temporal-file
:hint "unable to find a tempfile candidate")))) :hint "unable to find a tempfile candidate"))))
@@ -61,11 +73,12 @@
(defn stat (defn stat
[path] [path]
(-> (.stat fs/promises path) (->> (.stat fs/promises path)
(p/then (fn [data] (p/fmap (fn [data]
{:created-at (inst-ms (.-ctime ^js data)) {:created-at (inst-ms (.-ctime ^js data))
:size (.-size data)})) :size (.-size data)}))
(p/catch (constantly nil)))) (p/merr (fn [_cause]
(p/resolved nil)))))
(defn rmdir! (defn rmdir!
[path] [path]

1
frontend/src/app/config.cljs
View File

@@ -104,7 +104,6 @@
(def plugins-whitelist (into #{} (obj/get global "penpotPluginsWhitelist" []))) (def plugins-whitelist (into #{} (obj/get global "penpotPluginsWhitelist" [])))
(def templates-uri (obj/get global "penpotTemplatesUri" "https://penpot.github.io/penpot-files/")) (def templates-uri (obj/get global "penpotTemplatesUri" "https://penpot.github.io/penpot-files/"))
;; We set the current parsed flags under common for make ;; We set the current parsed flags under common for make
;; it available for common code without the need to pass ;; it available for common code without the need to pass
;; the flags all arround on parameters. ;; the flags all arround on parameters.

23
frontend/src/app/main/data/exports/assets.cljs
View File

@@ -99,16 +99,18 @@
(watch [_ state _] (watch [_ state _]
(let [file-id (:current-file-id state) (let [file-id (:current-file-id state)
page-id (:current-page-id state) page-id (:current-page-id state)
exports (for [frame frames] exports (mapv (fn [frame]
{:enabled true {:enabled true
:page-id page-id :page-id page-id
:file-id file-id :file-id file-id
:object-id (:id frame) :object-id (:id frame)
:shape frame :shape frame
:name (:name frame)})] :name (:name frame)})
frames)]
(rx/of (modal/show :export-frames (rx/of (modal/show :export-frames
{:exports (vec exports) :origin "workspace:menu"})))))) {:exports exports
:origin "workspace:menu"}))))))
(defn- initialize-export-status (defn- initialize-export-status
[exports cmd resource] [exports cmd resource]
@@ -127,7 +129,7 @@
:cmd cmd})))) :cmd cmd}))))
(defn- update-export-status (defn- update-export-status
[{:keys [done status resource-id filename] :as data}] [{:keys [done status resource-uri filename mtype] :as data}]
(ptk/reify ::update-export-status (ptk/reify ::update-export-status
ptk/UpdateEvent ptk/UpdateEvent
(update [_ state] (update [_ state]
@@ -146,9 +148,7 @@
ptk/WatchEvent ptk/WatchEvent
(watch [_ _ _] (watch [_ _ _]
(when (= status "ended") (when (= status "ended")
(->> (rp/cmd! :export {:cmd :get-resource :blob? true :id resource-id}) (dom/trigger-download-uri filename mtype resource-uri)))))
(rx/delay 500)
(rx/map #(dom/trigger-download filename %)))))))
(defn request-simple-export (defn request-simple-export
[{:keys [export]}] [{:keys [export]}]
@@ -174,17 +174,14 @@
(rx/timeout 400 (rx/empty))) (rx/timeout 400 (rx/empty)))
(->> (rp/cmd! :export params) (->> (rp/cmd! :export params)
(rx/mapcat (fn [{:keys [id filename]}] (rx/map (fn [{:keys [filename mtype uri]}]
(->> (rp/cmd! :export {:cmd :get-resource :blob? true :id id}) (dom/trigger-download-uri filename mtype uri)
(rx/map (fn [data] (clear-export-state uuid/zero)))
(dom/trigger-download filename data)
(clear-export-state uuid/zero))))))
(rx/catch (fn [cause] (rx/catch (fn [cause]
(rx/concat (rx/concat
(rx/of (clear-export-state uuid/zero)) (rx/of (clear-export-state uuid/zero))
(rx/throw cause)))))))))) (rx/throw cause))))))))))
(defn request-multiple-export (defn request-multiple-export
[{:keys [exports cmd] [{:keys [exports cmd]
:or {cmd :export-shapes} :or {cmd :export-shapes}

7
frontend/src/app/main/ui/exports/assets.cljs
View File

@@ -34,7 +34,8 @@
(def ^:private close-icon (def ^:private close-icon
(deprecated-icon/icon-xref :close (stl/css :close-icon))) (deprecated-icon/icon-xref :close (stl/css :close-icon)))
(mf/defc export-multiple-dialog (mf/defc export-multiple-dialog*
{::mf/private true}
[{:keys [exports title cmd no-selection origin]}] [{:keys [exports title cmd no-selection origin]}]
(let [lstate (mf/deref refs/export) (let [lstate (mf/deref refs/export)
in-progress? (:in-progress lstate) in-progress? (:in-progress lstate)
@@ -186,7 +187,7 @@
::mf/register-as :export-shapes} ::mf/register-as :export-shapes}
[{:keys [exports origin]}] [{:keys [exports origin]}]
(let [title (tr "dashboard.export-shapes.title")] (let [title (tr "dashboard.export-shapes.title")]
[:& export-multiple-dialog [:> export-multiple-dialog*
{:exports exports {:exports exports
:title title :title title
:cmd :export-shapes :cmd :export-shapes
@@ -198,7 +199,7 @@
::mf/register-as :export-frames} ::mf/register-as :export-frames}
[{:keys [exports origin]}] [{:keys [exports origin]}]
(let [title (tr "dashboard.export-frames.title")] (let [title (tr "dashboard.export-frames.title")]
[:& export-multiple-dialog [:> export-multiple-dialog*
{:exports exports {:exports exports
:title title :title title
:cmd :export-frames :cmd :export-frames

3
frontend/src/app/main/ui/workspace/sidebar/options/menus/exports.cljs
View File

@@ -101,8 +101,7 @@
:name sname :name sname
:object-id (:id (first shapes-with-exports))} :object-id (:id (first shapes-with-exports))}
full-export (merge export defaults)] full-export (merge export defaults)]
(st/emit! (st/emit! (de/request-simple-export {:export full-export})
(de/request-simple-export {:export full-export})
(de/export-shapes-event [full-export] "workspace:sidebar"))) (de/export-shapes-event [full-export] "workspace:sidebar")))
(st/emit! (st/emit!
(de/show-workspace-export-dialog {:selected (reverse ids) :origin "workspace:sidebar"}))) (de/show-workspace-export-dialog {:selected (reverse ids) :origin "workspace:sidebar"})))