From f71f491590fd1e769e9e78331c6fa32d1a524a14 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 20 Nov 2025 12:18:35 +0100 Subject: [PATCH 1/4] :bug: Fix incorrect bearer token decoding --- backend/src/app/http/middleware.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/app/http/middleware.clj b/backend/src/app/http/middleware.clj index 648950c30b..1e8ac15039 100644 --- a/backend/src/app/http/middleware.clj +++ b/backend/src/app/http/middleware.clj @@ -274,7 +274,7 @@ (fn [request] (if-let [{:keys [type token] :as auth} (get-token request)] (let [decode-fn (get decoders type)] - (if (= type :cookie) + (if (or (= type :cookie) (= type :bearer)) (let [metadata (tokens/decode-header token)] ;; NOTE: we only proceed to decode claims on new ;; cookie tokens. The old cookies dont need to be From 57aa9a585b086c9b76c9ab6837699327886e10eb Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 17 Nov 2025 09:53:42 +0100 Subject: [PATCH 2/4] :wrench: Add explicit network alias for minio on devenv --- docker/devenv/docker-compose.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker/devenv/docker-compose.yaml b/docker/devenv/docker-compose.yaml index b546301643..33e2036ca8 100644 --- a/docker/devenv/docker-compose.yaml +++ b/docker/devenv/docker-compose.yaml @@ -82,6 +82,11 @@ services: - 9000:9000 - 9001:9001 + networks: + default: + aliases: + - minio + postgres: image: postgres:16.8 command: postgres -c config_file=/etc/postgresql.conf From 4fddf3d98635d695fafef3aea7688c7dcd1758e5 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Tue, 18 Nov 2025 15:06:58 +0100 Subject: [PATCH 3/4] :recycle: Make management key derivable from secret key Still preserves the ability to set management --- backend/scripts/_env | 3 +-- backend/src/app/config.clj | 3 +-- backend/src/app/http/management.clj | 34 +++++++++++++++------------ backend/src/app/http/middleware.clj | 15 +++++++----- backend/src/app/rpc.clj | 10 ++++---- backend/src/app/rpc/commands/demo.clj | 2 +- backend/src/app/rpc/cond.clj | 5 ++-- backend/src/app/setup.clj | 8 +++---- backend/src/app/setup/keys.clj | 4 ++-- 9 files changed, 44 insertions(+), 40 deletions(-) diff --git a/backend/scripts/_env b/backend/scripts/_env index 2709853db6..1e4408efe8 100644 --- a/backend/scripts/_env +++ b/backend/scripts/_env @@ -1,7 +1,6 @@ #!/usr/bin/env bash - -export PENPOT_MANAGEMENT_API_SHARED_KEY=super-secret-management-api-key +export PENPOT_MANAGEMENT_API_KEY=super-secret-management-api-key export PENPOT_SECRET_KEY=super-secret-devenv-key export PENPOT_HOST=devenv diff --git a/backend/src/app/config.clj b/backend/src/app/config.clj index 1fc3b0539d..de030f2e11 100644 --- a/backend/src/app/config.clj +++ b/backend/src/app/config.clj @@ -5,7 +5,6 @@ ;; Copyright (c) KALEIDOS INC (ns app.config - "A configuration management." (:refer-clojure :exclude [get]) (:require [app.common.data :as d] @@ -103,7 +102,7 @@ [:http-server-io-threads {:optional true} ::sm/int] [:http-server-max-worker-threads {:optional true} ::sm/int] - [:management-api-shared-key {:optional true} :string] + [:management-api-key {:optional true} :string] [:telemetry-uri {:optional true} :string] [:telemetry-with-taiga {:optional true} ::sm/boolean] ;; DELETE diff --git a/backend/src/app/http/management.clj b/backend/src/app/http/management.clj index 756ca7450c..b32b7f4077 100644 --- a/backend/src/app/http/management.clj +++ b/backend/src/app/http/management.clj @@ -50,23 +50,27 @@ (db/tx-run! cfg handler request)))))}) (defmethod ig/init-key ::routes - [_ cfg] - ["" {:middleware [[mw/shared-key-auth (cf/get :management-api-shared-key)] - [default-system cfg] - [transaction]]} - ["/authenticate" - {:handler authenticate - :allowed-methods #{:post}}] + [_ {:keys [::setup/props] :as cfg}] - ["/get-customer" - {:handler get-customer - :transaction true - :allowed-methods #{:post}}] + (let [management-key (or (cf/get :management-api-key) + (get props :management-key))] - ["/update-customer" - {:handler update-customer - :allowed-methods #{:post} - :transaction true}]]) + ["" {:middleware [[mw/shared-key-auth management-key] + [default-system cfg] + [transaction]]} + ["/authenticate" + {:handler authenticate + :allowed-methods #{:post}}] + + ["/get-customer" + {:handler get-customer + :transaction true + :allowed-methods #{:post}}] + + ["/update-customer" + {:handler update-customer + :allowed-methods #{:post} + :transaction true}]])) ;; ---- HELPERS diff --git a/backend/src/app/http/middleware.clj b/backend/src/app/http/middleware.clj index 1e8ac15039..e7b4b5c953 100644 --- a/backend/src/app/http/middleware.clj +++ b/backend/src/app/http/middleware.clj @@ -16,6 +16,7 @@ [app.http.errors :as errors] [app.tokens :as tokens] [app.util.pointer-map :as pmap] + [buddy.core.codecs :as bc] [cuerdas.core :as str] [yetti.adapter :as yt] [yetti.middleware :as ymw] @@ -243,7 +244,6 @@ (handler request) {::yres/status 405}))))))}) - (defn- wrap-auth [handler decoders] (let [token-re @@ -303,11 +303,14 @@ (defn- wrap-shared-key-auth [handler shared-key] (if shared-key - (fn [request] - (let [key (yreq/get-header request "x-shared-key")] - (if (= key shared-key) - (handler request) - {::yres/status 403}))) + (let [shared-key (if (string? shared-key) + shared-key + (bc/bytes->b64-str shared-key true))] + (fn [request] + (let [key (yreq/get-header request "x-shared-key")] + (if (= key shared-key) + (handler request) + {::yres/status 403})))) (fn [_ _] {::yres/status 403}))) diff --git a/backend/src/app/rpc.clj b/backend/src/app/rpc.clj index 5b7d4a0edd..bd9b8c8af9 100644 --- a/backend/src/app/rpc.clj +++ b/backend/src/app/rpc.clj @@ -346,14 +346,16 @@ (assert (valid-methods? (::management-methods params)) "expect valid methods map")) (defmethod ig/init-key ::routes - [_ {:keys [::methods ::management-methods] :as cfg}] + [_ {:keys [::methods ::management-methods ::setup/props] :as cfg}] + + (let [public-uri (cf/get :public-uri) + management-key (or (cf/get :management-api-key) + (get props :management-key))] - (let [public-uri (cf/get :public-uri)] ["/api" - ["/management" ["/methods/:type" - {:middleware [[mw/shared-key-auth (cf/get :management-api-shared-key)] + {:middleware [[mw/shared-key-auth management-key] [session/authz cfg]] :handler (make-rpc-handler management-methods)}] diff --git a/backend/src/app/rpc/commands/demo.clj b/backend/src/app/rpc/commands/demo.clj index e609d7ec79..d4f46e750b 100644 --- a/backend/src/app/rpc/commands/demo.clj +++ b/backend/src/app/rpc/commands/demo.clj @@ -39,7 +39,7 @@ fullname (str "Demo User " sem) password (-> (bn/random-bytes 16) - (bc/bytes->b64u) + (bc/bytes->b64 true) (bc/bytes->str)) params {:email email diff --git a/backend/src/app/rpc/cond.clj b/backend/src/app/rpc/cond.clj index 168b7f0c8a..aeb7f7d99d 100644 --- a/backend/src/app/rpc/cond.clj +++ b/backend/src/app/rpc/cond.clj @@ -39,9 +39,8 @@ (defn- encode [s] (-> s - bh/blake2b-256 - bc/bytes->b64u - bc/bytes->str)) + (bh/blake2b-256) + (bc/bytes->b64-str true))) (defn- fmt-key [s] diff --git a/backend/src/app/setup.clj b/backend/src/app/setup.clj index 4debccdad6..03bed90182 100644 --- a/backend/src/app/setup.clj +++ b/backend/src/app/setup.clj @@ -22,8 +22,7 @@ (defn- generate-random-key [] (-> (bn/random-bytes 64) - (bc/bytes->b64u) - (bc/bytes->str))) + (bc/bytes->b64-str true))) (defn- get-all-props [conn] @@ -85,12 +84,11 @@ (l/warn :hint (str "using autogenerated secret-key, it will change on each restart and will invalidate " "all sessions on each restart, it is highly recommended setting up the " "PENPOT_SECRET_KEY environment variable"))) - (let [secret (or key (generate-random-key))] (-> (get-all-props conn) (assoc :secret-key secret) (assoc :tokens-key (keys/derive secret :salt "tokens")) + (assoc :management-key (keys/derive secret :salt "management")) (update :instance-id handle-instance-id conn (db/read-only? pool))))))) -;; FIXME -(sm/register! ::props :any) +(sm/register! ::props [:map-of :keyword ::sm/any]) diff --git a/backend/src/app/setup/keys.clj b/backend/src/app/setup/keys.clj index bdc5ce45e3..125e1a59bc 100644 --- a/backend/src/app/setup/keys.clj +++ b/backend/src/app/setup/keys.clj @@ -8,13 +8,13 @@ "Keys derivation service." (:refer-clojure :exclude [derive]) (:require - [app.common.spec :as us] [buddy.core.kdf :as bk])) (defn derive "Derive a key from secret-key" [secret-key & {:keys [salt size] :or {size 32}}] - (us/assert! ::us/not-empty-string secret-key) + (assert (string? secret-key) "expect string") + (assert (seq secret-key) "expect string") (let [engine (bk/engine {:key secret-key :salt salt :alg :hkdf From 81632a03dd54fb22e2da851a10b6efac4f2fbdc6 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Tue, 18 Nov 2025 15:17:06 +0100 Subject: [PATCH 4/4] :recycle: Make exporter upload resources using backend management api Instead of custon shared fs approach. This commit fixes the main scalability issue of exporter removing the need of shared-fs for make it work with multiple instances. --- backend/src/app/rpc.clj | 3 +- backend/src/app/rpc/management/exporter.clj | 49 +++++++ exporter/scripts/run | 6 + exporter/scripts/wait-and-start.sh | 6 +- exporter/src/app/config.cljs | 18 ++- exporter/src/app/handlers.cljs | 5 +- exporter/src/app/handlers/export_frames.cljs | 124 ++++++++---------- exporter/src/app/handlers/export_shapes.cljs | 96 +++++--------- exporter/src/app/handlers/resources.cljs | 66 +++++----- exporter/src/app/renderer/bitmap.cljs | 5 +- exporter/src/app/renderer/pdf.cljs | 2 +- exporter/src/app/util/shell.cljs | 57 ++++---- frontend/src/app/config.cljs | 1 - .../src/app/main/data/exports/assets.cljs | 33 +++-- frontend/src/app/main/ui/exports/assets.cljs | 7 +- .../sidebar/options/menus/exports.cljs | 5 +- 16 files changed, 265 insertions(+), 218 deletions(-) create mode 100644 backend/src/app/rpc/management/exporter.clj create mode 100755 exporter/scripts/run diff --git a/backend/src/app/rpc.clj b/backend/src/app/rpc.clj index bd9b8c8af9..782c91b042 100644 --- a/backend/src/app/rpc.clj +++ b/backend/src/app/rpc.clj @@ -295,7 +295,8 @@ [cfg] (let [cfg (assoc cfg ::type "management" ::metrics-id :rpc-management-timing)] (->> (sv/scan-ns - 'app.rpc.management.subscription) + 'app.rpc.management.subscription + 'app.rpc.management.exporter) (map (partial process-method cfg "management" wrap-management)) (into {})))) diff --git a/backend/src/app/rpc/management/exporter.clj b/backend/src/app/rpc/management/exporter.clj new file mode 100644 index 0000000000..daefd2ef6d --- /dev/null +++ b/backend/src/app/rpc/management/exporter.clj @@ -0,0 +1,49 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC + +(ns app.rpc.management.exporter + (:require + [app.common.schema :as sm] + [app.common.time :as ct] + [app.common.uri :as u] + [app.config :as cf] + [app.media :refer [schema:upload]] + [app.rpc :as-alias rpc] + [app.rpc.doc :as doc] + [app.storage :as sto] + [app.util.services :as sv])) + +;; ---- RPC METHOD: UPLOAD-TEMPFILE + +(def ^:private + schema:upload-tempfile-params + [:map {:title "upload-templfile-params"} + [:content schema:upload]]) + +(def ^:private + schema:upload-tempfile-result + [:map {:title "upload-templfile-result"}]) + +(sv/defmethod ::upload-tempfile + {::doc/added "2.12" + ::sm/params schema:upload-tempfile-params + ::sm/result schema:upload-tempfile-result} + [cfg {:keys [::rpc/profile-id content]}] + (let [storage (sto/resolve cfg) + hash (sto/calculate-hash (:path content)) + data (-> (sto/content (:path content)) + (sto/wrap-with-hash hash)) + content {::sto/content data + ::sto/deduplicate? true + ::sto/touched-at (ct/in-future {:minutes 10}) + :profile-id profile-id + :content-type (:mtype content) + :bucket "tempfile"} + object (sto/put-object! storage content)] + {:id (:id object) + :uri (-> (cf/get :public-uri) + (u/join "/assets/by-id/") + (u/join (str (:id object))))})) diff --git a/exporter/scripts/run b/exporter/scripts/run new file mode 100755 index 0000000000..4bb272fc6a --- /dev/null +++ b/exporter/scripts/run @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +SCRIPT_DIR=$(dirname $0); +source $SCRIPT_DIR/../../backend/scripts/_env; + +exec node target/app.js diff --git a/exporter/scripts/wait-and-start.sh b/exporter/scripts/wait-and-start.sh index c049353672..f9638eb06d 100755 --- a/exporter/scripts/wait-and-start.sh +++ b/exporter/scripts/wait-and-start.sh @@ -1,6 +1,10 @@ #!/usr/bin/env bash +SCRIPT_DIR=$(dirname $0); +source $SCRIPT_DIR/../../backend/scripts/_env; + bb -i '(babashka.wait/wait-for-port "localhost" 9630)'; bb -i '(babashka.wait/wait-for-path "target/app.js")'; sleep 2; -node target/app.js + +exec node target/app.js diff --git a/exporter/src/app/config.cljs b/exporter/src/app/config.cljs index 6ca84f584c..03f2b56b8a 100644 --- a/exporter/src/app/config.cljs +++ b/exporter/src/app/config.cljs @@ -7,7 +7,9 @@ (ns app.config (:refer-clojure :exclude [get]) (:require - ["process" :as process] + ["node:buffer" :as buffer] + ["node:crypto" :as crypto] + ["node:process" :as process] [app.common.data :as d] [app.common.flags :as flags] [app.common.schema :as sm] @@ -21,13 +23,14 @@ :host "localhost" :http-server-port 6061 :http-server-host "0.0.0.0" - :tempdir "/tmp/penpot-exporter" + :tempdir "/tmp/penpot" :redis-uri "redis://redis/0"}) -(def ^:private - schema:config +(def ^:private schema:config [:map {:title "config"} + [:secret-key :string] [:public-uri {:optional true} ::sm/uri] + [:management-api-key {:optional true} :string] [:host {:optional true} :string] [:tenant {:optional true} :string] [:flags {:optional true} [::sm/set :keyword]] @@ -93,3 +96,10 @@ (c/get config key)) ([key default] (c/get config key default))) + +(def management-key + (or (c/get config :management-api-key) + (let [secret-key (c/get config :secret-key) + derived-key (crypto/hkdfSync "blake2b512" secret-key, "management" "" 32)] + (-> (.from buffer/Buffer derived-key) + (.toString "base64url"))))) diff --git a/exporter/src/app/handlers.cljs b/exporter/src/app/handlers.cljs index ec1253b4c4..5a1b9a11c0 100644 --- a/exporter/src/app/handlers.cljs +++ b/exporter/src/app/handlers.cljs @@ -12,7 +12,6 @@ [app.common.spec :as us] [app.handlers.export-frames :as export-frames] [app.handlers.export-shapes :as export-shapes] - [app.handlers.resources :as resources] [app.util.transit :as t] [clojure.spec.alpha :as s] [cuerdas.core :as str])) @@ -54,7 +53,7 @@ :else (let [data {:type :server-error - :code type + :code code :hint (ex-message error) :data data}] (l/error :hint "unexpected internal error" :cause error) @@ -71,7 +70,6 @@ (defmethod command-spec :export-shapes [_] ::export-shapes/params) (defmethod command-spec :export-frames [_] ::export-frames/params) -(defmethod command-spec :get-resource [_] (s/keys :req-un [::id])) (s/def ::params (s/and (s/keys :req-un [::cmd] @@ -83,7 +81,6 @@ (let [{:keys [cmd] :as params} (us/conform ::params params)] (l/debug :hint "process-request" :cmd cmd) (case cmd - :get-resource (resources/handler exchange) :export-shapes (export-shapes/handler exchange params) :export-frames (export-frames/handler exchange params) (ex/raise :type :internal diff --git a/exporter/src/app/handlers/export_frames.cljs b/exporter/src/app/handlers/export_frames.cljs index ef437e00e0..3148a3fc7d 100644 --- a/exporter/src/app/handlers/export_frames.cljs +++ b/exporter/src/app/handlers/export_frames.cljs @@ -43,90 +43,78 @@ ;; datastructure preparation uses it for creating the groups. (let [exports (-> (map #(assoc % :type :pdf :scale 1 :suffix "") exports) (prepare-exports auth-token))] + (handle-export exchange (assoc params :exports exports)))) (defn handle-export - [exchange {:keys [exports wait name profile-id] :as params}] - (let [total (count exports) - topic (str profile-id) - resource (rsc/create :pdf (or name (-> exports first :name))) + [{:keys [:request/auth-token] :as exchange} {:keys [exports name profile-id] :as params}] + (let [topic (str profile-id) + file-id (-> exports first :file-id) - on-progress (fn [{:keys [done]}] - (when-not wait - (let [data {:type :export-update - :resource-id (:id resource) - :name (:name resource) - :filename (:filename resource) - :status "running" - :total total - :done done}] - (redis/pub! topic data)))) + resource + (rsc/create :pdf (or name (-> exports first :name))) - on-complete (fn [] - (when-not wait - (let [data {:type :export-update - :resource-id (:id resource) - :name (:name resource) - :filename (:filename resource) - :status "ended"}] - (redis/pub! topic data)))) + on-progress + (fn [done] + (let [data {:type :export-update + :resource-id (:id resource) + :status "running" + :done done}] + (redis/pub! topic data))) - on-error (fn [cause] - (l/error :hint "unexpected error on frames exportation" :cause cause) - (if wait - (p/rejected cause) - (let [data {:type :export-update - :resource-id (:id resource) - :name (:name resource) - :filename (:filename resource) - :status "error" - :cause (ex-message cause)}] - (redis/pub! topic data)))) + on-complete + (fn [resource] + (let [data {:type :export-update + :resource-id (:id resource) + :resource-uri (:uri resource) + :name (:name resource) + :filename (:filename resource) + :mtype (:mtype resource) + :status "ended"}] + (redis/pub! topic data))) - proc (create-pdf :resource resource - :exports exports - :on-progress on-progress - :on-complete on-complete - :on-error on-error)] - (if wait - (p/then proc #(assoc exchange :response/body (dissoc % :path))) - (assoc exchange :response/body (dissoc resource :path))))) + on-error + (fn [cause] + (l/error :hint "unexpected error on frames exportation" :cause cause) + (let [data {:type :export-update + :resource-id (:id resource) + :name (:name resource) + :filename (:filename resource) + :status "error" + :cause (ex-message cause)}] + (redis/pub! topic data))) -(defn create-pdf - [& {:keys [resource exports on-progress on-complete on-error] - :or {on-progress (constantly nil) - on-complete (constantly nil) - on-error p/rejected}}] - - (let [file-id (-> exports first :file-id) - result (atom []) + result-cache + (atom []) on-object (fn [{:keys [path] :as object}] - (let [res (swap! result conj path)] - (on-progress {:done (count res)})))] + (let [res (swap! result-cache conj path)] + (on-progress (count res)))) - (-> (p/loop [exports (seq exports)] - (when-let [export (first exports)] - (p/do - (rd/render export on-object) - (p/recur (rest exports))))) + procs + (->> (seq exports) + (map #(rd/render % on-object)))] - (p/then (fn [_] (deref result))) - (p/then (partial join-pdf file-id)) - (p/then (partial move-file resource)) - (p/then (constantly resource)) - (p/then (fn [resource] - (-> (sh/stat (:path resource)) - (p/then #(merge resource %))))) - (p/catch on-error) - (p/finally (fn [_ cause] - (when-not cause - (on-complete))))))) + (->> (p/all procs) + (p/fmap (fn [] @result-cache)) + (p/mcat (partial join-pdf file-id)) + (p/mcat (partial move-file resource)) + (p/fmap (constantly resource)) + (p/mcat (partial rsc/upload-resource auth-token)) + (p/mcat (fn [resource] + (->> (sh/stat (:path resource)) + (p/fmap #(merge resource %))))) + (p/merr on-error) + (p/fnly (fn [resource cause] + (when-not cause + (on-complete resource))))) + + (assoc exchange :response/body (dissoc resource :path)))) (defn- join-pdf [file-id paths] - (p/let [prefix (str/concat "penpot.tmp.pdfunite." file-id ".") + (p/let [prefix (str/concat "penpot.pdfunite." file-id ".") path (sh/tempfile :prefix prefix :suffix ".pdf")] (sh/run-cmd! (str "pdfunite " (str/join " " paths) " " path)) path)) diff --git a/exporter/src/app/handlers/export_shapes.cljs b/exporter/src/app/handlers/export_shapes.cljs index 0acfda287c..c254aba5b5 100644 --- a/exporter/src/app/handlers/export_shapes.cljs +++ b/exporter/src/app/handlers/export_shapes.cljs @@ -60,46 +60,26 @@ (handle-multiple-export exchange (assoc params :exports exports))))) (defn- handle-single-export - [exchange {:keys [export wait profile-id name skip-children] :as params}] - (let [topic (str profile-id) - resource (rsc/create (:type export) (or name (:name export))) + [{:keys [:request/auth-token] :as exchange} {:keys [export name skip-children] :as params}] + (let [resource (rsc/create (:type export) (or name (:name export))) + export (assoc export :skip-children skip-children)] - on-progress (fn [{:keys [path] :as object}] - (p/do - ;; Move the generated path to the resource - ;; path destination. - (sh/move! path (:path resource)) - - (when-not wait - (redis/pub! topic {:type :export-update - :resource-id (:id resource) - :status "running" - :total 1 - :done 1}) - (redis/pub! topic {:type :export-update - :resource-id (:id resource) - :filename (:filename resource) - :name (:name resource) - :status "ended"})))) - on-error (fn [cause] - (l/error :hint "unexpected error on export multiple" - :cause cause) - (if wait - (p/rejected cause) - (redis/pub! topic {:type :export-update - :resource-id (:id resource) - :status "error" - :cause (ex-message cause)}))) - export (assoc export :skip-children skip-children) - proc (-> (rd/render export on-progress) - (p/then (constantly resource)) - (p/catch on-error))] - (if wait - (p/then proc #(assoc exchange :response/body (dissoc % :path))) - (assoc exchange :response/body (dissoc resource :path))))) + (->> (rd/render export + (fn [{:keys [path] :as object}] + (sh/move! path (:path resource)))) + (p/fmap (constantly resource)) + (p/mcat (partial rsc/upload-resource auth-token)) + (p/fmap (fn [resource] + (dissoc resource :path))) + (p/fmap (fn [resource] + (assoc exchange :response/body resource))) + (p/merr (fn [cause] + (l/error :hint "unexpected error on export multiple" + :cause cause) + (p/rejected cause)))))) (defn- handle-multiple-export - [exchange {:keys [exports wait profile-id name skip-children] :as params}] + [{:keys [:request/auth-token] :as exchange} {:keys [exports wait profile-id name] :as params}] (let [resource (rsc/create :zip (or name (-> exports first :name))) total (count exports) topic (str profile-id) @@ -113,15 +93,6 @@ :done done}] (redis/pub! topic data)))) - on-complete (fn [] - (when-not wait - (let [data {:type :export-update - :name (:name resource) - :filename (:filename resource) - :resource-id (:id resource) - :status "ended"}] - (redis/pub! topic data)))) - on-error (fn [cause] (l/error :hint "unexpected error on multiple exportation" :cause cause) (if wait @@ -132,30 +103,35 @@ :cause (ex-message cause)}))) zip (rsc/create-zip :resource resource - :on-complete on-complete :on-error on-error :on-progress on-progress) - append (fn [{:keys [filename path] :as object}] + append (fn [{:keys [filename path] :as resource}] (rsc/add-to-zip! zip path (str/replace filename sanitize-file-regex "_"))) - proc (-> (p/do - (p/loop [exports (seq exports)] - (when-let [export (some-> (first exports) - (assoc :skip-children skip-children))] - (p/do - (rd/render export append) - (p/recur (rest exports))))) - (.finalize zip)) - (p/then (constantly resource)) - (p/catch on-error))] + proc (->> exports + (map (fn [export] (rd/render export append))) + (p/all) + (p/fnly (fn [_] (.finalize zip))) + (p/fmap (constantly resource)) + (p/mcat (partial rsc/upload-resource auth-token)) + (p/fmap (fn [resource] + (let [data {:type :export-update + :name (:name resource) + :filename (:filename resource) + :resource-id (:id resource) + :resource-uri (:uri resource) + :mtype (:mtype resource) + :status "ended"}] + (p/do (redis/pub! topic data) + (assoc exchange :response/body resource))))) + (p/merr on-error))] (if wait (p/then proc #(assoc exchange :response/body (dissoc % :path))) (assoc exchange :response/body (dissoc resource :path))))) - (defn- assoc-file-name - "A transducer that assocs a candidate filename and avoid duplicates." + "A transducer that assocs a candidate filename and avoid duplicates" [] (letfn [(find-candidate [params used] (loop [index 0] diff --git a/exporter/src/app/handlers/resources.cljs b/exporter/src/app/handlers/resources.cljs index 8d78f861ae..32c968c0fb 100644 --- a/exporter/src/app/handlers/resources.cljs +++ b/exporter/src/app/handlers/resources.cljs @@ -9,9 +9,13 @@ (:require ["archiver$default" :as arc] ["node:fs" :as fs] + ["node:fs/promises" :as fsp] ["node:path" :as path] [app.common.exceptions :as ex] + [app.common.transit :as t] + [app.common.uri :as u] [app.common.uuid :as uuid] + [app.config :as cf] [app.util.mime :as mime] [app.util.shell :as sh] [cljs.core :as c] @@ -25,44 +29,20 @@ (defn create "Generates ephimeral resource object." [type name] - (let [task-id (uuid/next)] - {:path (get-path type task-id) + (let [task-id (uuid/next) + path (-> (get-path type task-id) + (sh/schedule-deletion))] + {:path path :mtype (mime/get type) :name name :filename (str/concat name (mime/get-extension type)) - :id (str/concat (c/name type) "." task-id)})) - -(defn- lookup - [id] - (p/let [[type task-id] (str/split id "." 2) - path (get-path type task-id) - mtype (mime/get (keyword type)) - stat (sh/stat path)] - - (when-not stat - (ex/raise :type :not-found)) - - {:stream (fs/createReadStream path) - :headers {"content-type" mtype - "content-length" (:size stat)}})) - -(defn handler - [{:keys [:request/params] :as exchange}] - (when-not (contains? params :id) - (ex/raise :type :validation - :code :missing-id)) - - (-> (lookup (get params :id)) - (p/then (fn [{:keys [stream headers] :as resource}] - (-> exchange - (assoc :response/status 200) - (assoc :response/body stream) - (assoc :response/headers headers)))))) + :id task-id})) (defn create-zip [& {:keys [resource on-complete on-progress on-error]}] (let [^js zip (arc/create "zip") ^js out (fs/createWriteStream (:path resource)) + on-complete (or on-complete (constantly nil)) progress (atom 0)] (.on zip "error" on-error) (.on zip "end" on-complete) @@ -80,3 +60,29 @@ (defn close-zip! [zip] (.finalize ^js zip)) + +(defn upload-resource + [auth-token resource] + (->> (fsp/readFile (:path resource)) + (p/fmap (fn [buffer] + (new js/Blob #js [buffer] #js {:type (:mtype resource)}))) + (p/mcat (fn [blob] + (let [fdata (new js/FormData) + uri (-> (cf/get :public-uri) + (u/ensure-path-slash) + (u/join "api/management/methods/upload-tempfile") + (str))] + (.append fdata "content" blob (:filename resource)) + (js/fetch uri #js {:headers #js {"X-Shared-Key" cf/management-key + "Authorization" (str "Bearer " auth-token)} + :method "POST" + :body fdata})))) + (p/mcat (fn [response] + (if (not= (.-status response) 200) + (ex/raise :type :internal + :code :unable-to-upload-resource + :response-status (.-status response)) + (.text response)))) + (p/fmap t/decode-str) + (p/fmap (fn [result] + (merge resource (dissoc result :id)))))) diff --git a/exporter/src/app/renderer/bitmap.cljs b/exporter/src/app/renderer/bitmap.cljs index 54335be501..b1df4a7447 100644 --- a/exporter/src/app/renderer/bitmap.cljs +++ b/exporter/src/app/renderer/bitmap.cljs @@ -29,13 +29,13 @@ :userAgent bw/default-user-agent}) (render-object [page {:keys [id] :as object}] - (p/let [path (sh/tempfile :prefix "penpot.tmp.render.bitmap." :suffix (mime/get-extension type)) + (p/let [path (sh/tempfile :prefix "penpot.tmp.bitmap." :suffix (mime/get-extension type)) node (bw/select page (str/concat "#screenshot-" id))] (bw/wait-for node) (case type :png (bw/screenshot node {:omit-background? true :type type :path path}) :jpeg (bw/screenshot node {:omit-background? false :type type :path path}) - :webp (p/let [png-path (sh/tempfile :prefix "penpot.tmp.render.bitmap." :suffix ".png")] + :webp (p/let [png-path (sh/tempfile :prefix "penpot.tmp.bitmap." :suffix ".png")] ;; playwright only supports jpg and png, we need to convert it afterwards (bw/screenshot node {:omit-background? true :type :png :path png-path}) (sh/run-cmd! (str "convert " png-path " -quality 100 WEBP:" path)))) @@ -52,6 +52,7 @@ ;; take the screnshot of requested objects, one by one (p/run (partial render-object page) objects) nil))] + (p/let [params {:file-id file-id :page-id page-id :share-id share-id diff --git a/exporter/src/app/renderer/pdf.cljs b/exporter/src/app/renderer/pdf.cljs index 1464f62f3c..c7558184c4 100644 --- a/exporter/src/app/renderer/pdf.cljs +++ b/exporter/src/app/renderer/pdf.cljs @@ -40,7 +40,7 @@ (render-object [page base-uri {:keys [id] :as object}] (p/let [uri (prepare-uri base-uri id) - path (sh/tempfile :prefix "penpot.tmp.render.pdf." :suffix (mime/get-extension type))] + path (sh/tempfile :prefix "penpot.tmp.pdf." :suffix (mime/get-extension type))] (l/info :uri uri) (bw/nav! page uri) (p/let [dom (bw/select page (dm/str "#screenshot-" id))] diff --git a/exporter/src/app/util/shell.cljs b/exporter/src/app/util/shell.cljs index 1d2030b8ef..717ca21073 100644 --- a/exporter/src/app/util/shell.cljs +++ b/exporter/src/app/util/shell.cljs @@ -7,11 +7,12 @@ (ns app.util.shell "Shell & FS utilities." (:require - ["child_process" :as proc] - ["fs" :as fs] - ["path" :as path] + ["node:child_process" :as proc] + ["node:fs" :as fs] + ["node:path" :as path] [app.common.exceptions :as ex] [app.common.logging :as l] + [app.common.time :as ct] [app.common.uuid :as uuid] [app.config :as cf] [cuerdas.core :as str] @@ -19,7 +20,8 @@ (l/set-level! :trace) -(def tempfile-minage (* 1000 60 60 1)) ;; 1h +(def ^:const default-deletion-delay + (* 60 60 1)) ;; 1h (def tmpdir (let [path (cf/get :tempdir)] @@ -28,16 +30,28 @@ (fs/mkdirSync path #js {:recursive true})) path)) -(defn- schedule-deletion! - [path] - (letfn [(remote-tempfile [] - (when (fs/existsSync path) - (l/trace :hint "permanently remove tempfile" :path path) - (fs/rmSync path #js {:recursive true})))] - (l/trace :hint "schedule tempfile deletion" - :path path - :scheduled-at (.. (js/Date. (+ (js/Date.now) tempfile-minage)) toString)) - (js/setTimeout remote-tempfile tempfile-minage))) +(defn schedule-deletion + ([path] (schedule-deletion path default-deletion-delay)) + ([path delay] + (let [remove-path + (fn [] + (try + (when (fs/existsSync path) + (fs/rmSync path #js {:recursive true}) + (l/trc :hint "tempfile permanently deleted" :path path)) + (catch :default cause + (l/err :hint "error on deleting temporal file" + :path path + :cause cause)))) + scheduled-at + (-> (ct/now) (ct/plus #js {:seconds delay}))] + + (l/trc :hint "schedule tempfile deletion" + :path path + :scheduled-at (ct/format-inst scheduled-at)) + + (js/setTimeout remove-path (* delay 1000)) + path))) (defn tempfile [& {:keys [prefix suffix] @@ -48,9 +62,7 @@ (let [path (path/join tmpdir (str/concat prefix (uuid/next) "-" i suffix))] (if (fs/existsSync path) (recur (inc i)) - (do - (schedule-deletion! path) - path))) + (schedule-deletion path))) (ex/raise :type :internal :code :unable-to-locate-temporal-file :hint "unable to find a tempfile candidate")))) @@ -61,11 +73,12 @@ (defn stat [path] - (-> (.stat fs/promises path) - (p/then (fn [data] - {:created-at (inst-ms (.-ctime ^js data)) - :size (.-size data)})) - (p/catch (constantly nil)))) + (->> (.stat fs/promises path) + (p/fmap (fn [data] + {:created-at (inst-ms (.-ctime ^js data)) + :size (.-size data)})) + (p/merr (fn [_cause] + (p/resolved nil))))) (defn rmdir! [path] diff --git a/frontend/src/app/config.cljs b/frontend/src/app/config.cljs index 0f1d2c41e6..76879756c7 100644 --- a/frontend/src/app/config.cljs +++ b/frontend/src/app/config.cljs @@ -104,7 +104,6 @@ (def plugins-whitelist (into #{} (obj/get global "penpotPluginsWhitelist" []))) (def templates-uri (obj/get global "penpotTemplatesUri" "https://penpot.github.io/penpot-files/")) - ;; We set the current parsed flags under common for make ;; it available for common code without the need to pass ;; the flags all arround on parameters. diff --git a/frontend/src/app/main/data/exports/assets.cljs b/frontend/src/app/main/data/exports/assets.cljs index dcd6cc1f6f..4355ad7ef9 100644 --- a/frontend/src/app/main/data/exports/assets.cljs +++ b/frontend/src/app/main/data/exports/assets.cljs @@ -99,16 +99,18 @@ (watch [_ state _] (let [file-id (:current-file-id state) page-id (:current-page-id state) - exports (for [frame frames] - {:enabled true - :page-id page-id - :file-id file-id - :object-id (:id frame) - :shape frame - :name (:name frame)})] + exports (mapv (fn [frame] + {:enabled true + :page-id page-id + :file-id file-id + :object-id (:id frame) + :shape frame + :name (:name frame)}) + frames)] (rx/of (modal/show :export-frames - {:exports (vec exports) :origin "workspace:menu"})))))) + {:exports exports + :origin "workspace:menu"})))))) (defn- initialize-export-status [exports cmd resource] @@ -127,7 +129,7 @@ :cmd cmd})))) (defn- update-export-status - [{:keys [done status resource-id filename] :as data}] + [{:keys [done status resource-uri filename mtype] :as data}] (ptk/reify ::update-export-status ptk/UpdateEvent (update [_ state] @@ -146,9 +148,7 @@ ptk/WatchEvent (watch [_ _ _] (when (= status "ended") - (->> (rp/cmd! :export {:cmd :get-resource :blob? true :id resource-id}) - (rx/delay 500) - (rx/map #(dom/trigger-download filename %))))))) + (dom/trigger-download-uri filename mtype resource-uri))))) (defn request-simple-export [{:keys [export]}] @@ -174,17 +174,14 @@ (rx/timeout 400 (rx/empty))) (->> (rp/cmd! :export params) - (rx/mapcat (fn [{:keys [id filename]}] - (->> (rp/cmd! :export {:cmd :get-resource :blob? true :id id}) - (rx/map (fn [data] - (dom/trigger-download filename data) - (clear-export-state uuid/zero)))))) + (rx/map (fn [{:keys [filename mtype uri]}] + (dom/trigger-download-uri filename mtype uri) + (clear-export-state uuid/zero))) (rx/catch (fn [cause] (rx/concat (rx/of (clear-export-state uuid/zero)) (rx/throw cause)))))))))) - (defn request-multiple-export [{:keys [exports cmd] :or {cmd :export-shapes} diff --git a/frontend/src/app/main/ui/exports/assets.cljs b/frontend/src/app/main/ui/exports/assets.cljs index 6dab22b7f1..d444b56139 100644 --- a/frontend/src/app/main/ui/exports/assets.cljs +++ b/frontend/src/app/main/ui/exports/assets.cljs @@ -34,7 +34,8 @@ (def ^:private close-icon (deprecated-icon/icon-xref :close (stl/css :close-icon))) -(mf/defc export-multiple-dialog +(mf/defc export-multiple-dialog* + {::mf/private true} [{:keys [exports title cmd no-selection origin]}] (let [lstate (mf/deref refs/export) in-progress? (:in-progress lstate) @@ -186,7 +187,7 @@ ::mf/register-as :export-shapes} [{:keys [exports origin]}] (let [title (tr "dashboard.export-shapes.title")] - [:& export-multiple-dialog + [:> export-multiple-dialog* {:exports exports :title title :cmd :export-shapes @@ -198,7 +199,7 @@ ::mf/register-as :export-frames} [{:keys [exports origin]}] (let [title (tr "dashboard.export-frames.title")] - [:& export-multiple-dialog + [:> export-multiple-dialog* {:exports exports :title title :cmd :export-frames diff --git a/frontend/src/app/main/ui/workspace/sidebar/options/menus/exports.cljs b/frontend/src/app/main/ui/workspace/sidebar/options/menus/exports.cljs index 7c32bcfd19..5e7d5c9127 100644 --- a/frontend/src/app/main/ui/workspace/sidebar/options/menus/exports.cljs +++ b/frontend/src/app/main/ui/workspace/sidebar/options/menus/exports.cljs @@ -101,9 +101,8 @@ :name sname :object-id (:id (first shapes-with-exports))} full-export (merge export defaults)] - (st/emit! - (de/request-simple-export {:export full-export}) - (de/export-shapes-event [full-export] "workspace:sidebar"))) + (st/emit! (de/request-simple-export {:export full-export}) + (de/export-shapes-event [full-export] "workspace:sidebar"))) (st/emit! (de/show-workspace-export-dialog {:selected (reverse ids) :origin "workspace:sidebar"})))