Merge remote-tracking branch 'origin/staging' into develop

backend/src/app/binfile/common.clj

@@ -550,7 +550,7 @@
   [cfg data file-id]
   (let [library-ids (get-libraries cfg [file-id])]
     (reduce (fn [data library-id]
-              (if-let [library (get-file cfg library-id)]
+              (if-let [library (get-file cfg library-id :include-deleted? true)]
                 (ctf/absorb-assets data (:data library))
                 data))
             data

backend/src/app/binfile/v3.clj

@@ -228,6 +228,7 @@
     (db/tx-run! cfg (fn [cfg]
                       (cond-> (bfc/get-file cfg file-id
                                             {:realize? true
+                                             :include-deleted? true
                                              :lock-for-update? true})
                         detach?
                         (-> (ctf/detach-external-references file-id)
@@ -285,14 +286,12 @@
 
     (let [file (cond-> (select-keys file bfc/file-attrs)
                  (:options data)
-                 (assoc :options (:options data))
+                 (assoc :options (:options data)))
 
-                 :always
-                 (dissoc :data))
-
-          file (cond-> file
-                 :always
-                 (encode-file))
+          file (-> file
+                   (dissoc :data)
+                   (dissoc :deleted-at)
+                   (encode-file))
 
           path (str "files/" file-id ".json")]
       (write-entry! output path file))

backend/src/app/config.clj

@@ -319,5 +319,9 @@
   ([key default]
    (c/get config key default)))
 
+(defn logging-context
+  []
+  {:version/backend (:full version)})
+
 ;; Set value for all new threads bindings.
 (alter-var-root #'*assert* (constantly (contains? flags :backend-asserts)))

backend/src/app/http/errors.clj

@@ -25,15 +25,14 @@
   (let [claims (-> {}
                    (into (::session/token-claims request))
                    (into (::actoken/token-claims request)))]
-    {:request/path       (:path request)
-     :request/method     (:method request)
-     :request/params     (:params request)
-     :request/user-agent (yreq/get-header request "user-agent")
-     :request/ip-addr    (inet/parse-request request)
-     :request/profile-id (:uid claims)
-     :version/frontend   (or (yreq/get-header request "x-frontend-version") "unknown")
-     :version/backend    (:full cf/version)}))
-
+    (-> (cf/logging-context)
+        (assoc :request/path (:path request))
+        (assoc :request/method (:method request))
+        (assoc :request/params (:params request))
+        (assoc :request/user-agent (yreq/get-header request "user-agent"))
+        (assoc :request/ip-addr (inet/parse-request request))
+        (assoc :request/profile-id (:uid claims))
+        (assoc :version/frontend (or (yreq/get-header request "x-frontend-version") "unknown")))))
 
 (defmulti handle-error
   (fn [cause _ _]

backend/src/app/rpc.clj

@@ -64,9 +64,13 @@
   (let [mdata    (meta result)
         response (if (fn? result)
                    (result request)
-                   (let [result (rph/unwrap result)]
-                     {::yres/status  (::http/status mdata 200)
-                      ::yres/headers (::http/headers mdata {})
+                   (let [result  (rph/unwrap result)
+                         status  (::http/status mdata 200)
+                         headers (cond-> (::http/headers mdata {})
+                                   (yres/stream-body? result)
+                                   (assoc "content-type" "application/octet-stream"))]
+                     {::yres/status  status
+                      ::yres/headers headers
                       ::yres/body    result}))]
     (-> response
         (handle-response-transformation request mdata)

backend/src/app/rpc/commands/binfile.clj

@@ -25,10 +25,10 @@
    [app.rpc.commands.projects :as projects]
    [app.rpc.commands.teams :as teams]
    [app.rpc.doc :as-alias doc]
+   [app.rpc.helpers :as rph]
    [app.tasks.file-gc]
    [app.util.services :as sv]
-   [app.worker :as-alias wrk]
-   [yetti.response :as yres]))
+   [app.worker :as-alias wrk]))
 
 (set! *warn-on-reflection* true)
 
@@ -44,7 +44,7 @@
 
 (defn stream-export-v1
   [cfg {:keys [file-id include-libraries embed-assets] :as params}]
-  (yres/stream-body
+  (rph/stream
    (fn [_ output-stream]
      (try
        (-> cfg
@@ -59,7 +59,7 @@
 
 (defn stream-export-v3
   [cfg {:keys [file-id include-libraries embed-assets] :as params}]
-  (yres/stream-body
+  (rph/stream
    (fn [_ output-stream]
      (try
        (-> cfg
@@ -79,16 +79,11 @@
    ::sm/params schema:export-binfile}
   [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id version file-id] :as params}]
   (files/check-read-permissions! pool profile-id file-id)
-  (fn [_]
-    (let [version (or version 1)
-          body    (case (int version)
-                    1 (stream-export-v1 cfg params)
-                    2 (throw (ex-info "not-implemented" {}))
-                    3 (stream-export-v3 cfg params))]
-
-      {::yres/status 200
-       ::yres/headers {"content-type" "application/octet-stream"}
-       ::yres/body body})))
+  (let [version (or version 1)]
+    (case (int version)
+      1 (stream-export-v1 cfg params)
+      2 (throw (ex-info "not-implemented" {}))
+      3 (stream-export-v3 cfg params))))
 
 ;; --- Command: import-binfile
 

backend/src/app/rpc/helpers.clj

@@ -11,7 +11,7 @@
    [app.common.data.macros :as dm]
    [app.http :as-alias http]
    [app.rpc :as-alias rpc]
-   [yetti.response :as-alias yres]))
+   [yetti.response :as yres]))
 
 ;; A utilty wrapper object for wrap service responses that does not
 ;; implements the IObj interface that make possible attach metadata to
@@ -78,3 +78,8 @@
                (let [exp (if (integer? max-age) max-age (inst-ms max-age))
                      val (dm/fmt "max-age=%" (int (/ exp 1000.0)))]
                  (update response ::yres/headers assoc "cache-control" val)))))
+
+(defn stream
+  "A convenience allias for yetti.response/stream-body"
+  [f]
+  (yres/stream-body f))

backend/src/app/worker/runner.clj

@@ -13,6 +13,7 @@
    [app.common.schema :as sm]
    [app.common.time :as ct]
    [app.common.transit :as t]
+   [app.config :as cf]
    [app.db :as db]
    [app.metrics :as mtx]
    [app.redis :as rds]
@@ -60,7 +61,8 @@
 
 (defn get-error-context
   [_ item]
-  {:params item})
+  (-> (cf/logging-context)
+      (assoc :params item)))
 
 (defn- get-task
   [{:keys [::db/pool]} task-id]
@@ -131,6 +133,11 @@
   [{:keys [::id ::timeout] :as cfg} task-id scheduled-at]
   (loop [task (get-task cfg task-id)]
     (cond
+      (nil? task)
+      (l/wrn :hint "no task found on the database"
+             :runner-id id
+             :task-id task-id)
+
       (ex/exception? task)
       (if (or (db/connection-error? task)
               (db/serialization-error? task))
@@ -153,11 +160,6 @@
              :task-id task-id
              :runner-id id)
 
-      (nil? task)
-      (l/wrn :hint "no task found on the database"
-             :runner-id id
-             :task-id task-id)
-
       :else
       (let [result (run-task cfg task)]
         (with-meta result
@@ -213,6 +215,7 @@
                          :payload payload)))
               (catch Throwable cause
                 (l/err :hint "unable to decode payload"
+                       ::l/context (cf/logging-context)
                        :payload payload
                        :length (alength ^String/1 payload)
                        :cause cause))))
@@ -224,11 +227,11 @@
                "failed"    (handle-task-failure result)
                "completed" (handle-task-completion result)
                (throw (IllegalArgumentException.
-                       (str "invalid status received: " status))))))
+                       (str "invalid status received: '" status "'"))))))
 
           (run-task-loop [[task-id scheduled-at]]
             (loop [result (run-task! cfg task-id scheduled-at)]
-              (when-let [cause (process-result result)]
+              (when-let [cause (some-> result process-result)]
                 (if (or (db/connection-error? cause)
                         (db/serialization-error? cause))
                   (do
@@ -236,9 +239,9 @@
                            :cause cause)
                     (px/sleep timeout)
                     (recur result))
-                  (do
-                    (l/err :hint "unhandled exception on processing task result"
-                           :cause cause))))))]
+                  (l/err :hint "unhandled exception on processing task result"
+                         ::l/context (cf/logging-context)
+                         :cause cause)))))]
 
     (try
       (let [key         (str/ffmt "penpot.worker.queue:%" queue)
@@ -254,11 +257,14 @@
         (if (rds/timeout-exception? cause)
           (do
             (l/err :hint "redis pop operation timeout, consider increasing redis timeout (will retry in some instants)"
+                   ::l/context (cf/logging-context)
                    :timeout timeout
                    :cause cause)
             (px/sleep timeout))
 
-          (l/err :hint "unhandled exception" :cause cause))))))
+          (l/err :hint "unhandled exception"
+                 ::l/context (cf/logging-context)
+                 :cause cause))))))
 
 (defn- start-thread!
   [{:keys [::id ::queue ::wrk/tenant] :as cfg}]
@@ -284,6 +290,7 @@
                  :queue queue))
         (catch Throwable cause
           (l/err :hint "unexpected exception"
+                 ::l/context (cf/logging-context)
                  :id id
                  :queue queue
                  :cause cause))