🎉 Add .penpot (binfile-v3) support for library

library/package.json

@@ -33,7 +33,6 @@
   "dependencies": {
     "@zip.js/zip.js": "patch:@zip.js/zip.js@npm%3A2.7.60#~/.yarn/patches/@zip.js-zip.js-npm-2.7.60-b6b814410b.patch",
     "luxon": "^3.6.1",
-    "sax": "^1.4.1",
     "source-map-support": "^0.5.21"
   }
 }

library/playground/sample1.js

@@ -1,47 +1,87 @@
 import * as penpot from "../target/library/penpot.js";
-import { writeFile } from 'fs/promises';
+import { writeFile, readFile } from 'fs/promises';
 import { createWriteStream } from 'fs';
 import { Writable } from "stream";
 
-console.log(penpot);
+// console.log(penpot);
 
 (async function() {
-  const file = penpot.createFile({name: "Test"});
-
-  file.addPage({name: "Foo Page"})
-  const boardId = file.addArtboard({name: "Foo Board"})
-  const rectId  = file.addRect({name: "Foo Rect", width:100, height: 200})
-
-  file.addLibraryColor({color: "#fabada", opacity: 0.5})
-
-  // console.log("created board", boardId);
-  // console.log("created rect", rectId);
-
-  // const board = file.getShape(boardId);
-  // console.log("=========== BOARD =============")
-  // console.dir(board, {depth: 10});
-
-  // const rect = file.getShape(rectId);
-  // console.log("=========== RECT =============")
-  // console.dir(rect, {depth: 10});
+  const context = penpot.createBuildContext();
 
   {
-    let result = await penpot.exportAsBytes(file)
+    context.addFile({name: "Test File 1"});
+    context.addPage({name: "Foo Page"})
+
+    // Add image media
+    const buffer = await readFile("./playground/sample.jpg");
+    const blob = new Blob([buffer], { type: 'image/jpeg' });
+
+    const mediaId = context.addFileMedia({
+      name: "avatar.jpg",
+      width: 512,
+      height: 512
+    }, blob);
+
+    // Add image color asset
+    const assetColorId = context.addLibraryColor({
+      name: "Avatar",
+      opacity: 1,
+      image: {
+        ...context.getMediaAsImage(mediaId),
+        keepAspectRatio: true
+      }
+    });
+
+    const boardId = context.addBoard({
+      name: "Foo Board",
+      x: 0,
+      y: 0,
+      width: 500,
+      height: 300,
+    })
+
+    const fill = {
+      fillColorRefId: assetColorId,
+      fillColorRefFile: context.currentFileId,
+      fillImage: {
+        ...context.getMediaAsImage(mediaId),
+        keepAspectRatio: true
+      }
+    };
+
+    context.addRect({
+      name: "Rect 1",
+      x: 20,
+      y: 20,
+      width:100,
+      height:200,
+      fills: [fill]
+    });
+
+    context.closeBoard();
+    context.closeFile();
+  }
+
+  {
+    let result = await penpot.exportAsBytes(context)
     await writeFile("sample-sync.zip", result);
   }
 
-  {
-    // Create a file stream to write the zip to
-    const output = createWriteStream('sample-stream.zip');
-
-    // Wrap Node's stream in a WHATWG WritableStream
-    const writable = Writable.toWeb(output);
-
-    await penpot.exportStream(file, writable);
-  }
+  // {
+  //   // Create a file stream to write the zip to
+  //   const output = createWriteStream('sample-stream.zip');
+  //   // Wrap Node's stream in a WHATWG WritableStream
+  //   const writable = Writable.toWeb(output);
+  //   await penpot.exportStream(context, writable);
+  // }
 
 })().catch((cause) => {
-  console.log(cause);
+  console.error(cause);
+
+  const innerCause = cause.cause;
+  if (innerCause) {
+    console.error("Inner cause:", innerCause);
+  }
   process.exit(-1);
 }).finally(() => {
   process.exit(0);

library/shadow-cljs.edn

@@ -25,7 +25,7 @@
    :modules
    {:penpot
     {:exports {BuilderError lib.builder/BuilderError
-               createFile lib.builder/create-file
+               createBuildContext lib.builder/create-build-context
                exportAsBytes lib.export/export-bytes
                exportAsBlob lib.export/export-blob
                exportStream lib.export/export-stream
@@ -34,7 +34,7 @@
    :js-options
    {:entry-keys ["module" "browser" "main"]
     :export-conditions ["module" "import", "browser" "require" "default"]
-    :js-provider :import
+    ;; :js-provider :import
     ;; :external-index "target/library/dependencies.js"
     ;; :external-index-format :esm
     }

library/src/lib/builder.cljs

@@ -55,201 +55,235 @@
 (defn- decode-params
   [params]
   (if (obj/plain-object? params)
-    (json/->js params)
+    (json/->clj params)
     params))
 
-(defn- create-file-api
-  [file]
-  (let [state* (volatile! file)
-        api    (obj/reify {:name "File"}
-                 :id
-                 {:get #(dm/str (:id @state*))}
+(defn- get-current-page-id
+  [state]
+  (dm/str (get state ::fb/current-page-id)))
 
-                 :currentFrameId
-                 {:get #(dm/str (::fb/current-frame-id @state*))}
+(defn- get-last-id
+  [state]
+  (dm/str (get state ::fb/last-id)))
 
-                 :currentPageId
-                 {:get #(dm/str (::fb/current-page-id @state*))}
+(defn- create-builder-api
+  [state]
+  (obj/reify {:name "File"}
+    :currentFileId
+    {:get #(dm/str (get @state ::fb/current-file-id))}
 
-                 :lastId
-                 {:get #(dm/str (::fb/last-id @state*))}
+    :currentFrameId
+    {:get #(dm/str (get @state ::fb/current-frame-id))}
 
-                 :addPage
-                 (fn [params]
-                   (try
-                     (let [params (-> params
-                                      (decode-params)
-                                      (fb/decode-page))]
-                       (vswap! state* fb/add-page params)
-                       (dm/str (::fb/current-page-id @state*)))
-                     (catch :default cause
-                       (handle-exception cause))))
+    :currentPageId
+    {:get #(get-current-page-id @state)}
 
-                 :closePage
-                 (fn []
-                   (vswap! state* fb/close-page))
+    :lastId
+    {:get #(get-last-id @state)}
 
-                 :addArtboard
-                 (fn [params]
-                   (try
-                     (let [params (-> params
-                                      (json/->clj)
-                                      (assoc :type :frame)
-                                      (fb/decode-shape))]
-                       (vswap! state* fb/add-artboard params)
-                       (dm/str (::fb/last-id @state*)))
-                     (catch :default cause
-                       (handle-exception cause))))
+    :addFile
+    (fn [params]
+      (try
+        (let [params (-> params decode-params fb/decode-file)]
+          (-> (swap! state fb/add-file params)
+              (get ::fb/current-file-id)))
+        (catch :default cause
+          (handle-exception cause))))
 
-                 :closeArtboard
-                 (fn []
-                   (vswap! state* fb/close-artboard))
+    :closeFile
+    (fn []
+      (swap! state fb/close-file)
+      nil)
 
-                 :addGroup
-                 (fn [params]
-                   (try
-                     (let [params (-> params
-                                      (json/->clj)
-                                      (assoc :type :group)
-                                      (fb/decode-shape))]
-                       (vswap! state* fb/add-group params)
-                       (dm/str (::fb/last-id @state*)))
-                     (catch :default cause
-                       (handle-exception cause))))
+    :addPage
+    (fn [params]
+      (try
+        (let [params (-> (decode-params params)
+                         (fb/decode-page))]
 
-                 :closeGroup
-                 (fn []
-                   (vswap! state* fb/close-group))
+          (-> (swap! state fb/add-page params)
+              (get-current-page-id)))
 
-                 :addBool
-                 (fn [params]
-                   (try
-                     (let [params (-> params
-                                      (json/->clj)
-                                      (fb/decode-add-bool))]
-                       (vswap! state* fb/add-bool params)
-                       (dm/str (::fb/last-id @state*)))
-                     (catch :default cause
-                       (handle-exception cause))))
+        (catch :default cause
+          (handle-exception cause))))
 
-                 :addRect
-                 (fn [params]
-                   (try
-                     (let [params (-> params
-                                      (json/->clj)
-                                      (assoc :type :rect)
-                                      (fb/decode-shape))]
-                       (vswap! state* fb/add-shape params)
-                       (dm/str (::fb/last-id @state*)))
-                     (catch :default cause
-                       (handle-exception cause))))
+    :closePage
+    (fn []
+      (swap! state fb/close-page)
+      nil)
 
-                 :addCircle
-                 (fn [params]
-                   (try
-                     (let [params (-> params
-                                      (json/->clj)
-                                      (assoc :type :circle)
-                                      (fb/decode-shape))]
-                       (vswap! state* fb/add-shape params)
-                       (dm/str (::fb/last-id @state*)))
-                     (catch :default cause
-                       (handle-exception cause))))
+    :addBoard
+    (fn [params]
+      (try
+        (let [params (-> (decode-params params)
+                         (assoc :type :frame)
+                         (fb/decode-shape))]
+          (-> (swap! state fb/add-board params)
+              (get-last-id)))
+        (catch :default cause
+          (handle-exception cause))))
 
-                 :addPath
-                 (fn [params]
-                   (try
-                     (let [params (-> params
-                                      (json/->clj)
-                                      (assoc :type :path)
-                                      (fb/decode-shape))]
-                       (vswap! state* fb/add-shape params)
-                       (dm/str (::fb/last-id @state*)))
-                     (catch :default cause
-                       (handle-exception cause))))
+    :closeBoard
+    (fn []
+      (swap! state fb/close-board)
+      nil)
 
-                 :addText
-                 (fn [params]
-                   (try
-                     (let [params (-> params
-                                      (json/->clj)
-                                      (assoc :type :text)
-                                      (fb/decode-shape))]
-                       (vswap! state* fb/add-shape params)
-                       (dm/str (::fb/last-id @state*)))
-                     (catch :default cause
-                       (handle-exception cause))))
+    :addGroup
+    (fn [params]
+      (try
+        (let [params (-> (decode-params params)
+                         (assoc :type :group)
+                         (fb/decode-shape))]
+          (-> (swap! state fb/add-group params)
+              (get-last-id)))
+        (catch :default cause
+          (handle-exception cause))))
 
-                 :addLibraryColor
-                 (fn [params]
-                   (try
-                     (let [params (-> params
-                                      (json/->clj)
-                                      (fb/decode-library-color)
-                                      (d/without-nils))]
-                       (vswap! state* fb/add-library-color params)
-                       (dm/str (::fb/last-id @state*)))
-                     (catch :default cause
-                       (handle-exception cause))))
+    :closeGroup
+    (fn []
+      (swap! state fb/close-group)
+      nil)
 
-                 :addLibraryTypography
-                 (fn [params]
-                   (try
-                     (let [params (-> params
-                                      (json/->clj)
-                                      (fb/decode-library-typography)
-                                      (d/without-nils))]
-                       (vswap! state* fb/add-library-typography params)
-                       (dm/str (::fb/last-id @state*)))
-                     (catch :default cause
-                       (handle-exception cause))))
+    :addBool
+    (fn [params]
+      (try
+        (let [params (-> (decode-params params)
+                         (fb/decode-add-bool))]
+          (-> (swap! state fb/add-bool params)
+              (get-last-id)))
+        (catch :default cause
+          (handle-exception cause))))
 
-                 :addComponent
-                 (fn [params]
-                   (try
-                     (let [params (-> params
-                                      (json/->clj)
-                                      (fb/decode-component)
-                                      (d/without-nils))]
-                       (vswap! state* fb/add-component params)
-                       (dm/str (::fb/last-id @state*)))
-                     (catch :default cause
-                       (handle-exception cause))))
+    :addRect
+    (fn [params]
+      (try
+        (let [params (-> (decode-params params)
+                         (assoc :type :rect)
+                         (fb/decode-shape))]
+          (-> (swap! state fb/add-shape params)
+              (get-last-id)))
+        (catch :default cause
+          (handle-exception cause))))
 
-                 :addComponentInstance
-                 (fn [params]
-                   (try
-                     (let [params (-> params
-                                      (json/->clj)
-                                      (fb/decode-add-component-instance)
-                                      (d/without-nils))]
-                       (vswap! state* fb/add-component-instance params)
-                       (dm/str (::fb/last-id @state*)))
-                     (catch :default cause
-                       (handle-exception cause))))
+    :addCircle
+    (fn [params]
+      (try
+        (let [params (-> (decode-params params)
+                         (assoc :type :circle)
+                         (fb/decode-shape))]
+          (-> (swap! state fb/add-shape params)
+              (get-last-id)))
+        (catch :default cause
+          (handle-exception cause))))
 
-                 :getShape
-                 (fn [shape-id]
-                   (let [shape-id (uuid/parse shape-id)]
-                     (some-> (fb/lookup-shape @state* shape-id)
-                             (json/->js))))
+    :addPath
+    (fn [params]
+      (try
+        (let [params (-> (decode-params params)
+                         (assoc :type :path)
+                         (fb/decode-shape))]
+          (-> (swap! state fb/add-shape params)
+              (get-last-id)))
+        (catch :default cause
+          (handle-exception cause))))
 
-                 :toMap
-                 (fn []
-                   (-> @state*
-                       (d/without-qualified)
-                       (json/->js))))]
+    :addText
+    (fn [params]
+      (try
+        (let [params (-> (decode-params params)
+                         (assoc :type :text)
+                         (fb/decode-shape))]
+          (-> (swap! state fb/add-shape params)
+              (get-last-id)))
+        (catch :default cause
+          (handle-exception cause))))
+
+    :addLibraryColor
+    (fn [params]
+      (try
+        (let [params (-> (decode-params params)
+                         (fb/decode-library-color)
+                         (d/without-nils))]
+          (-> (swap! state fb/add-library-color params)
+              (get-last-id)))
+        (catch :default cause
+          (handle-exception cause))))
+
+    :addLibraryTypography
+    (fn [params]
+      (try
+        (let [params (-> (decode-params params)
+                         (fb/decode-library-typography)
+                         (d/without-nils))]
+          (-> (swap! state fb/add-library-typography params)
+              (get-last-id)))
+        (catch :default cause
+          (handle-exception cause))))
+
+    :addComponent
+    (fn [params]
+      (try
+        (let [params (-> (decode-params params)
+                         (fb/decode-add-component))]
+          (-> (swap! state fb/add-component params)
+              (get-last-id)))
+        (catch :default cause
+          (handle-exception cause))))
+
+    :addComponentInstance
+    (fn [params]
+      (try
+        (let [params (-> (decode-params params)
+                         (fb/decode-add-component-instance))]
+          (-> (swap! state fb/add-component-instance params)
+              (get-last-id)))
+        (catch :default cause
+          (handle-exception cause))))
+
+    :addFileMedia
+    (fn [params blob]
+
+      (when-not (instance? js/Blob blob)
+        (throw (BuilderError. "validation"
+                              "invalid-media"
+                              "only Blob instance are soported")))
+      (try
+        (let [blob (fb/map->BlobWrapper
+                    {:size (.-size ^js blob)
+                     :mtype (.-type ^js blob)
+                     :blob blob})
+              params
+              (-> (decode-params params)
+                  (fb/decode-add-file-media))]
+
+          (-> (swap! state fb/add-file-media params blob)
+              (get-last-id)))
+
+        (catch :default cause
+          (handle-exception cause))))
+
+    :getMediaAsImage
+    (fn [id]
+      (let [id (uuid/parse id)]
+        (when-let [fmedia (get-in @state [::fb/file-media id])]
+          (let [image {:id (get fmedia :id)
+                       :width (get fmedia :width)
+                       :height (get fmedia :height)
+                       :name (get fmedia :name)
+                       :mtype (get fmedia :mtype)}]
+            (json/->js (d/without-nils image))))))
+
+    :genId
+    (fn []
+      (dm/str (uuid/next)))))
+
+
+(defn create-build-context
+  "Create an empty builder state context."
+  []
+  (let [state (atom {})
+        api   (create-builder-api state)]
 
     (specify! api
       cljs.core/IDeref
-      (-deref [_]
-        (d/without-qualified @state*)))))
-
-(defn create-file
-  [params]
-  (try
-    (let [params (-> params json/->clj fb/decode-file)
-          file   (fb/create-file params)]
-      (create-file-api file))
-    (catch :default cause
-      (handle-exception cause))))
+      (-deref [_] @state))))

library/src/lib/export.cljs

@@ -8,12 +8,10 @@
   "A .penpot export implementation"
   (:require
    [app.common.data :as d]
-   [app.common.data.macros :as dm]
    [app.common.files.builder :as fb]
    [app.common.json :as json]
+   [app.common.media :as media]
    [app.common.schema :as sm]
-   [app.common.uuid :as uuid]
-   [app.util.object :as obj]
    [app.common.types.color :as types.color]
    [app.common.types.component :as types.component]
    [app.common.types.file :as types.file]
@@ -22,8 +20,8 @@
    [app.common.types.shape :as types.shape]
    [app.common.types.tokens-lib :as types.tokens-lib]
    [app.common.types.typography :as types.typography]
-   [cuerdas.core :as str]
    [app.util.zip :as zip]
+   [cuerdas.core :as str]
    [promesa.core :as p]))
 
 (def ^:private schema:file
@@ -43,9 +41,6 @@
 (def ^:private encode-component
   (sm/encoder types.component/schema:component sm/json-transformer))
 
-;; (def encode-media
-;;   (sm/encoder ::ctf/media sm/json-transformer))
-
 (def encode-color
   (sm/encoder types.color/schema:color sm/json-transformer))
 
@@ -68,7 +63,6 @@
     "file-data-fragment"
     "file-change"})
 
-;; FIXME: move to types
 (def ^:private schema:storage-object
   [:map {:title "StorageObject"}
    [:id ::sm/uuid]
@@ -80,12 +74,6 @@
 (def encode-storage-object
   (sm/encoder schema:storage-object sm/json-transformer))
 
-;; (def encode-file-thumbnail
-;;   (sm/encoder schema:file-thumbnail sm/json-transformer))
-
-
-;; FIXME: naming
-
 (def ^:private file-attrs
   #{:id
     :name
@@ -96,8 +84,6 @@
 
 (defn- generate-file-export-procs
   [{:keys [id data] :as file}]
-  ;; (prn "generate-file-export-procs")
-  ;; (app.common.pprint/pprint file)
   (cons
    (let [file (cond-> (select-keys file file-attrs)
                 (:options data)
@@ -108,11 +94,11 @@
    (concat
     (let [pages       (get data :pages)
           pages-index (get data :pages-index)]
+
       (->> (d/enumerate pages)
            (mapcat
             (fn [[index page-id]]
-              (let [path    (str "files/" id "/pages/" page-id ".json")
-                    page    (get pages-index page-id)
+              (let [page    (get pages-index page-id)
                     objects (:objects page)
                     page    (-> page
                                 (dissoc :objects)
@@ -155,45 +141,74 @@
     (when-let [tokens-lib (get data :tokens-lib)]
       (list [(str "files/" id "/tokens.json")
              (delay (-> tokens-lib
-                        (encode-tokens-lib tokens-lib)
+                        encode-tokens-lib
                         json/encode))])))))
 
-(defn generate-manifest-procs
-  [file]
-  (let [mdata  {:id (:id file)
-                :name (:name file)
-                :features (:features file)}
+(defn- generate-files-export-procs
+  [state]
+  (->> (vals (get state ::fb/files))
+       (mapcat generate-file-export-procs)))
+
+(defn- generate-media-export-procs
+  [state]
+  (->> (get state ::fb/file-media)
+       (mapcat (fn [[file-media-id file-media]]
+                 (let [media-id (get file-media :media-id)
+                       media    (get-in state [::fb/media media-id])
+                       blob     (get-in state [::fb/blobs media-id])]
+                   (list
+                    [(str "objects/" media-id (media/mtype->extension (:content-type media)))
+                     (delay (get blob :blob))]
+
+                    [(str "objects/" media-id ".json")
+                     (delay (-> media
+                                ;; FIXME: proper encode?
+                                (json/encode)))]
+                    [(str "files/" (:file-id file-media) "/media/" file-media-id ".json")
+                     (delay (-> file-media
+                                (dissoc :file-id)
+                                (json/encode)))]))))))
+
+(defn- generate-manifest-procs
+  [state]
+  (let [files (->> (get state ::fb/files)
+                   (mapv (fn [[file-id file]]
+                           {:id file-id
+                            :name (:name file)
+                            :features (:features file)})))
         params {:type "penpot/export-files"
                 :version 1
+                ;; FIXME: set proper placeholder for replacement on build
                 :generated-by "penpot-lib/develop"
-                :files [mdata]
+                :files files
                 :relations []}]
-    (list
-     ["manifest.json" (delay (json/encode params))])))
+    ["manifest.json" (delay (json/encode params))]))
 
 (defn- export
-  [file writer]
-  (->> (p/reduce (fn [writer [path proc]]
-                   (let [data (deref proc)]
+  [state writer]
+  (->> (p/reduce (fn [writer [path data]]
+                   (let [data (if (delay? data) (deref data) data)]
                      (js/console.log "export" path)
                      (->> (zip/add writer path data)
                           (p/fmap (constantly writer)))))
 
                  writer
-                 (concat
-                  (generate-manifest-procs @file)
-                  (generate-file-export-procs @file)))
+                 (cons (generate-manifest-procs @state)
+                       (concat
+                        (generate-files-export-procs @state)
+                        (generate-media-export-procs @state))))
+
        (p/mcat (fn [writer]
                  (zip/close writer)))))
 
 (defn export-bytes
-  [file]
-  (export file (zip/writer (zip/bytes-writer))))
+  [state]
+  (export state (zip/writer (zip/bytes-writer))))
 
 (defn export-blob
-  [file]
-  (export file (zip/writer (zip/blob-writer))))
+  [state]
+  (export state (zip/writer (zip/blob-writer))))
 
 (defn export-stream
-  [file stream]
-  (export file (zip/writer stream)))
+  [state stream]
+  (export state (zip/writer stream)))

library/yarn.lock

@@ -59,7 +59,6 @@ __metadata:
     concurrently: "npm:^9.1.2"
     luxon: "npm:^3.6.1"
     nodemon: "npm:^3.1.9"
-    sax: "npm:^1.4.1"
     shadow-cljs: "npm:3.0.5"
     source-map-support: "npm:^0.5.21"
   languageName: unknown
@@ -73,11 +72,11 @@ __metadata:
   linkType: hard
 
 "@types/node@npm:^22.12.0":
-  version: 22.15.17
-  resolution: "@types/node@npm:22.15.17"
+  version: 22.15.18
+  resolution: "@types/node@npm:22.15.18"
   dependencies:
     undici-types: "npm:~6.21.0"
-  checksum: 10c0/fb92aa10b628683c5b965749f955bc2322485ecb0ea6c2f4cae5f2c7537a16834607e67083a9e9281faaae8d7dee9ada8d6a5c0de9a52c17d82912ef00c0fdd4
+  checksum: 10c0/e23178c568e2dc6b93b6aa3b8dfb45f9556e527918c947fe7406a4c92d2184c7396558912400c3b1b8d0fa952ec63819aca2b8e4d3545455fc6f1e9623e09ca6
   languageName: node
   linkType: hard
 
@@ -318,14 +317,14 @@ __metadata:
   linkType: hard
 
 "debug@npm:4, debug@npm:^4, debug@npm:^4.3.4":
-  version: 4.4.0
-  resolution: "debug@npm:4.4.0"
+  version: 4.4.1
+  resolution: "debug@npm:4.4.1"
   dependencies:
     ms: "npm:^2.1.3"
   peerDependenciesMeta:
     supports-color:
       optional: true
-  checksum: 10c0/db94f1a182bf886f57b4755f85b3a74c39b5114b9377b7ab375dc2cfa3454f09490cc6c30f829df3fc8042bc8b8995f6567ce5cd96f3bc3688bd24027197d9de
+  checksum: 10c0/d2b44bc1afd912b49bb7ebb0d50a860dc93a4dd7d946e8de94abc957bb63726b7dd5aa48c18c2386c379ec024c46692e15ed3ed97d481729f929201e671fcd55
   languageName: node
   linkType: hard
 
@@ -962,13 +961,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"sax@npm:^1.4.1":
-  version: 1.4.1
-  resolution: "sax@npm:1.4.1"
-  checksum: 10c0/6bf86318a254c5d898ede6bd3ded15daf68ae08a5495a2739564eb265cd13bcc64a07ab466fb204f67ce472bb534eb8612dac587435515169593f4fffa11de7c
-  languageName: node
-  linkType: hard
-
 "semver@npm:^7.3.5, semver@npm:^7.5.3":
   version: 7.7.2
   resolution: "semver@npm:7.7.2"