Merge pull request #7366 from penpot/niwinz-develop-page-data-type

⚡ Add several enhancements for reduce workspace file load time

backend/dev/user.clj

@@ -35,6 +35,7 @@
    [app.util.blob :as blob]
    [clj-async-profiler.core :as prof]
    [clojure.contrib.humanize :as hum]
+   [clojure.datafy :refer [datafy]]
    [clojure.java.io :as io]
    [clojure.pprint :refer [pprint print-table]]
    [clojure.repl :refer :all]

backend/src/app/features/fdata.clj

@@ -12,12 +12,13 @@
    [app.common.files.helpers :as cfh]
    [app.common.files.migrations :as fmg]
    [app.common.logging :as l]
+   [app.common.types.objects-map :as omap]
    [app.common.types.path :as path]
    [app.db :as db]
    [app.db.sql :as-alias sql]
    [app.storage :as sto]
    [app.util.blob :as blob]
-   [app.util.objects-map :as omap]
+   [app.util.objects-map :as omap.legacy]
    [app.util.pointer-map :as pmap]))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -36,10 +37,7 @@
   [file & _opts]
   (let [update-page
         (fn [page]
-          (if (and (pmap/pointer-map? page)
-                   (not (pmap/loaded? page)))
-            page
-            (update page :objects omap/wrap)))
+          (update page :objects omap/wrap))
 
         update-data
         (fn [fdata]
@@ -58,7 +56,8 @@
             (fn [page]
               (update page :objects
                       (fn [objects]
-                        (if (omap/objects-map? objects)
+                        (if (or (omap/objects-map? objects)
+                                (omap.legacy/objects-map? objects))
                           (update-fn objects)
                           objects)))))
     fdata))

backend/src/app/rpc/commands/files.clj

@@ -336,14 +336,24 @@
           (cfeat/check-client-features! (:features params))
           (cfeat/check-file-features! (:features file)))
 
-      ;; This operation is needed for backward comapatibility with frontends that
-      ;; does not support pointer-map resolution mechanism; this just resolves the
-      ;; pointers on backend and return a complete file.
-      (if (and (contains? (:features file) "fdata/pointer-map")
-               (not (contains? (:features params) "fdata/pointer-map")))
-        (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
-          (update file :data feat.fdata/process-pointers deref))
-        file))))
+      (as-> file file
+        ;; This operation is needed for backward comapatibility with
+        ;; frontends that does not support pointer-map resolution
+        ;; mechanism; this just resolves the pointers on backend and
+        ;; return a complete file
+        (if (and (contains? (:features file) "fdata/pointer-map")
+                 (not (contains? (:features params) "fdata/pointer-map")))
+          (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
+            (update file :data feat.fdata/process-pointers deref))
+          file)
+
+        ;; This operation is needed for backward comapatibility with
+        ;; frontends that does not support objects-map mechanism; this
+        ;; just converts all objects map instaces to plain maps
+        (if (and (contains? (:features file) "fdata/objects-map")
+                 (not (contains? (:features params) "fdata/objects-map")))
+          (update file :data feat.fdata/process-objects (partial into {}))
+          file)))))
 
 ;; --- COMMAND QUERY: get-file-fragment (by id)
 

backend/src/app/rpc/commands/files_create.clj

@@ -112,14 +112,15 @@
     ;; FIXME: IMPORTANT: this code can have race conditions, because
     ;; we have no locks for updating team so, creating two files
     ;; concurrently can lead to lost team features updating
-
     (when-let [features (-> features
                             (set/difference (:features team))
                             (set/difference cfeat/no-team-inheritable-features)
                             (not-empty))]
-      (let [features (->> features
-                          (set/union (:features team))
-                          (db/create-array conn "text"))]
+      (let [features (-> features
+                         (set/union (:features team))
+                         (set/difference cfeat/no-team-inheritable-features)
+                         (into-array))]
+
         (db/update! conn :team
                     {:features features}
                     {:id (:id team)}

backend/src/app/rpc/commands/files_update.clj

@@ -158,7 +158,6 @@
 
                           tpoint   (ct/tpoint)]
 
-
                       (when (not= (:vern params)
                                   (:vern file))
                         (ex/raise :type :validation
@@ -181,15 +180,15 @@
                                               (set/difference (:features team))
                                               (set/difference cfeat/no-team-inheritable-features)
                                               (not-empty))]
-                        (let [features (->> features
-                                            (set/union (:features team))
-                                            (db/create-array conn "text"))]
+                        (let [features (-> features
+                                           (set/union (:features team))
+                                           (set/difference cfeat/no-team-inheritable-features)
+                                           (into-array))]
                           (db/update! conn :team
                                       {:features features}
                                       {:id (:id team)}
                                       {::db/return-keys false})))
 
-
                       (mtx/run! metrics {:id :update-file-changes :inc (count changes)})
 
                       (binding [l/*context* (some-> (meta params)

backend/src/app/rpc/commands/teams.clj

@@ -503,7 +503,7 @@
 
   (let [features (-> (cfeat/get-enabled-features cf/flags)
                      (set/difference cfeat/frontend-only-features)
-                     (cfeat/check-client-features! (:features params)))
+                     (set/difference cfeat/no-team-inheritable-features))
         params   (-> params
                      (assoc :profile-id profile-id)
                      (assoc :features features))

backend/src/app/srepl/main.clj

@@ -40,6 +40,7 @@
    [app.util.blob :as blob]
    [app.util.pointer-map :as pmap]
    [app.worker :as wrk]
+   [clojure.datafy :refer [datafy]]
    [clojure.java.io :as io]
    [clojure.pprint :refer [print-table]]
    [clojure.stacktrace :as strace]

common/src/app/common/buffer.cljc

@@ -50,6 +50,13 @@
     (let [target (with-meta target {:tag 'java.nio.ByteBuffer})]
       `(long (.getInt ~target (unchecked-int ~offset))))))
 
+(defmacro read-long
+  [target offset]
+  (if (:ns &env)
+    `(.getInt64 ~target ~offset true)
+    (let [target (with-meta target {:tag 'java.nio.ByteBuffer})]
+      `(.getLong ~target (unchecked-int ~offset)))))
+
 (defmacro read-float
   [target offset]
   (if (:ns &env)
@@ -75,6 +82,40 @@
          (finally
            (.order ~target ByteOrder/LITTLE_ENDIAN))))))
 
+(defmacro read-bytes
+  "Get a byte array from buffer. It is potentially unsafe because on
+  JS/CLJS it returns a subarray without doing any copy of data."
+  [target offset size]
+  (if (:ns &env)
+    `(new js/Uint8Array
+          (.-buffer ~target)
+          (+ (.-byteOffset ~target) ~offset)
+          ~size)
+    (let [target (with-meta target {:tag 'java.nio.ByteBuffer})
+          bbuf   (with-meta (gensym "bbuf") {:tag bytes})]
+      `(let [~bbuf (byte-array ~size)]
+         (.get ~target
+               (unchecked-int ~offset)
+               ~bbuf
+               0
+               ~size)
+         ~bbuf))))
+
+;; FIXME: implement in cljs
+(defmacro write-bytes
+  ([target offset src size]
+   `(write-bytes ~target ~offset ~src 0 ~size))
+  ([target offset src src-offset size]
+   (if (:ns &env)
+     (throw (ex-info "not implemented" {}))
+     (let [target (with-meta target {:tag 'java.nio.ByteBuffer})
+           src    (with-meta src {:tag 'bytes})]
+       `(.put ~target
+              (unchecked-int ~offset)
+              ~src
+              (unchecked-int ~src-offset)
+              (unchecked-int ~size))))))
+
 (defmacro write-byte
   [target offset value]
   (if (:ns &env)
@@ -144,13 +185,15 @@
        (.setUint32 ~target (+ ~offset 12) (aget barray# 3) true))
 
     (let [target (with-meta target {:tag 'java.nio.ByteBuffer})
-          value  (with-meta value {:tag 'java.util.UUID})]
-      `(try
-         (.order ~target ByteOrder/BIG_ENDIAN)
-         (.putLong ~target (unchecked-int (+ ~offset 0)) (.getMostSignificantBits ~value))
-         (.putLong ~target (unchecked-int (+ ~offset 8)) (.getLeastSignificantBits ~value))
-         (finally
-           (.order ~target ByteOrder/LITTLE_ENDIAN))))))
+          value  (with-meta value {:tag 'java.util.UUID})
+          prev   (with-meta (gensym "prev-") {:tag 'java.nio.ByteOrder})]
+      `(let [~prev (.order ~target)]
+         (try
+           (.order ~target ByteOrder/BIG_ENDIAN)
+           (.putLong ~target (unchecked-int (+ ~offset 0)) (.getMostSignificantBits ~value))
+           (.putLong ~target (unchecked-int (+ ~offset 8)) (.getLeastSignificantBits ~value))
+           (finally
+             (.order ~target ~prev)))))))
 
 (defn wrap
   [data]
@@ -160,7 +203,7 @@
 
 (defn allocate
   [size]
-  #?(:clj (let [buffer (ByteBuffer/allocate (int size))]
+  #?(:clj (let [buffer (ByteBuffer/allocate (unchecked-int size))]
             (.order buffer ByteOrder/LITTLE_ENDIAN))
      :cljs (new js/DataView (new js/ArrayBuffer size))))
 
@@ -181,6 +224,14 @@
        (.set dst-view src-view)
        (js/DataView. dst-buff))))
 
+;; FIXME: cljs impl
+#?(:clj
+   (defn copy-bytes
+     [src src-offset size dst dst-offset]
+     (let [tmp (byte-array size)]
+       (.get ^ByteBuffer src src-offset tmp 0 size)
+       (.put ^ByteBuffer dst dst-offset tmp 0 size))))
+
 (defn equals?
   [buffer-a buffer-b]
   #?(:clj
@@ -208,3 +259,18 @@
   [o]
   #?(:clj (instance? ByteBuffer o)
      :cljs (instance? js/DataView o)))
+
+(defn slice
+  [buffer offset size]
+  #?(:cljs
+     (let [offset (+ (.-byteOffset buffer) offset)]
+       (new js/DataView (.-buffer buffer) offset size))
+
+     :clj
+     (-> (.slice ^ByteBuffer buffer (unchecked-int offset) (unchecked-int size))
+         (.order ByteOrder/LITTLE_ENDIAN))))
+
+(defn size
+  [o]
+  #?(:cljs (.-byteLength ^js o)
+     :clj  (.capacity ^ByteBuffer o)))

common/src/app/common/features.cljc

@@ -68,11 +68,6 @@
     "design-tokens/v1"
     "variants/v1"})
 
-;; A set of features that should not be propagated to team on creating
-;; or modifying a file
-(def no-team-inheritable-features
-  #{"fdata/path-data"})
-
 ;; A set of features which only affects on frontend and can be enabled
 ;; and disabled freely by the user any time. This features does not
 ;; persist on file features field but can be permanently enabled on
@@ -87,8 +82,14 @@
 ;; Features that are mainly backend only or there are a proper
 ;; fallback when frontend reports no support for it
 (def backend-only-features
-  #{"fdata/objects-map"
-    "fdata/pointer-map"})
+  #{"fdata/pointer-map"
+    "fdata/objects-map"})
+
+;; A set of features that should not be propagated to team on creating
+;; or modifying a file or creating or modifying a team
+(def no-team-inheritable-features
+  #{"fdata/path-data"
+    "fdata/shape-data-type"})
 
 ;; This is a set of features that does not require an explicit
 ;; migration like components/v2 or the migration is not mandatory to
@@ -226,8 +227,6 @@
                 :hint (str/ffmt "enabled feature '%' not present in file (missing migration)"
                                 not-supported)))
 
-    (check-supported-features! file-features)
-
     ;; Components v1 is deprecated
     (when-not (contains? file-features "components/v2")
       (ex/raise :type :restriction

common/src/app/common/files/indices.cljc

@@ -6,14 +6,29 @@
 
 (ns app.common.files.indices
   (:require
+   [app.common.data :as d]
+   [app.common.data.macros :as dm]
    [app.common.files.helpers :as cfh]
    [app.common.uuid :as uuid]))
 
+(defn- generate-index
+  "An optimized algorithm for calculate parents index that walk from top
+  to down starting from a provided shape-id. Usefull when you want to
+  create an index for the whole objects or subpart of the tree."
+  [index objects shape-id parents]
+  (let [shape   (get objects shape-id)
+        index   (assoc index shape-id parents)
+        parents (cons shape-id parents)]
+    (reduce (fn [index shape-id]
+              (generate-index index objects shape-id parents))
+            index
+            (:shapes shape))))
+
 (defn generate-child-all-parents-index
   "Creates an index where the key is the shape id and the value is a set
   with all the parents"
   ([objects]
-   (generate-child-all-parents-index objects (vals objects)))
+   (generate-index {} objects uuid/zero []))
 
   ([objects shapes]
    (let [shape->entry
@@ -24,24 +39,25 @@
 (defn create-clip-index
   "Retrieves the mask information for an object"
   [objects parents-index]
-  (let [retrieve-clips
+  (let [get-clip-parents
+        (fn [shape]
+          (let [shape-id (dm/get-prop shape :id)]
+            (cond-> []
+              (or (and (cfh/frame-shape? shape)
+                       (not (:show-content shape))
+                       (not= uuid/zero shape-id))
+                  (cfh/bool-shape? shape))
+              (conj shape)
+
+              (:masked-group shape)
+              (conj (get objects (->> shape :shapes first))))))
+
+        xform
+        (comp (map (d/getf objects))
+              (mapcat get-clip-parents))
+
+        populate-with-clips
         (fn [parents]
-          (let [lookup-object (fn [id] (get objects id))
-                get-clip-parents
-                (fn [shape]
-                  (cond-> []
-                    (or (and (= :frame (:type shape))
-                             (not (:show-content shape))
-                             (not= uuid/zero (:id shape)))
-                        (cfh/bool-shape? shape))
-                    (conj shape)
+          (into [] xform parents))]
 
-                    (:masked-group shape)
-                    (conj (get objects (->> shape :shapes first)))))]
-
-            (into []
-                  (comp (map lookup-object)
-                        (mapcat get-clip-parents))
-                  parents)))]
-    (-> parents-index
-        (update-vals retrieve-clips))))
+    (d/update-vals parents-index populate-with-clips)))

common/src/app/common/logging.cljc

@@ -49,6 +49,7 @@
    [app.common.exceptions :as ex]
    [app.common.pprint :as pp]
    [app.common.schema :as sm]
+   [app.common.time :as ct]
    [app.common.uuid :as uuid]
    [cuerdas.core :as str]
    [promesa.exec :as px]
@@ -221,36 +222,42 @@
   #?(:clj (inst-ms (java.time.Instant/now))
      :cljs (js/Date.now)))
 
+(defn emit-log
+  [props cause context logger level sync?]
+  (let [props    (cond-> props sync? deref)
+        ts       (current-timestamp)
+        gcontext *context*
+        logfn    (fn []
+                   (let [props   (if sync? props (deref props))
+                         props   (into (d/ordered-map) props)
+                         context (if (and (empty? gcontext)
+                                          (empty? context))
+                                   {}
+                                   (d/without-nils (merge gcontext context)))
+
+                         lrecord {::id (uuid/next)
+                                  ::timestamp ts
+                                  ::message (delay (build-message props))
+                                  ::props props
+                                  ::context context
+                                  ::level level
+                                  ::logger logger}
+                         lrecord (cond-> lrecord
+                                   (some? cause)
+                                   (assoc ::cause cause
+                                          ::trace (delay (build-stack-trace cause))))]
+                     (swap! log-record (constantly lrecord))))]
+    (if sync?
+      (logfn)
+      (px/exec! *default-executor* logfn))))
+
 (defmacro log!
   "Emit a new log record to the global log-record state (asynchronously). "
   [& props]
   (let [{:keys [::level ::logger ::context ::sync? cause] :or {sync? false}} props
         props (into [] msg-props-xf props)]
     `(when (enabled? ~logger ~level)
-       (let [props#   (cond-> (delay ~props) ~sync? deref)
-             ts#      (current-timestamp)
-             context# *context*
-             logfn#   (fn []
-                        (let [props#   (if ~sync? props# (deref props#))
-                              props#   (into (d/ordered-map) props#)
-                              cause#   ~cause
-                              context# (d/without-nils
-                                        (merge context# ~context))
-                              lrecord# {::id (uuid/next)
-                                        ::timestamp ts#
-                                        ::message (delay (build-message props#))
-                                        ::props props#
-                                        ::context context#
-                                        ::level ~level
-                                        ::logger ~logger}
-                              lrecord# (cond-> lrecord#
-                                         (some? cause#)
-                                         (assoc ::cause cause#
-                                                ::trace (delay (build-stack-trace cause#))))]
-                          (swap! log-record (constantly lrecord#))))]
-         (if ~sync?
-           (logfn#)
-           (px/exec! *default-executor* logfn#))))))
+       (emit-log (delay ~props) ~cause ~context ~logger ~level ~sync?))))
 
 #?(:clj
    (defn slf4j-log-handler
@@ -276,7 +283,8 @@
      (when (enabled? logger level)
        (let [hstyles (str/ffmt "font-weight: 600; color: %" (level->color level))
              mstyles (str/ffmt "font-weight: 300; color: %" (level->color level))
-             header  (str/concat "%c" (level->name level) " [" logger "] ")
+             ts      (ct/format-inst (ct/now) "kk:mm:ss.SSSS")
+             header  (str/concat "%c" (level->name level) " " ts  " [" logger "] ")
              message (str/concat header "%c" @message)]
 
          (js/console.group message hstyles mstyles)

common/src/app/common/schema/generators.cljc

@@ -5,7 +5,7 @@
 ;; Copyright (c) KALEIDOS INC
 
 (ns app.common.schema.generators
-  (:refer-clojure :exclude [set subseq uuid filter map let boolean vector keyword int double])
+  (:refer-clojure :exclude [set subseq uuid filter map let boolean vector keyword int double not-empty])
   #?(:cljs (:require-macros [app.common.schema.generators]))
   (:require
    [app.common.math :as mth]
@@ -146,3 +146,5 @@
 
 (def any
   (tg/one-of [text boolean double int keyword]))
+
+(def not-empty tg/not-empty)

common/src/app/common/types/objects_map.cljc

@@ -0,0 +1,521 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.common.types.objects-map
+  "Implements a specialized map-like data structure for store an UUID =>
+  OBJECT mappings. The main purpose of this data structure is be able
+  to serialize it on fressian as byte-array and have the ability to
+  decode each field separatelly without the need to decode the whole
+  map from the byte-array.
+
+  It works transparently, so no aditional dynamic vars are needed.  It
+  only works by reference equality and the hash-code is calculated
+  properly from each value."
+
+  (:require
+   #?(:clj [app.common.fressian :as fres])
+   #?(:clj [clojure.data.json :as json])
+   [app.common.transit :as t]
+   [clojure.core :as c]
+   [clojure.core.protocols :as cp])
+  #?(:clj
+     (:import
+      clojure.lang.Murmur3
+      clojure.lang.RT
+      java.util.Iterator)))
+
+#?(:clj (set! *warn-on-reflection* true))
+
+(declare create)
+(declare ^:private do-compact)
+
+(defprotocol IObjectsMap
+  (^:no-doc compact [this])
+  (^:no-doc get-data [this] "retrieve internal data")
+  (^:no-doc -hash-for-key [this key] "retrieve a hash for a key"))
+
+#?(:cljs
+   (deftype ObjectsMapEntry [key omap]
+     c/IMapEntry
+     (-key [_] key)
+     (-val [_] (get omap key))
+
+     c/IHash
+     (-hash [_]
+       (-hash-for-key omap key))
+
+     c/IEquiv
+     (-equiv [this other]
+       (and (c/map-entry? other)
+            (= (key this)
+               (key other))
+            (= (val this)
+               (val other))))
+
+     c/ISequential
+     c/ISeqable
+     (-seq [this]
+       (cons key (lazy-seq (cons (c/-val this) nil))))
+
+     c/ICounted
+     (-count [_] 2)
+
+     c/IIndexed
+     (-nth [node n]
+       (cond (== n 0) key
+             (== n 1) (c/-val node)
+             :else    (throw (js/Error. "Index out of bounds"))))
+
+     (-nth [node n not-found]
+       (cond (== n 0) key
+             (== n 1) (c/-val node)
+             :else    not-found))
+
+     c/ILookup
+     (-lookup [node k]
+       (c/-nth node k nil))
+     (-lookup [node k not-found]
+       (c/-nth node k not-found))
+
+     c/IFn
+     (-invoke [node k]
+       (c/-nth node k))
+
+     (-invoke [node k not-found]
+       (c/-nth node k not-found))
+
+     c/IPrintWithWriter
+     (-pr-writer [this writer opts]
+       (c/pr-sequential-writer
+        writer
+        (fn [item w _]
+          (c/-write w (pr-str item)))
+        "[" ", " "]"
+        opts
+        this)))
+
+   :clj
+   (deftype ObjectsMapEntry [key omap]
+     clojure.lang.IMapEntry
+     (key [_] key)
+     (getKey [_] key)
+
+     (val [_]
+       (get omap key))
+     (getValue [_]
+       (get omap key))
+
+     clojure.lang.Indexed
+     (nth [node n]
+       (cond
+         (== n 0) key
+         (== n 1) (val node)
+         :else    (throw (IllegalArgumentException. "Index out of bounds"))))
+
+     (nth [node n not-found]
+       (cond
+         (== n 0) key
+         (== n 1) (val node)
+         :else    not-found))
+
+     clojure.lang.IPersistentCollection
+     (empty [_] [])
+     (count [_] 2)
+     (seq [this]
+       (cons key (lazy-seq (cons (val this) nil))))
+     (cons [this item]
+       (.cons ^clojure.lang.IPersistentCollection (vec this) item))
+
+     clojure.lang.IHashEq
+     (hasheq [_]
+       (-hash-for-key omap key))))
+
+#?(:cljs
+   (deftype ObjectMapIterator [iterator omap]
+     Object
+     (hasNext [_]
+       (.hasNext ^js iterator))
+
+     (next [_]
+       (let [entry (.next iterator)]
+         (ObjectsMapEntry. (key entry) omap)))
+
+     (remove [_]
+       (js/Error. "Unsupported operation")))
+
+   :clj
+   (deftype ObjectsMapIterator [^Iterator iterator omap]
+     Iterator
+     (hasNext [_]
+       (.hasNext iterator))
+
+     (next [_]
+       (let [entry (.next iterator)]
+         (ObjectsMapEntry. (key entry) omap)))))
+
+#?(:cljs
+   (deftype ObjectsMap [metadata cache
+                        ^:mutable data
+                        ^:mutable modified
+                        ^:mutable hash]
+     Object
+     (toString [this]
+       (pr-str* this))
+     (equiv [this other]
+       (c/-equiv this other))
+     (keys [this]
+       (c/es6-iterator (keys this)))
+     (entries [this]
+       (c/es6-entries-iterator (seq this)))
+     (values [this]
+       (es6-iterator (vals this)))
+     (has [this k]
+       (c/contains? this k))
+     (get [this k not-found]
+       (c/-lookup this k not-found))
+     (forEach [this f]
+       (run! (fn [[k v]] (f v k)) this))
+
+     cp/Datafiable
+     (datafy [_]
+       {:data data
+        :cache cache
+        :modified modified
+        :hash hash})
+
+     IObjectsMap
+     (compact [this]
+       (when modified
+         (do-compact data cache
+                     (fn [data']
+                       (set! (.-modified this) false)
+                       (set! (.-data this) data'))))
+       this)
+
+     (get-data [this]
+       (compact this)
+       data)
+
+     (-hash-for-key [this key]
+       (if (c/-contains-key? cache key)
+         (c/-hash (c/-lookup cache key))
+         (c/-hash (c/-lookup this key))))
+
+     c/IWithMeta
+     (-with-meta [this new-meta]
+       (if (identical? new-meta meta)
+         this
+         (ObjectsMap. new-meta
+                      cache
+                      data
+                      modified
+                      hash)))
+
+     c/IMeta
+     (-meta [_] metadata)
+
+     c/ICloneable
+     (-clone [this]
+       (compact this)
+       (ObjectsMap. metadata {} data false nil))
+
+     c/IIterable
+     (-iterator [this]
+       (c/seq-iter this))
+
+     c/ICollection
+     (-conj [this entry]
+       (cond
+         (map-entry? entry)
+         (c/-assoc this (c/-key entry) (c/-val entry))
+
+         (vector? entry)
+         (c/-assoc this (c/-nth entry 0) (c/-nth entry 1))
+
+         :else
+         (loop [ret this es (seq entry)]
+           (if (nil? es)
+             ret
+             (let [e (first es)]
+               (if (vector? e)
+                 (recur (c/-assoc ret (c/-nth e 0) (c/-nth e 1))
+                        (next es))
+                 (throw (js/Error. "conj on a map takes map entries or seqables of map entries"))))))))
+
+     c/IEmptyableCollection
+     (-empty [_]
+       (create))
+
+     c/IEquiv
+     (-equiv [this other]
+       (equiv-map this other))
+
+     c/IHash
+     (-hash [this]
+       (when-not hash
+         (set! hash (hash-unordered-coll this)))
+       hash)
+
+     c/ISeqable
+     (-seq [this]
+       (->> (keys data)
+            (map (fn [id] (new ObjectsMapEntry id this)))
+            (seq)))
+
+     c/ICounted
+     (-count [_]
+       (c/-count data))
+
+     c/ILookup
+     (-lookup [this k]
+       (or (c/-lookup cache k)
+           (if (c/-contains-key? data k)
+             (let [v (c/-lookup data k)
+                   v (t/decode-str v)]
+               (set! (.-cache this) (c/-assoc cache k v))
+               v)
+             (do
+               (set! (.-cache this) (assoc cache key nil))
+               nil))))
+
+     (-lookup [this k not-found]
+       (if (c/-contains-key? data k)
+         (c/-lookup this k)
+         not-found))
+
+     c/IAssociative
+     (-assoc [_ k v]
+       (ObjectsMap. metadata
+                    (c/-assoc cache k v)
+                    (c/-assoc data k nil)
+                    true
+                    nil))
+
+     (-contains-key? [_ k]
+       (c/-contains-key? data k))
+
+     c/IFind
+     (-find [this k]
+       (when (c/-contains-key? data k)
+         (new ObjectsMapEntry k this)))
+
+     c/IMap
+     (-dissoc [_ k]
+       (ObjectsMap. metadata
+                    (c/-dissoc cache k)
+                    (c/-dissoc data k)
+                    true
+                    nil))
+
+     c/IKVReduce
+     (-kv-reduce [this f init]
+       (c/-kv-reduce data
+                     (fn [init k _]
+                       (f init k (c/-lookup this k)))
+                     init))
+
+     c/IFn
+     (-invoke [this k]
+       (c/-lookup this k))
+     (-invoke [this k not-found]
+       (c/-lookup this k not-found))
+
+     c/IPrintWithWriter
+     (-pr-writer [this writer opts]
+       (c/pr-sequential-writer
+        writer
+        (fn [item w _]
+          (c/-write w (pr-str (c/-key item)))
+          (c/-write w \space)
+          (c/-write w (pr-str (c/-val item))))
+        "#penpot/objects-map {" ", " "}"
+        opts
+        (seq this))))
+
+   :clj
+   (deftype ObjectsMap [metadata cache
+                        ^:unsynchronized-mutable data
+                        ^:unsynchronized-mutable modified
+                        ^:unsynchronized-mutable hash]
+
+     Object
+     (hashCode [this]
+       (.hasheq ^clojure.lang.IHashEq this))
+
+     cp/Datafiable
+     (datafy [_]
+       {:data data
+        :cache cache
+        :modified modified
+        :hash hash})
+
+     IObjectsMap
+     (compact [this]
+       (locking this
+         (when modified
+           (do-compact data cache
+                       (fn [data']
+                         (set! (.-modified this) false)
+                         (set! (.-data this) data')))))
+       this)
+
+     (get-data [this]
+       (compact this)
+       data)
+
+     (-hash-for-key [this key]
+       (if (contains? cache key)
+         (c/hash (get cache key))
+         (c/hash (get this key))))
+
+     json/JSONWriter
+     (-write [this writter options]
+       (json/-write (into {} this) writter options))
+
+     clojure.lang.IHashEq
+     (hasheq [this]
+       (when-not hash
+         (set! hash (Murmur3/hashUnordered this)))
+       hash)
+
+     clojure.lang.Seqable
+     (seq [this]
+       (RT/chunkIteratorSeq (.iterator ^Iterable this)))
+
+     java.lang.Iterable
+     (iterator [this]
+       (ObjectsMapIterator. (.iterator ^Iterable data) this))
+
+     clojure.lang.IPersistentCollection
+     (equiv [this other]
+       (and (instance? ObjectsMap other)
+            (= (count this) (count other))
+            (reduce-kv (fn [_ id _]
+                         (let [this-val  (get this id)
+                               other-val (get other id)
+                               result    (= this-val other-val)]
+                           (or result
+                               (reduced false))))
+                       true
+                       data)))
+
+     clojure.lang.IPersistentMap
+     (cons [this o]
+       (if (map-entry? o)
+         (assoc this (key o) (val o))
+         (if (vector? o)
+           (assoc this (nth o 0) (nth o 1))
+           (throw (UnsupportedOperationException. "invalid arguments to cons")))))
+
+     (empty [_]
+       (create))
+
+     (containsKey [_ key]
+       (.containsKey ^clojure.lang.IPersistentMap data key))
+
+     (entryAt [this key]
+       (ObjectsMapEntry. this key))
+
+     (valAt [this key]
+       (or (get cache key)
+           (locking this
+             (if (contains? data key)
+               (let [value (get data key)
+                     value (t/decode-str value)]
+                 (set! (.-cache this) (assoc cache key value))
+                 value)
+               (do
+                 (set! (.-cache this) (assoc cache key nil))
+                 nil)))))
+
+     (valAt [this key not-found]
+       (if (.containsKey ^clojure.lang.IPersistentMap data key)
+         (.valAt this key)
+         not-found))
+
+     (assoc [_ key val]
+       (ObjectsMap. metadata
+                    (assoc cache key val)
+                    (assoc data key nil)
+                    true
+                    nil))
+
+
+     (assocEx [_ _ _]
+       (throw (UnsupportedOperationException. "method not implemented")))
+
+     (without [_ key]
+       (ObjectsMap. metadata
+                    (dissoc cache key)
+                    (dissoc data key)
+                    true
+                    nil))
+
+     clojure.lang.Counted
+     (count [_]
+       (count data))))
+
+#?(:cljs (es6-iterable ObjectsMap))
+
+
+(defn- do-compact
+  [data cache update-fn]
+  (let [new-data
+        (persistent!
+         (reduce-kv (fn [data id obj]
+                      (if (nil? obj)
+                        (assoc! data id (t/encode-str (get cache id)))
+                        data))
+                    (transient data)
+                    data))]
+    (update-fn new-data)
+    nil))
+
+(defn from-data
+  [data]
+  (ObjectsMap. {} {}
+               data
+               false
+               nil))
+
+(defn objects-map?
+  [o]
+  (instance? ObjectsMap o))
+
+(defn create
+  ([] (from-data {}))
+  ([other]
+   (cond
+     (objects-map? other)
+     (-> other get-data from-data)
+
+     :else
+     (throw #?(:clj (UnsupportedOperationException. "invalid arguments")
+               :cljs (js/Error. "invalid arguments"))))))
+
+(defn wrap
+  [objects]
+  (if (instance? ObjectsMap objects)
+    objects
+    (->> objects
+         (into (create))
+         (compact))))
+
+#?(:clj
+   (fres/add-handlers!
+    {:name "penpot/objects-map/v2"
+     :class ObjectsMap
+     :wfn (fn [n w o]
+            (fres/write-tag! w n)
+            (fres/write-object! w (get-data o)))
+     :rfn (fn [r]
+            (-> r fres/read-object! from-data))}))
+
+(t/add-handlers!
+ {:id "penpot/objects-map/v2"
+  :class ObjectsMap
+  :wfn get-data
+  :rfn from-data})

common/src/app/common/weak.cljc

@@ -22,7 +22,7 @@
 
 #?(:cljs
    (defn weak-map
-     "Create a WeakMap like instance what uses clojure equality
+     "Create a WeakMap-like instance what uses clojure equality
   semantics."
      []
      (new wm/WeakEqMap #js {:hash hash :equals =})))

common/test/common_tests/runner.cljc

@@ -41,6 +41,7 @@
    [common-tests.types.components-test]
    [common-tests.types.fill-test]
    [common-tests.types.modifiers-test]
+   [common-tests.types.objects-map-test]
    [common-tests.types.path-data-test]
    [common-tests.types.shape-decode-encode-test]
    [common-tests.types.shape-interactions-test]
@@ -90,9 +91,10 @@
    'common-tests.time-test
    'common-tests.types.absorb-assets-test
    'common-tests.types.components-test
-   'common-tests.types.modifiers-test
-   'common-tests.types.path-data-test
    'common-tests.types.fill-test
+   'common-tests.types.modifiers-test
+   'common-tests.types.objects-map-test
+   'common-tests.types.path-data-test
    'common-tests.types.shape-decode-encode-test
    'common-tests.types.shape-interactions-test
    'common-tests.types.tokens-lib-test

common/test/common_tests/types/objects_map_test.cljc

@@ -0,0 +1,133 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns common-tests.types.objects-map-test
+  (:require
+   #?(:clj [app.common.fressian :as fres])
+   [app.common.json :as json]
+   [app.common.pprint :as pp]
+   [app.common.schema :as sm]
+   [app.common.schema.generators :as sg]
+   [app.common.schema.test :as smt]
+   [app.common.transit :as transit]
+   [app.common.types.objects-map :as omap]
+   [app.common.types.path :as path]
+   [app.common.types.plugins :refer [schema:plugin-data]]
+   [app.common.types.shape :as cts]
+   [app.common.uuid :as uuid]
+   [clojure.datafy :refer [datafy]]
+   [clojure.test :as t]))
+
+(t/deftest basic-operations
+  (t/testing "assoc"
+    (let [id  (uuid/custom 0 1)
+          id' (uuid/custom 0 2)
+          obj (-> (omap/create) (assoc id {:foo 1}))]
+      (t/is (not= id id'))
+      (t/is (not (contains? obj id')))
+      (t/is (contains? obj id))))
+
+  (t/testing "assoc-with-non-uuid-keys"
+    (let [obj (-> (omap/create)
+                  (assoc :a {:foo 1})
+                  (assoc :b {:bar 1}))]
+      (t/is (not (contains? obj :c)))
+      (t/is (contains? obj :a))
+      (t/is (contains? obj :b))))
+
+  (t/testing "dissoc"
+    (let [id  (uuid/custom 0 1)
+          obj (-> (omap/create) (assoc id {:foo 1}))]
+      (t/is (contains? obj id))
+      (let [obj (dissoc obj id)]
+        (t/is (not (contains? obj id))))))
+
+  (t/testing "seq"
+    (let [id  (uuid/custom 0 1)
+          obj (-> (omap/create) (assoc id 1))]
+      (t/is (contains? obj id))
+      (let [[entry] (seq obj)]
+        (t/is (map-entry? entry))
+        (t/is (= (key entry) id))
+        (t/is (= (val entry) 1)))))
+
+  (t/testing "cons & count"
+    (let [obj (into (omap/create) [[uuid/zero 1]])]
+      (t/is (contains? obj uuid/zero))
+      (t/is (= 1 (count obj)))
+      (t/is (omap/objects-map? obj))))
+
+  (t/testing "wrap"
+    (let [obj1 (omap/wrap {})
+          tmp  (omap/create)
+          obj2 (omap/wrap tmp)]
+      (t/is (omap/objects-map? obj1))
+      (t/is (omap/objects-map? obj2))
+      (t/is (identical? tmp obj2))
+      (t/is (= 0 (count obj1)))
+      (t/is (= 0 (count obj2))))))
+
+(t/deftest internal-state
+  (t/testing "modified & compact"
+    (let [obj (-> (omap/create)
+                  (assoc :a 1)
+                  (assoc :b 2))]
+      (t/is (= 2 (count obj)))
+      (t/is (-> obj datafy :modified))
+      (let [obj (omap/compact obj)]
+        (t/is (not (-> obj datafy :modified))))))
+
+  (t/testing "create from other"
+    (let [obj1 (-> (omap/create)
+                   (assoc :a {:foo 1})
+                   (assoc :b {:bar 2}))
+          obj2 (omap/create obj1)]
+
+      (t/is (not (identical? obj1 obj2)))
+      (t/is (= obj1 obj2))
+      (t/is (= (hash obj1) (hash obj2)))
+      (t/is (= (get obj1 :a) (get obj2 :a)))
+      (t/is (= (get obj1 :b) (get obj2 :b))))))
+
+(t/deftest creation-and-duplication
+  (smt/check!
+   (smt/for [data (->> (sg/map-of (sg/uuid) (sg/generator cts/schema:shape))
+                       (sg/not-empty))]
+     (let [obj1 (omap/wrap data)
+           obj2 (omap/create obj1)]
+       (and (= (hash obj1) (hash obj2))
+            (= obj1 obj2))))
+   {:num 100}))
+
+#?(:clj
+   (t/deftest fressian-encode-decode
+     (smt/check!
+      (smt/for [data (->> (sg/map-of (sg/uuid) (sg/generator cts/schema:shape))
+                          (sg/not-empty)
+                          (sg/fmap omap/wrap)
+                          (sg/fmap (fn [o] {:objects o})))]
+
+        (let [res (-> data fres/encode fres/decode)]
+          (and (contains? res :objects)
+               (omap/objects-map? (:objects res))
+               (= res data))))
+      {:num 100})))
+
+(t/deftest transit-encode-decode
+  (smt/check!
+   (smt/for [data (->> (sg/map-of (sg/uuid) (sg/generator cts/schema:shape))
+                       (sg/not-empty)
+                       (sg/fmap omap/wrap)
+                       (sg/fmap (fn [o] {:objects o})))]
+     (let [res (-> data transit/encode-str transit/decode-str)]
+       ;; (app.common.pprint/pprint data)
+       ;; (app.common.pprint/pprint res)
+       (and (every? (fn [[k v]]
+                      (= v (get-in data [:objects k])))
+                    (:objects res))
+            (omap/objects-map? (:objects data))
+            (omap/objects-map? (:objects res)))))
+   {:num 100}))

frontend/src/app/main.cljs

@@ -8,6 +8,7 @@
   (:require
    [app.common.data.macros :as dm]
    [app.common.logging :as log]
+   [app.common.types.objects-map]
    [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.main.data.auth :as da]

frontend/src/app/main/data/changes.cljs

@@ -52,7 +52,7 @@
         (->> (rx/from changes)
              (rx/merge-map (fn [[page-id changes]]
                              (log/debug :hint "update-indexes" :page-id page-id :changes (count changes))
-                             (mw/ask! {:cmd :index/update-page-index
+                             (mw/ask! {:cmd :index/update
                                        :page-id page-id
                                        :changes changes})))
              (rx/catch (fn [cause]

frontend/src/app/main/data/workspace.cljs

@@ -22,7 +22,6 @@
    [app.common.types.component :as ctc]
    [app.common.types.fills :as types.fills]
    [app.common.types.shape :as cts]
-   [app.common.types.shape-tree :as ctst]
    [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.main.data.changes :as dch]
@@ -44,7 +43,6 @@
    [app.main.data.workspace.common :as dwc]
    [app.main.data.workspace.drawing :as dwd]
    [app.main.data.workspace.edition :as dwe]
-   [app.main.data.workspace.fix-broken-shapes :as fbs]
    [app.main.data.workspace.fix-deleted-fonts :as fdf]
    [app.main.data.workspace.groups :as dwg]
    [app.main.data.workspace.guides :as dwgu]
@@ -71,7 +69,6 @@
    [app.main.features.pointer-map :as fpmap]
    [app.main.repo :as rp]
    [app.main.router :as rt]
-   [app.main.worker :as mw]
    [app.render-wasm :as wasm]
    [app.render-wasm.api :as api]
    [app.util.dom :as dom]
@@ -85,7 +82,7 @@
    [cuerdas.core :as str]
    [potok.v2.core :as ptk]))
 
-(log/set-level! :debug)
+(log/set-level! :trace)
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; Workspace Initialization
@@ -159,18 +156,9 @@
   (->> (fpmap/resolve-file file)
        (rx/map :data)
        (rx/map process-fills)
-       (rx/mapcat
-        (fn [{:keys [pages-index] :as data}]
-          (->> (rx/from (seq pages-index))
-               (rx/mapcat
-                (fn [[id page]]
-                  (let [page (update page :objects ctst/start-page-index)]
-                    (->> (mw/ask! {:cmd :index/initialize-page-index :page page})
-                         (rx/map (fn [_] [id page]))))))
-               (rx/reduce conj {})
-               (rx/map (fn [pages-index]
-                         (let [data (assoc data :pages-index pages-index)]
-                           (assoc file :data (d/removem (comp t/pointer? val) data))))))))))
+       (rx/map
+        (fn [data]
+          (assoc file :data (d/removem (comp t/pointer? val) data))))))
 
 (defn- check-libraries-synchronozation
   [file-id libraries]
@@ -244,8 +232,7 @@
     ptk/WatchEvent
     (watch [_ _ _]
       (rx/of (dp/check-open-plugin)
-             (fdf/fix-deleted-fonts)
-             (fbs/fix-broken-shapes)))))
+             (fdf/fix-deleted-fonts-for-local-library file-id)))))
 
 (defn- bundle-fetched
   [{:keys [file file-id thumbnails] :as bundle}]
@@ -281,6 +268,8 @@
   (ptk/reify ::fetch-bundle
     ptk/WatchEvent
     (watch [_ _ stream]
+      (log/debug :hint "fetch bundle" :file-id (dm/str file-id))
+
       (let [stopper-s (rx/filter (ptk/type? ::finalize-workspace) stream)]
         (->> (rx/zip (rp/cmd! :get-file {:id file-id :features features})
                      (get-file-object-thumbnails file-id))
@@ -289,6 +278,7 @@
               (fn [[file thumbnails]]
                 (->> (resolve-file file)
                      (rx/map (fn [file]
+                               (log/trace :hint "file resolved" :file-id file-id)
                                {:file file
                                 :file-id file-id
                                 :features features
@@ -358,6 +348,10 @@
                    (rx/map deref)
                    (rx/mapcat
                     (fn [{:keys [file]}]
+                      (log/debug :hint "bundle fetched"
+                                 :team-id (dm/str team-id)
+                                 :file-id (dm/str file-id))
+
                       (rx/of (dpj/initialize-project (:project-id file))
                              (dwn/initialize team-id file-id)
                              (dwsl/initialize-shape-layout)

frontend/src/app/main/data/workspace/drawing/common.cljs

@@ -83,7 +83,7 @@
               (rx/of (dwsh/add-shape shape {:no-select? (= tool :curve)}))
               (if (cfh/frame-shape? shape)
                 (rx/concat
-                 (->> (mw/ask! {:cmd :selection/query
+                 (->> (mw/ask! {:cmd :index/query-selection
                                 :page-id page-id
                                 :rect (:selrect shape)
                                 :include-frames? true

frontend/src/app/main/data/workspace/fix_broken_shapes.cljs

@@ -1,56 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) KALEIDOS INC
-
-(ns app.main.data.workspace.fix-broken-shapes
-  (:require
-   [app.main.data.changes :as dch]
-   [app.main.data.helpers :as dsh]
-   [beicon.v2.core :as rx]
-   [potok.v2.core :as ptk]))
-
-(defn- generate-broken-link-changes
-  [attr {:keys [objects id] :as container}]
-  (let [base      {:type :fix-obj :fix :broken-children attr id}
-        contains? (partial contains? objects)
-        xform     (comp
-                   ;; FIXME: Ensure all obj have id field (this is needed
-                   ;; because some bug adds an ephimeral shape with id ZERO,
-                   ;; with a single attr `:shapes` having a vector of ids
-                   ;; pointing to not existing shapes). That happens on
-                   ;; components. THIS IS A WORKAOURD
-                   (map (fn [[id obj]]
-                          (if (some? (:id obj))
-                            obj
-                            (assoc obj :id id))))
-
-                   ;; Remove all valid shapes
-                   (remove (fn [obj]
-                             (every? contains? (:shapes obj))))
-
-                   (map (fn [obj]
-                          (assoc base :id (:id obj)))))]
-
-    (sequence xform objects)))
-
-(defn fix-broken-shapes
-  []
-  (ptk/reify ::fix-broken-shapes
-    ptk/WatchEvent
-    (watch [it state _]
-      (let [fdata   (dsh/lookup-file-data state)
-            changes (concat
-                     (mapcat (partial generate-broken-link-changes :page-id)
-                             (vals (:pages-index fdata)))
-                     (mapcat (partial generate-broken-link-changes :component-id)
-                             (vals (:components fdata))))]
-
-        (if (seq changes)
-          (rx/of (dch/commit-changes
-                  {:origin it
-                   :redo-changes (vec changes)
-                   :undo-changes []
-                   :save-undo? false}))
-          (rx/empty))))))

frontend/src/app/main/data/workspace/fix_deleted_fonts.cljs

@@ -14,8 +14,9 @@
    [beicon.v2.core :as rx]
    [potok.v2.core :as ptk]))
 
-;; This event will update the file so the texts with non existing custom fonts try to be fixed.
-;; This can happen when:
+;; This event will update the file so the texts with non existing
+;; custom fonts try to be fixed. This can happen when:
+;;
 ;; - Exporting/importing files to different teams or penpot instances
 ;; - Moving files from one team to another in the same instance
 ;; - Custom fonts are explicitly deleted in the team area
@@ -23,112 +24,99 @@
 (defn- calculate-alternative-font-id
   [value]
   (let [fonts (deref fonts/fontsdb)]
-    (->> (vals fonts)
-         (filter #(= (:family %) value))
-         (first)
-         :id)))
+    (reduce-kv (fn [_ _ font]
+                 (if (= (:family font) value)
+                   (reduced (:id font))
+                   nil))
+               nil
+               fonts)))
 
 (defn- has-invalid-font-family?
   [node]
-  (let [fonts               (deref fonts/fontsdb)
-        font-family         (:font-family node)
-        alternative-font-id (calculate-alternative-font-id font-family)]
+  (let [fonts (deref fonts/fontsdb)
+        font-family (:font-family node)]
     (and (some? font-family)
-         (nil? (get fonts (:font-id node)))
-         (some? alternative-font-id))))
+         (nil? (get fonts (:font-id node))))))
 
-(defn- should-fix-deleted-font-shape?
+(defn- shape-has-invalid-font-family??
   [shape]
-  (let [text-nodes (txt/node-seq txt/is-text-node? (:content shape))]
-    (and (cfh/text-shape? shape)
-         (some has-invalid-font-family? text-nodes))))
-
-(defn- should-fix-deleted-font-component?
-  [component]
-  (let [xf (comp (map val)
-                 (filter should-fix-deleted-font-shape?))]
-    (first (sequence xf (:objects component)))))
+  (and (cfh/text-shape? shape)
+       (some has-invalid-font-family?
+             (txt/node-seq txt/is-text-node? (:content shape)))))
 
 (defn- fix-deleted-font
   [node]
-  (let [alternative-font-id (calculate-alternative-font-id (:font-family node))]
-    (cond-> node
-      (some? alternative-font-id) (assoc :font-id alternative-font-id))))
+  (if-let [alternative-font-id (calculate-alternative-font-id (:font-family node))]
+    (assoc node :font-id alternative-font-id)
+    node))
 
-(defn- fix-deleted-font-shape
+(defn- fix-shape-content
   [shape]
-  (let [transform (partial txt/transform-nodes has-invalid-font-family? fix-deleted-font)]
-    (update shape :content transform)))
+  (txt/transform-nodes has-invalid-font-family? fix-deleted-font
+                       (:content shape)))
 
-(defn- fix-deleted-font-component
-  [component]
-  (update component
-          :objects
-          (fn [objects]
-            (update-vals objects fix-deleted-font-shape))))
-
-(defn fix-deleted-font-typography
+(defn- fix-typography
   [typography]
-  (let [alternative-font-id (calculate-alternative-font-id (:font-family typography))]
-    (cond-> typography
-      (some? alternative-font-id) (assoc :font-id alternative-font-id))))
+  (if-let [alternative-font-id (calculate-alternative-font-id (:font-family typography))]
+    (assoc typography :font-id alternative-font-id)
+    typography))
 
-(defn- generate-deleted-font-shape-changes
+(defn- generate-page-changes
   [{:keys [objects id]}]
-  (sequence
-   (comp (map val)
-         (filter should-fix-deleted-font-shape?)
-         (map (fn [shape]
-                {:type :mod-obj
-                 :id (:id shape)
-                 :page-id id
-                 :operations [{:type :set
-                               :attr :content
-                               :val (:content (fix-deleted-font-shape shape))}
-                              {:type :set
-                               :attr :position-data
-                               :val nil}]})))
-   objects))
+  (reduce-kv (fn [changes shape-id shape]
+               (if (shape-has-invalid-font-family?? shape)
+                 (conj changes {:type :mod-obj
+                                :id shape-id
+                                :page-id id
+                                :operations [{:type :set
+                                              :attr :content
+                                              :val (fix-shape-content shape)}
+                                             {:type :set
+                                              :attr :position-data
+                                              :val nil}]})
+                 changes))
+             []
+             objects))
 
-(defn- generate-deleted-font-components-changes
+(defn- generate-library-changes
   [fdata]
-  (sequence
-   (comp (map val)
-         (filter should-fix-deleted-font-component?)
-         (map (fn [component]
-                {:type :mod-component
-                 :id (:id component)
-                 :objects (-> (fix-deleted-font-component component) :objects)})))
-   (:components fdata)))
+  (reduce-kv (fn [changes _ typography]
+               (if (has-invalid-font-family? typography)
+                 (conj changes {:type :mod-typography
+                                :typography (fix-typography typography)})
+                 changes))
+             []
+             (:typographies fdata)))
 
-(defn- generate-deleted-font-typography-changes
-  [fdata]
-  (sequence
-   (comp (map val)
-         (filter has-invalid-font-family?)
-         (map (fn [typography]
-                {:type :mod-typography
-                 :typography (fix-deleted-font-typography typography)})))
-   (:typographies fdata)))
-
-(defn fix-deleted-fonts
-  []
-  (ptk/reify ::fix-deleted-fonts
+(defn fix-deleted-fonts-for-local-library
+  "Looks the file local library for deleted fonts and emit changes if
+  invalid but fixable typographyes found."
+  [file-id]
+  (ptk/reify ::fix-deleted-fonts-for-local-library
     ptk/WatchEvent
     (watch [it state _]
-      (let [fdata              (dsh/lookup-file-data state)
-            pages              (:pages-index fdata)
-
-            shape-changes      (mapcat generate-deleted-font-shape-changes (vals pages))
-            components-changes (generate-deleted-font-components-changes fdata)
-            typography-changes (generate-deleted-font-typography-changes fdata)
-            changes            (concat shape-changes
-                                       components-changes
-                                       typography-changes)]
-        (if (seq changes)
+      (let [fdata (dsh/lookup-file-data state file-id)]
+        (when-let [changes (-> (generate-library-changes fdata)
+                               (not-empty))]
           (rx/of (dwc/commit-changes
                   {:origin it
-                   :redo-changes (vec changes)
+                   :redo-changes changes
                    :undo-changes []
-                   :save-undo? false}))
-          (rx/empty))))))
+                   :save-undo? false})))))))
+
+;; FIXME: would be nice to not execute this code twice per page in the
+;; same working session, maybe some local memoization can improve that
+
+(defn fix-deleted-fonts-for-page
+  [file-id page-id]
+  (ptk/reify ::fix-deleted-fonts-for-page
+    ptk/WatchEvent
+    (watch [it state _]
+      (let [page (dsh/lookup-page state file-id page-id)]
+        (when-let [changes (-> (generate-page-changes page)
+                               (not-empty))]
+          (rx/of (dwc/commit-changes
+                  {:origin it
+                   :redo-changes changes
+                   :undo-changes []
+                   :save-undo? false})))))))

frontend/src/app/main/data/workspace/pages.cljs

@@ -15,6 +15,7 @@
    [app.common.types.components-list :as ctkl]
    [app.common.types.container :as ctn]
    [app.common.types.page :as ctp]
+   [app.common.types.shape-tree :as ctst]
    [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.main.data.changes :as dch]
@@ -23,12 +24,14 @@
    [app.main.data.helpers :as dsh]
    [app.main.data.persistence :as-alias dps]
    [app.main.data.workspace.drawing :as dwd]
+   [app.main.data.workspace.fix-deleted-fonts :as fdf]
    [app.main.data.workspace.layout :as layout]
    [app.main.data.workspace.libraries :as dwl]
    [app.main.data.workspace.thumbnails :as dwth]
    [app.main.errors]
    [app.main.features :as features]
    [app.main.router :as rt]
+   [app.main.worker :as mw]
    [app.render-wasm.shape :as wasm.shape]
    [app.util.http :as http]
    [app.util.i18n :as i18n :refer [tr]]
@@ -56,16 +59,21 @@
                   (some? metadata)   (cf/resolve-file-media metadata)
                   (some? fill-image) (cf/resolve-file-media fill-image))))))
 
+(defn- get-page-cache
+  [state file-id page-id]
+  (dm/get-in state [:workspace-cache [file-id page-id]]))
 
 (defn- initialize-page*
   "Second phase of page initialization, once we know the page is
   available in the state"
-  [file-id page-id page]
+  [file-id page-id]
   (ptk/reify ::initialize-page*
     ptk/UpdateEvent
     (update [_ state]
-      ;; selection; when user abandon the current page, the selection is lost
-      (let [local (dm/get-in state [:workspace-cache [file-id page-id]] default-workspace-local)]
+      (let [state (dsh/update-page state file-id page-id #(update % :objects ctst/start-page-index))
+            page  (dsh/lookup-page state file-id page-id)
+            local (or (get-page-cache state file-id page-id) default-workspace-local)]
+
         (-> state
             (assoc :current-page-id page-id)
             (assoc :workspace-local (assoc local :selected (d/ordered-set)))
@@ -75,11 +83,16 @@
             (update :workspace-layout layout/load-layout-flags)
             (update :workspace-global layout/load-layout-state))))
 
-    ptk/EffectEvent
-    (effect [_ _ _]
-      (let [uris  (into #{} xf:collect-file-media (:objects page))]
-        (->> (rx/from uris)
-             (rx/subs! #(http/fetch-data-uri % false)))))))
+    ptk/WatchEvent
+    (watch [_ state _]
+      (let [page (dsh/lookup-page state file-id page-id)
+            uris (into #{} xf:collect-file-media (:objects page))]
+        (rx/merge
+         (->> (rx/from uris)
+              (rx/map #(http/fetch-data-uri % false))
+              (rx/ignore))
+         (->> (mw/ask! {:cmd :index/initialize :page page})
+              (rx/ignore)))))))
 
 (defn initialize-page
   [file-id page-id]
@@ -89,9 +102,10 @@
   (ptk/reify ::initialize-page
     ptk/WatchEvent
     (watch [_ state _]
-      (if-let [page (dsh/lookup-page state file-id page-id)]
+      (if (dsh/lookup-page state file-id page-id)
         (rx/concat
-         (rx/of (initialize-page* file-id page-id page)
+         (rx/of (initialize-page* file-id page-id)
+                (fdf/fix-deleted-fonts-for-page file-id page-id)
                 (dwth/watch-state-changes file-id page-id)
                 (dwl/watch-component-changes))
          (let [profile (:profile state)

frontend/src/app/main/data/workspace/selection.cljs

@@ -344,7 +344,7 @@
 
          (if (some? selrect)
            (->> (ask-worker
-                 {:cmd :selection/query
+                 {:cmd :index/query-selection
                   :page-id page-id
                   :rect selrect
                   :include-frames? true

frontend/src/app/main/repo.cljs

@@ -8,6 +8,8 @@
   (:require
    [app.common.data :as d]
    [app.common.exceptions :as ex]
+   [app.common.logging :as log]
+   [app.common.time :as ct]
    [app.common.transit :as t]
    [app.common.uri :as u]
    [app.config :as cf]
@@ -17,6 +19,8 @@
    [beicon.v2.core :as rx]
    [cuerdas.core :as str]))
 
+(log/set-level! :info)
+
 (defn handle-response
   [{:keys [status body headers uri] :as response}]
   (cond
@@ -126,13 +130,21 @@
                     (select-keys params query-params)
                     nil))
          :response-type
-         (if stream? nil response-type)}]
+         (if stream? nil response-type)}
+
+        tpoint
+        (ct/tpoint-ms)]
+
+    (log/trc :hint "make request" :id id)
 
     (->> (http/fetch request)
          (rx/map http/response->map)
          (rx/mapcat (fn [{:keys [headers body] :as response}]
+                      (log/trc :hint "response received" :id id :elapsed (tpoint))
+
                       (let [ctype (get headers "content-type")
-                            response-stream? (str/starts-with? ctype "text/event-stream")]
+                            response-stream? (str/starts-with? ctype "text/event-stream")
+                            tpoint (ct/tpoint-ms)]
 
                         (when (and response-stream? (not stream?))
                           (ex/raise :type :internal
@@ -148,6 +160,8 @@
                           (->> response
                                (http/process-response-type response-type)
                                (rx/map decode-fn)
+                               (rx/tap (fn [_]
+                                         (log/trc :hint "response decoded" :id id :elapsed (tpoint))))
                                (rx/mapcat handle-response)))))))))
 
 (defmulti cmd! (fn [id _] id))

frontend/src/app/main/snap.cljs

@@ -84,7 +84,7 @@
   (let [value (get point coord)
         vbox @refs/vbox
         ranges [[(- value (/ 0.5 zoom)) (+ value (/ 0.5 zoom))]]]
-    (->> (mw/ask! {:cmd :snaps/range-query
+    (->> (mw/ask! {:cmd :index/query-snap
                    :page-id page-id
                    :frame-id frame-id
                    :axis coord
@@ -101,7 +101,7 @@
                     (mapv #(vector (- % snap-accuracy)
                                    (+ % snap-accuracy))))
         vbox @refs/vbox]
-    (->> (mw/ask! {:cmd :snaps/range-query
+    (->> (mw/ask! {:cmd :index/query-snap
                    :page-id page-id
                    :frame-id frame-id
                    :axis coord
@@ -217,7 +217,7 @@
 
 (defn select-shapes-area
   [page-id frame-id selected objects area]
-  (->> (mw/ask! {:cmd :selection/query
+  (->> (mw/ask! {:cmd :index/query-selection
                  :page-id page-id
                  :frame-id frame-id
                  :include-frames? true

frontend/src/app/main/ui/workspace/viewport/hooks.cljs

@@ -194,7 +194,7 @@
              (if (mf/ref-val hover-disabled-ref)
                (rx/of nil)
                (->> (mw/ask-buffered!
-                     {:cmd :selection/query
+                     {:cmd :index/query-selection
                       :page-id page-id
                       :rect rect
                       :include-frames? true

frontend/src/app/main/worker.cljs

@@ -21,7 +21,6 @@
                      :config {:public-uri cf/public-uri
                               :build-data cf/build-date
                               :version cf/version}})
-
     (set! instance worker)))
 
 (defn ask!
@@ -34,6 +33,16 @@
      (uw/ask! instance message transfer)
      (rx/empty))))
 
+(defn emit!
+  ([message]
+   (if instance
+     (uw/emit! instance message)
+     (rx/empty)))
+  ([message transfer]
+   (if instance
+     (uw/emit! instance message transfer)
+     (rx/empty))))
+
 (defn ask-buffered!
   ([message]
    (if instance

frontend/src/app/util/worker.cljs

@@ -19,7 +19,7 @@
   ([worker message]
    (send-message! worker message nil))
 
-  ([worker {sender-id :sender-id :as message} {:keys [many?] :or {many? false}}]
+  ([worker {sender-id :sender-id :as message} {:keys [many? ignore-response?] :or {many? false ignore-response? false}}]
    (let [take-messages
          (fn [ob]
            (if many?
@@ -34,11 +34,13 @@
 
      (if (some? instance)
        (do (.postMessage instance data transfer)
-           (->> (:stream worker)
-                (rx/filter #(= (:reply-to %) sender-id))
-                (take-messages)
-                (rx/filter (complement :dropped))
-                (rx/map handle-response)))
+           (if (not ignore-response?)
+             (->> (:stream worker)
+                  (rx/filter #(= (:reply-to %) sender-id))
+                  (take-messages)
+                  (rx/filter (complement :dropped))
+                  (rx/map handle-response))
+             (rx/empty)))
        (rx/empty)))))
 
 (defn ask!
@@ -51,6 +53,17 @@
      :payload message
      :transfer transfer})))
 
+(defn emit!
+  ([worker message]
+   (emit! worker message nil))
+  ([worker message transfer]
+   (send-message!
+    worker
+    {:sender-id (uuid/next)
+     :payload message
+     :transfer transfer}
+    {:ignore-response? true})))
+
 (defn ask-many!
   ([worker message]
    (ask-many! worker message nil))

frontend/src/app/worker.cljs

@@ -9,14 +9,13 @@
    [app.common.data.macros :as dm]
    [app.common.logging :as log]
    [app.common.schema :as sm]
+   [app.common.types.objects-map]
    [app.util.object :as obj]
    [app.worker.export]
    [app.worker.impl :as impl]
    [app.worker.import]
    [app.worker.index]
    [app.worker.messages :as wm]
-   [app.worker.selection]
-   [app.worker.snaps]
    [app.worker.thumbnails]
    [beicon.v2.core :as rx]
    [promesa.core :as p]))

frontend/src/app/worker/index.cljs

@@ -9,26 +9,69 @@
   (:require
    [app.common.data.macros :as dm]
    [app.common.files.changes :as ch]
+   [app.common.geom.rect :as grc]
+   [app.common.logging :as log]
+   [app.common.time :as ct]
    [app.worker.impl :as impl]
+   [app.worker.selection :as selection]
+   [app.worker.snap :as snap]
    [okulary.core :as l]))
 
+(log/set-level! :info)
+
 (defonce state (l/atom {:pages-index {}}))
 
-(defmethod impl/handler :index/initialize-page-index
+(defmethod impl/handler :index/initialize
   [{:keys [page] :as message}]
-  (swap! state update :pages-index assoc (:id page) page)
-  (impl/handler (assoc message :cmd :selection/initialize-page-index))
-  (impl/handler (assoc message :cmd :snaps/initialize-page-index)))
+  (let [tpoint (ct/tpoint-ms)]
+    (try
+      (swap! state update :pages-index assoc (:id page) page)
+      (swap! state update ::selection selection/add-page page)
+      (swap! state update ::snap snap/add-page page)
 
-(defmethod impl/handler :index/update-page-index
+      (finally
+        (let [elapsed (tpoint)]
+          (log/dbg :hint "page indexed" :id (:id page) :elapsed elapsed ::log/sync? true))))
+    nil))
+
+(defmethod impl/handler :index/update
   [{:keys [page-id changes] :as message}]
+  (let [tpoint (ct/tpoint-ms)]
+    (try
+      (let [old-page (dm/get-in @state [:pages-index page-id])
+            new-page (-> state
+                         (swap! ch/process-changes changes false)
+                         (dm/get-in [:pages-index page-id]))]
 
-  (let [old-page (dm/get-in @state [:pages-index page-id])
-        new-page (-> state
-                     (swap! ch/process-changes changes false)
-                     (dm/get-in [:pages-index page-id]))
-        message (assoc message
-                       :old-page old-page
-                       :new-page new-page)]
-    (impl/handler (assoc message :cmd :selection/update-page-index))
-    (impl/handler (assoc message :cmd :snaps/update-page-index))))
+        (swap! state update ::snap snap/update-page old-page new-page)
+        (swap! state update ::selection selection/update-page old-page new-page))
+      (finally
+        (let [elapsed (tpoint)]
+          (log/dbg :hint "page index updated" :id page-id :elapsed elapsed ::log/sync? true))))
+    nil))
+
+;; FIXME: schema
+
+(defmethod impl/handler :index/query-snap
+  [{:keys [page-id frame-id axis ranges bounds] :as message}]
+  (if-let [index (get @state ::snap)]
+    (let [match-bounds?
+          (fn [[_ data]]
+            (some #(or (= :guide (:type %))
+                       (= :layout (:type %))
+                       (grc/contains-point? bounds (:pt %))) data))
+
+          xform
+          (comp (mapcat #(snap/query index page-id frame-id axis %))
+                (distinct)
+                (filter match-bounds?))]
+      (into [] xform ranges))
+    []))
+
+;; FIXME: schema
+
+(defmethod impl/handler :index/query-selection
+  [message]
+  (if-let [index (get @state ::selection)]
+    (selection/query index message)
+    []))

frontend/src/app/worker/selection.cljs

@@ -17,46 +17,62 @@
    [app.common.types.modifiers :as ctm]
    [app.common.uuid :as uuid]
    [app.util.quadtree :as qdt]
-   [app.worker.impl :as impl]
-   [clojure.set :as set]
-   [okulary.core :as l]))
+   [clojure.set :as set]))
 
-;; FIXME: performance shape & rect static props
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; IMPL
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (def ^:const padding-percent 0.10)
 
-(defonce state (l/atom {}))
+(defn- index-shape
+  "A reducing function that ads a shape to the index"
+  [objects parents-index clip-index index shape]
+  (let [bounds
+        (cond
+          (and ^boolean (cfh/text-shape? shape)
+               ^boolean (some? (:position-data shape))
+               ^boolean (d/not-empty? (:position-data shape)))
+          (gst/shape->bounds shape)
 
-(defn make-index-shape
-  [objects parents-index clip-parents-index]
-  (fn [index shape]
-    (let [{:keys [x y width height]}
-          (cond
-            (and ^boolean (cfh/text-shape? shape)
-                 ^boolean (some? (:position-data shape))
-                 ^boolean (d/not-empty? (:position-data shape)))
-            (gst/shape->bounds shape)
+          :else
+          (grc/points->rect (:points shape)))
 
-            :else
-            (grc/points->rect (:points shape)))
+        bound
+        #js {:x (dm/get-prop bounds :x)
+             :y (dm/get-prop bounds :y)
+             :width (dm/get-prop bounds :width)
+             :height (dm/get-prop bounds :height)}
 
-          shape-bound #js {:x x :y y :width width :height height}
+        shape-id
+        (dm/get-prop shape :id)
 
-          parents      (get parents-index (:id shape))
-          clip-parents (get clip-parents-index (:id shape))
+        frame-id
+        (dm/get-prop shape :frame-id)
 
-          frame   (when (and (not= :frame (:type shape))
-                             (not= (:frame-id shape) uuid/zero))
-                    (get objects (:frame-id shape)))]
-      (qdt/insert index
-                  (:id shape)
-                  shape-bound
-                  (assoc shape
-                         :frame frame
-                         :clip-parents clip-parents
-                         :parents parents)))))
+        shape-type
+        (dm/get-prop shape :type)
 
-(defn objects-bounds
+        parents
+        (get parents-index shape-id)
+
+        clip-parents
+        (get clip-index shape-id)
+
+        frame
+        (when (and (not= :frame shape-type)
+                   (not= frame-id uuid/zero))
+          (get objects frame-id))]
+
+    (qdt/insert index
+                shape-id
+                bound
+                (assoc shape
+                       :frame frame
+                       :clip-parents clip-parents
+                       :parents parents))))
+
+(defn- objects-bounds
   "Calculates the bounds of the quadtree given a objects map."
   [objects]
   (-> objects
@@ -64,7 +80,7 @@
       vals
       gsh/shapes->rect))
 
-(defn add-padding-bounds
+(defn- add-padding-bounds
   "Adds a padding to the bounds defined as a percent in the constant `padding-percent`.
   For a value of 0.1 will add a 20% width increase (2 x padding)"
   [bounds]
@@ -81,41 +97,48 @@
 
 (defn- create-index
   [objects]
-  (let [shapes             (-> objects (dissoc uuid/zero) vals)
-        parents-index      (cfi/generate-child-all-parents-index objects)
-        clip-parents-index (cfi/create-clip-index objects parents-index)
-
-        root-shapes        (cfh/get-immediate-children objects uuid/zero)
-        bounds             (-> root-shapes gsh/shapes->rect add-padding-bounds)
-
-        index-shape        (make-index-shape objects parents-index clip-parents-index)
-        initial-quadtree   (qdt/create (clj->js bounds))
-
-        index              (reduce index-shape initial-quadtree shapes)]
+  (let [parents-index (cfi/generate-child-all-parents-index objects)
+        clip-index    (cfi/create-clip-index objects parents-index)
+        root-shapes   (cfh/get-immediate-children objects uuid/zero)
+        bounds        (-> root-shapes gsh/shapes->rect add-padding-bounds)
 
+        index         (reduce-kv #(index-shape objects parents-index clip-index %1 %3)
+                                 (qdt/create (clj->js bounds))
+                                 (dissoc objects uuid/zero))]
     {:index index :bounds bounds}))
 
+;; FIXME: optimize
 (defn- update-index
   [{index :index :as data} old-objects new-objects]
-  (let [changes? (fn [id]
-                   (not= (get old-objects id)
-                         (get new-objects id)))
+  (let [object-changed?
+        (fn [id]
+          (not= (get old-objects id)
+                (get new-objects id)))
 
-        changed-ids (into #{}
-                          (comp (filter #(not= % uuid/zero))
-                                (filter changes?)
-                                (mapcat #(into [%] (cfh/get-children-ids new-objects %))))
-                          (set/union (set (keys old-objects))
-                                     (set (keys new-objects))))
+        changed-ids
+        (into #{}
+              (comp (filter #(not= % uuid/zero))
+                    (filter object-changed?)
+                    (mapcat #(into [%] (cfh/get-children-ids new-objects %))))
 
-        shapes             (->> changed-ids (mapv #(get new-objects %)) (filterv (comp not nil?)))
-        parents-index      (cfi/generate-child-all-parents-index new-objects shapes)
-        clip-parents-index (cfi/create-clip-index new-objects parents-index)
+              (set/union (set (keys old-objects))
+                         (set (keys new-objects))))
 
-        new-index (qdt/remove-all index changed-ids)
+        shapes
+        (->> changed-ids
+             (map #(get new-objects %))
+             (filterv (comp not nil?)))
 
-        index-shape (make-index-shape new-objects parents-index clip-parents-index)
-        index (reduce index-shape new-index shapes)]
+        parents-index
+        (cfi/generate-child-all-parents-index new-objects shapes)
+
+        clip-index
+        (cfi/create-clip-index new-objects parents-index)
+
+        index
+        (reduce #(index-shape new-objects parents-index clip-index %1 %2)
+                (qdt/remove-all index changed-ids)
+                shapes)]
 
     (assoc data :index index)))
 
@@ -231,35 +254,36 @@
                 (map :id))
           result)))
 
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; PUBLIC API
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(defmethod impl/handler :selection/initialize-page-index
-  [{:keys [page] :as message}]
-  (letfn [(add-page [state {:keys [id objects] :as page}]
-            (assoc state id (create-index objects)))]
-    (swap! state add-page page)
-    nil))
+(defn add-page
+  "Add a page index to the state"
+  [state {:keys [id objects] :as page}]
+  (assoc state id (create-index objects)))
 
-(defmethod impl/handler :selection/update-page-index
-  [{:keys [page-id old-page new-page] :as message}]
-  (swap! state update page-id
-         (fn [index]
-           (let [old-objects (:objects old-page)
-                 new-objects (:objects new-page)
-                 old-bounds  (:bounds index)
-                 new-bounds  (objects-bounds new-objects)]
+(defn update-page
+  "Update page index on the state"
+  [state old-page new-page]
+  (let [page-id (get old-page :id)]
+    (update state page-id
+            (fn [index]
+              (let [old-objects (:objects old-page)
+                    new-objects (:objects new-page)
+                    old-bounds  (:bounds index)
+                    new-bounds  (objects-bounds new-objects)]
 
-             ;; If the new bounds are contained within the old bounds
-             ;; we can update the index. Otherwise we need to
-             ;; re-create it.
-             (if (and (some? index)
-                      (grc/contains-rect? old-bounds new-bounds))
-               (update-index index old-objects new-objects)
-               (create-index new-objects)))))
-  nil)
+                ;; If the new bounds are contained within the old bounds
+                ;; we can update the index. Otherwise we need to
+                ;; re-create it.
+                (if (and (some? index)
+                         (grc/contains-rect? old-bounds new-bounds))
+                  (update-index index old-objects new-objects)
+                  (create-index new-objects)))))))
 
-(defmethod impl/handler :selection/query
-  [{:keys [page-id rect frame-id full-frame? include-frames? ignore-groups? clip-children? using-selrect?]
-    :or {full-frame? false include-frames? false clip-children? true using-selrect? false}
-    :as message}]
-  (when-let [index (get @state page-id)]
+(defn query
+  [index {:keys [page-id rect frame-id full-frame? include-frames? ignore-groups? clip-children? using-selrect?]
+          :or {full-frame? false include-frames? false clip-children? true using-selrect? false}}]
+  (when-let [index (get index page-id)]
     (query-index index rect frame-id full-frame? include-frames? ignore-groups? clip-children? using-selrect?)))

frontend/src/app/worker/snap.cljs

@@ -4,7 +4,7 @@
 ;;
 ;; Copyright (c) KALEIDOS INC
 
-(ns app.util.snap-data
+(ns app.worker.snap
   "Data structure that holds and retrieves the data to make the snaps.
    Internally is implemented with a balanced binary tree that queries by range.
    https://en.wikipedia.org/wiki/Range_tree"

frontend/src/app/worker/snaps.cljs

@@ -1,40 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) KALEIDOS INC
-
-(ns app.worker.snaps
-  (:require
-   [app.common.geom.rect :as grc]
-   [app.util.snap-data :as sd]
-   [app.worker.impl :as impl]
-   [okulary.core :as l]))
-
-(defonce state (l/atom {}))
-
-;; Public API
-(defmethod impl/handler :snaps/initialize-page-index
-  [{:keys [page] :as message}]
-  (swap! state sd/add-page page)
-  nil)
-
-(defmethod impl/handler :snaps/update-page-index
-  [{:keys [old-page new-page] :as message}]
-  (swap! state sd/update-page old-page new-page)
-  nil)
-
-(defmethod impl/handler :snaps/range-query
-  [{:keys [page-id frame-id axis ranges bounds] :as message}]
-  (let [match-bounds?
-        (fn [[_ data]]
-          (some #(or (= :guide (:type %))
-                     (= :layout (:type %))
-                     (grc/contains-point? bounds (:pt %))) data))]
-    (->> (into []
-               (comp (mapcat #(sd/query @state page-id frame-id axis %))
-                     (distinct))
-               ranges)
-         (filter match-bounds?))))
-
-

frontend/test/frontend_tests/runner.cljs

@@ -18,7 +18,7 @@
    [frontend-tests.tokens.token-form-test]
    [frontend-tests.util-range-tree-test]
    [frontend-tests.util-simple-math-test]
-   [frontend-tests.util-snap-data-test]))
+   [frontend-tests.worker-snap-test]))
 
 (enable-console-print!)
 
@@ -30,6 +30,8 @@
 (defn init
   []
   (t/run-tests
+   'frontend-tests.basic-shapes-test
+   'frontend-tests.data.workspace-colors-test
    'frontend-tests.helpers-shapes-test
    'frontend-tests.logic.comp-remove-swap-slots-test
    'frontend-tests.logic.components-and-tokens
@@ -38,13 +40,11 @@
    'frontend-tests.logic.groups-test
    'frontend-tests.logic.pasting-in-containers-test
    'frontend-tests.plugins.context-shapes-test
-   'frontend-tests.util-range-tree-test
-   'frontend-tests.util-snap-data-test
-   'frontend-tests.util-simple-math-test
-   'frontend-tests.basic-shapes-test
-   'frontend-tests.data.workspace-colors-test
+   'frontend-tests.tokens.import-export-test
    'frontend-tests.tokens.logic.token-actions-test
    'frontend-tests.tokens.logic.token-data-test
-   'frontend-tests.tokens.import-export-test
    'frontend-tests.tokens.style-dictionary-test
-   'frontend-tests.tokens.token-form-test))
+   'frontend-tests.tokens.token-form-test
+   'frontend-tests.util-range-tree-test
+   'frontend-tests.util-simple-math-test
+   'frontend-tests.worker-snap-test))

frontend/test/frontend_tests/worker_snap_test.cljs

@@ -4,12 +4,12 @@
 ;;
 ;; Copyright (c) KALEIDOS INC
 
-(ns frontend-tests.util-snap-data-test
+(ns frontend-tests.worker-snap-test
   (:require
    [app.common.files.builder :as fb]
    [app.common.types.shape :as cts]
    [app.common.uuid :as uuid]
-   [app.util.snap-data :as sd]
+   [app.worker.snap :as snap]
    [cljs.pprint :refer [pprint]]
    [cljs.test :as t :include-macros true]))
 
@@ -21,9 +21,9 @@
     (fn []
       (uuid/custom 123456789 (swap! counter inc)))))
 
-(t/deftest test-create-index
+(t/deftest create-index
   (t/testing "Create empty data"
-    (let [data (sd/make-snap-data)]
+    (let [data (snap/make-snap-data)]
       (t/is (some? data))))
 
   (t/testing "Add empty page (only root-frame)"
@@ -32,8 +32,8 @@
                    (fb/add-page {:name "Page 1"})
                    (fb/get-current-page))
 
-          data (-> (sd/make-snap-data)
-                   (sd/add-page page))]
+          data (-> (snap/make-snap-data)
+                   (snap/add-page page))]
       (t/is (some? data))))
 
   (t/testing "Create simple shape on root"
@@ -48,10 +48,10 @@
                       :height 100}))
           page  (fb/get-current-page state)
 
-          data  (-> (sd/make-snap-data)
-                    (sd/add-page page))
+          data  (-> (snap/make-snap-data)
+                    (snap/add-page page))
 
-          result-x (sd/query data (:id page) uuid/zero :x [0 100])]
+          result-x (snap/query data (:id page) uuid/zero :x [0 100])]
 
       (t/is (some? data))
 
@@ -82,11 +82,11 @@
           page     (fb/get-current-page state)
 
           ;; frame-id (::fb/last-id file)
-          data (-> (sd/make-snap-data)
-                   (sd/add-page page))
+          data (-> (snap/make-snap-data)
+                   (snap/add-page page))
 
-          result-zero-x (sd/query data (:id page) uuid/zero :x [0 100])
-          result-frame-x (sd/query data (:id page) frame-id :x [0 100])]
+          result-zero-x (snap/query data (:id page) uuid/zero :x [0 100])
+          result-frame-x (snap/query data (:id page) frame-id :x [0 100])]
 
       (t/is (some? data))
       (t/is (= (count result-zero-x) 3))
@@ -116,11 +116,11 @@
 
             page     (fb/get-current-page state)
 
-            data (-> (sd/make-snap-data)
-                     (sd/add-page page))
+            data (-> (snap/make-snap-data)
+                     (snap/add-page page))
 
-            result-zero-x (sd/query data (:id page) uuid/zero :x [0 100])
-            result-frame-x (sd/query data (:id page) frame-id :x [0 100])]
+            result-zero-x (snap/query data (:id page) uuid/zero :x [0 100])
+            result-frame-x (snap/query data (:id page) frame-id :x [0 100])]
 
         (t/is (some? data))
         (t/is (= (count result-zero-x) 3))
@@ -137,13 +137,13 @@
           frame-id (::fb/last-id state)
           page     (fb/get-current-page state)
 
-          data     (-> (sd/make-snap-data)
-                       (sd/add-page page))
+          data     (-> (snap/make-snap-data)
+                       (snap/add-page page))
 
-          result-zero-x  (sd/query data (:id page) uuid/zero :x [0 100])
-          result-zero-y  (sd/query data (:id page) uuid/zero :y [0 100])
-          result-frame-x (sd/query data (:id page) frame-id :x [0 100])
-          result-frame-y (sd/query data (:id page) frame-id :y [0 100])]
+          result-zero-x  (snap/query data (:id page) uuid/zero :x [0 100])
+          result-zero-y  (snap/query data (:id page) uuid/zero :y [0 100])
+          result-frame-x (snap/query data (:id page) frame-id :x [0 100])
+          result-frame-y (snap/query data (:id page) frame-id :y [0 100])]
 
       (t/is (some? data))
       ;; We can snap in the root
@@ -168,13 +168,13 @@
 
           page     (fb/get-current-page state)
 
-          data     (-> (sd/make-snap-data)
-                       (sd/add-page page))
+          data     (-> (snap/make-snap-data)
+                       (snap/add-page page))
 
-          result-zero-x (sd/query data (:id page) uuid/zero :x [0 100])
-          result-zero-y (sd/query data (:id page) uuid/zero :y [0 100])
-          result-frame-x (sd/query data (:id page) frame-id :x [0 100])
-          result-frame-y (sd/query data (:id page) frame-id :y [0 100])]
+          result-zero-x (snap/query data (:id page) uuid/zero :x [0 100])
+          result-zero-y (snap/query data (:id page) uuid/zero :y [0 100])
+          result-frame-x (snap/query data (:id page) frame-id :x [0 100])
+          result-frame-y (snap/query data (:id page) frame-id :y [0 100])]
 
       (t/is (some? data))
       ;; We can snap in the root
@@ -185,7 +185,7 @@
       (t/is (= (count result-frame-x) 1))
       (t/is (= (count result-frame-y) 0)))))
 
-(t/deftest test-update-index
+(t/deftest update-index
   (t/testing "Create frame on root and then remove it."
     (let [state   (-> (fb/create-state)
                       (fb/add-file {:name "Test"})
@@ -200,17 +200,17 @@
           shape-id (::fb/last-id state)
           page     (fb/get-current-page state)
 
-          data     (-> (sd/make-snap-data)
-                       (sd/add-page page))
+          data     (-> (snap/make-snap-data)
+                       (snap/add-page page))
 
           state     (-> state
                         (fb/delete-shape shape-id))
 
           new-page (fb/get-current-page state)
-          data     (sd/update-page data page new-page)
+          data     (snap/update-page data page new-page)
 
-          result-x (sd/query data (:id page) uuid/zero :x [0 100])
-          result-y (sd/query data (:id page) uuid/zero :y [0 100])]
+          result-x (snap/query data (:id page) uuid/zero :x [0 100])
+          result-y (snap/query data (:id page) uuid/zero :y [0 100])]
 
       (t/is (some? data))
       (t/is (= (count result-x) 0))
@@ -231,16 +231,16 @@
           page     (fb/get-current-page state)
 
           ;; frame-id (::fb/last-id state)
-          data (-> (sd/make-snap-data)
-                   (sd/add-page page))
+          data (-> (snap/make-snap-data)
+                   (snap/add-page page))
 
           state (fb/delete-shape state shape-id)
 
           new-page (fb/get-current-page state)
-          data (sd/update-page data page new-page)
+          data (snap/update-page data page new-page)
 
-          result-x (sd/query data (:id page) uuid/zero :x [0 100])
-          result-y (sd/query data (:id page) uuid/zero :y [0 100])]
+          result-x (snap/query data (:id page) uuid/zero :x [0 100])
+          result-y (snap/query data (:id page) uuid/zero :y [0 100])]
 
       (t/is (some? data))
       (t/is (= (count result-x) 0))
@@ -263,16 +263,16 @@
           state    (fb/close-board state)
 
           page     (fb/get-current-page state)
-          data     (-> (sd/make-snap-data)
-                       (sd/add-page page))
+          data     (-> (snap/make-snap-data)
+                       (snap/add-page page))
 
           state    (fb/delete-shape state shape-id)
           new-page (fb/get-current-page state)
 
-          data     (sd/update-page data page new-page)
+          data     (snap/update-page data page new-page)
 
-          result-zero-x  (sd/query data (:id page) uuid/zero :x [0 100])
-          result-frame-x (sd/query data (:id page) frame-id :x [0 100])]
+          result-zero-x  (snap/query data (:id page) uuid/zero :x [0 100])
+          result-frame-x (snap/query data (:id page) frame-id :x [0 100])]
 
       (t/is (some? data))
       (t/is (= (count result-zero-x) 3))
@@ -291,18 +291,18 @@
 
           frame-id (::fb/last-id state)
           page     (fb/get-current-page state)
-          data     (-> (sd/make-snap-data)
-                       (sd/add-page page))
+          data     (-> (snap/make-snap-data)
+                       (snap/add-page page))
 
           new-page (-> (fb/delete-guide state guide-id)
                        (fb/get-current-page))
 
-          data     (sd/update-page data page new-page)
+          data     (snap/update-page data page new-page)
 
-          result-zero-x  (sd/query data (:id page) uuid/zero :x [0 100])
-          result-zero-y  (sd/query data (:id page) uuid/zero :y [0 100])
-          result-frame-x (sd/query data (:id page) frame-id :x [0 100])
-          result-frame-y (sd/query data (:id page) frame-id :y [0 100])]
+          result-zero-x  (snap/query data (:id page) uuid/zero :x [0 100])
+          result-zero-y  (snap/query data (:id page) uuid/zero :y [0 100])
+          result-frame-x (snap/query data (:id page) frame-id :x [0 100])
+          result-frame-y (snap/query data (:id page) frame-id :y [0 100])]
 
       (t/is (some? data))
       ;; We can snap in the root
@@ -325,17 +325,17 @@
           guide-id (::fb/last-id file)
 
           page     (fb/get-current-page file)
-          data (-> (sd/make-snap-data) (sd/add-page page))
+          data (-> (snap/make-snap-data) (snap/add-page page))
 
           new-page (-> (fb/delete-guide file guide-id)
                        (fb/get-current-page))
 
-          data (sd/update-page data page new-page)
+          data (snap/update-page data page new-page)
 
-          result-zero-x (sd/query data (:id page) uuid/zero :x [0 100])
-          result-zero-y (sd/query data (:id page) uuid/zero :y [0 100])
-          result-frame-x (sd/query data (:id page) frame-id :x [0 100])
-          result-frame-y (sd/query data (:id page) frame-id :y [0 100])]
+          result-zero-x (snap/query data (:id page) uuid/zero :x [0 100])
+          result-zero-y (snap/query data (:id page) uuid/zero :y [0 100])
+          result-frame-x (snap/query data (:id page) frame-id :x [0 100])
+          result-frame-y (snap/query data (:id page) frame-id :y [0 100])]
       (t/is (some? data))
       ;; We can snap in the root
       (t/is (= (count result-zero-x) 0))
@@ -358,8 +358,8 @@
 
           frame-id (::fb/last-id state)
           page     (fb/get-current-page state)
-          data     (-> (sd/make-snap-data)
-                       (sd/add-page page))
+          data     (-> (snap/make-snap-data)
+                       (snap/add-page page))
 
           state    (fb/update-shape state frame-id
                                     (fn [shape]
@@ -370,12 +370,12 @@
 
 
           new-page (fb/get-current-page state)
-          data     (sd/update-page data page new-page)
+          data     (snap/update-page data page new-page)
 
-          result-zero-x-1 (sd/query data (:id page) uuid/zero :x [0 100])
-          result-frame-x-1 (sd/query data (:id page) frame-id :x [0 100])
-          result-zero-x-2 (sd/query data (:id page) uuid/zero :x [200 300])
-          result-frame-x-2 (sd/query data (:id page) frame-id :x [200 300])]
+          result-zero-x-1 (snap/query data (:id page) uuid/zero :x [0 100])
+          result-frame-x-1 (snap/query data (:id page) frame-id :x [0 100])
+          result-zero-x-2 (snap/query data (:id page) uuid/zero :x [200 300])
+          result-frame-x-2 (snap/query data (:id page) frame-id :x [200 300])]
 
       (t/is (some? data))
       (t/is (= (count result-zero-x-1) 0))
@@ -396,8 +396,8 @@
 
           shape-id (::fb/last-id state)
           page     (fb/get-current-page state)
-          data     (-> (sd/make-snap-data)
-                       (sd/add-page page))
+          data     (-> (snap/make-snap-data)
+                       (snap/add-page page))
 
           state     (fb/update-shape state shape-id
                                      (fn [shape]
@@ -408,10 +408,10 @@
 
           new-page (fb/get-current-page state)
           ;; FIXME: update
-          data     (sd/update-page data page new-page)
+          data     (snap/update-page data page new-page)
 
-          result-zero-x-1 (sd/query data (:id page) uuid/zero :x [0 100])
-          result-zero-x-2 (sd/query data (:id page) uuid/zero :x [200 300])]
+          result-zero-x-1 (snap/query data (:id page) uuid/zero :x [0 100])
+          result-zero-x-2 (snap/query data (:id page) uuid/zero :x [200 300])]
 
       (t/is (some? data))
       (t/is (= (count result-zero-x-1) 0))
@@ -432,22 +432,22 @@
 
           frame-id (::fb/last-id state)
           page     (fb/get-current-page state)
-          data     (-> (sd/make-snap-data) (sd/add-page page))
+          data     (-> (snap/make-snap-data) (snap/add-page page))
 
           new-page (-> (fb/update-guide state (assoc guide :position 150))
                        (fb/get-current-page))
 
-          data     (sd/update-page data page new-page)
+          data     (snap/update-page data page new-page)
 
-          result-zero-x-1 (sd/query data (:id page) uuid/zero :x [0 100])
-          result-zero-y-1 (sd/query data (:id page) uuid/zero :y [0 100])
-          result-frame-x-1 (sd/query data (:id page) frame-id :x [0 100])
-          result-frame-y-1 (sd/query data (:id page) frame-id :y [0 100])
+          result-zero-x-1 (snap/query data (:id page) uuid/zero :x [0 100])
+          result-zero-y-1 (snap/query data (:id page) uuid/zero :y [0 100])
+          result-frame-x-1 (snap/query data (:id page) frame-id :x [0 100])
+          result-frame-y-1 (snap/query data (:id page) frame-id :y [0 100])
 
-          result-zero-x-2 (sd/query data (:id page) uuid/zero :x [0 200])
-          result-zero-y-2 (sd/query data (:id page) uuid/zero :y [0 200])
-          result-frame-x-2 (sd/query data (:id page) frame-id :x [0 200])
-          result-frame-y-2 (sd/query data (:id page) frame-id :y [0 200])]
+          result-zero-x-2 (snap/query data (:id page) uuid/zero :x [0 200])
+          result-zero-y-2 (snap/query data (:id page) uuid/zero :y [0 200])
+          result-frame-x-2 (snap/query data (:id page) frame-id :x [0 200])
+          result-frame-y-2 (snap/query data (:id page) frame-id :y [0 200])]
 
       (t/is (some? data))