🎉 Add loadable weak map impl for libraries loading on validation and migration

backend/src/app/binfile/common.clj

@@ -19,6 +19,7 @@
    [app.common.time :as ct]
    [app.common.types.file :as ctf]
    [app.common.uuid :as uuid]
+   [app.common.weak :as weak]
    [app.config :as cf]
    [app.db :as db]
    [app.db.sql :as sql]
@@ -606,15 +607,19 @@
          (map decode-row))
         (db/exec! conn [sql:get-file-libraries file-id])))
 
-;; FIXME: this will use a lot of memory if file uses too many big
-;; libraries, we should load required libraries on demand
 (defn get-resolved-file-libraries
-  "A helper for preload file libraries"
-  [{:keys [::db/conn] :as cfg} file]
-  (->> (get-file-libraries conn (:id file))
-       ;; WARNING: we don't migrate the libraries for avoid cascade
-       ;; migration; it is not ideal but it reduces the total of the
-       ;; required memory needed for process a single file migration
-       ;; that requires libraries to be loaded.
-       (into [file] (map #(get-file cfg (:id %) :migrate? false)))
-       (d/index-by :id)))
+  "Get all file libraries including itself. Returns an instance of
+  LoadableWeakValueMap that allows do not have strong references to
+  the loaded libraries and reduce possible memory pressure on having
+  all this libraries loaded at same time on processing file validation
+  or file migration.
+
+  This still requires at least one library at time to be loaded while
+  access to it is performed, but it improves considerable not having
+  the need of loading all the libraries at the same time."
+  [{:keys [::db/conn] :as cfg} {:keys [id] :as file}]
+  (let [library-ids (->> (get-file-libraries conn (:id file))
+                         (map :id)
+                         (cons (:id file)))
+        load-fn     #(get-file cfg % :migrate? false)]
+    (weak/loadable-weak-value-map library-ids load-fn {id file})))

backend/src/app/rpc/commands/files_update.clj

@@ -408,7 +408,6 @@
                    (not skip-validate))
           (bfc/get-resolved-file-libraries cfg file))
 
-
         ;; The main purpose of this atom is provide a contextual state
         ;; for the changes subsystem where optionally some hints can
         ;; be provided for the changes processing. Right now we are

common/src/app/common/types/fills/impl.cljc

@@ -443,7 +443,7 @@
                           :code :invalid-fill
                           :hint "found invalid fill on encoding fills to binary format")))))
 
-        #?(:cljs (Fills. total dbuffer mbuffer image-ids (weak/create-weak-value-map) nil)
+        #?(:cljs (Fills. total dbuffer mbuffer image-ids (weak/weak-value-map) nil)
            :clj  (Fills. total dbuffer mbuffer nil))))))
 
 (defn fills?

common/src/app/common/types/path/impl.cljc

@@ -379,7 +379,7 @@
      (-transform [this m]
        (let [buffer (buf/clone buffer)]
          (impl-transform buffer m size)
-         (PathData. size buffer (weak/create-weak-value-map) nil)))
+         (PathData. size buffer (weak/weak-value-map) nil)))
 
      (-walk [_ f initial]
        (impl-walk buffer f initial size))
@@ -600,14 +600,14 @@
              count (long (/ size SEGMENT-U8-SIZE))]
          (PathData. count
                     (js/DataView. buffer)
-                    (weak/create-weak-value-map)
+                    (weak/weak-value-map)
                     nil))
 
        (instance? js/DataView buffer)
        (let [buffer' (.-buffer ^js/DataView buffer)
              size    (.-byteLength ^js/ArrayBuffer buffer')
              count   (long (/ size SEGMENT-U8-SIZE))]
-         (PathData. count buffer (weak/create-weak-value-map) nil))
+         (PathData. count buffer (weak/weak-value-map) nil))
 
        (instance? js/Uint8Array buffer)
        (from-bytes (.-buffer buffer))

common/src/app/common/weak.cljc

@@ -0,0 +1,79 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.common.weak
+  "A collection of helpers for work with weak references and weak
+  data structures on JS runtime."
+  (:refer-clojure :exclude [memoize])
+  (:require
+   #?@(:cljs [["./weak/impl_weak_map.js" :as wm]
+              ["./weak/impl_weak_value_map.js" :as wvm]]
+       :clj  [[app.common.weak.impl-loadable-weak-value-map :as lwvm]])))
+
+#?(:cljs
+   (defn weak-value-map
+     "Creates a WeakMap instance where values are held by soft
+  references and keys are held by hard references."
+     []
+     (new wvm/WeakValueMap.)))
+
+#?(:cljs
+   (defn weak-map
+     "Create a WeakMap like instance what uses clojure equality
+  semantics."
+     []
+     (new wm/WeakEqMap #js {:hash hash :equals =})))
+
+#?(:clj
+   (defn loadable-weak-value-map
+     "Creates an instance of a LoadableWeakValueMap. It gives you a clojure-like,
+  map instance with fixed number of keys and fixed preload data (for
+  the provided keys) where not preload data is lazy loadable. It
+  internally uses soft-like references, leaving the runtime to collect
+  values that are not in use (no hard references keeps on the runtime)."
+     ([keys load-fn]
+      (lwvm/loadable-weak-value-map keys load-fn {}))
+     ([keys load-fn preload-data]
+      (lwvm/loadable-weak-value-map keys load-fn preload-data))))
+
+#?(:cljs (def ^:private state (new js/WeakMap)))
+#?(:cljs (def ^:private global-counter 0))
+
+#?(:cljs
+   (defn weak-key
+     "A simple helper that returns a stable key string for an object while
+  that object remains in memory and is not collected by the GC.
+
+  Mainly used for assign temporal IDs/keys for react children
+  elements when the element has no specific id."
+     [o]
+     (let [key (.get ^js/WeakMap state o)]
+       (if (some? key)
+         key
+         (let [key (str "weak-key" (js* "~{}++" global-counter))]
+           (.set ^js/WeakMap state o key)
+           key)))))
+
+#?(:cljs
+   (defn memoize
+     "Returns a memoized version of a referentially transparent
+  function. The memoized version of the function keeps a cache of the
+  mapping from arguments to results and, when calls with the same
+  arguments are repeated often, has higher performance at the expense
+  of higher memory use.
+
+  The main difference with clojure.core/memoize, is that this function
+  uses weak-map, so cache is cleared once GC is passed and cached keys
+  are collected"
+     [f]
+     (let [mem (weak-map)]
+       (fn [& args]
+         (let [v (.get mem args)]
+           (if (undefined? v)
+             (let [ret (apply f args)]
+               (.set ^js mem args ret)
+               ret)
+             v))))))

common/src/app/common/weak.cljs

@@ -1,59 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) KALEIDOS INC
-
-(ns app.common.weak
-  "A collection of helpers for work with weak references and weak
-  data structures on JS runtime."
-  (:refer-clojure :exclude [memoize])
-  (:require
-   ["./weak/impl_weak_map.js" :as wm]
-   ["./weak/impl_weak_value_map.js" :as wvm]))
-
-(defn create-weak-value-map
-  []
-  (new wvm/WeakValueMap.))
-
-(defn create-weak-map
-  []
-  (new wm/WeakEqMap #js {:hash hash :equals =}))
-
-(def ^:private state (new js/WeakMap))
-(def ^:private global-counter 0)
-
-(defn weak-key
-  "A simple helper that returns a stable key string for an object
-  while that object remains in memory and is not collected by the GC.
-
-  Mainly used for assign temporal IDs/keys for react children
-  elements when the element has no specific id."
-  [o]
-  (let [key (.get ^js/WeakMap state o)]
-    (if (some? key)
-      key
-      (let [key (str "weak-key" (js* "~{}++" global-counter))]
-        (.set ^js/WeakMap state o key)
-        key))))
-
-(defn memoize
-  "Returns a memoized version of a referentially transparent function. The
-  memoized version of the function keeps a cache of the mapping from arguments
-  to results and, when calls with the same arguments are repeated often, has
-  higher performance at the expense of higher memory use.
-
-  The main difference with clojure.core/memoize, is that this function
-  uses weak-map, so cache is cleared once GC is passed and cached keys
-  are collected"
-  [f]
-  (let [mem (create-weak-map)]
-    (fn [& args]
-      (let [v (.get mem args)]
-        (if (undefined? v)
-          (let [ret (apply f args)]
-            (.set ^js mem args ret)
-            ret)
-          v)))))
-
-

common/src/app/common/weak/impl_loadable_weak_value_map.clj

@@ -0,0 +1,51 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.common.weak.impl-loadable-weak-value-map
+  (:import
+   java.lang.ref.SoftReference
+   java.util.HashMap))
+
+(deftype LoadableWeakValueMap [^HashMap data load-fn]
+  clojure.lang.ILookup
+  (valAt [_ key]
+    (when-let [reference (.get data key)]
+      (if (instance? SoftReference reference)
+        (or (.get ^SoftReference reference)
+            (let [value (load-fn key)]
+              (.put data key (SoftReference. value))
+              value))
+        reference)))
+
+  (valAt [_ key default]
+    (if-let [reference (.get data key)]
+      (if (instance? SoftReference reference)
+        (or (.get ^SoftReference reference)
+            (let [value (load-fn key)]
+              (.put data key (SoftReference. value))
+              value))
+        reference)
+      default))
+
+  clojure.lang.Counted
+  (count [_]
+    (.size data))
+
+  clojure.lang.Seqable
+  (seq [this]
+    (->> (seq (.keySet data))
+         (map (fn [key]
+                (clojure.lang.MapEntry. key (.valAt this key)))))))
+
+(defn loadable-weak-value-map
+  [keys load-fn preload-data]
+  (let [^HashMap hmap (HashMap. (count keys))]
+    (run! (fn [key]
+            (if-let [value (get preload-data key)]
+              (.put hmap key value)
+              (.put hmap key (SoftReference. nil))))
+          keys)
+    (LoadableWeakValueMap. hmap load-fn)))