diff --git a/backend/src/app/binfile/common.clj b/backend/src/app/binfile/common.clj index 380ea34306..9e614cc148 100644 --- a/backend/src/app/binfile/common.clj +++ b/backend/src/app/binfile/common.clj @@ -19,6 +19,7 @@ [app.common.time :as ct] [app.common.types.file :as ctf] [app.common.uuid :as uuid] + [app.common.weak :as weak] [app.config :as cf] [app.db :as db] [app.db.sql :as sql] @@ -606,15 +607,19 @@ (map decode-row)) (db/exec! conn [sql:get-file-libraries file-id]))) -;; FIXME: this will use a lot of memory if file uses too many big -;; libraries, we should load required libraries on demand (defn get-resolved-file-libraries - "A helper for preload file libraries" - [{:keys [::db/conn] :as cfg} file] - (->> (get-file-libraries conn (:id file)) - ;; WARNING: we don't migrate the libraries for avoid cascade - ;; migration; it is not ideal but it reduces the total of the - ;; required memory needed for process a single file migration - ;; that requires libraries to be loaded. - (into [file] (map #(get-file cfg (:id %) :migrate? false))) - (d/index-by :id))) + "Get all file libraries including itself. Returns an instance of + LoadableWeakValueMap that allows do not have strong references to + the loaded libraries and reduce possible memory pressure on having + all this libraries loaded at same time on processing file validation + or file migration. + + This still requires at least one library at time to be loaded while + access to it is performed, but it improves considerable not having + the need of loading all the libraries at the same time." + [{:keys [::db/conn] :as cfg} {:keys [id] :as file}] + (let [library-ids (->> (get-file-libraries conn (:id file)) + (map :id) + (cons (:id file))) + load-fn #(get-file cfg % :migrate? false)] + (weak/loadable-weak-value-map library-ids load-fn {id file}))) diff --git a/backend/src/app/rpc/commands/files_update.clj b/backend/src/app/rpc/commands/files_update.clj index 9d52b7a2c4..e499ea2642 100644 --- a/backend/src/app/rpc/commands/files_update.clj +++ b/backend/src/app/rpc/commands/files_update.clj @@ -408,7 +408,6 @@ (not skip-validate)) (bfc/get-resolved-file-libraries cfg file)) - ;; The main purpose of this atom is provide a contextual state ;; for the changes subsystem where optionally some hints can ;; be provided for the changes processing. Right now we are diff --git a/common/src/app/common/types/fills/impl.cljc b/common/src/app/common/types/fills/impl.cljc index 30228a5a1c..26f3757c6e 100644 --- a/common/src/app/common/types/fills/impl.cljc +++ b/common/src/app/common/types/fills/impl.cljc @@ -443,7 +443,7 @@ :code :invalid-fill :hint "found invalid fill on encoding fills to binary format"))))) - #?(:cljs (Fills. total dbuffer mbuffer image-ids (weak/create-weak-value-map) nil) + #?(:cljs (Fills. total dbuffer mbuffer image-ids (weak/weak-value-map) nil) :clj (Fills. total dbuffer mbuffer nil)))))) (defn fills? diff --git a/common/src/app/common/types/path/impl.cljc b/common/src/app/common/types/path/impl.cljc index 1dfdf31593..a32e4465ab 100644 --- a/common/src/app/common/types/path/impl.cljc +++ b/common/src/app/common/types/path/impl.cljc @@ -379,7 +379,7 @@ (-transform [this m] (let [buffer (buf/clone buffer)] (impl-transform buffer m size) - (PathData. size buffer (weak/create-weak-value-map) nil))) + (PathData. size buffer (weak/weak-value-map) nil))) (-walk [_ f initial] (impl-walk buffer f initial size)) @@ -600,14 +600,14 @@ count (long (/ size SEGMENT-U8-SIZE))] (PathData. count (js/DataView. buffer) - (weak/create-weak-value-map) + (weak/weak-value-map) nil)) (instance? js/DataView buffer) (let [buffer' (.-buffer ^js/DataView buffer) size (.-byteLength ^js/ArrayBuffer buffer') count (long (/ size SEGMENT-U8-SIZE))] - (PathData. count buffer (weak/create-weak-value-map) nil)) + (PathData. count buffer (weak/weak-value-map) nil)) (instance? js/Uint8Array buffer) (from-bytes (.-buffer buffer)) diff --git a/common/src/app/common/weak.cljc b/common/src/app/common/weak.cljc new file mode 100644 index 0000000000..87ae6cbc4b --- /dev/null +++ b/common/src/app/common/weak.cljc @@ -0,0 +1,79 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC + +(ns app.common.weak + "A collection of helpers for work with weak references and weak + data structures on JS runtime." + (:refer-clojure :exclude [memoize]) + (:require + #?@(:cljs [["./weak/impl_weak_map.js" :as wm] + ["./weak/impl_weak_value_map.js" :as wvm]] + :clj [[app.common.weak.impl-loadable-weak-value-map :as lwvm]]))) + +#?(:cljs + (defn weak-value-map + "Creates a WeakMap instance where values are held by soft + references and keys are held by hard references." + [] + (new wvm/WeakValueMap.))) + +#?(:cljs + (defn weak-map + "Create a WeakMap like instance what uses clojure equality + semantics." + [] + (new wm/WeakEqMap #js {:hash hash :equals =}))) + +#?(:clj + (defn loadable-weak-value-map + "Creates an instance of a LoadableWeakValueMap. It gives you a clojure-like, + map instance with fixed number of keys and fixed preload data (for + the provided keys) where not preload data is lazy loadable. It + internally uses soft-like references, leaving the runtime to collect + values that are not in use (no hard references keeps on the runtime)." + ([keys load-fn] + (lwvm/loadable-weak-value-map keys load-fn {})) + ([keys load-fn preload-data] + (lwvm/loadable-weak-value-map keys load-fn preload-data)))) + +#?(:cljs (def ^:private state (new js/WeakMap))) +#?(:cljs (def ^:private global-counter 0)) + +#?(:cljs + (defn weak-key + "A simple helper that returns a stable key string for an object while + that object remains in memory and is not collected by the GC. + + Mainly used for assign temporal IDs/keys for react children + elements when the element has no specific id." + [o] + (let [key (.get ^js/WeakMap state o)] + (if (some? key) + key + (let [key (str "weak-key" (js* "~{}++" global-counter))] + (.set ^js/WeakMap state o key) + key))))) + +#?(:cljs + (defn memoize + "Returns a memoized version of a referentially transparent + function. The memoized version of the function keeps a cache of the + mapping from arguments to results and, when calls with the same + arguments are repeated often, has higher performance at the expense + of higher memory use. + + The main difference with clojure.core/memoize, is that this function + uses weak-map, so cache is cleared once GC is passed and cached keys + are collected" + [f] + (let [mem (weak-map)] + (fn [& args] + (let [v (.get mem args)] + (if (undefined? v) + (let [ret (apply f args)] + (.set ^js mem args ret) + ret) + v)))))) diff --git a/common/src/app/common/weak.cljs b/common/src/app/common/weak.cljs deleted file mode 100644 index 58ae9aad72..0000000000 --- a/common/src/app/common/weak.cljs +++ /dev/null @@ -1,59 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; Copyright (c) KALEIDOS INC - -(ns app.common.weak - "A collection of helpers for work with weak references and weak - data structures on JS runtime." - (:refer-clojure :exclude [memoize]) - (:require - ["./weak/impl_weak_map.js" :as wm] - ["./weak/impl_weak_value_map.js" :as wvm])) - -(defn create-weak-value-map - [] - (new wvm/WeakValueMap.)) - -(defn create-weak-map - [] - (new wm/WeakEqMap #js {:hash hash :equals =})) - -(def ^:private state (new js/WeakMap)) -(def ^:private global-counter 0) - -(defn weak-key - "A simple helper that returns a stable key string for an object - while that object remains in memory and is not collected by the GC. - - Mainly used for assign temporal IDs/keys for react children - elements when the element has no specific id." - [o] - (let [key (.get ^js/WeakMap state o)] - (if (some? key) - key - (let [key (str "weak-key" (js* "~{}++" global-counter))] - (.set ^js/WeakMap state o key) - key)))) - -(defn memoize - "Returns a memoized version of a referentially transparent function. The - memoized version of the function keeps a cache of the mapping from arguments - to results and, when calls with the same arguments are repeated often, has - higher performance at the expense of higher memory use. - - The main difference with clojure.core/memoize, is that this function - uses weak-map, so cache is cleared once GC is passed and cached keys - are collected" - [f] - (let [mem (create-weak-map)] - (fn [& args] - (let [v (.get mem args)] - (if (undefined? v) - (let [ret (apply f args)] - (.set ^js mem args ret) - ret) - v))))) - - diff --git a/common/src/app/common/weak/impl_loadable_weak_value_map.clj b/common/src/app/common/weak/impl_loadable_weak_value_map.clj new file mode 100644 index 0000000000..ea72fea211 --- /dev/null +++ b/common/src/app/common/weak/impl_loadable_weak_value_map.clj @@ -0,0 +1,51 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC + +(ns app.common.weak.impl-loadable-weak-value-map + (:import + java.lang.ref.SoftReference + java.util.HashMap)) + +(deftype LoadableWeakValueMap [^HashMap data load-fn] + clojure.lang.ILookup + (valAt [_ key] + (when-let [reference (.get data key)] + (if (instance? SoftReference reference) + (or (.get ^SoftReference reference) + (let [value (load-fn key)] + (.put data key (SoftReference. value)) + value)) + reference))) + + (valAt [_ key default] + (if-let [reference (.get data key)] + (if (instance? SoftReference reference) + (or (.get ^SoftReference reference) + (let [value (load-fn key)] + (.put data key (SoftReference. value)) + value)) + reference) + default)) + + clojure.lang.Counted + (count [_] + (.size data)) + + clojure.lang.Seqable + (seq [this] + (->> (seq (.keySet data)) + (map (fn [key] + (clojure.lang.MapEntry. key (.valAt this key))))))) + +(defn loadable-weak-value-map + [keys load-fn preload-data] + (let [^HashMap hmap (HashMap. (count keys))] + (run! (fn [key] + (if-let [value (get preload-data key)] + (.put hmap key value) + (.put hmap key (SoftReference. nil)))) + keys) + (LoadableWeakValueMap. hmap load-fn)))