🎉 Use textures directly for images

frontend/src/app/render_wasm/api.cljs

@@ -34,8 +34,6 @@
    [app.render-wasm.wasm :as wasm]
    [app.util.debug :as dbg]
    [app.util.functions :as fns]
-   [app.util.http :as http]
-   [app.util.webapi :as wapi]
    [beicon.v2.core :as rx]
    [promesa.core :as p]
    [rumext.v2 :as mf]))
@@ -234,49 +232,90 @@
   [string]
   (+ (count string) 1))
 
+(defn- create-webgl-texture-from-image
+  "Creates a WebGL texture from an HTMLImageElement or ImageBitmap and returns the texture object"
+  [gl image-element]
+  (let [texture (.createTexture ^js gl)]
+    (.bindTexture ^js gl (.-TEXTURE_2D ^js gl) texture)
+    (.texParameteri ^js gl (.-TEXTURE_2D ^js gl) (.-TEXTURE_WRAP_S ^js gl) (.-CLAMP_TO_EDGE ^js gl))
+    (.texParameteri ^js gl (.-TEXTURE_2D ^js gl) (.-TEXTURE_WRAP_T ^js gl) (.-CLAMP_TO_EDGE ^js gl))
+    (.texParameteri ^js gl (.-TEXTURE_2D ^js gl) (.-TEXTURE_MIN_FILTER ^js gl) (.-LINEAR ^js gl))
+    (.texParameteri ^js gl (.-TEXTURE_2D ^js gl) (.-TEXTURE_MAG_FILTER ^js gl) (.-LINEAR ^js gl))
+    (.texImage2D ^js gl (.-TEXTURE_2D ^js gl) 0 (.-RGBA ^js gl) (.-RGBA ^js gl) (.-UNSIGNED_BYTE ^js gl) image-element)
+    (.bindTexture ^js gl (.-TEXTURE_2D ^js gl) nil)
+    texture))
+
+(defn- get-webgl-context
+  "Gets the WebGL context from the WASM module"
+  []
+  (when wasm/context-initialized?
+    (let [gl-obj (unchecked-get wasm/internal-module "GL")]
+      (when gl-obj
+        ;; Get the current WebGL context from Emscripten
+        ;; The GL object has a currentContext property that contains the context handle
+        (let [current-ctx (.-currentContext ^js gl-obj)]
+          (when current-ctx
+            (.-GLctx ^js current-ctx)))))))
+
+(defn- get-texture-id-for-gl-object
+  "Registers a WebGL texture with Emscripten's GL object system and returns its ID"
+  [texture]
+  (let [gl-obj (unchecked-get wasm/internal-module "GL")
+        textures (.-textures ^js gl-obj)
+        new-id (.getNewId ^js gl-obj textures)]
+    (aset textures new-id texture)
+    new-id))
 
 (defn- fetch-image
+  "Loads an image and creates a WebGL texture from it, passing the texture ID to WASM.
+   This avoids decoding the image twice (once in browser, once in WASM)."
   [shape-id image-id thumbnail?]
   (let [url (cf/resolve-file-media {:id image-id} thumbnail?)]
     {:key url
      :thumbnail? thumbnail?
-     :callback #(->> (http/send! {:method :get
+     :callback #(->> (p/create
-                                  :uri url
+                      (fn [resolve reject]
-                                  :response-type :blob})
+                        (let [img (js/Image.)
-                     (rx/map :body)
+                              on-load (fn []
-                     (rx/mapcat wapi/read-file-as-array-buffer)
+                                        (resolve img))
-                     (rx/map (fn [image]
+                              on-error (fn [err]
-                               (let [size        (.-byteLength image)
+                                         (reject err))]
-                                     padded-size (if (zero? (mod size 4)) size (+ size (- 4 (mod size 4))))
+                          (set! (.-crossOrigin img) "anonymous")
-                                      ;; 36 bytes header (32 for UUIDs + 4 for thumbnail flag) + padded image
+                          (.addEventListener img "load" on-load)
-                                     total-bytes (+ 36 padded-size)
+                          (.addEventListener img "error" on-error)
-                                     offset      (mem/alloc->offset-32 total-bytes)
+                          (set! (.-src img) url))))
-                                     heap32      (mem/get-heap-u32)
+                     (rx/from)
-                                     data        (js/Uint8Array. image)
+                     (rx/map (fn [img]
-                                     padded      (js/Uint8Array. padded-size)]
+                               (when-let [gl (get-webgl-context)]
+                                 (let [texture (create-webgl-texture-from-image gl img)
+                                       texture-id (get-texture-id-for-gl-object texture)
+                                       width  (.-width ^js img)
+                                       height (.-height ^js img)
+                                       ;; Header: 32 bytes (2 UUIDs) + 4 bytes (thumbnail) + 4 bytes (texture ID) + 8 bytes (dimensions)
+                                       total-bytes 48
+                                       offset (mem/alloc->offset-32 total-bytes)
+                                       heap32 (mem/get-heap-u32)]
 
-                                 ;; 1. Set shape id (offset + 0 to offset + 3)
+                                   ;; 1. Set shape id (offset + 0 to offset + 3)
-                                 (mem.h32/write-uuid offset heap32 shape-id)
+                                   (mem.h32/write-uuid offset heap32 shape-id)
 
-                                 ;; 2. Set image id (offset + 4 to offset + 7)
+                                   ;; 2. Set image id (offset + 4 to offset + 7)
-                                 (mem.h32/write-uuid (+ offset 4) heap32 image-id)
+                                   (mem.h32/write-uuid (+ offset 4) heap32 image-id)
 
-                                 ;; 3. Set thumbnail flag as u32 (offset + 8)
+                                   ;; 3. Set thumbnail flag as u32 (offset + 8)
-                                 (aset heap32 (+ offset 8) thumbnail?)
+                                   (aset heap32 (+ offset 8) (if thumbnail? 1 0))
 
-                                 ;; 4. Adjust padding on image data
+                                   ;; 4. Set texture ID (offset + 9)
-                                 (.set padded data)
+                                   (aset heap32 (+ offset 9) texture-id)
-                                 (when (< size padded-size)
-                                   (dotimes [i (- padded-size size)]
-                                     (aset padded (+ size i) 0)))
 
-                                 ;; 5. Set image data (starting at offset + 9)
+                                   ;; 5. Set width (offset + 10)
-                                 (let [u32view (js/Uint32Array. (.-buffer padded))
+                                   (aset heap32 (+ offset 10) width)
-                                       image-u32-offset (+ offset 9)]
-                                   (.set heap32 u32view image-u32-offset))
 
-                                 (h/call wasm/internal-module "_store_image")
+                                   ;; 6. Set height (offset + 11)
-                                 true))))}))
+                                   (aset heap32 (+ offset 11) height)
+
+                                   (h/call wasm/internal-module "_store_image_from_texture")
+                                   true)))))}))
 
 (defn- get-fill-images
   [leaf]

render-wasm/src/render.rs

@@ -329,6 +329,19 @@ impl RenderState {
         self.images.add(id, is_thumbnail, image_data)
     }
 
+    /// Adds an image from an existing WebGL texture, avoiding re-decoding
+    pub fn add_image_from_gl_texture(
+        &mut self,
+        id: Uuid,
+        is_thumbnail: bool,
+        texture_id: u32,
+        width: i32,
+        height: i32,
+    ) -> Result<(), String> {
+        self.images
+            .add_image_from_gl_texture(id, is_thumbnail, texture_id, width, height)
+    }
+
     pub fn has_image(&self, id: &Uuid, is_thumbnail: bool) -> bool {
         self.images.contains(id, is_thumbnail)
     }

render-wasm/src/render/images.rs

@@ -62,6 +62,48 @@ pub struct ImageStore {
     context: Box<DirectContext>,
 }
 
+/// Creates a Skia image from an existing WebGL texture.
+/// This avoids re-decoding the image, as the browser has already decoded
+/// and uploaded it to the GPU.
+fn create_image_from_gl_texture(
+    context: &mut Box<DirectContext>,
+    texture_id: u32,
+    width: i32,
+    height: i32,
+) -> Result<Image, String> {
+    use skia_safe::gpu;
+    use skia_safe::gpu::gl::TextureInfo;
+
+    // Create a TextureInfo describing the existing GL texture
+    let texture_info = TextureInfo {
+        target: gl::TEXTURE_2D,
+        id: texture_id,
+        format: gl::RGBA8,
+        protected: gpu::Protected::No,
+    };
+
+    // Create a backend texture from the GL texture using the new API
+    let label = format!("shared_texture_{}", texture_id);
+    let backend_texture = unsafe {
+        gpu::backend_textures::make_gl((width, height), gpu::Mipmapped::No, texture_info, label)
+    };
+
+    // Create a Skia image from the backend texture
+    // Use TopLeft origin because HTML images have their origin at top-left,
+    // while WebGL textures traditionally use bottom-left
+    let image = Image::from_texture(
+        context.as_mut(),
+        &backend_texture,
+        gpu::SurfaceOrigin::TopLeft,
+        skia::ColorType::RGBA8888,
+        skia::AlphaType::Premul,
+        None,
+    )
+    .ok_or("Failed to create Skia image from GL texture")?;
+
+    Ok(image)
+}
+
 // Decode and upload to GPU
 fn decode_image(context: &mut Box<DirectContext>, raw_data: &[u8]) -> Option<Image> {
     let data = unsafe { skia::Data::new_bytes(raw_data) };
@@ -122,6 +164,30 @@ impl ImageStore {
         Ok(())
     }
 
+    /// Creates a Skia image from an existing WebGL texture, avoiding re-decoding.
+    /// This is much more efficient as it reuses the texture that was already
+    /// decoded and uploaded to GPU by the browser.
+    pub fn add_image_from_gl_texture(
+        &mut self,
+        id: Uuid,
+        is_thumbnail: bool,
+        texture_id: u32,
+        width: i32,
+        height: i32,
+    ) -> Result<(), String> {
+        let key = (id, is_thumbnail);
+
+        if self.images.contains_key(&key) {
+            return Err("Image already exists".to_string());
+        }
+
+        // Create a Skia image from the existing GL texture
+        let image = create_image_from_gl_texture(&mut self.context, texture_id, width, height)?;
+        self.images.insert(key, StoredImage::Gpu(image));
+
+        Ok(())
+    }
+
     pub fn contains(&self, id: &Uuid, is_thumbnail: bool) -> bool {
         self.images.contains_key(&(*id, is_thumbnail))
     }

render-wasm/src/wasm/fills/image.rs

@@ -89,3 +89,55 @@ pub extern "C" fn store_image() {
 
     mem::free_bytes();
 }
+
+/// Stores an image from an existing WebGL texture, avoiding re-decoding
+/// Expected memory layout:
+/// - bytes 0-15: shape UUID
+/// - bytes 16-31: image UUID  
+/// - bytes 32-35: is_thumbnail flag (u32)
+/// - bytes 36-39: GL texture ID (u32)
+/// - bytes 40-43: width (i32)
+/// - bytes 44-47: height (i32)
+#[no_mangle]
+pub extern "C" fn store_image_from_texture() {
+    let bytes = mem::bytes();
+
+    if bytes.len() < 48 {
+        eprintln!("store_image_from_texture: insufficient data");
+        mem::free_bytes();
+        return;
+    }
+
+    let ids = ShapeImageIds::try_from(bytes[0..IMAGE_IDS_SIZE].to_vec()).unwrap();
+
+    // Read is_thumbnail flag (4 bytes as u32)
+    let is_thumbnail_bytes = &bytes[IMAGE_IDS_SIZE..IMAGE_HEADER_SIZE];
+    let is_thumbnail_value = u32::from_le_bytes(is_thumbnail_bytes.try_into().unwrap());
+    let is_thumbnail = is_thumbnail_value != 0;
+
+    // Read GL texture ID (4 bytes as u32)
+    let texture_id_bytes = &bytes[36..40];
+    let texture_id = u32::from_le_bytes(texture_id_bytes.try_into().unwrap());
+
+    // Read width and height (8 bytes as two i32s)
+    let width_bytes = &bytes[40..44];
+    let width = i32::from_le_bytes(width_bytes.try_into().unwrap());
+
+    let height_bytes = &bytes[44..48];
+    let height = i32::from_le_bytes(height_bytes.try_into().unwrap());
+
+    with_state_mut!(state, {
+        if let Err(msg) = state.render_state_mut().add_image_from_gl_texture(
+            ids.image_id,
+            is_thumbnail,
+            texture_id,
+            width,
+            height,
+        ) {
+            eprintln!("store_image_from_texture error: {}", msg);
+        }
+        state.touch_shape(ids.shape_id);
+    });
+
+    mem::free_bytes();
+}