✨ Make the dist bundle use consistent and cache-aware uris (#7911)

common/src/app/common/version.cljc

@@ -14,7 +14,8 @@
 (defn parse
   [data]
   (cond
-    (str/starts-with? data "%")
+    (or (str/starts-with? data "%")
+        (= data "develop"))
     {:full "develop"
      :branch "develop"
      :base "0.0.0"

docker/devenv/files/nginx.conf

@@ -38,11 +38,11 @@ http {
 
     gzip_vary on;
     gzip_proxied any;
-    gzip_comp_level 3;
+    gzip_comp_level 6;
     gzip_buffers 16 8k;
     gzip_http_version 1.1;
 
-    gzip_types text/plain text/css text/javascript application/javascript application/json application/transit+json image/svg+xml;
+    gzip_types text/plain text/css text/javascript application/javascript application/json application/transit+json image/svg+xml application/wasm;
 
     map $http_upgrade $connection_upgrade {
         default upgrade;
@@ -223,16 +223,15 @@ http {
                 add_header X-Cache-Status $upstream_cache_status;
             }
 
-            location ~ ^/(/|css|fonts|images|js|wasm|mjs|map) {
+            location ~* \.(js|css|jpg|png|svg|ttf|woff|woff2|wasm)$ {
+                add_header Cache-Control "public, max-age=604800" always; # 7 days
             }
 
             location ~ ^/[^/]+/(.*)$ {
                 return 301 " /404";
             }
 
-            add_header Cache-Control "no-store";
+            add_header Cache-Control "no-store, no-cache, max-age=0" always;
-            # This header is what we need to use on prod
-            # add_header Cache-Control "public, must-revalidate, max-age=0";
             try_files $uri /index.html$is_args$args /index.html =404;
         }
     }

docker/images/files/nginx.conf.template

@@ -42,11 +42,11 @@ http {
     gzip_vary on;
     gzip_proxied any;
     gzip_static on;
-    gzip_comp_level 4;
+    gzip_comp_level 6;
     gzip_buffers 16 8k;
     gzip_http_version 1.1;
 
-    gzip_types text/plain text/css text/javascript application/javascript application/json application/transit+json image/svg+xml;
+    gzip_types text/plain text/css text/javascript application/javascript application/json application/transit+json image/svg+xml application/wasm;
 
     proxy_buffer_size 16k;
     proxy_busy_buffers_size 24k; # essentially, proxy_buffer_size + 2 small buffers of 4k
@@ -142,15 +142,15 @@ http {
         location / {
             include /etc/nginx/overrides/location.d/*.conf;
 
-
+            location ~* \.(js|css|jpg|png|svg|ttf|woff|woff2|wasm)$ {
-            location ~ ^/(/|css|fonts|images|js|wasm|mjs|map) {
+                add_header Cache-Control "public, max-age=604800" always; # 7 days
             }
 
             location ~ ^/[^/]+/(.*)$ {
                 return 301 " /404";
             }
 
-            add_header Cache-Control "public, must-revalidate, max-age=0";
+            add_header Cache-Control "no-store, no-cache, max-age=0" always;
             try_files $uri /index.html$is_args$args /index.html =404;
         }
     }

frontend/playwright/ui/pages/BasePage.js

@@ -73,7 +73,7 @@ export class BasePage {
   }
 
   static async mockConfigFlags(page, flags) {
-    const url = "**/js/config.js";
+    const url = "**/js/config.js*";
     return await page.route(url, (route) =>
       route.fulfill({
         status: 200,

frontend/resources/templates/index.mustache

@@ -17,22 +17,25 @@
     <meta name="twitter:site" content="@penpotapp">
     <meta name="twitter:creator" content="@penpotapp">
     <meta name="theme-color" content="#FFFFFF" media="(prefers-color-scheme: light)">
-    <link id="theme" href="css/main.css?ts={{& ts}}" rel="stylesheet" type="text/css" />
+    <link id="theme" href="css/main.css?version={{& version}}" rel="stylesheet" type="text/css" />
     {{#isDebug}}
-    <link href="css/debug.css?ts={{& ts}}" rel="stylesheet" type="text/css" />
+    <link href="css/debug.css?version={{& version}}" rel="stylesheet" type="text/css" />
     {{/isDebug}}
 
     <link rel="icon" href="images/favicon.png" />
-    {{# manifest}}
-    <script>window.penpotWorkerURI="{{& worker_main}}"</script>
-    <script src="{{& config}}"></script>
-    <script src="{{& polyfills}}"></script>
-    {{/manifest}}
 
     <script type="module">
-      globalThis.penpotVersion = "%version%";
+      globalThis.penpotVersion = "{{& version}}";
-      globalThis.penpotBuildDate = "%buildDate%";
+      globalThis.penpotBuildDate = "{{& build_date}}";
+      globalThis.penpotWorkerURI = "{{& manifest.worker_main}}";
     </script>
+
+    {{# manifest}}
+    <script src="{{& config}}"></script>
+    <script src="{{& polyfills}}"></script>
+    <script type="importmap">{{& importmap }}</script>
+    {{/manifest}}
+
     <!--cookie-consent-->
   </head>
   <body>
@@ -45,7 +48,7 @@
     {{# manifest}}
     <script type="module" src="{{& libs}}"></script>
     <script type="module">
-      import { init } from "{{& main}}";
+      import { init } from "{{& app_main}}";
       init();
     </script>
     {{/manifest}}

frontend/resources/templates/preview-head.mustache

@@ -1,4 +1,4 @@
-<link href="./css/ds.css?ts={{& ts}}" rel="stylesheet" type="text/css" />
+<link href="./css/ds.css?version={{& version}}" rel="stylesheet" type="text/css" />
 
 <style>
   body {

frontend/resources/templates/rasterizer.mustache

@@ -6,22 +6,22 @@
     <link rel="icon" href="images/favicon.png" />
 
     <script>
-      window.penpotVersion = "%version%";
+      globalThis.penpotVersion = "{{& version}}";
-      window.penpotBuildDate = "%buildDate%";
+      globalThis.penpotBuildDate = "{{& build_date}}";
+      globalThis.penpotWorkerURI = "{{& manifest.worker_main}}";
     </script>
 
     {{# manifest}}
-    <script>window.penpotWorkerURI="{{& worker_main}}"</script>
     <script src="{{& config}}"></script>
     <script src="{{& polyfills}}"></script>
+    <script type="importmap">{{& importmap }}</script>
     {{/manifest}}
-
   </head>
   <body>
     {{# manifest}}
     <script type="module" src="{{& libs}}"></script>
     <script type="module">
-      import { init } from "{{& rasterizer}}";
+      import { init } from "{{& rasterizer_main}}";
       init();
     </script>
     {{/manifest}}

frontend/resources/templates/render.mustache

@@ -7,12 +7,14 @@
     <link rel="icon" href="images/favicon.png" />
 
     <script>
-      window.penpotVersion = "%version%";
+      globalThis.penpotVersion = "{{& version}}";
+      globalThis.penpotBuildDate = "{{& build_date}}";
     </script>
 
     {{# manifest}}
     <script src="{{& config}}"></script>
     <script src="{{& polyfills}}"></script>
+    <script type="importmap">{{& importmap }}</script>
     {{/manifest}}
   </head>
   <body>
@@ -20,7 +22,7 @@
     {{# manifest}}
     <script type="module" src="{{& libs}}"></script>
     <script type="module">
-      import { init } from "{{& render}}";
+      import { init } from "{{& render_main}}";
       init();
     </script>
     {{/manifest}}

frontend/scripts/_helpers.js

@@ -28,6 +28,8 @@ export function startWorker() {
 }
 
 export const isDebug = process.env.NODE_ENV !== "production";
+export const CURRENT_VERSION = process.env.CURRENT_VERSION || "develop";
+export const BUILD_DATE = process.env.BUILD_DATE || "" + new Date();
 
 async function findFiles(basePath, predicate, options = {}) {
   predicate =
@@ -47,8 +49,7 @@ async function findFiles(basePath, predicate, options = {}) {
 function syncDirs(originPath, destPath) {
   const command = `rsync -ar --delete ${originPath} ${destPath}`;
 
-  return new Promise((resolve, reject) => {
+  return new Promise((resolve, reject) => {proc.exec(command, (cause, stdout) => {
-    proc.exec(command, (cause, stdout) => {
       if (cause) {
         reject(cause);
       } else {
@@ -187,18 +188,31 @@ async function readManifestFile() {
 }
 
 async function readShadowManifest() {
-  const ts = Date.now();
-
   const index = {
-    ts: ts,
+    app_main: "./js/main.js",
-    config: "./js/config.js",
+    render_main: "./js/render.js",
-    polyfills: "./js/polyfills.js",
+    rasterizer_main: "./js/rasterizer.js",
-    main: "./js/main.js",
+
-    shared: "./js/shared.js",
+    config: "./js/config.js?version=" + CURRENT_VERSION,
-    render: "./js/render.js",
+    polyfills: "./js/polyfills.js?version=" + CURRENT_VERSION,
-    worker_main: "./js/worker/main.js",
+    libs: "./js/libs.js?version=" + CURRENT_VERSION,
-    rasterizer: "./js/rasterizer.js",
+    worker_main: "./js/worker/main.js?version=" + CURRENT_VERSION,
-    libs: "./js/libs.js",
+
+    importmap: JSON.stringify({
+      "imports": {
+        "./js/shared.js": "./js/shared.js?version=" + CURRENT_VERSION,
+        "./js/main.js": "./js/main.js?version=" + CURRENT_VERSION,
+        "./js/render.js": "./js/render.js?version=" + CURRENT_VERSION,
+        "./js/render-wasm.js": "./js/render-wasm.js?version=" + CURRENT_VERSION,
+        "./js/rasterizer.js": "./js/rasterizer.js?version=" + CURRENT_VERSION,
+        "./js/main-dashboard.js": "./js/main-dashboard.js?version=" + CURRENT_VERSION,
+        "./js/main-auth.js": "./js/main-auth.js?version=" + CURRENT_VERSION,
+        "./js/main-viewer.js": "./js/main-viewer.js?version=" + CURRENT_VERSION,
+        "./js/main-settings.js": "./js/main-settings.js?version=" + CURRENT_VERSION,
+        "./js/main-workspace.js": "./js/main-workspace.js?version=" + CURRENT_VERSION,
+        "./js/util-highlight.js": "./js/util-highlight.js?version=" + CURRENT_VERSION
+      }
+    })
   };
 
   return index;
@@ -398,12 +412,16 @@ async function generateTemplates() {
     "../public/images/sprites/assets.svg": assetsSprite,
   };
 
+  const context = {
+    manifest: manifest,
+    version: CURRENT_VERSION,
+    build_date: BUILD_DATE,
+    isDebug,
+  };
+
   content = await renderTemplate(
     "resources/templates/index.mustache",
-    {
+    context,
-      manifest: manifest,
-      isDebug,
-    },
     partials,
   );
 
@@ -411,38 +429,30 @@ async function generateTemplates() {
 
   content = await renderTemplate(
     "resources/templates/challenge.mustache",
-    {},
+    context,
     partials,
   );
   await fs.writeFile("./resources/public/challenge.html", content);
 
   content = await renderTemplate(
     "resources/templates/preview-body.mustache",
-    {
+    context,
-      manifest: manifest,
-    },
     partials,
   );
   await fs.writeFile("./.storybook/preview-body.html", content);
 
   content = await renderTemplate(
     "resources/templates/preview-head.mustache",
-    {
+    context,
-      manifest: manifest,
-    },
     partials,
   );
   await fs.writeFile("./.storybook/preview-head.html", content);
 
-  content = await renderTemplate("resources/templates/render.mustache", {
+  content = await renderTemplate("resources/templates/render.mustache", context);
-    manifest: manifest,
-  });
 
   await fs.writeFile("./resources/public/render.html", content);
 
-  content = await renderTemplate("resources/templates/rasterizer.mustache", {
+  content = await renderTemplate("resources/templates/rasterizer.mustache", context);
-    manifest: manifest,
-  });
 
   await fs.writeFile("./resources/public/rasterizer.html", content);
 }

frontend/scripts/build

@@ -38,15 +38,7 @@ fi
 yarn run build:app:libs || exit 1;
 yarn run build:app:assets || exit 1;
 
-sed -i -re "s/\%version\%/$CURRENT_VERSION/g" ./resources/public/index.html;
+sed -i "s/render-wasm.js/render-wasm.js?version=$CURRENT_VERSION/g" ./resources/public/js/worker/main.js;
-sed -i -re "s/\%version\%/$CURRENT_VERSION/g" ./resources/public/render.html;
-sed -i -re "s/\%version\%/$CURRENT_VERSION/g" ./resources/public/rasterizer.html;
-sed -i -re "s/\%buildDate\%/$BUILD_DATE/g" ./resources/public/index.html;
-sed -i -re "s/\%buildDate\%/$BUILD_DATE/g" ./resources/public/rasterizer.html;
-
-if [ "$INCLUDE_WASM" = "yes" ];  then
-    sed -i "s/version=develop/version=$CURRENT_VERSION/g" ./resources/public/js/render_wasm.js;
-fi
 
 rsync -avr resources/public/ target/dist/;
 

frontend/shadow-cljs.edn

@@ -23,28 +23,28 @@
 
     :util-highlight
     {:entries [app.util.code-highlight]
-     :depends-on #{:main}}
+     :depends-on #{:shared}}
 
     :main-auth
     {:entries [app.main.ui.auth
                app.main.ui.auth.verify-token]
-     :depends-on #{:main}}
+     :depends-on #{:shared}}
 
     :main-viewer
     {:entries [app.main.ui.viewer]
-     :depends-on #{:main :main-auth}}
+     :depends-on #{:shared :main-auth}}
 
     :main-workspace
     {:entries [app.main.ui.workspace]
-     :depends-on #{:main}}
+     :depends-on #{:shared}}
 
     :main-dashboard
     {:entries [app.main.ui.dashboard]
-     :depends-on #{:main}}
+     :depends-on #{:shared}}
 
     :main-settings
     {:entries [app.main.ui.settings]
-     :depends-on #{:main}}
+     :depends-on #{:shared}}
 
     :render
     {:entries [app.render]

frontend/src/app/config.cljs

@@ -190,5 +190,4 @@
 
 (defn resolve-static-asset
   [path]
-  (let [uri (u/join public-uri path)]
+  (u/join public-uri path))
-    (assoc uri :query (dm/str "version=" (:full version)))))

frontend/src/app/render_wasm/api.cljs

@@ -1361,7 +1361,7 @@
 (defonce module
   (delay
     (if (exists? js/dynamicImport)
-      (let [uri (cf/resolve-static-asset "js/render_wasm.js")]
+      (let [uri (cf/resolve-static-asset "js/render-wasm.js")]
         (->> (mod/import uri)
              (p/mcat init-wasm-module)
              (p/fmap

frontend/src/app/util/i18n.cljs

@@ -116,7 +116,7 @@
 
 (defn- load
   [locale]
-  (let [path (str "./translation." locale ".js")]
+  (let [path (str "./translation." locale ".js?version=" (:full cf/version))]
     (->> (mod/import path)
          (p/fmap (fn [result] (unchecked-get result "default")))
          (p/fnly (fn [data cause]

frontend/src/app/worker/thumbnails.cljs

@@ -100,7 +100,7 @@
 
 (def init-wasm
   (delay
-    (let [uri (cf/resolve-static-asset "js/render_wasm.js")]
+    (let [uri (cf/resolve-static-asset "js/render-wasm.js")]
       (-> (mod/import (str uri))
           (p/then #(wasm.api/init-wasm-module %))
           (p/then #(set! wasm/internal-module %))))))

render-wasm/_build_env

@@ -1,11 +1,15 @@
 #!/usr/bin/env bash
 
+CURRENT_VERSION=${CURRENT_VERSION:-develop};
+
 if [ "$NODE_ENV" = "production" ];  then
-    export _BUILD_MODE="release";
+    export BUILD_MODE="release";
 else
-    export _BUILD_MODE=${1:-debug};
+    export BUILD_MODE=${1:-debug};
 fi
 
+BUILD_NAME="${BUILD_NAME:-render-wasm}"
+
 # 256 MB of initial heap to perform less
 # initial calls to memory grow.
 EM_INITIAL_HEAP=$((256 * 1024 * 1024))
@@ -40,10 +44,10 @@ EMCC_CFLAGS="--no-entry \
 
 export EM_CACHE="/tmp/emsdk_cache";
 
-_CARGO_PARAMS="${@:2}";
+CARGO_PARAMS="${@:2}";
 
-if [ "$_BUILD_MODE" = "release" ]; then
+if [ "$BUILD_MODE" = "release" ]; then
-    _CARGO_PARAMS="--release $_CARGO_PARAMS"
+    CARGO_PARAMS="--release $CARGO_PARAMS"
     EMCC_CFLAGS="-Os $EMCC_CFLAGS"
 else
     # TODO: Extra parameters that could be good to look into:
@@ -54,5 +58,5 @@ else
 fi
 
 export EMCC_CFLAGS;
-export _CARGO_PARAMS;
+export CARGO_PARAMS;
 

render-wasm/build

@@ -2,26 +2,23 @@
 
 EMSDK_QUIET=1 . /opt/emsdk/emsdk_env.sh
 
-set -x
-
-_BUILD_NAME="${_BUILD_NAME:-render_wasm}"
-
 _SCRIPT_DIR=$(dirname $0);
 pushd $_SCRIPT_DIR;
 
 . ./_build_env
 
+set -x
+
 export CARGO_BUILD_TARGET=${CARGO_BUILD_TARGET:-"wasm32-unknown-emscripten"};
 export SKIA_BINARIES_URL=${SKIA_BINARIES_URL:-"https://github.com/penpot/skia-binaries/releases/download/0.87.0/skia-binaries-e551f334ad5cbdf43abf-wasm32-unknown-emscripten-gl-svg-textlayout-binary-cache-webp.tar.gz"}
 
-cargo build $_CARGO_PARAMS
+cargo build $CARGO_PARAMS
 
 _SHARED_FILE=$(find target/wasm32-unknown-emscripten -name render_wasm_shared.js | head -n 1);
 
-cat target/wasm32-unknown-emscripten/$_BUILD_MODE/render_wasm.js "$_SHARED_FILE" > ../frontend/resources/public/js/$_BUILD_NAME.js
+cat target/wasm32-unknown-emscripten/$BUILD_MODE/render_wasm.js "$_SHARED_FILE" > ../frontend/resources/public/js/$BUILD_NAME.js
-cp target/wasm32-unknown-emscripten/$_BUILD_MODE/render_wasm.wasm ../frontend/resources/public/js/$_BUILD_NAME.wasm
+cp target/wasm32-unknown-emscripten/$BUILD_MODE/render_wasm.wasm ../frontend/resources/public/js/$BUILD_NAME.wasm
-
+sed -i "s/render_wasm.wasm/$BUILD_NAME.wasm?version=$CURRENT_VERSION/g" ../frontend/resources/public/js/$BUILD_NAME.js;
-sed -i "s/render_wasm.wasm/$_BUILD_NAME.wasm?version=develop/g" ../frontend/resources/public/js/$_BUILD_NAME.js;
 
 exit $?
 

render-wasm/watch

@@ -1,25 +1,26 @@
 #!/usr/bin/env bash
-set -x
 
 _SCRIPT_DIR=$(dirname $0);
 pushd $_SCRIPT_DIR;
 
 . ./_build_env
 
+set -x
+
 export CARGO_BUILD_TARGET=${CARGO_BUILD_TARGET:-"wasm32-unknown-emscripten"};
 export SKIA_BINARIES_URL=${SKIA_BINARIES_URL:-"https://github.com/penpot/skia-binaries/releases/download/0.87.0/skia-binaries-e551f334ad5cbdf43abf-wasm32-unknown-emscripten-gl-svg-textlayout-binary-cache-webp.tar.gz"}
 
-_SHARED_FILE=$(find target/wasm32-unknown-emscripten -name render_wasm_shared.js | head -n 1);
+SHARED_FILE=$(find target/wasm32-unknown-emscripten -name render_wasm_shared.js | head -n 1);
 
-cat target/wasm32-unknown-emscripten/$_BUILD_MODE/render_wasm.js "$_SHARED_FILE" > ../frontend/resources/public/js/$_BUILD_NAME.js
+cat target/wasm32-unknown-emscripten/$BUILD_MODE/render_wasm.js "$SHARED_FILE" > ../frontend/resources/public/js/$BUILD_NAME.js
-cp target/wasm32-unknown-emscripten/$_BUILD_MODE/render_wasm.wasm ../frontend/resources/public/js/$_BUILD_NAME.wasm
+cp target/wasm32-unknown-emscripten/$BUILD_MODE/render_wasm.wasm ../frontend/resources/public/js/$BUILD_NAME.wasm
 
 pushd $_SCRIPT_DIR;
 cargo watch \
-      -x "build $_CARGO_PARAMS" \
+      -x "build $CARGO_PARAMS" \
-      -s "cat target/wasm32-unknown-emscripten/$_BUILD_MODE/render_wasm.js \"$_SHARED_FILE\" > ../frontend/resources/public/js/$_BUILD_NAME.js" \
+      -s "cat target/wasm32-unknown-emscripten/$BUILD_MODE/render_wasm.js \"$SHARED_FILE\" > ../frontend/resources/public/js/$BUILD_NAME.js" \
-      -s "cp target/wasm32-unknown-emscripten/$_BUILD_MODE/render_wasm.wasm ../frontend/resources/public/js/" \
+      -s "cp target/wasm32-unknown-emscripten/$BUILD_MODE/render_wasm.wasm ../frontend/resources/public/js/$BUILD_NAME.wasm" \
-      -s "sed -i 's/render_wasm.wasm/render_wasm.wasm?version=develop/g' ../frontend/resources/public/js/render_wasm.js" \
+      -s "sed -i 's/render_wasm.wasm/$BUILD_NAME.wasm?version=$CURRENT_VERSION/g' ../frontend/resources/public/js/$BUILD_NAME.js" \
       -s "echo 'DONE\n'";
 
 popd