working on better orchestration

WebReflection · WebReflection · commit 97e72ff783e3 · 2025-10-03T20:45:25.000+02:00
diff --git a/docs/index.js b/docs/index.js
diff --git a/docs/index.js.map b/docs/index.js.map
diff --git a/esm/interpreter/_remote_package.js b/esm/interpreter/_remote_package.js
@@ -1,41 +1,74 @@
-// hackity hack hack for PyScript FUN
+import fetch from '@webreflection/fetch';
 
 import { toml } from '../3rd-party.js';
 
-const remote = async (config, packages) => {
+const { parse } = JSON;
+
+const href = (key, pkg) => new URL(key, pkg).href;
+
+const addPath = (target, key, value) => {
+  if (key in target)
+    throw new Error(`Duplicated path: ${key}`);
+  target[key] = value;
+};
+
+const addPaths = (target, source, pkg) => {
+  for (const key in source)
+    addPath(target, href(key, pkg), source[key]);
+};
+
+const pollute = (t_js_modules, s_js_modules, name, pkg) => {
+  const source = s_js_modules[name];
+  if (source) {
+    t_js_modules[name] ??= {};
+    addPaths(t_js_modules[name], source, pkg);
+  }
+};
+
+const remote = async (
+  config,
+  packages = config.packages,
+  set = new Set(),
+) => {
   const repackaged = [];
   for (const pkg of packages) {
-    if (pkg.endsWith('.toml')) {
-      const text = await (await fetch(pkg)).text();
-      const { files, js_modules, packages } = await toml(text);
-      if (packages)
-        repackaged.push(...(await remote(config, packages)));
+    // avoid re-processing already processed packages
+    if (set.has(pkg)) continue;
+    set.add(pkg);
+    const isTOML = pkg.endsWith('.toml');
+    if (isTOML || pkg.endsWith('.json')) {
+      const text = await fetch(pkg).text();
+      const {
+        name,
+        files,
+        js_modules,
+        packages,
+      } = isTOML ? await toml(text) : parse(text);
+
+      if (set.has(name))
+        throw new Error(`Unable to process ${name} @ ${pkg}`);
+
+      set.add(name);
+
+      if (packages) {
+        // process nested packages from the remote config
+        repackaged.push(...(await remote(config, packages, set)));
+      }
+
       if (js_modules) {
-        if (!config.js_modules) config.js_modules = {};
-        const { main, worker } = js_modules;
-        if (main) {
-          if (!config.js_modules.main) config.js_modules.main = {};
-          for (const key in main) {
-            config.js_modules.main[new URL(key, pkg).href] = main[key];
-          }
-        }
-        if (worker) {
-          if (!config.js_modules.worker) config.js_modules.worker = {};
-          for (const key in worker) {
-            config.js_modules.worker[new URL(key, pkg).href] = worker[key];
-          }
-        }
+        config.js_modules ??= {};
+        pollute(config.js_modules, js_modules, 'main', pkg);
+        pollute(config.js_modules, js_modules, 'worker', pkg);
       }
+
       if (files) {
-        if (!config.files) config.files = {};
-        for (const key in files) {
-          config.files[new URL(key, pkg).href] = files[key];
-        }
+        config.files ??= {};
+        addPaths(config.files, files, pkg);
       }
     }
     else repackaged.push(pkg);
   }
-  return [...new Set(repackaged)];
+  return repackaged;
 };
 
 export default remote;
diff --git a/esm/interpreter/micropython.js b/esm/interpreter/micropython.js
@@ -8,6 +8,7 @@ import mip from '../python/mip.js';
 import { zip } from '../3rd-party.js';
 
 import { initializeNativeFS } from './_nativefs.js';
+import _remote_package from './_remote_package.js';
 
 const type = 'micropython';
 
@@ -44,6 +45,11 @@ export default {
         // Install Micropython Package
         this.writeFile(interpreter, './mip.py', mip);
         if (config.packages) {
+            if (config.experimental_remote_packages) {
+                progress('Loading remote packages');
+                config.packages = await _remote_package(config);
+                progress('Loaded remote packages');
+            }
             progress('Loading packages');
             await py_imports(config.packages.map(fixedRelative, baseURL));
             progress('Loaded packages');
diff --git a/esm/interpreter/pyodide.js b/esm/interpreter/pyodide.js
@@ -85,47 +85,55 @@ export default {
         // https://github.com/pyodide/pyodide/issues/5736
         const save = config.packages_cache !== 'never' && version !== '0.28.0';
         await storage.sync();
+        progress('Loaded Storage');
         // packages_cache = 'never' means: erase the whole DB
         if (!save) storage.clear();
         // otherwise check if cache is known
-        else if (packages) {
-            config.packages = packages = await _remote_package(config, packages);
-            // packages_cache = 'passthrough' means: do not use micropip.install
-            if (config.packages_cache === 'passthrough') {
-                options.packages = packages;
-                packages = null;
-                storage.clear();
+        if (packages) {
+            if (config.experimental_remote_packages) {
+                progress('Loading remote packages');
+                config.packages = (packages = await _remote_package(config, packages));
+                progress('Loaded remote packages');
             }
-            else {
-                packages = packages.sort();
-                // packages are uniquely stored as JSON key
-                const key = stringify(packages);
-                if (storage.has(key)) {
-                    const value = storage.get(key);
+            if (save) {
+                // packages_cache = 'passthrough' means: do not use micropip.install
+                if (config.packages_cache === 'passthrough') {
+                    options.packages = packages;
+                    packages = null;
+                    storage.clear();
+                }
+                else {
+                    packages = packages.sort();
+                    // packages are uniquely stored as JSON key
+                    const key = stringify(packages);
+                    if (storage.has(key)) {
+                        const value = storage.get(key);
 
-                    // versions are not currently understood by pyodide when
-                    // a lockFileURL is used instead of micropip.install(packages)
-                    // https://github.com/pyodide/pyodide/issues/5135#issuecomment-2441038644
-                    // https://github.com/pyscript/pyscript/issues/2245
-                    options.packages = packages.map(name => name.split(/[>=<]=/)[0]);
+                        // versions are not currently understood by pyodide when
+                        // a lockFileURL is used instead of micropip.install(packages)
+                        // https://github.com/pyodide/pyodide/issues/5135#issuecomment-2441038644
+                        // https://github.com/pyscript/pyscript/issues/2245
+                        options.packages = packages.map(name => name.split(/[>=<]=/)[0]);
 
-                    if (version.startsWith('0.27')) {
-                        const blob = new Blob([value], { type: 'application/json' });
-                        options.lockFileURL = URL.createObjectURL(blob);
-                    }
-                    else {
-                      options.lockFileContents = value;
-                    }
+                        if (version.startsWith('0.27')) {
+                            const blob = new Blob([value], { type: 'application/json' });
+                            options.lockFileURL = URL.createObjectURL(blob);
+                        }
+                        else {
+                        options.lockFileContents = value;
+                        }
 
-                    packages = null;
+                        packages = null;
+                    }
                 }
             }
         }
-        progress('Loaded Storage');
         const { stderr, stdout, get } = stdio();
+        progress('Loading interpreter');
         const interpreter = await get(
             loadPyodide({ stderr, stdout, ...options }),
         );
+        progress('Loaded interpreter');
         globalThis[js_modules].set('-T-', this.transform.bind(this, interpreter));
         if (config.debug) interpreter.setDebug(true);
         const py_imports = importPackages.bind(interpreter);
diff --git a/package.json b/package.json
@@ -95,6 +95,6 @@
         "to-json-callback": "^0.1.1"
     },
     "worker": {
-        "blob": "sha256-or/eVGAwbAdeOd7ha5qhvvgTfdnS92Z0Vg6qnwmIgBI="
+        "blob": "sha256-+m8Z7z7HsdruPyWKjobeopewUgExRgfFbMBvuIJoLaU="
     }
 }

Original file line number	Diff line number	Diff line change
`@@ -95,6 +95,6 @@`
`95`	`95`	`"to-json-callback": "^0.1.1"`
`96`	`96`	`},`
`97`	`97`	`"worker": {`
`98`		`- "blob": "sha256-or/eVGAwbAdeOd7ha5qhvvgTfdnS92Z0Vg6qnwmIgBI="`
	`98`	`+ "blob": "sha256-+m8Z7z7HsdruPyWKjobeopewUgExRgfFbMBvuIJoLaU="`
`99`	`99`	`}`
`100`	`100`	`}`