From 692b7cd199fd5a8ba66e28d1a4d4d17a78da1e2c Mon Sep 17 00:00:00 2001
From: "Jason J. Gullickson" <mr@jasongullickson.com>
Date: Tue, 9 Apr 2024 16:01:51 -0500
Subject: [PATCH 01/41] wip

---
 docs/examples/jsfsx/hello.js |  1 +
 docs/jsfsx.md                | 32 ++++++++++++++++++++++
 lib/constants.js             |  6 +++--
 lib/inode.js                 |  3 +++
 server.js                    | 52 +++++++++++++++++++++++++++++-------
 5 files changed, 82 insertions(+), 12 deletions(-)
 create mode 100644 docs/examples/jsfsx/hello.js
 create mode 100644 docs/jsfsx.md

diff --git a/docs/examples/jsfsx/hello.js b/docs/examples/jsfsx/hello.js
new file mode 100644
index 0000000..7ce67f4
--- /dev/null
+++ b/docs/examples/jsfsx/hello.js
@@ -0,0 +1 @@
+console.log("Hack the Planet!");
diff --git a/docs/jsfsx.md b/docs/jsfsx.md
new file mode 100644
index 0000000..519cb5a
--- /dev/null
+++ b/docs/jsfsx.md
@@ -0,0 +1,32 @@
+# jsfsx
+
+## curl to store an executable file
+```bash
+curl -X POST -H "content-type: text/javascript" -H "x-access-key: jjg" -H "x-executable: true" --data-binary @hello.js "http://localhost:7302/bin/hello.js"
+```
+
+### result
+```json
+{
+  "url": "/localhost/bin/hello.js",
+  "created": 1712694738665,
+  "version": 0,
+  "private": false,
+  "encrypted": false,
+  "fingerprint": "438754d26cf1daaf69f9a0e6421b3053e4c00f75",
+  "access_key": "jjg",
+  "content_type": "text/javascript",
+  "file_size": 33,
+  "block_size": 1048576,
+  "blocks_replicated": 0,
+  "inode_replicated": 0,
+  "blocks": [
+    {
+      "block_hash": "a3b622a18ce02eb4d6e609f842964f430325e3d4",
+      "last_seen": "./blocks/"
+    }
+  ],
+  "executable": true,
+  "media_type": "unknown"
+}
+```
diff --git a/lib/constants.js b/lib/constants.js
index 7262427..693c4df 100644
--- a/lib/constants.js
+++ b/lib/constants.js
@@ -20,7 +20,8 @@ module.exports.ALLOWED_HEADERS = ["Accept",
                                   "X-Encrypted",
                                   "X-Private",
                                   "X-Replacement-Access-Key",
-                                  "X-Requested-With"];
+                                  "X-Requested-With",
+                                  "X-Executable"];
 
 module.exports.EXPOSED_HEADERS = ["X-Media-Bitrate",
                                   "X-Media-Channels",
@@ -38,4 +39,5 @@ module.exports.ACCEPTED_PARAMS = [{"access_key": "x"},
                                   {"inode_only": "x"},
                                   {"private": "x"},
                                   {"replacement_access_key": "x"},
-                                  {"version": "x"}];
+                                  {"version": "x"},
+                                  {"executable":"x"}];
diff --git a/lib/inode.js b/lib/inode.js
index 041af2f..b46431c 100644
--- a/lib/inode.js
+++ b/lib/inode.js
@@ -33,6 +33,9 @@ var Inode = {
 
     // use fingerprint as default key
     this.file_metadata.access_key = this.file_metadata.fingerprint;
+
+    // experimental executable support (jsfsx)
+    this.file_metadata.executable = false;
   },
   write: function(chunk, req, callback){
     this.input_buffer = new Buffer.concat([this.input_buffer, chunk]);
diff --git a/server.js b/server.js
index df63e04..4cb4996 100644
--- a/server.js
+++ b/server.js
@@ -178,20 +178,48 @@ http.createServer(function(req, res){
           read_file(block_filename, try_compressed);
         };
 
-        var send_blocks = function send_blocks(){
+        // If the file is marked executable, run it and return the result
+        // instead of sending the file itself back to the caller
+        if(requested_file.executable){
+          // TODO: execute!
+          log.message(log.WARN, "File is executable, we should execute it, but we dont know how!");
 
-          if (idx === total_blocks) { // we're done
-            return;
-          } else {
-            if (requested_file.blocks[idx].last_seen) {
-              load_from_last_seen(true);
+          // DEBUG
+          console.log(requested_file);
+
+          var read_blocks = function read_blocks(){
+
+            if (idx === total_blocks) { // we're done
+              return;
             } else {
-              search_for_block(0);
+              if (requested_file.blocks[idx].last_seen) {
+                load_from_last_seen(true);
+              } else {
+                search_for_block(0);
+              }
             }
-          }
-        };
+          };
+
+          read_blocks();
+
+        } else {
+          log.message(log.WARN, "File is not executable, just return the data.");
+
+          var send_blocks = function send_blocks(){
+
+            if (idx === total_blocks) { // we're done
+              return;
+            } else {
+              if (requested_file.blocks[idx].last_seen) {
+                load_from_last_seen(true);
+              } else {
+                search_for_block(0);
+              }
+            }
+          };
 
-        send_blocks();
+          send_blocks();
+        }
 
       });
 
@@ -235,6 +263,10 @@ http.createServer(function(req, res){
           new_file.file_metadata.encrypted = true;
         }
 
+        if(params.executable){
+          new_file.file_metadata.executable = true;
+        }
+
         // if access_key is supplied with update, replace the default one
         if(params.access_key){
           new_file.file_metadata.access_key = params.access_key;

From b744956be0114ae31f8e4b5e3c9c2388053c34d2 Mon Sep 17 00:00:00 2001
From: "Jason J. Gullickson" <mr@jasongullickson.com>
Date: Wed, 10 Apr 2024 12:19:13 -0500
Subject: [PATCH 02/41] wip: executing code in vm.

---
 docs/jsfsx.md | 27 +++++++++++++++++++
 server.js     | 72 ++++++++++++++++++++++++---------------------------
 2 files changed, 61 insertions(+), 38 deletions(-)

diff --git a/docs/jsfsx.md b/docs/jsfsx.md
index 519cb5a..6814d01 100644
--- a/docs/jsfsx.md
+++ b/docs/jsfsx.md
@@ -1,5 +1,12 @@
 # jsfsx
 
+"The simplest thing that could possibly work."
+
+## TODO
+[x] Add executable flag
+[] Execute executable files on GET
+[] Execute executable files on POST
+
 ## curl to store an executable file
 ```bash
 curl -X POST -H "content-type: text/javascript" -H "x-access-key: jjg" -H "x-executable: true" --data-binary @hello.js "http://localhost:7302/bin/hello.js"
@@ -30,3 +37,23 @@ curl -X POST -H "content-type: text/javascript" -H "x-access-key: jjg" -H "x-exe
   "media_type": "unknown"
 }
 ```
+
+## flow
+```
+case "GET"
+send_blocks()
+load_from_last_seen(true)
+read_file()
+read_stream = operations.stream_read(path)
+read_stream.pipe(unzipper).pipe(decryptor).pipe(res)
+on_end()
+read_stream shutdown
+send_blocks() (until all blocks are sent)
+- or -
+search_for_block(0)
+read_file()
+```
+
+
+## References
+* https://nodejs.org/api/vm.html
diff --git a/server.js b/server.js
index 4cb4996..1ec7f3c 100644
--- a/server.js
+++ b/server.js
@@ -1,4 +1,5 @@
 "use strict";
+
 /* globals require */
 
 /// jsfs - Javascript filesystem with a REST interface
@@ -18,6 +19,9 @@ var utils      = require("./lib/utils.js");
 var validate   = require("./lib/validate.js");
 var operations = require("./lib/" + (config.CONFIGURED_STORAGE || "fs") + "/disk-operations.js");
 
+// needed for exec support 
+const vm = require('node:vm');
+
 // base storage object
 var Inode = require("./lib/inode.js");
 
@@ -81,6 +85,25 @@ http.createServer(function(req, res){
           }
         }
 
+        // check executable
+        if(requested_file.executable){
+
+          // TODO: Actually execute the file
+          // TODO: Load the file's blocks into a variable
+          // TODO: eval() the variable
+          // TODO: return the output of the eval()
+          //log.message(log.WARN, "File is executable, but we don't know how to execute yet (Result: 501)!");
+          //res.statusCode = 501;
+          //return res.end();
+
+          const context = {x_out:""};
+          vm.createContext(context);
+          const code = "x_out = 'Hack the Planet!';";
+          vm.runInContext(code, context)
+
+          return res.end(context.x_out);
+        }
+
         var create_decryptor = function create_decryptor(options){
           return options.encrypted ? crypto.createDecipher("aes-256-cbc", options.key) : through();
         };
@@ -158,6 +181,7 @@ http.createServer(function(req, res){
             if (res.getMaxListeners !== undefined) {
               res.setMaxListeners(res.getMaxListeners() - 1);
             }
+
             send_blocks();
           }
 
@@ -178,48 +202,20 @@ http.createServer(function(req, res){
           read_file(block_filename, try_compressed);
         };
 
-        // If the file is marked executable, run it and return the result
-        // instead of sending the file itself back to the caller
-        if(requested_file.executable){
-          // TODO: execute!
-          log.message(log.WARN, "File is executable, we should execute it, but we dont know how!");
-
-          // DEBUG
-          console.log(requested_file);
-
-          var read_blocks = function read_blocks(){
-
-            if (idx === total_blocks) { // we're done
-              return;
-            } else {
-              if (requested_file.blocks[idx].last_seen) {
-                load_from_last_seen(true);
-              } else {
-                search_for_block(0);
-              }
-            }
-          };
-
-          read_blocks();
-
-        } else {
-          log.message(log.WARN, "File is not executable, just return the data.");
-
-          var send_blocks = function send_blocks(){
+        var send_blocks = function send_blocks(){
 
-            if (idx === total_blocks) { // we're done
-              return;
+          if (idx === total_blocks) { // we're done
+            return;
+          } else {
+            if (requested_file.blocks[idx].last_seen) {
+              load_from_last_seen(true);
             } else {
-              if (requested_file.blocks[idx].last_seen) {
-                load_from_last_seen(true);
-              } else {
-                search_for_block(0);
-              }
+              search_for_block(0);
             }
-          };
+          }
+        };
 
-          send_blocks();
-        }
+        send_blocks();
 
       });
 

From e38d9865d274cae8e5e21130567e882468b4493a Mon Sep 17 00:00:00 2001
From: "Jason J. Gullickson" <mr@jasongullickson.com>
Date: Wed, 10 Apr 2024 15:59:29 -0500
Subject: [PATCH 03/41] wip

---
 server.js | 67 ++++++++++++++++++++++++++++++++++++++++++++++---------
 1 file changed, 56 insertions(+), 11 deletions(-)

diff --git a/server.js b/server.js
index 1ec7f3c..b282a5f 100644
--- a/server.js
+++ b/server.js
@@ -21,6 +21,7 @@ var operations = require("./lib/" + (config.CONFIGURED_STORAGE || "fs") + "/disk
 
 // needed for exec support 
 const vm = require('node:vm');
+const Stream = require('stream');
 
 // base storage object
 var Inode = require("./lib/inode.js");
@@ -84,25 +85,31 @@ http.createServer(function(req, res){
             return res.end();
           }
         }
-
+        
+        /*
         // check executable
         if(requested_file.executable){
 
-          // TODO: Actually execute the file
-          // TODO: Load the file's blocks into a variable
-          // TODO: eval() the variable
-          // TODO: return the output of the eval()
-          //log.message(log.WARN, "File is executable, but we don't know how to execute yet (Result: 501)!");
-          //res.statusCode = 501;
-          //return res.end();
+          log.message(log.INFO, "Executing " + requested_file.url);
+
+          // TODO: Read the code out of the file instead of hardcoding it.
+          const code = "x_out = 'Hack the planet!';";
+
+          // TODO: Ideally this would handle things like `console.log()` automatically,
+          // but for now we'll just define some sort of unix-like standard.
+          const context = {
+            x_in:"",
+            x_out:"",
+            x_err:""
+          };
 
-          const context = {x_out:""};
           vm.createContext(context);
-          const code = "x_out = 'Hack the Planet!';";
           vm.runInContext(code, context)
+          log.message(log.INFO, "Execution complete!");
 
           return res.end(context.x_out);
         }
+        */
 
         var create_decryptor = function create_decryptor(options){
           return options.encrypted ? crypto.createDecipher("aes-256-cbc", options.key) : through();
@@ -112,6 +119,27 @@ http.createServer(function(req, res){
           return compressed ? zlib.createGunzip() : through();
         };
 
+        // Try using streams to construct executor...
+        class exec_stream extends Stream.Writable {
+          _write(chunk, enc, next){
+            console.log(chunk.toString());
+            next();
+          }
+        };
+
+        var xstream = new exec_stream();
+
+        //const exec_stream = new Stream.Writeable()
+        /*
+        exec_stream.exec_code = "";
+        exec_stream._write = (chunk, encoding, next) => {
+          log.message(log.INFO, "Got exec chunk!");
+
+          this.exec_code = this.exec_code + chunk;
+          next();
+        }
+        */
+
         // return status
         res.statusCode = 200;
 
@@ -185,14 +213,31 @@ http.createServer(function(req, res){
             send_blocks();
           }
 
+          function on_exec_end(){
+            console.log("got exec end");
+          }
+
           if (res.getMaxListeners !== undefined) {
             res.setMaxListeners(res.getMaxListeners() + 1);
           } else {
             res.setMaxListeners(0);
           }
+
           read_stream.on("end", on_end);
           read_stream.on("error", on_error);
-          read_stream.pipe(unzipper).pipe(decryptor).pipe(res, {end: should_end});
+
+          // TODO: cross fingers
+          if(requested_file.executable){
+            log.message(log.INFO, "Got executable");
+
+            // try wiring-up an end handler
+            xstream.on("end", on_exec_end);
+
+            // pump it to the exec stream
+            read_stream.pipe(unzipper).pipe(decryptor).pipe(xstream, {end: should_end});
+          } else {
+            read_stream.pipe(unzipper).pipe(decryptor).pipe(res, {end: should_end});
+          }
         };
 
         var load_from_last_seen = function load_from_last_seen(try_compressed){

From 39bcff61c7ed46115a16cb5fa36c9bb85750b9a4 Mon Sep 17 00:00:00 2001
From: "Jason J. Gullickson" <mr@jasongullickson.com>
Date: Thu, 11 Apr 2024 09:25:37 -0500
Subject: [PATCH 04/41] wip

---
 server.js | 82 +++++++++++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 76 insertions(+), 6 deletions(-)

diff --git a/server.js b/server.js
index b282a5f..f0063c7 100644
--- a/server.js
+++ b/server.js
@@ -21,7 +21,7 @@ var operations = require("./lib/" + (config.CONFIGURED_STORAGE || "fs") + "/disk
 
 // needed for exec support 
 const vm = require('node:vm');
-const Stream = require('stream');
+const { Writable } = require('node:stream');
 
 // base storage object
 var Inode = require("./lib/inode.js");
@@ -120,14 +120,69 @@ http.createServer(function(req, res){
         };
 
         // Try using streams to construct executor...
+        class ExecutableStream extends Writable {
+          constructor() {
+            super();
+            this.code = null;
+          }
+          _construct(callback) {
+            console.log("Got _construct");
+            this.code = "";
+            callback();
+          }
+          _write(chunk, encoding, callback){
+            console.log("Got _write");
+            this.code = this.code + chunk.toString();
+            callback();
+          }
+          _final(callback){
+            console.log("Got _final");
+            callback();
+          }
+          _destroy(err, callback){
+            console.log("Got _destroy");
+            callback(null);
+          }
+        }
+
+        var xstream = new ExecutableStream();
+
+        /*
+        var xstream = new Stream.Writable();
+        xstream.on("write", () => {
+          console.log("got write");
+        });
+        xstream.on("error", () => {
+          console.log("got error");
+        });
+        xstream.on("pipe", () => {
+          console.log("got pipe");
+        });
+        xstream.on("finish", () => {
+          console.log("got finish");
+        });
+        */
+        /*
+        var xstream = new Stream.Writable({
+          write: function(chunk, encoding, next){
+            console.log(chunk.toString());
+            next();
+          },
+          end: function(){
+            console.log("got end");
+          }
+        });
+        */
+        /*
         class exec_stream extends Stream.Writable {
           _write(chunk, enc, next){
             console.log(chunk.toString());
             next();
           }
         };
-
+       
         var xstream = new exec_stream();
+        */
 
         //const exec_stream = new Stream.Writeable()
         /*
@@ -193,6 +248,9 @@ http.createServer(function(req, res){
           var should_end  = (idx + 1) === total_blocks;
 
           function on_error(){
+
+            console.log("Got on_error");
+
             if (try_compressed) {
               log.message(log.WARN, "Cannot locate compressed block in last_seen location, trying uncompressed");
               return load_from_last_seen(false);
@@ -213,8 +271,17 @@ http.createServer(function(req, res){
             send_blocks();
           }
 
-          function on_exec_end(){
-            console.log("got exec end");
+          function x_on_end(){
+            console.log("Got x_on_end");
+
+            idx++;
+            xstream.removeListener("end", on_end);
+            xstream.removeListener("error", on_error);
+            if (res.getMaxListeners !== undefined) {
+              res.setMaxListeners(res.getMaxListeners() - 1);
+            }
+
+            send_blocks();
           }
 
           if (res.getMaxListeners !== undefined) {
@@ -226,15 +293,18 @@ http.createServer(function(req, res){
           read_stream.on("end", on_end);
           read_stream.on("error", on_error);
 
+          xstream.on("end", x_on_end);
+          xstream.on("error", on_error);
+
           // TODO: cross fingers
           if(requested_file.executable){
             log.message(log.INFO, "Got executable");
 
             // try wiring-up an end handler
-            xstream.on("end", on_exec_end);
+            //xstream.on("end", on_exec_end);
 
             // pump it to the exec stream
-            read_stream.pipe(unzipper).pipe(decryptor).pipe(xstream, {end: should_end});
+            read_stream.pipe(unzipper).pipe(decryptor).pipe(xstream).pipe(res, {end: should_end});
           } else {
             read_stream.pipe(unzipper).pipe(decryptor).pipe(res, {end: should_end});
           }

From 03833701aee25ae0365fcadf8503ec495637ea19 Mon Sep 17 00:00:00 2001
From: "Jason J. Gullickson" <mr@jasongullickson.com>
Date: Thu, 11 Apr 2024 10:48:14 -0500
Subject: [PATCH 05/41] finally got the pipeline thing figured out.

---
 docs/examples/jsfsx/hello.js |  2 +-
 server.js                    | 26 +++++++++++++-------------
 2 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/docs/examples/jsfsx/hello.js b/docs/examples/jsfsx/hello.js
index 7ce67f4..838e23f 100644
--- a/docs/examples/jsfsx/hello.js
+++ b/docs/examples/jsfsx/hello.js
@@ -1 +1 @@
-console.log("Hack the Planet!");
+x_out = "Hack the Planet!";
diff --git a/server.js b/server.js
index f0063c7..b1ce427 100644
--- a/server.js
+++ b/server.js
@@ -21,7 +21,7 @@ var operations = require("./lib/" + (config.CONFIGURED_STORAGE || "fs") + "/disk
 
 // needed for exec support 
 const vm = require('node:vm');
-const { Writable } = require('node:stream');
+const { Transform } = require('node:stream');
 
 // base storage object
 var Inode = require("./lib/inode.js");
@@ -120,7 +120,7 @@ http.createServer(function(req, res){
         };
 
         // Try using streams to construct executor...
-        class ExecutableStream extends Writable {
+        class ExecutableStream extends Transform {
           constructor() {
             super();
             this.code = null;
@@ -130,17 +130,17 @@ http.createServer(function(req, res){
             this.code = "";
             callback();
           }
-          _write(chunk, encoding, callback){
-            console.log("Got _write");
-            this.code = this.code + chunk.toString();
-            callback();
-          }
-          _final(callback){
-            console.log("Got _final");
+          _flush(callback) {
+            console.log("Got _flush");
+            // TODO: is this where we execute the code and emit the result,
+            // maybe using this.push()?
+            this.push(this.code);
             callback();
           }
-          _destroy(err, callback){
-            console.log("Got _destroy");
+          _transform(chunk, encoding, callback){
+            console.log("Got _transform");
+            console.log("chunk: " + chunk.toString());
+            this.code = this.code + chunk.toString();
             callback(null);
           }
         }
@@ -293,8 +293,8 @@ http.createServer(function(req, res){
           read_stream.on("end", on_end);
           read_stream.on("error", on_error);
 
-          xstream.on("end", x_on_end);
-          xstream.on("error", on_error);
+          //xstream.on("end", x_on_end);
+          //xstream.on("error", on_error);
 
           // TODO: cross fingers
           if(requested_file.executable){

From 1ce10cf76dadbd0f7b2114a206a29b40ba01522c Mon Sep 17 00:00:00 2001
From: "Jason J. Gullickson" <mr@jasongullickson.com>
Date: Thu, 11 Apr 2024 10:51:44 -0500
Subject: [PATCH 06/41] First sucessful execution inside JSFS of code stored
 within JSFS!

---
 server.js | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/server.js b/server.js
index b1ce427..4240146 100644
--- a/server.js
+++ b/server.js
@@ -132,9 +132,20 @@ http.createServer(function(req, res){
           }
           _flush(callback) {
             console.log("Got _flush");
-            // TODO: is this where we execute the code and emit the result,
-            // maybe using this.push()?
-            this.push(this.code);
+
+            // TODO: Ideally this would handle things like `console.log()` automatically,
+            // but for now we'll just define some sort of unix-like standard.
+            const context = {
+              x_in:"",
+              x_out:"",
+              x_err:""
+            };
+
+            vm.createContext(context);
+            vm.runInContext(this.code, context)
+            log.message(log.INFO, "Execution complete!");
+
+            this.push(context.x_out);
             callback();
           }
           _transform(chunk, encoding, callback){

From 5659ec40291501242d5e01b7fee6195857d0f757 Mon Sep 17 00:00:00 2001
From: "Jason J. Gullickson" <mr@jasongullickson.com>
Date: Thu, 11 Apr 2024 11:01:10 -0500
Subject: [PATCH 07/41] update docs

---
 docs/jsfsx.md | 52 +++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 52 insertions(+)

diff --git a/docs/jsfsx.md b/docs/jsfsx.md
index 6814d01..92d0a51 100644
--- a/docs/jsfsx.md
+++ b/docs/jsfsx.md
@@ -54,6 +54,58 @@ search_for_block(0)
 read_file()
 ```
 
+## it works!
+
+### source file
+`hello.js`
+```javascript
+x_out = "Hack the Planet!";
+```
+
+### upload
+```bash
+curl -X POST -H "content-type: text/javascript" -H "x-access-key: jjg" -H "x-executable: true" --data-binary @hello.js "http://localhost:7302/bin/hello.js"
+```
+
+### execute
+```bash
+curl "http://localhost:7302/bin/hello.js"
+```
+
+### output
+```bash
+Hack the Planet!Hack the Pla
+```
+
+There's clearly bugs, but I think it proves the concept.
+
+```bash
+curl -v "http://localhost:7302/bin/hello.js"
+*   Trying 127.0.0.1:7302...
+* Connected to localhost (127.0.0.1) port 7302 (#0)
+> GET /bin/hello.js HTTP/1.1
+> Host: localhost:7302
+> User-Agent: curl/7.81.0
+> Accept: */*
+> 
+* Mark bundle as not supporting multiuse
+< HTTP/1.1 200 OK
+< Access-Control-Allow-Methods: GET,POST,PUT,DELETE,OPTIONS
+< Access-Control-Allow-Headers: Accept,Accept-Version,Api-Version,Content-Type,Origin,Range,X_FILENAME,X-Access-Key,X-Access-Token,X-Append,X-Encrypted,X-Private,X-Replacement-Access-Key,X-Requested-With,X-Executable
+< Access-Control-Allow-Origin: *
+< Access-Control-Expose-Headers: X-Media-Bitrate,X-Media-Channels,X-Media-Duration,X-Media-Resolution,X-Media-Size,X-Media-Type
+< Content-Type: text/javascript
+< Content-Length: 28
+< Date: Thu, 11 Apr 2024 15:50:10 GMT
+< Connection: keep-alive
+< Keep-Alive: timeout=5
+< 
+* Excess found in a read: excess = 4, size = 28, maxdownload = 28, bytecount = 0
+* Closing connection 0
+Hack the Planet!Hack the Pla
+```
+
 
 ## References
 * https://nodejs.org/api/vm.html
+* https://nodejs.org/docs/latest/api/stream.html#stream_implementing_a_transform_stream

From cba5c5342b96ab3aa28f22dc5e5f8aee192aa29e Mon Sep 17 00:00:00 2001
From: "Jason J. Gullickson" <mr@jasongullickson.com>
Date: Thu, 11 Apr 2024 11:04:28 -0500
Subject: [PATCH 08/41] formatting

---
 docs/jsfsx.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/jsfsx.md b/docs/jsfsx.md
index 92d0a51..389ede5 100644
--- a/docs/jsfsx.md
+++ b/docs/jsfsx.md
@@ -3,9 +3,9 @@
 "The simplest thing that could possibly work."
 
 ## TODO
-[x] Add executable flag
-[] Execute executable files on GET
-[] Execute executable files on POST
+ [x] Add executable flag
+ [x] Execute executable files on GET
+ [] Execute executable files on POST
 
 ## curl to store an executable file
 ```bash

From 9a18fa8fbeb312173f031c4c46b7fd42874b7cc6 Mon Sep 17 00:00:00 2001
From: "Jason J. Gullickson" <mr@jasongullickson.com>
Date: Thu, 11 Apr 2024 11:05:37 -0500
Subject: [PATCH 09/41] formatting

---
 docs/jsfsx.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/jsfsx.md b/docs/jsfsx.md
index 389ede5..bff22ee 100644
--- a/docs/jsfsx.md
+++ b/docs/jsfsx.md
@@ -3,9 +3,9 @@
 "The simplest thing that could possibly work."
 
 ## TODO
- [x] Add executable flag
- [x] Execute executable files on GET
- [] Execute executable files on POST
+- [x] Add executable flag
+- [x] Execute executable files on GET
+- [] Execute executable files on POST
 
 ## curl to store an executable file
 ```bash

From 9b658a4dd7c81ae1cd37ab4421d7e7fd26caad35 Mon Sep 17 00:00:00 2001
From: "Jason J. Gullickson" <mr@jasongullickson.com>
Date: Thu, 11 Apr 2024 11:15:31 -0500
Subject: [PATCH 10/41] Cleanup and more docs.

---
 docs/jsfsx.md |  9 ++++--
 server.js     | 84 +--------------------------------------------------
 2 files changed, 8 insertions(+), 85 deletions(-)

diff --git a/docs/jsfsx.md b/docs/jsfsx.md
index bff22ee..ed2faad 100644
--- a/docs/jsfsx.md
+++ b/docs/jsfsx.md
@@ -2,10 +2,15 @@
 
 "The simplest thing that could possibly work."
 
-## TODO
+## TODO (in no strict order)
 - [x] Add executable flag
 - [x] Execute executable files on GET
-- [] Execute executable files on POST
+- [ ] Fix duplicated output error
+- [ ] Standardize i/o interface (`x_in`, `x_out`, etc.)
+  - [ ] Expose some/all request input to the executing code
+- [ ] Refactor (code structure, logging, error handling, etc.)
+- [ ] Experiment with `vm` settings to maximize stability, performance, security
+- [ ] Execute executable files on POST
 
 ## curl to store an executable file
 ```bash
diff --git a/server.js b/server.js
index 4240146..9fda3d2 100644
--- a/server.js
+++ b/server.js
@@ -86,31 +86,6 @@ http.createServer(function(req, res){
           }
         }
         
-        /*
-        // check executable
-        if(requested_file.executable){
-
-          log.message(log.INFO, "Executing " + requested_file.url);
-
-          // TODO: Read the code out of the file instead of hardcoding it.
-          const code = "x_out = 'Hack the planet!';";
-
-          // TODO: Ideally this would handle things like `console.log()` automatically,
-          // but for now we'll just define some sort of unix-like standard.
-          const context = {
-            x_in:"",
-            x_out:"",
-            x_err:""
-          };
-
-          vm.createContext(context);
-          vm.runInContext(code, context)
-          log.message(log.INFO, "Execution complete!");
-
-          return res.end(context.x_out);
-        }
-        */
-
         var create_decryptor = function create_decryptor(options){
           return options.encrypted ? crypto.createDecipher("aes-256-cbc", options.key) : through();
         };
@@ -158,54 +133,6 @@ http.createServer(function(req, res){
 
         var xstream = new ExecutableStream();
 
-        /*
-        var xstream = new Stream.Writable();
-        xstream.on("write", () => {
-          console.log("got write");
-        });
-        xstream.on("error", () => {
-          console.log("got error");
-        });
-        xstream.on("pipe", () => {
-          console.log("got pipe");
-        });
-        xstream.on("finish", () => {
-          console.log("got finish");
-        });
-        */
-        /*
-        var xstream = new Stream.Writable({
-          write: function(chunk, encoding, next){
-            console.log(chunk.toString());
-            next();
-          },
-          end: function(){
-            console.log("got end");
-          }
-        });
-        */
-        /*
-        class exec_stream extends Stream.Writable {
-          _write(chunk, enc, next){
-            console.log(chunk.toString());
-            next();
-          }
-        };
-       
-        var xstream = new exec_stream();
-        */
-
-        //const exec_stream = new Stream.Writeable()
-        /*
-        exec_stream.exec_code = "";
-        exec_stream._write = (chunk, encoding, next) => {
-          log.message(log.INFO, "Got exec chunk!");
-
-          this.exec_code = this.exec_code + chunk;
-          next();
-        }
-        */
-
         // return status
         res.statusCode = 200;
 
@@ -304,17 +231,8 @@ http.createServer(function(req, res){
           read_stream.on("end", on_end);
           read_stream.on("error", on_error);
 
-          //xstream.on("end", x_on_end);
-          //xstream.on("error", on_error);
-
-          // TODO: cross fingers
+          // if file is executable, run it before returning the data 
           if(requested_file.executable){
-            log.message(log.INFO, "Got executable");
-
-            // try wiring-up an end handler
-            //xstream.on("end", on_exec_end);
-
-            // pump it to the exec stream
             read_stream.pipe(unzipper).pipe(decryptor).pipe(xstream).pipe(res, {end: should_end});
           } else {
             read_stream.pipe(unzipper).pipe(decryptor).pipe(res, {end: should_end});

From 8a9a27ab08e19dbe854212b3004f5086c442a0aa Mon Sep 17 00:00:00 2001
From: "Jason J. Gullickson" <mr@jasongullickson.com>
Date: Thu, 11 Apr 2024 12:21:19 -0500
Subject: [PATCH 11/41] more todos

---
 docs/jsfsx.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/jsfsx.md b/docs/jsfsx.md
index ed2faad..6725b2f 100644
--- a/docs/jsfsx.md
+++ b/docs/jsfsx.md
@@ -11,6 +11,7 @@
 - [ ] Refactor (code structure, logging, error handling, etc.)
 - [ ] Experiment with `vm` settings to maximize stability, performance, security
 - [ ] Execute executable files on POST
+- [ ] Preserve `executable` bit through `PUT`s
 
 ## curl to store an executable file
 ```bash

From 9374aa9461cfa4b6b39122d6b0ca646bfb944ed6 Mon Sep 17 00:00:00 2001
From: "Jason J. Gullickson" <mr@jasongullickson.com>
Date: Thu, 11 Apr 2024 14:13:59 -0500
Subject: [PATCH 12/41] wip

---
 docs/jsfsx.md |  1 +
 server.js     | 12 +++++++-----
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/docs/jsfsx.md b/docs/jsfsx.md
index ed2faad..c3e99c3 100644
--- a/docs/jsfsx.md
+++ b/docs/jsfsx.md
@@ -11,6 +11,7 @@
 - [ ] Refactor (code structure, logging, error handling, etc.)
 - [ ] Experiment with `vm` settings to maximize stability, performance, security
 - [ ] Execute executable files on POST
+- [ ] Come up with a way to fetch the source of an executable w/o running it
 
 ## curl to store an executable file
 ```bash
diff --git a/server.js b/server.js
index 9fda3d2..448ab6e 100644
--- a/server.js
+++ b/server.js
@@ -94,6 +94,10 @@ http.createServer(function(req, res){
           return compressed ? zlib.createGunzip() : through();
         };
 
+        // TODO: Maybe we can follow the pattern above for the x stream
+        // and get rid of the need for the conditional pipeline below?
+
+        // TODO: This class almost certainly should be declared elsewhere...
         // Try using streams to construct executor...
         class ExecutableStream extends Transform {
           constructor() {
@@ -101,12 +105,13 @@ http.createServer(function(req, res){
             this.code = null;
           }
           _construct(callback) {
-            console.log("Got _construct");
             this.code = "";
             callback();
           }
           _flush(callback) {
-            console.log("Got _flush");
+
+            // TODO: Find the right way to include the filename in the log below
+            log.message(log.INFO, "Beginning execution of ...");
 
             // TODO: Ideally this would handle things like `console.log()` automatically,
             // but for now we'll just define some sort of unix-like standard.
@@ -124,13 +129,10 @@ http.createServer(function(req, res){
             callback();
           }
           _transform(chunk, encoding, callback){
-            console.log("Got _transform");
-            console.log("chunk: " + chunk.toString());
             this.code = this.code + chunk.toString();
             callback(null);
           }
         }
-
         var xstream = new ExecutableStream();
 
         // return status

From 795deec1003599a8cad3caa0d3cfd569cc852432 Mon Sep 17 00:00:00 2001
From: "Jason J. Gullickson" <mr@jasongullickson.com>
Date: Thu, 11 Apr 2024 14:53:26 -0500
Subject: [PATCH 13/41] cleanup

---
 server.js | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/server.js b/server.js
index 448ab6e..82a8801 100644
--- a/server.js
+++ b/server.js
@@ -188,9 +188,6 @@ http.createServer(function(req, res){
           var should_end  = (idx + 1) === total_blocks;
 
           function on_error(){
-
-            console.log("Got on_error");
-
             if (try_compressed) {
               log.message(log.WARN, "Cannot locate compressed block in last_seen location, trying uncompressed");
               return load_from_last_seen(false);

From a29b40bfb3e4ddccbf1605485ca6143dbd6352ea Mon Sep 17 00:00:00 2001
From: "Jason J. Gullickson" <mr@jasongullickson.com>
Date: Thu, 11 Apr 2024 15:00:54 -0500
Subject: [PATCH 14/41] more cleanup

---
 server.js | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/server.js b/server.js
index 82a8801..64d208d 100644
--- a/server.js
+++ b/server.js
@@ -208,19 +208,6 @@ http.createServer(function(req, res){
             send_blocks();
           }
 
-          function x_on_end(){
-            console.log("Got x_on_end");
-
-            idx++;
-            xstream.removeListener("end", on_end);
-            xstream.removeListener("error", on_error);
-            if (res.getMaxListeners !== undefined) {
-              res.setMaxListeners(res.getMaxListeners() - 1);
-            }
-
-            send_blocks();
-          }
-
           if (res.getMaxListeners !== undefined) {
             res.setMaxListeners(res.getMaxListeners() + 1);
           } else {

From 32cff6c033ef725634245d03f05598664db4daf9 Mon Sep 17 00:00:00 2001
From: "Jason J. Gullickson" <mr@jasongullickson.com>
Date: Thu, 11 Apr 2024 15:28:28 -0500
Subject: [PATCH 15/41] notes

---
 server.js | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/server.js b/server.js
index 64d208d..1a2dcc7 100644
--- a/server.js
+++ b/server.js
@@ -98,7 +98,6 @@ http.createServer(function(req, res){
         // and get rid of the need for the conditional pipeline below?
 
         // TODO: This class almost certainly should be declared elsewhere...
-        // Try using streams to construct executor...
         class ExecutableStream extends Transform {
           constructor() {
             super();
@@ -218,7 +217,16 @@ http.createServer(function(req, res){
           read_stream.on("error", on_error);
 
           // if file is executable, run it before returning the data 
+          // TODO: Try to consolidate this exec-specific stuff instead
+          // of having to have all these conditional checks everywhere.
           if(requested_file.executable){
+
+            // TODO: Maybe this can be set by the code to something more specific? 
+            res.removeHeader("Content-Type");
+
+            // TODO: Can we get the Content-Length from the x_out value of the xstream? 
+            res.removeHeader("Content-Length");
+
             read_stream.pipe(unzipper).pipe(decryptor).pipe(xstream).pipe(res, {end: should_end});
           } else {
             read_stream.pipe(unzipper).pipe(decryptor).pipe(res, {end: should_end});

From e32a135e243ac11d1dd3e1f3827e7ad66323621a Mon Sep 17 00:00:00 2001
From: "Jason J. Gullickson" <mr@jasongullickson.com>
Date: Thu, 11 Apr 2024 16:00:24 -0500
Subject: [PATCH 16/41] wip

---
 server.js | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/server.js b/server.js
index 1a2dcc7..8171490 100644
--- a/server.js
+++ b/server.js
@@ -181,6 +181,7 @@ http.createServer(function(req, res){
         };
 
         var read_file = function read_file(path, try_compressed){
+
           var read_stream = operations.stream_read(path);
           var decryptor   = create_decryptor({ encrypted : requested_file.encrypted, key : requested_file.access_key});
           var unzipper    = create_unzipper(try_compressed);
@@ -221,6 +222,8 @@ http.createServer(function(req, res){
           // of having to have all these conditional checks everywhere.
           if(requested_file.executable){
 
+            console.log("Running pipeline");
+
             // TODO: Maybe this can be set by the code to something more specific? 
             res.removeHeader("Content-Type");
 

From 6f81c3c243a86e461a2171a0c5798856348d5476 Mon Sep 17 00:00:00 2001
From: "Jason J. Gullickson" <mr@jasongullickson.com>
Date: Thu, 11 Apr 2024 20:59:29 -0500
Subject: [PATCH 17/41] wip

---
 server.js | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/server.js b/server.js
index 8171490..5c0abc2 100644
--- a/server.js
+++ b/server.js
@@ -182,6 +182,9 @@ http.createServer(function(req, res){
 
         var read_file = function read_file(path, try_compressed){
 
+          // DEBUG
+          console.log("Got read_file");
+
           var read_stream = operations.stream_read(path);
           var decryptor   = create_decryptor({ encrypted : requested_file.encrypted, key : requested_file.access_key});
           var unzipper    = create_unzipper(try_compressed);

From 3aacd1bc228905034c3ec38b55a43b97a71cc749 Mon Sep 17 00:00:00 2001
From: "Jason J. Gullickson" <mr@jasongullickson.com>
Date: Fri, 12 Apr 2024 10:14:51 -0500
Subject: [PATCH 18/41] Refactored and fixed duplicated output (somehow).

---
 docs/jsfsx.md |  4 +--
 server.js     | 79 +++++++++++----------------------------------------
 2 files changed, 19 insertions(+), 64 deletions(-)

diff --git a/docs/jsfsx.md b/docs/jsfsx.md
index af34321..5a50fc4 100644
--- a/docs/jsfsx.md
+++ b/docs/jsfsx.md
@@ -5,10 +5,10 @@
 ## TODO (in no strict order)
 - [x] Add executable flag
 - [x] Execute executable files on GET
-- [ ] Fix duplicated output error
+- [x] Fix duplicated output error
 - [ ] Standardize i/o interface (`x_in`, `x_out`, etc.)
   - [ ] Expose some/all request input to the executing code
-- [ ] Refactor (code structure, logging, error handling, etc.)
+- [x] Refactor (code structure, logging, error handling, etc.)
 - [ ] Experiment with `vm` settings to maximize stability, performance, security
 - [ ] Execute executable files on POST
 - [ ] Come up with a way to fetch the source of an executable w/o running it
diff --git a/server.js b/server.js
index 8171490..ba6fef3 100644
--- a/server.js
+++ b/server.js
@@ -19,13 +19,12 @@ var utils      = require("./lib/utils.js");
 var validate   = require("./lib/validate.js");
 var operations = require("./lib/" + (config.CONFIGURED_STORAGE || "fs") + "/disk-operations.js");
 
-// needed for exec support 
-const vm = require('node:vm');
-const { Transform } = require('node:stream');
-
 // base storage object
 var Inode = require("./lib/inode.js");
 
+// executable support 
+var XStream = require("./lib/xstream.js");
+
 // get this now, rather than at several other points
 var TOTAL_LOCATIONS = config.STORAGE_LOCATIONS.length;
 
@@ -94,52 +93,23 @@ http.createServer(function(req, res){
           return compressed ? zlib.createGunzip() : through();
         };
 
-        // TODO: Maybe we can follow the pattern above for the x stream
-        // and get rid of the need for the conditional pipeline below?
-
-        // TODO: This class almost certainly should be declared elsewhere...
-        class ExecutableStream extends Transform {
-          constructor() {
-            super();
-            this.code = null;
-          }
-          _construct(callback) {
-            this.code = "";
-            callback();
-          }
-          _flush(callback) {
-
-            // TODO: Find the right way to include the filename in the log below
-            log.message(log.INFO, "Beginning execution of ...");
-
-            // TODO: Ideally this would handle things like `console.log()` automatically,
-            // but for now we'll just define some sort of unix-like standard.
-            const context = {
-              x_in:"",
-              x_out:"",
-              x_err:""
-            };
-
-            vm.createContext(context);
-            vm.runInContext(this.code, context)
-            log.message(log.INFO, "Execution complete!");
-
-            this.push(context.x_out);
-            callback();
-          }
-          _transform(chunk, encoding, callback){
-            this.code = this.code + chunk.toString();
-            callback(null);
-          }
-        }
-        var xstream = new ExecutableStream();
+        var create_executor = function create_executor(executable){
+          return executable ? new XStream : through();
+        };
 
         // return status
         res.statusCode = 200;
 
         // return file metadata as HTTP headers
-        res.setHeader("Content-Type", requested_file.content_type);
-        res.setHeader("Content-Length", requested_file.file_size);
+        // TODO: These can change for executable files, 
+        // so for now only set them if we're not executing
+        // (a better solution would be to count the output
+        // from the running code, but I don't know how to
+        // do that yet...)
+        if(!requested_file.executable){
+          res.setHeader("Content-Type", requested_file.content_type);
+          res.setHeader("Content-Length", requested_file.file_size);
+        }
 
         var total_blocks = requested_file.blocks.length;
         var idx = 0;
@@ -185,6 +155,7 @@ http.createServer(function(req, res){
           var read_stream = operations.stream_read(path);
           var decryptor   = create_decryptor({ encrypted : requested_file.encrypted, key : requested_file.access_key});
           var unzipper    = create_unzipper(try_compressed);
+          var executor = create_executor(requested_file.executable);
           var should_end  = (idx + 1) === total_blocks;
 
           function on_error(){
@@ -217,23 +188,7 @@ http.createServer(function(req, res){
           read_stream.on("end", on_end);
           read_stream.on("error", on_error);
 
-          // if file is executable, run it before returning the data 
-          // TODO: Try to consolidate this exec-specific stuff instead
-          // of having to have all these conditional checks everywhere.
-          if(requested_file.executable){
-
-            console.log("Running pipeline");
-
-            // TODO: Maybe this can be set by the code to something more specific? 
-            res.removeHeader("Content-Type");
-
-            // TODO: Can we get the Content-Length from the x_out value of the xstream? 
-            res.removeHeader("Content-Length");
-
-            read_stream.pipe(unzipper).pipe(decryptor).pipe(xstream).pipe(res, {end: should_end});
-          } else {
-            read_stream.pipe(unzipper).pipe(decryptor).pipe(res, {end: should_end});
-          }
+          read_stream.pipe(unzipper).pipe(decryptor).pipe(executor).pipe(res, {end: should_end});
         };
 
         var load_from_last_seen = function load_from_last_seen(try_compressed){

From 42b93b97d87e8c864a9fae8d0423dbaeb115d122 Mon Sep 17 00:00:00 2001
From: "Jason J. Gullickson" <mr@jasongullickson.com>
Date: Fri, 12 Apr 2024 10:15:39 -0500
Subject: [PATCH 19/41] Add xstream module.

---
 lib/xstream.js | 38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)
 create mode 100644 lib/xstream.js

diff --git a/lib/xstream.js b/lib/xstream.js
new file mode 100644
index 0000000..27775c3
--- /dev/null
+++ b/lib/xstream.js
@@ -0,0 +1,38 @@
+const vm = require('node:vm');
+const { Transform } = require('node:stream');
+
+var log = require("../jlog.js");
+
+class ExecutableStream extends Transform {
+  constructor() {
+    super();
+    this.code = "";
+  }
+  _flush(callback) {
+
+    log.message(log.INFO, "Executing " + this.code.length + " bytes of Javascript...");
+
+    // TODO: Ideally this would handle things like `console.log()` automatically,
+    // but for now we'll just define some sort of unix-like standard.
+    const context = {
+      x_in:"",
+      x_out:"",
+      x_err:""
+    };
+
+    vm.createContext(context);
+    vm.runInContext(this.code, context)
+    log.message(log.INFO, "Execution complete!");
+
+    log.message(log.INFO, "Returning " + context.x_out.length + " bytes of data from executing Javascript");
+    this.push(context.x_out);
+
+    callback();
+  }
+  _transform(chunk, encoding, callback){
+    this.code = this.code + chunk.toString();
+    callback(null);
+  }
+}
+
+module.exports = ExecutableStream;

From 32e516969956a13bffabb640e1b3ba61d48b05c0 Mon Sep 17 00:00:00 2001
From: "Jason J. Gullickson" <mr@jasongullickson.com>
Date: Fri, 12 Apr 2024 10:20:54 -0500
Subject: [PATCH 20/41] cleanup

---
 docs/jsfsx.md | 2 +-
 server.js     | 6 +-----
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/docs/jsfsx.md b/docs/jsfsx.md
index 5a50fc4..b0235c6 100644
--- a/docs/jsfsx.md
+++ b/docs/jsfsx.md
@@ -12,7 +12,7 @@
 - [ ] Experiment with `vm` settings to maximize stability, performance, security
 - [ ] Execute executable files on POST
 - [ ] Come up with a way to fetch the source of an executable w/o running it
-- [ ] Preserve `executable` bit through `PUT`s
+- [ ] Preserve `executable` bit through `PUT`s (note: this might impact other properties...)
 
 ## curl to store an executable file
 ```bash
diff --git a/server.js b/server.js
index ba6fef3..8e76eab 100644
--- a/server.js
+++ b/server.js
@@ -1,5 +1,4 @@
 "use strict";
-
 /* globals require */
 
 /// jsfs - Javascript filesystem with a REST interface
@@ -84,7 +83,7 @@ http.createServer(function(req, res){
             return res.end();
           }
         }
-        
+
         var create_decryptor = function create_decryptor(options){
           return options.encrypted ? crypto.createDecipher("aes-256-cbc", options.key) : through();
         };
@@ -151,7 +150,6 @@ http.createServer(function(req, res){
         };
 
         var read_file = function read_file(path, try_compressed){
-
           var read_stream = operations.stream_read(path);
           var decryptor   = create_decryptor({ encrypted : requested_file.encrypted, key : requested_file.access_key});
           var unzipper    = create_unzipper(try_compressed);
@@ -175,7 +173,6 @@ http.createServer(function(req, res){
             if (res.getMaxListeners !== undefined) {
               res.setMaxListeners(res.getMaxListeners() - 1);
             }
-
             send_blocks();
           }
 
@@ -187,7 +184,6 @@ http.createServer(function(req, res){
 
           read_stream.on("end", on_end);
           read_stream.on("error", on_error);
-
           read_stream.pipe(unzipper).pipe(decryptor).pipe(executor).pipe(res, {end: should_end});
         };
 

From ada84c88ddda0f9b81f6fce1fe0af64bdcb2037c Mon Sep 17 00:00:00 2001
From: "Jason J. Gullickson" <mr@jasongullickson.com>
Date: Fri, 12 Apr 2024 10:26:30 -0500
Subject: [PATCH 21/41] docs.

---
 docs/jsfsx.md | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/docs/jsfsx.md b/docs/jsfsx.md
index b0235c6..35ea7fc 100644
--- a/docs/jsfsx.md
+++ b/docs/jsfsx.md
@@ -12,7 +12,7 @@
 - [ ] Experiment with `vm` settings to maximize stability, performance, security
 - [ ] Execute executable files on POST
 - [ ] Come up with a way to fetch the source of an executable w/o running it
-- [ ] Preserve `executable` bit through `PUT`s (note: this might impact other properties...)
+- [ ] Preserve `executable` bit through `PUT`s (note: this might be an existing bug, other properties appear to behave the same way...)
 
 ## curl to store an executable file
 ```bash
@@ -112,6 +112,12 @@ curl -v "http://localhost:7302/bin/hello.js"
 Hack the Planet!Hack the Pla
 ```
 
+## Refactoring and bugs
+
+For whatever reason refactoring the code a bit made the duplicate output bug go away ("IT'S MAGIC!").  Now executable `GET` requests work as expected, so it's probably time to think more about the interface between requests, reponses and the executable code.
+
+It's tempting to move on to `POST` but probably better to figure out the I/O first..
+
 
 ## References
 * https://nodejs.org/api/vm.html

From 4b834afa555de15a0f47fb0b54040267a581dbad Mon Sep 17 00:00:00 2001
From: "Jason J. Gullickson" <mr@jasongullickson.com>
Date: Fri, 12 Apr 2024 11:23:28 -0500
Subject: [PATCH 22/41] input working

---
 docs/examples/jsfsx/input.js | 1 +
 docs/jsfsx.md                | 3 ++-
 lib/xstream.js               | 5 +++--
 server.js                    | 4 +++-
 4 files changed, 9 insertions(+), 4 deletions(-)
 create mode 100644 docs/examples/jsfsx/input.js

diff --git a/docs/examples/jsfsx/input.js b/docs/examples/jsfsx/input.js
new file mode 100644
index 0000000..8f927cb
--- /dev/null
+++ b/docs/examples/jsfsx/input.js
@@ -0,0 +1 @@
+x_out = "The request method is: " + x_in.method;
diff --git a/docs/jsfsx.md b/docs/jsfsx.md
index 35ea7fc..6d1f6ff 100644
--- a/docs/jsfsx.md
+++ b/docs/jsfsx.md
@@ -7,12 +7,13 @@
 - [x] Execute executable files on GET
 - [x] Fix duplicated output error
 - [ ] Standardize i/o interface (`x_in`, `x_out`, etc.)
-  - [ ] Expose some/all request input to the executing code
+  - [x] Expose some/all request input to the executing code
 - [x] Refactor (code structure, logging, error handling, etc.)
 - [ ] Experiment with `vm` settings to maximize stability, performance, security
 - [ ] Execute executable files on POST
 - [ ] Come up with a way to fetch the source of an executable w/o running it
 - [ ] Preserve `executable` bit through `PUT`s (note: this might be an existing bug, other properties appear to behave the same way...)
+- [ ] Don't execute if an access-key/token is presented
 
 ## curl to store an executable file
 ```bash
diff --git a/lib/xstream.js b/lib/xstream.js
index 27775c3..ea52b08 100644
--- a/lib/xstream.js
+++ b/lib/xstream.js
@@ -4,8 +4,9 @@ const { Transform } = require('node:stream');
 var log = require("../jlog.js");
 
 class ExecutableStream extends Transform {
-  constructor() {
+  constructor(x_in) {
     super();
+    this.x_in = x_in;
     this.code = "";
   }
   _flush(callback) {
@@ -15,7 +16,7 @@ class ExecutableStream extends Transform {
     // TODO: Ideally this would handle things like `console.log()` automatically,
     // but for now we'll just define some sort of unix-like standard.
     const context = {
-      x_in:"",
+      x_in: this.x_in,
       x_out:"",
       x_err:""
     };
diff --git a/server.js b/server.js
index 8e76eab..b29880d 100644
--- a/server.js
+++ b/server.js
@@ -93,7 +93,9 @@ http.createServer(function(req, res){
         };
 
         var create_executor = function create_executor(executable){
-          return executable ? new XStream : through();
+          // TODO: We're passing the whole damn request in for now,
+          // but it might be a good ideal to pair this down at some point.
+          return executable ? new XStream(req) : through();
         };
 
         // return status

From 112b42ba1843812132c9a9c97d7d1101120a730b Mon Sep 17 00:00:00 2001
From: "Jason J. Gullickson" <mr@jasongullickson.com>
Date: Fri, 12 Apr 2024 11:26:23 -0500
Subject: [PATCH 23/41] docs

---
 docs/jsfsx.md | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/docs/jsfsx.md b/docs/jsfsx.md
index 6d1f6ff..5a79b6a 100644
--- a/docs/jsfsx.md
+++ b/docs/jsfsx.md
@@ -120,6 +120,22 @@ For whatever reason refactoring the code a bit made the duplicate output bug go
 It's tempting to move on to `POST` but probably better to figure out the I/O first..
 
 
+## input
+
+Let's start by passing the whole `request` into the executor, what's the worst that can happen?
+
+Then this uploaded code:
+
+```javascript
+x_out = "The request method is: " + x_in.method;
+```
+
+...yeilds this result:
+```bash
+curl "http://localhost:7302/bin/input.js"
+The request method is: GET
+```
+
 ## References
 * https://nodejs.org/api/vm.html
 * https://nodejs.org/docs/latest/api/stream.html#stream_implementing_a_transform_stream

From 270f8d5bc2e7bd0610d630e1df89e38c7d431dc7 Mon Sep 17 00:00:00 2001
From: "Jason J. Gullickson" <mr@jasongullickson.com>
Date: Fri, 12 Apr 2024 12:57:15 -0500
Subject: [PATCH 24/41] x_err & logging example.

---
 docs/examples/jsfsx/logit.js |  2 ++
 docs/jsfsx.md                | 19 +++++++++++++++++--
 lib/xstream.js               | 12 ++++++++++--
 3 files changed, 29 insertions(+), 4 deletions(-)
 create mode 100644 docs/examples/jsfsx/logit.js

diff --git a/docs/examples/jsfsx/logit.js b/docs/examples/jsfsx/logit.js
new file mode 100644
index 0000000..51f1c36
--- /dev/null
+++ b/docs/examples/jsfsx/logit.js
@@ -0,0 +1,2 @@
+x_err = x_err + "Something to log";
+x_out = x_err;
diff --git a/docs/jsfsx.md b/docs/jsfsx.md
index 5a79b6a..fc6443c 100644
--- a/docs/jsfsx.md
+++ b/docs/jsfsx.md
@@ -6,14 +6,17 @@
 - [x] Add executable flag
 - [x] Execute executable files on GET
 - [x] Fix duplicated output error
-- [ ] Standardize i/o interface (`x_in`, `x_out`, etc.)
+- [x] Standardize i/o interface (`x_in`, `x_out`, etc.)
   - [x] Expose some/all request input to the executing code
 - [x] Refactor (code structure, logging, error handling, etc.)
 - [ ] Experiment with `vm` settings to maximize stability, performance, security
 - [ ] Execute executable files on POST
 - [ ] Come up with a way to fetch the source of an executable w/o running it
+  - [ ] Don't execute if an access-key/token is presented
 - [ ] Preserve `executable` bit through `PUT`s (note: this might be an existing bug, other properties appear to behave the same way...)
-- [ ] Don't execute if an access-key/token is presented
+- [ ] Figure out how to set the `content-length`, `content-type` headers when executing code
+- [ ] Consider finding a way for a client to access `x_err` data (maybe a `debug` flag that dumps the entire context to `response`?)
+- [ ] Consider the impact of executables that emit large amounts of data (or continuous streams)
 
 ## curl to store an executable file
 ```bash
@@ -136,6 +139,18 @@ curl "http://localhost:7302/bin/input.js"
 The request method is: GET
 ```
 
+## X runtime environment/interface
+
+There will be a lot more to explore here in the future, but in the spirit of MVP or whatever here's what we're going to do for now.
+
+Files marked executable will be run as Javascript in a [Node.js VM](https://nodejs.org/api/vm.html#vm-executing-javascript).  At startup three variables will be initialized: `x_in`, `x_out` and `x_err`.  These map loosely to the `stdin`, `stdout` and `stderr` unix convention.
+* `x_in` is the entire `request` object sent by the user agent (for now)
+* `x_out` is returned to the user agent in the `response` object
+* `x_err` is written to the JSFS log
+
+It would be useful if `x_err` was more accessible by the user agent, and I have some ideas for this (maybe a `x-jsfs-debug` header that dumps the entire `context` to `response`?), but for now I'm just going to let it write to the log (if it's needed for debugging you can always do that using a local JSFS instance right?).
+
+
 ## References
 * https://nodejs.org/api/vm.html
 * https://nodejs.org/docs/latest/api/stream.html#stream_implementing_a_transform_stream
diff --git a/lib/xstream.js b/lib/xstream.js
index ea52b08..f939791 100644
--- a/lib/xstream.js
+++ b/lib/xstream.js
@@ -13,8 +13,12 @@ class ExecutableStream extends Transform {
 
     log.message(log.INFO, "Executing " + this.code.length + " bytes of Javascript...");
 
-    // TODO: Ideally this would handle things like `console.log()` automatically,
-    // but for now we'll just define some sort of unix-like standard.
+    // This provides a basic unix-like in/out/error interface to executable code.
+    // * x_in is currently the entire `request` object sent by the client
+    // * x_out is written back to the client via the `response` object
+    // * x_err is written to JSFS logs
+    // A better solution would provide the client a way to access the logs,
+    // but I don't have a good answer for that yet.
     const context = {
       x_in: this.x_in,
       x_out:"",
@@ -25,6 +29,10 @@ class ExecutableStream extends Transform {
     vm.runInContext(this.code, context)
     log.message(log.INFO, "Execution complete!");
 
+    if(context.x_err.length > 0) {
+      log.message(log.INFO, "Execution error logs: " + context.x_err);
+    }
+
     log.message(log.INFO, "Returning " + context.x_out.length + " bytes of data from executing Javascript");
     this.push(context.x_out);
 

From c590f734b2a62c24e862fa2f38fa5e3487bd15ba Mon Sep 17 00:00:00 2001
From: "Jason J. Gullickson" <mr@jasongullickson.com>
Date: Fri, 12 Apr 2024 13:25:38 -0500
Subject: [PATCH 25/41] Return file contents if access info is presented.

---
 docs/jsfsx.md | 4 ++--
 server.js     | 3 ++-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/docs/jsfsx.md b/docs/jsfsx.md
index fc6443c..d86dd2d 100644
--- a/docs/jsfsx.md
+++ b/docs/jsfsx.md
@@ -11,8 +11,8 @@
 - [x] Refactor (code structure, logging, error handling, etc.)
 - [ ] Experiment with `vm` settings to maximize stability, performance, security
 - [ ] Execute executable files on POST
-- [ ] Come up with a way to fetch the source of an executable w/o running it
-  - [ ] Don't execute if an access-key/token is presented
+- [x] Come up with a way to fetch the source of an executable w/o running it
+  - [x] Don't execute if an access-key/token is presented
 - [ ] Preserve `executable` bit through `PUT`s (note: this might be an existing bug, other properties appear to behave the same way...)
 - [ ] Figure out how to set the `content-length`, `content-type` headers when executing code
 - [ ] Consider finding a way for a client to access `x_err` data (maybe a `debug` flag that dumps the entire context to `response`?)
diff --git a/server.js b/server.js
index b29880d..bcc075a 100644
--- a/server.js
+++ b/server.js
@@ -155,7 +155,8 @@ http.createServer(function(req, res){
           var read_stream = operations.stream_read(path);
           var decryptor   = create_decryptor({ encrypted : requested_file.encrypted, key : requested_file.access_key});
           var unzipper    = create_unzipper(try_compressed);
-          var executor = create_executor(requested_file.executable);
+          // If access auth is present, don't execute
+          var executor = create_executor(requested_file.executable && !params.access_key && !params.access_token);
           var should_end  = (idx + 1) === total_blocks;
 
           function on_error(){

From 55f16153244aeaffd0c4b9ed1b7bdff5747e5a1e Mon Sep 17 00:00:00 2001
From: "Jason J. Gullickson" <mr@jasongullickson.com>
Date: Fri, 12 Apr 2024 13:35:30 -0500
Subject: [PATCH 26/41] docs

---
 docs/jsfsx.md | 11 +++++++++++
 server.js     |  2 ++
 2 files changed, 13 insertions(+)

diff --git a/docs/jsfsx.md b/docs/jsfsx.md
index d86dd2d..492c6d7 100644
--- a/docs/jsfsx.md
+++ b/docs/jsfsx.md
@@ -150,6 +150,17 @@ Files marked executable will be run as Javascript in a [Node.js VM](https://node
 
 It would be useful if `x_err` was more accessible by the user agent, and I have some ideas for this (maybe a `x-jsfs-debug` header that dumps the entire `context` to `response`?), but for now I'm just going to let it write to the log (if it's needed for debugging you can always do that using a local JSFS instance right?).
 
+## view source
+Now you can retrieve the data (as opposed to executing the code) in a stored file that is marked as executable:
+
+```bash
+curl "http://localhost:7302/bin/viewsource2.js"
+Call me with an access-key to view my sourcecode!
+
+curl -H "x-access-key: jjg" "http://localhost:7302/bin/viewsource2.js"
+// If you're seeing this, the execute override worked!
+x_out = "Call me with an access-key to view my sourcecode!";
+```
 
 ## References
 * https://nodejs.org/api/vm.html
diff --git a/server.js b/server.js
index bcc075a..82fbe45 100644
--- a/server.js
+++ b/server.js
@@ -224,6 +224,8 @@ http.createServer(function(req, res){
 
         if (inode){
 
+          // TODO: If the file is executable, and access credentials are not present, execute the file.
+
           // check authorization
           if (validate.is_authorized(inode, req.method, params)){
             log.message(log.INFO, "File update request authorized");

From 576bf535773cb9c4022cea1e68334b6e53275eee Mon Sep 17 00:00:00 2001
From: "Jason J. Gullickson" <mr@jasongullickson.com>
Date: Fri, 12 Apr 2024 13:39:15 -0500
Subject: [PATCH 27/41] docs

---
 docs/jsfsx.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/jsfsx.md b/docs/jsfsx.md
index 492c6d7..9597e98 100644
--- a/docs/jsfsx.md
+++ b/docs/jsfsx.md
@@ -143,7 +143,7 @@ The request method is: GET
 
 There will be a lot more to explore here in the future, but in the spirit of MVP or whatever here's what we're going to do for now.
 
-Files marked executable will be run as Javascript in a [Node.js VM](https://nodejs.org/api/vm.html#vm-executing-javascript).  At startup three variables will be initialized: `x_in`, `x_out` and `x_err`.  These map loosely to the `stdin`, `stdout` and `stderr` unix convention.
+Files marked executable are run as Javascript in a [Node.js VM](https://nodejs.org/api/vm.html#vm-executing-javascript).  At startup three variables will be initialized: `x_in`, `x_out` and `x_err`.  These map loosely to the `stdin`, `stdout` and `stderr` unix convention.
 * `x_in` is the entire `request` object sent by the user agent (for now)
 * `x_out` is returned to the user agent in the `response` object
 * `x_err` is written to the JSFS log

From 3b4556e77c160208c20173213f5d3823877b2122 Mon Sep 17 00:00:00 2001
From: "Jason J. Gullickson" <mr@jasongullickson.com>
Date: Fri, 12 Apr 2024 13:48:37 -0500
Subject: [PATCH 28/41] docs

---
 docs/jsfsx.md | 1 +
 server.js     | 1 +
 2 files changed, 2 insertions(+)

diff --git a/docs/jsfsx.md b/docs/jsfsx.md
index 9597e98..81b425d 100644
--- a/docs/jsfsx.md
+++ b/docs/jsfsx.md
@@ -17,6 +17,7 @@
 - [ ] Figure out how to set the `content-length`, `content-type` headers when executing code
 - [ ] Consider finding a way for a client to access `x_err` data (maybe a `debug` flag that dumps the entire context to `response`?)
 - [ ] Consider the impact of executables that emit large amounts of data (or continuous streams)
+- [ ] Handle errors generated by uploaded code so they don't crash the whole server
 
 ## curl to store an executable file
 ```bash
diff --git a/server.js b/server.js
index 82fbe45..e8e6402 100644
--- a/server.js
+++ b/server.js
@@ -225,6 +225,7 @@ http.createServer(function(req, res){
         if (inode){
 
           // TODO: If the file is executable, and access credentials are not present, execute the file.
+          // This is really just GET, so maybe there is a way to branch-out to GET and avoid a lot of duplication?
 
           // check authorization
           if (validate.is_authorized(inode, req.method, params)){

From 3f172c90ef5b30df011501df36147dc868aea020 Mon Sep 17 00:00:00 2001
From: "Jason J. Gullickson" <mr@jasongullickson.com>
Date: Fri, 12 Apr 2024 13:50:03 -0500
Subject: [PATCH 29/41] view source example.

---
 docs/examples/jsfsx/viewsource.js | 2 ++
 1 file changed, 2 insertions(+)
 create mode 100644 docs/examples/jsfsx/viewsource.js

diff --git a/docs/examples/jsfsx/viewsource.js b/docs/examples/jsfsx/viewsource.js
new file mode 100644
index 0000000..61d9628
--- /dev/null
+++ b/docs/examples/jsfsx/viewsource.js
@@ -0,0 +1,2 @@
+// If you're seeing this, the execute override worked!
+x_out = "Call me with an access-key to view my sourcecode!";

From f23fa764165ed5b75c2420df75f5491dced5febe Mon Sep 17 00:00:00 2001
From: "Jason J. Gullickson" <mr@jasongullickson.com>
Date: Fri, 12 Apr 2024 13:53:52 -0500
Subject: [PATCH 30/41] docs

---
 docs/jsfsx.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/jsfsx.md b/docs/jsfsx.md
index 81b425d..6c50a6a 100644
--- a/docs/jsfsx.md
+++ b/docs/jsfsx.md
@@ -18,6 +18,7 @@
 - [ ] Consider finding a way for a client to access `x_err` data (maybe a `debug` flag that dumps the entire context to `response`?)
 - [ ] Consider the impact of executables that emit large amounts of data (or continuous streams)
 - [ ] Handle errors generated by uploaded code so they don't crash the whole server
+- [ ] Consider adding "internal primatives" that let X code access JSFS data w/o HTTP overhead
 
 ## curl to store an executable file
 ```bash

From 838fbd6d2a2dc95fe79755c96d1d30052db40548 Mon Sep 17 00:00:00 2001
From: "Jason J. Gullickson" <mr@jasongullickson.com>
Date: Fri, 12 Apr 2024 14:00:54 -0500
Subject: [PATCH 31/41] docs

---
 docs/jsfsx.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/jsfsx.md b/docs/jsfsx.md
index 6c50a6a..1f7e8e8 100644
--- a/docs/jsfsx.md
+++ b/docs/jsfsx.md
@@ -15,6 +15,7 @@
   - [x] Don't execute if an access-key/token is presented
 - [ ] Preserve `executable` bit through `PUT`s (note: this might be an existing bug, other properties appear to behave the same way...)
 - [ ] Figure out how to set the `content-length`, `content-type` headers when executing code
+  - [ ] Allow the executable code to set these values somehow?
 - [ ] Consider finding a way for a client to access `x_err` data (maybe a `debug` flag that dumps the entire context to `response`?)
 - [ ] Consider the impact of executables that emit large amounts of data (or continuous streams)
 - [ ] Handle errors generated by uploaded code so they don't crash the whole server

From b2e0bef871c7132c216f7da33e1585421d8704d0 Mon Sep 17 00:00:00 2001
From: "Jason J. Gullickson" <mr@jasongullickson.com>
Date: Fri, 12 Apr 2024 14:53:01 -0500
Subject: [PATCH 32/41] docs

---
 docs/jsfsx.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/docs/jsfsx.md b/docs/jsfsx.md
index 1f7e8e8..cb1d306 100644
--- a/docs/jsfsx.md
+++ b/docs/jsfsx.md
@@ -10,6 +10,11 @@
   - [x] Expose some/all request input to the executing code
 - [x] Refactor (code structure, logging, error handling, etc.)
 - [ ] Experiment with `vm` settings to maximize stability, performance, security
+  - [ ] Caching/compiling scripts for faster startup time
+  - [ ] Other languages?
+  - [ ] How to handle modules/dependencies?
+  - [ ] Threading/eventloop impact/tuning
+  - [ ] Related: other workloads (external binaries, shell scripts, etc.)
 - [ ] Execute executable files on POST
 - [x] Come up with a way to fetch the source of an executable w/o running it
   - [x] Don't execute if an access-key/token is presented

From ecc51198e61ae67e84a8639626c66d5a7a374fb8 Mon Sep 17 00:00:00 2001
From: "Jason J. Gullickson" <mr@jasongullickson.com>
Date: Mon, 15 Apr 2024 08:25:07 -0500
Subject: [PATCH 33/41] docs

---
 docs/jsfsx.md | 33 +++++++++++++++++++++++----------
 1 file changed, 23 insertions(+), 10 deletions(-)

diff --git a/docs/jsfsx.md b/docs/jsfsx.md
index cb1d306..76f8e79 100644
--- a/docs/jsfsx.md
+++ b/docs/jsfsx.md
@@ -3,28 +3,33 @@
 "The simplest thing that could possibly work."
 
 ## TODO (in no strict order)
+
+### poc
 - [x] Add executable flag
 - [x] Execute executable files on GET
 - [x] Fix duplicated output error
 - [x] Standardize i/o interface (`x_in`, `x_out`, etc.)
   - [x] Expose some/all request input to the executing code
 - [x] Refactor (code structure, logging, error handling, etc.)
+- [x] Come up with a way to fetch the source of an executable w/o running it
+  - [x] Don't execute if an access-key/token is presented
+- [ ] Consider the impact of executables that emit large amounts of data (or continuous streams)
+- [ ] Handle errors generated by uploaded code so they don't crash the whole server
+- [ ] Finalize X interface (`x_in`, `x_out`, etc.)
+- [ ] Execute executable files on POST
+
+### post-poc
+- [ ] Preserve `executable` bit through `PUT`s (note: this might be an existing bug, other properties appear to behave the same way...)
+- [ ] Consider adding "internal primatives" that let X code access JSFS data w/o HTTP overhead
+- [ ] Consider finding a way for a client to access `x_err` data (maybe a `debug` flag that dumps the entire context to `response`?)
+- [ ] Figure out how to set the `content-length`, `content-type` headers when executing code
+  - [ ] Allow the executable code to set these values somehow?
 - [ ] Experiment with `vm` settings to maximize stability, performance, security
   - [ ] Caching/compiling scripts for faster startup time
   - [ ] Other languages?
   - [ ] How to handle modules/dependencies?
   - [ ] Threading/eventloop impact/tuning
   - [ ] Related: other workloads (external binaries, shell scripts, etc.)
-- [ ] Execute executable files on POST
-- [x] Come up with a way to fetch the source of an executable w/o running it
-  - [x] Don't execute if an access-key/token is presented
-- [ ] Preserve `executable` bit through `PUT`s (note: this might be an existing bug, other properties appear to behave the same way...)
-- [ ] Figure out how to set the `content-length`, `content-type` headers when executing code
-  - [ ] Allow the executable code to set these values somehow?
-- [ ] Consider finding a way for a client to access `x_err` data (maybe a `debug` flag that dumps the entire context to `response`?)
-- [ ] Consider the impact of executables that emit large amounts of data (or continuous streams)
-- [ ] Handle errors generated by uploaded code so they don't crash the whole server
-- [ ] Consider adding "internal primatives" that let X code access JSFS data w/o HTTP overhead
 
 ## curl to store an executable file
 ```bash
@@ -170,6 +175,14 @@ curl -H "x-access-key: jjg" "http://localhost:7302/bin/viewsource2.js"
 x_out = "Call me with an access-key to view my sourcecode!";
 ```
 
+## streaming
+What if an X generates output for a long time on purpose (imagine an audio stream, just as a random example...)?
+
+
+
 ## References
 * https://nodejs.org/api/vm.html
 * https://nodejs.org/docs/latest/api/stream.html#stream_implementing_a_transform_stream
+* https://v8.dev/docs
+* https://v8.dev/features
+* 

From 60e5d3b4074c00245b37a3740e80c4f7a73c59bc Mon Sep 17 00:00:00 2001
From: "Jason J. Gullickson" <mr@jasongullickson.com>
Date: Mon, 15 Apr 2024 08:26:35 -0500
Subject: [PATCH 34/41] Beginning of streaming example

---
 docs/examples/jsfsx/stream.js | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 docs/examples/jsfsx/stream.js

diff --git a/docs/examples/jsfsx/stream.js b/docs/examples/jsfsx/stream.js
new file mode 100644
index 0000000..06cc923
--- /dev/null
+++ b/docs/examples/jsfsx/stream.js
@@ -0,0 +1,10 @@
+x_out = "";
+loop_count = 10;
+
+i = setInterval(function(){
+  if(loop_count > 0){
+    x_out = x_out + "drip\n";
+  } else {
+    clearInterval(i);
+  }
+},1000)

From de503ed645a205fe4a38711516dcd5411fb3e558 Mon Sep 17 00:00:00 2001
From: "Jason J. Gullickson" <mr@jasongullickson.com>
Date: Mon, 15 Apr 2024 12:00:51 -0500
Subject: [PATCH 35/41] Experimental streaming support WIP

---
 docs/examples/jsfsx/stream.js | 12 +++++-------
 docs/jsfsx.md                 | 19 +++++++++++++++++++
 lib/xstream.js                |  8 +++++++-
 3 files changed, 31 insertions(+), 8 deletions(-)

diff --git a/docs/examples/jsfsx/stream.js b/docs/examples/jsfsx/stream.js
index 06cc923..b2c0474 100644
--- a/docs/examples/jsfsx/stream.js
+++ b/docs/examples/jsfsx/stream.js
@@ -1,10 +1,8 @@
 x_out = "";
 loop_count = 10;
 
-i = setInterval(function(){
-  if(loop_count > 0){
-    x_out = x_out + "drip\n";
-  } else {
-    clearInterval(i);
-  }
-},1000)
+for(var i=0;i<loop_count;i++){
+  x_out = x_out + "drip\n";
+  gong(x_out.length); 
+  loop_count--;
+}
diff --git a/docs/jsfsx.md b/docs/jsfsx.md
index 76f8e79..250ce12 100644
--- a/docs/jsfsx.md
+++ b/docs/jsfsx.md
@@ -178,6 +178,25 @@ x_out = "Call me with an access-key to view my sourcecode!";
 ## streaming
 What if an X generates output for a long time on purpose (imagine an audio stream, just as a random example...)?
 
+This is tricky, and the more I think about it, the more I'm not sure it makes sense for code executing in this context to behave this way.  Let me try to explain...
+
+An HTTP request is made to an X file, the file is loaded, passed the request data and run.  When the run is complete, any output is passed back to the requestor.  This is pipelined using a Node.js stream but since the relationship between input and output is asymetrical the stream can't work in a "pipelined" fashion like passing a regular file back to the caller.  The `Transform` component used to execute the code can't "yield" output outside of fixed events which consist of either one per "chunk" passed to the component (not an option because the chunks are the sourcecode that can't be executed piecemeal) or at the end of receiving all the chunks (how we do it now).
+
+So this means that if an X needs all of it's code to run, it can only send data back to the caller in one shot.  That means that the output must be buffered in memory, unless there is some non-obvious way to "drain" the output downstream in the pipeline that I'm not finding in the docs...
+
+This might be overcome using a `Duplex` stream, however we also need a way to stream data out of the X running under `vm.runInContext()`.  Right now I don't see a way to do this as running the X in `vm` is a blocking operation, but maybe there's a way to get events out if it?  Maybe the `context` that gets passed-in could include a function that could somehow cross that barrier?
+
+OK, so if I pass a function via the `context` object, I can invoke that function from inside the `vm` that is running the X.  So this *could* be used to send a signal to the `ExecutableStream` that there's data to send along the pipeline.  I'm not sure yet if this is a *good* idea (it's starting to feel like entangling the X-world too much in the outer world) but let's just go with it for awhile and see if it actually works.
+
+So what's it going to take to test this end-to-end?  I think `ExecutableStream` needs to be re-written as a `Duplex` stream provider, so let's checkpoint all this in git before we break everything...
+
+
+
+
+
+
+This isn't all bad, in fact it hints at a better way to write X code.  Let me see if I can summarize that...
+
 
 
 ## References
diff --git a/lib/xstream.js b/lib/xstream.js
index f939791..215e3f7 100644
--- a/lib/xstream.js
+++ b/lib/xstream.js
@@ -13,6 +13,11 @@ class ExecutableStream extends Transform {
 
     log.message(log.INFO, "Executing " + this.code.length + " bytes of Javascript...");
 
+    // Experimental streaming event thing...
+    var gong = function gong(n) {
+      console.log("Got gong: " + n);
+    };
+
     // This provides a basic unix-like in/out/error interface to executable code.
     // * x_in is currently the entire `request` object sent by the client
     // * x_out is written back to the client via the `response` object
@@ -22,7 +27,8 @@ class ExecutableStream extends Transform {
     const context = {
       x_in: this.x_in,
       x_out:"",
-      x_err:""
+      x_err:"",
+      gong: gong
     };
 
     vm.createContext(context);

From 61d77056cd2778be1d0db0fbc3ef5e26ca0123b6 Mon Sep 17 00:00:00 2001
From: "Jason J. Gullickson" <mr@jasongullickson.com>
Date: Mon, 15 Apr 2024 14:27:02 -0500
Subject: [PATCH 36/41] Streaming WIP: working as Duplex.

---
 docs/jsfsx.md  |  2 +-
 lib/xstream.js | 40 ++++++++++++++++++++++++++++++++++++----
 2 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/docs/jsfsx.md b/docs/jsfsx.md
index 250ce12..91db50d 100644
--- a/docs/jsfsx.md
+++ b/docs/jsfsx.md
@@ -190,7 +190,7 @@ OK, so if I pass a function via the `context` object, I can invoke that function
 
 So what's it going to take to test this end-to-end?  I think `ExecutableStream` needs to be re-written as a `Duplex` stream provider, so let's checkpoint all this in git before we break everything...
 
-
+So it looks like I can cram what I need into the `_write()` and `_read()` methods of the `Duplex` stream, but I need a way to know when all the X code has been `_write()`-end, and I also need to know how to tell callers of `_read()` that the X is done generating output.
 
 
 
diff --git a/lib/xstream.js b/lib/xstream.js
index 215e3f7..4f1a6cd 100644
--- a/lib/xstream.js
+++ b/lib/xstream.js
@@ -1,19 +1,28 @@
 const vm = require('node:vm');
-const { Transform } = require('node:stream');
+const { Duplex } = require('node:stream');
 
 var log = require("../jlog.js");
 
-class ExecutableStream extends Transform {
+class ExecutableStream extends Duplex {
   constructor(x_in) {
     super();
     this.x_in = x_in;
+    //this.x_out = "";  // TODO: maybe this should be buffer, etc.?
     this.code = "";
+    this.x_done = false;
   }
-  _flush(callback) {
+  //_flush(callback) {
+  _write(chunk, encoding, callback) {
+
+    console.log("Got _write()");
+
+    // TODO: Figure out a way to buffer chunks until we have the whole program.
+    this.code = this.code + chunk.toString();
 
     log.message(log.INFO, "Executing " + this.code.length + " bytes of Javascript...");
 
     // Experimental streaming event thing...
+    // TODO: Rename this and use it to stream data from inside the X
     var gong = function gong(n) {
       console.log("Got gong: " + n);
     };
@@ -32,7 +41,7 @@ class ExecutableStream extends Transform {
     };
 
     vm.createContext(context);
-    vm.runInContext(this.code, context)
+    vm.runInContext(this.code, context) // x_out is set by code inside here
     log.message(log.INFO, "Execution complete!");
 
     if(context.x_err.length > 0) {
@@ -40,14 +49,37 @@ class ExecutableStream extends Transform {
     }
 
     log.message(log.INFO, "Returning " + context.x_out.length + " bytes of data from executing Javascript");
+    //this.x_out = context.x_out;
+    this.x_done = true;
+
+    // Why not?
     this.push(context.x_out);
 
     callback();
   }
+  _read(size){
+
+    console.log("Got _read()");
+    console.log("this.x_out: " + this.x_out);
+    console.log("this.x_done: " + this.x_done);
+
+    if(this.x_done){
+      this.push(null);
+    } else {
+      //this.push(this.x_out);
+    }
+
+    //return this.x_out.length;
+  }
+
+
+
+  /*
   _transform(chunk, encoding, callback){
     this.code = this.code + chunk.toString();
     callback(null);
   }
+  */
 }
 
 module.exports = ExecutableStream;

From 4179e25279412f507ebc5aa75eb66e54a295509f Mon Sep 17 00:00:00 2001
From: "Jason J. Gullickson" <mr@jasongullickson.com>
Date: Mon, 15 Apr 2024 15:26:39 -0500
Subject: [PATCH 37/41] Streaming works.

---
 docs/examples/jsfsx/stream.js |  7 ++---
 docs/jsfsx.md                 | 24 +++++++++++++--
 lib/xstream.js                | 56 +++++++++++++----------------------
 3 files changed, 44 insertions(+), 43 deletions(-)

diff --git a/docs/examples/jsfsx/stream.js b/docs/examples/jsfsx/stream.js
index b2c0474..b7936e5 100644
--- a/docs/examples/jsfsx/stream.js
+++ b/docs/examples/jsfsx/stream.js
@@ -1,8 +1,5 @@
-x_out = "";
-loop_count = 10;
+var loop_count = 5;
 
 for(var i=0;i<loop_count;i++){
-  x_out = x_out + "drip\n";
-  gong(x_out.length); 
-  loop_count--;
+  x_push("drip\n");
 }
diff --git a/docs/jsfsx.md b/docs/jsfsx.md
index 91db50d..8f0da08 100644
--- a/docs/jsfsx.md
+++ b/docs/jsfsx.md
@@ -13,9 +13,10 @@
 - [x] Refactor (code structure, logging, error handling, etc.)
 - [x] Come up with a way to fetch the source of an executable w/o running it
   - [x] Don't execute if an access-key/token is presented
-- [ ] Consider the impact of executables that emit large amounts of data (or continuous streams)
+- [x] Consider the impact of executables that emit large amounts of data (or continuous streams)
 - [ ] Handle errors generated by uploaded code so they don't crash the whole server
 - [ ] Finalize X interface (`x_in`, `x_out`, etc.)
+- [ ] Find a way to tell when all X source chunks are ready
 - [ ] Execute executable files on POST
 
 ### post-poc
@@ -30,6 +31,7 @@
   - [ ] How to handle modules/dependencies?
   - [ ] Threading/eventloop impact/tuning
   - [ ] Related: other workloads (external binaries, shell scripts, etc.)
+- [ ] Fix bug that allows invalid `access-key` to view source
 
 ## curl to store an executable file
 ```bash
@@ -192,11 +194,29 @@ So what's it going to take to test this end-to-end?  I think `ExecutableStream`
 
 So it looks like I can cram what I need into the `_write()` and `_read()` methods of the `Duplex` stream, but I need a way to know when all the X code has been `_write()`-end, and I also need to know how to tell callers of `_read()` that the X is done generating output.
 
+### it works, but it's buggy
 
+`stream.js`
+```javascript
+var loop_count = 5;
+
+for(var i=0;i<loop_count;i++){
+  x_push("drip\n");
+}
+```
 
+```bash
+curl "http://localhost:7302/bin/stream.js"
+drip
+drip
+drip
+drip
+drip
+```
 
-This isn't all bad, in fact it hints at a better way to write X code.  Let me see if I can summarize that...
+This change introduces a new piece to the X interface: `x_push()`.  When called, this will immediately stream the data passed to the function to the client.  Right now this is a string, and it should probably be something more generic.  I've also preserved the previous "batch-mode" `x_out` interface but off the top of my head I can't think of why everything can't just use the `x_push()` interface other than it sounds kind of weird.
 
+The "buggy" part is that right now there's no way to run more than one block's worth of X code because I can't find a way for the `Duplex` stream to know when it's got all the blocks from the previous step in the pipeline.  I assume there is a way, but I haven't figured it out yet.
 
 
 ## References
diff --git a/lib/xstream.js b/lib/xstream.js
index 4f1a6cd..4ab7e9e 100644
--- a/lib/xstream.js
+++ b/lib/xstream.js
@@ -7,79 +7,63 @@ class ExecutableStream extends Duplex {
   constructor(x_in) {
     super();
     this.x_in = x_in;
-    //this.x_out = "";  // TODO: maybe this should be buffer, etc.?
     this.code = "";
     this.x_done = false;
   }
-  //_flush(callback) {
   _write(chunk, encoding, callback) {
 
-    console.log("Got _write()");
-
     // TODO: Figure out a way to buffer chunks until we have the whole program.
     this.code = this.code + chunk.toString();
 
     log.message(log.INFO, "Executing " + this.code.length + " bytes of Javascript...");
 
-    // Experimental streaming event thing...
-    // TODO: Rename this and use it to stream data from inside the X
-    var gong = function gong(n) {
-      console.log("Got gong: " + n);
-    };
-
     // This provides a basic unix-like in/out/error interface to executable code.
     // * x_in is currently the entire `request` object sent by the client
     // * x_out is written back to the client via the `response` object
     // * x_err is written to JSFS logs
     // A better solution would provide the client a way to access the logs,
     // but I don't have a good answer for that yet.
+    //
+    // x_push() can be used to stream output from X code, if called, the
+    // data passed to the function will in turn be passed down the pipeline
+    // to the response and ultimately the caller.
+
+    // NOTE: This nonsense is needed to give access to the Duplex `push()` method
+    function make_x_push(t) {
+      return function (s) {
+        t.push(s);
+      };
+    }
+
     const context = {
       x_in: this.x_in,
       x_out:"",
       x_err:"",
-      gong: gong
+      x_push: make_x_push(this)
     };
-
     vm.createContext(context);
-    vm.runInContext(this.code, context) // x_out is set by code inside here
+    log.message(log.INFO, "Execution context created.");
+
+    vm.runInContext(this.code, context)
     log.message(log.INFO, "Execution complete!");
 
     if(context.x_err.length > 0) {
       log.message(log.INFO, "Execution error logs: " + context.x_err);
     }
 
-    log.message(log.INFO, "Returning " + context.x_out.length + " bytes of data from executing Javascript");
-    //this.x_out = context.x_out;
+    // If the code used the x_out interface, write it out now (this may be depreciated)
+    if(context.x_out.length > 0){
+      this.push(context.x_out);
+    }
     this.x_done = true;
 
-    // Why not?
-    this.push(context.x_out);
-
     callback();
   }
   _read(size){
-
-    console.log("Got _read()");
-    console.log("this.x_out: " + this.x_out);
-    console.log("this.x_done: " + this.x_done);
-
     if(this.x_done){
       this.push(null);
-    } else {
-      //this.push(this.x_out);
     }
-
-    //return this.x_out.length;
-  }
-
-
-
-  /*
-  _transform(chunk, encoding, callback){
-    this.code = this.code + chunk.toString();
-    callback(null);
   }
-  */
 }
 
 module.exports = ExecutableStream;

From 30c199e4b6f95f543d6af11f191100f53f3cb92d Mon Sep 17 00:00:00 2001
From: "Jason J. Gullickson" <mr@jasongullickson.com>
Date: Mon, 15 Apr 2024 15:35:04 -0500
Subject: [PATCH 38/41] cleanup

---
 docs/jsfsx.md  |  1 +
 lib/xstream.js | 13 ++++---------
 2 files changed, 5 insertions(+), 9 deletions(-)

diff --git a/docs/jsfsx.md b/docs/jsfsx.md
index 8f0da08..faa4d12 100644
--- a/docs/jsfsx.md
+++ b/docs/jsfsx.md
@@ -33,6 +33,7 @@
   - [ ] Related: other workloads (external binaries, shell scripts, etc.)
 - [ ] Fix bug that allows invalid `access-key` to view source
 
+
 ## curl to store an executable file
 ```bash
 curl -X POST -H "content-type: text/javascript" -H "x-access-key: jjg" -H "x-executable: true" --data-binary @hello.js "http://localhost:7302/bin/hello.js"
diff --git a/lib/xstream.js b/lib/xstream.js
index 4ab7e9e..27ef62d 100644
--- a/lib/xstream.js
+++ b/lib/xstream.js
@@ -19,16 +19,11 @@ class ExecutableStream extends Duplex {
 
     // This provides a basic unix-like in/out/error interface to executable code.
     // * x_in is currently the entire `request` object sent by the client
-    // * x_out is written back to the client via the `response` object
+    // * x_out is written back to the client via the `response` object after X is done
+    // * x_push() streams output to the client via the `response` object while X is still running
     // * x_err is written to JSFS logs
-    // A better solution would provide the client a way to access the logs,
-    // but I don't have a good answer for that yet.
-    //
-    // x_push() can be used to stream output from X code, if called, the
-    // data passed to the function will in turn be passed down the pipeline
-    // to the response and ultimately the caller.
 
-    // NOTE: This nonsense is needed to give access to the Duplex `push()` method
+    // NOTE: This nonsense is needed to give the X access to the Duplex `push()` method
     function make_x_push(t) {
       return function (s) {
         t.push(s);
@@ -51,7 +46,7 @@ class ExecutableStream extends Duplex {
       log.message(log.INFO, "Execution error logs: " + context.x_err);
     }
 
-    // If the code used the x_out interface, write it out now (this may be depreciated)
+    // If the X used the x_out interface, write it out now
     if(context.x_out.length > 0){
       this.push(context.x_out);
     }

From 1de7e4fc5bc81fac47e206d33491cce375b18993 Mon Sep 17 00:00:00 2001
From: "Jason J. Gullickson" <mr@jasongullickson.com>
Date: Mon, 15 Apr 2024 15:36:23 -0500
Subject: [PATCH 39/41] more cleanup

---
 lib/xstream.js | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/lib/xstream.js b/lib/xstream.js
index 27ef62d..d5c8ca0 100644
--- a/lib/xstream.js
+++ b/lib/xstream.js
@@ -12,6 +12,13 @@ class ExecutableStream extends Duplex {
   }
   _write(chunk, encoding, callback) {
 
+    // NOTE: This nonsense is needed to give the X access to the Duplex `push()` method
+    function make_x_push(t) {
+      return function (s) {
+        t.push(s);
+      };
+    }
+
     // TODO: Figure out a way to buffer chunks until we have the whole program.
     this.code = this.code + chunk.toString();
 
@@ -22,20 +29,13 @@ class ExecutableStream extends Duplex {
     // * x_out is written back to the client via the `response` object after X is done
     // * x_push() streams output to the client via the `response` object while X is still running
     // * x_err is written to JSFS logs
-
-    // NOTE: This nonsense is needed to give the X access to the Duplex `push()` method
-    function make_x_push(t) {
-      return function (s) {
-        t.push(s);
-      };
-    }
-
     const context = {
       x_in: this.x_in,
       x_out:"",
       x_err:"",
       x_push: make_x_push(this)
     };
+
     vm.createContext(context);
     log.message(log.INFO, "Execution context created.");
 

From ae09367813f1a6f320b75364bf2b50336b97a408 Mon Sep 17 00:00:00 2001
From: "Jason J. Gullickson" <mr@jasongullickson.com>
Date: Mon, 15 Apr 2024 15:57:29 -0500
Subject: [PATCH 40/41] dont you blow your top

---
 docs/jsfsx.md  |  5 +++--
 lib/xstream.js | 20 ++++++++++++--------
 2 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/docs/jsfsx.md b/docs/jsfsx.md
index faa4d12..53116fd 100644
--- a/docs/jsfsx.md
+++ b/docs/jsfsx.md
@@ -14,9 +14,8 @@
 - [x] Come up with a way to fetch the source of an executable w/o running it
   - [x] Don't execute if an access-key/token is presented
 - [x] Consider the impact of executables that emit large amounts of data (or continuous streams)
-- [ ] Handle errors generated by uploaded code so they don't crash the whole server
+- [x] Handle errors generated by uploaded code so they don't crash the whole server
 - [ ] Finalize X interface (`x_in`, `x_out`, etc.)
-- [ ] Find a way to tell when all X source chunks are ready
 - [ ] Execute executable files on POST
 
 ### post-poc
@@ -32,6 +31,8 @@
   - [ ] Threading/eventloop impact/tuning
   - [ ] Related: other workloads (external binaries, shell scripts, etc.)
 - [ ] Fix bug that allows invalid `access-key` to view source
+- [ ] Review types and make sure we're using the right ones to pass things around
+- [ ] Find a way to tell when all X source chunks are ready
 
 
 ## curl to store an executable file
diff --git a/lib/xstream.js b/lib/xstream.js
index d5c8ca0..2c08daf 100644
--- a/lib/xstream.js
+++ b/lib/xstream.js
@@ -22,7 +22,7 @@ class ExecutableStream extends Duplex {
     // TODO: Figure out a way to buffer chunks until we have the whole program.
     this.code = this.code + chunk.toString();
 
-    log.message(log.INFO, "Executing " + this.code.length + " bytes of Javascript...");
+    log.message(log.INFO, "Xing " + this.code.length + " bytes of Javascript...");
 
     // This provides a basic unix-like in/out/error interface to executable code.
     // * x_in is currently the entire `request` object sent by the client
@@ -37,19 +37,23 @@ class ExecutableStream extends Duplex {
     };
 
     vm.createContext(context);
-    log.message(log.INFO, "Execution context created.");
+    log.message(log.INFO, "X context created.");
 
-    vm.runInContext(this.code, context)
-    log.message(log.INFO, "Execution complete!");
+    // TODO: There could be a lot more done here when an X blows-up, but
+    // for now let's just not crash the whole damn server...
+    try {
+      vm.runInContext(this.code, context)
+      log.message(log.INFO, "X complete!");
+    } catch(err) {
+      log.message(log.ERROR, "X exception: " + err);
+    }
 
     if(context.x_err.length > 0) {
-      log.message(log.INFO, "Execution error logs: " + context.x_err);
+      log.message(log.INFO, "X error logs: " + context.x_err);
     }
 
     // If the X used the x_out interface, write it out now
-    if(context.x_out.length > 0){
-      this.push(context.x_out);
-    }
+    this.push(context.x_out);
     this.x_done = true;
 
     callback();

From 6086e84d15bdbb316d06b6b1b0f8deed8d319b4b Mon Sep 17 00:00:00 2001
From: "Jason J. Gullickson" <mr@jasongullickson.com>
Date: Mon, 15 Apr 2024 15:58:56 -0500
Subject: [PATCH 41/41] Add exception-throwing example.

---
 docs/examples/jsfsx/bomb.js | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 docs/examples/jsfsx/bomb.js

diff --git a/docs/examples/jsfsx/bomb.js b/docs/examples/jsfsx/bomb.js
new file mode 100644
index 0000000..72cb27b
--- /dev/null
+++ b/docs/examples/jsfsx/bomb.js
@@ -0,0 +1 @@
+x_out = butts;