diff --git a/docs/examples/jsfsx/bomb.js b/docs/examples/jsfsx/bomb.js new file mode 100644 index 0000000..72cb27b --- /dev/null +++ b/docs/examples/jsfsx/bomb.js @@ -0,0 +1 @@ +x_out = butts; diff --git a/docs/examples/jsfsx/hello.js b/docs/examples/jsfsx/hello.js new file mode 100644 index 0000000..838e23f --- /dev/null +++ b/docs/examples/jsfsx/hello.js @@ -0,0 +1 @@ +x_out = "Hack the Planet!"; diff --git a/docs/examples/jsfsx/input.js b/docs/examples/jsfsx/input.js new file mode 100644 index 0000000..8f927cb --- /dev/null +++ b/docs/examples/jsfsx/input.js @@ -0,0 +1 @@ +x_out = "The request method is: " + x_in.method; diff --git a/docs/examples/jsfsx/logit.js b/docs/examples/jsfsx/logit.js new file mode 100644 index 0000000..51f1c36 --- /dev/null +++ b/docs/examples/jsfsx/logit.js @@ -0,0 +1,2 @@ +x_err = x_err + "Something to log"; +x_out = x_err; diff --git a/docs/examples/jsfsx/stream.js b/docs/examples/jsfsx/stream.js new file mode 100644 index 0000000..b7936e5 --- /dev/null +++ b/docs/examples/jsfsx/stream.js @@ -0,0 +1,5 @@ +var loop_count = 5; + +for(var i=0;i GET /bin/hello.js HTTP/1.1 +> Host: localhost:7302 +> User-Agent: curl/7.81.0 +> Accept: */* +> +* Mark bundle as not supporting multiuse +< HTTP/1.1 200 OK +< Access-Control-Allow-Methods: GET,POST,PUT,DELETE,OPTIONS +< Access-Control-Allow-Headers: Accept,Accept-Version,Api-Version,Content-Type,Origin,Range,X_FILENAME,X-Access-Key,X-Access-Token,X-Append,X-Encrypted,X-Private,X-Replacement-Access-Key,X-Requested-With,X-Executable +< Access-Control-Allow-Origin: * +< Access-Control-Expose-Headers: X-Media-Bitrate,X-Media-Channels,X-Media-Duration,X-Media-Resolution,X-Media-Size,X-Media-Type +< Content-Type: text/javascript +< Content-Length: 28 +< Date: Thu, 11 Apr 2024 15:50:10 GMT +< Connection: keep-alive +< Keep-Alive: timeout=5 +< +* Excess found in a read: excess = 4, size = 28, maxdownload = 28, bytecount = 0 +* Closing connection 0 +Hack the Planet!Hack the Pla +``` + +## Refactoring and bugs + +For whatever reason refactoring the code a bit made the duplicate output bug go away ("IT'S MAGIC!"). Now executable `GET` requests work as expected, so it's probably time to think more about the interface between requests, reponses and the executable code. + +It's tempting to move on to `POST` but probably better to figure out the I/O first.. + + +## input + +Let's start by passing the whole `request` into the executor, what's the worst that can happen? + +Then this uploaded code: + +```javascript +x_out = "The request method is: " + x_in.method; +``` + +...yeilds this result: +```bash +curl "http://localhost:7302/bin/input.js" +The request method is: GET +``` + +## X runtime environment/interface + +There will be a lot more to explore here in the future, but in the spirit of MVP or whatever here's what we're going to do for now. + +Files marked executable are run as Javascript in a [Node.js VM](https://nodejs.org/api/vm.html#vm-executing-javascript). At startup three variables will be initialized: `x_in`, `x_out` and `x_err`. These map loosely to the `stdin`, `stdout` and `stderr` unix convention. +* `x_in` is the entire `request` object sent by the user agent (for now) +* `x_out` is returned to the user agent in the `response` object +* `x_err` is written to the JSFS log + +It would be useful if `x_err` was more accessible by the user agent, and I have some ideas for this (maybe a `x-jsfs-debug` header that dumps the entire `context` to `response`?), but for now I'm just going to let it write to the log (if it's needed for debugging you can always do that using a local JSFS instance right?). + +## view source +Now you can retrieve the data (as opposed to executing the code) in a stored file that is marked as executable: + +```bash +curl "http://localhost:7302/bin/viewsource2.js" +Call me with an access-key to view my sourcecode! + +curl -H "x-access-key: jjg" "http://localhost:7302/bin/viewsource2.js" +// If you're seeing this, the execute override worked! +x_out = "Call me with an access-key to view my sourcecode!"; +``` + +## streaming +What if an X generates output for a long time on purpose (imagine an audio stream, just as a random example...)? + +This is tricky, and the more I think about it, the more I'm not sure it makes sense for code executing in this context to behave this way. Let me try to explain... + +An HTTP request is made to an X file, the file is loaded, passed the request data and run. When the run is complete, any output is passed back to the requestor. This is pipelined using a Node.js stream but since the relationship between input and output is asymetrical the stream can't work in a "pipelined" fashion like passing a regular file back to the caller. The `Transform` component used to execute the code can't "yield" output outside of fixed events which consist of either one per "chunk" passed to the component (not an option because the chunks are the sourcecode that can't be executed piecemeal) or at the end of receiving all the chunks (how we do it now). + +So this means that if an X needs all of it's code to run, it can only send data back to the caller in one shot. That means that the output must be buffered in memory, unless there is some non-obvious way to "drain" the output downstream in the pipeline that I'm not finding in the docs... + +This might be overcome using a `Duplex` stream, however we also need a way to stream data out of the X running under `vm.runInContext()`. Right now I don't see a way to do this as running the X in `vm` is a blocking operation, but maybe there's a way to get events out if it? Maybe the `context` that gets passed-in could include a function that could somehow cross that barrier? + +OK, so if I pass a function via the `context` object, I can invoke that function from inside the `vm` that is running the X. So this *could* be used to send a signal to the `ExecutableStream` that there's data to send along the pipeline. I'm not sure yet if this is a *good* idea (it's starting to feel like entangling the X-world too much in the outer world) but let's just go with it for awhile and see if it actually works. + +So what's it going to take to test this end-to-end? I think `ExecutableStream` needs to be re-written as a `Duplex` stream provider, so let's checkpoint all this in git before we break everything... + +So it looks like I can cram what I need into the `_write()` and `_read()` methods of the `Duplex` stream, but I need a way to know when all the X code has been `_write()`-end, and I also need to know how to tell callers of `_read()` that the X is done generating output. + +### it works, but it's buggy + +`stream.js` +```javascript +var loop_count = 5; + +for(var i=0;i 0) { + log.message(log.INFO, "X error logs: " + context.x_err); + } + + // If the X used the x_out interface, write it out now + this.push(context.x_out); + this.x_done = true; + + callback(); + } + _read(size){ + if(this.x_done){ + this.push(null); + } + } +} + +module.exports = ExecutableStream; diff --git a/server.js b/server.js index df63e04..e8e6402 100644 --- a/server.js +++ b/server.js @@ -21,6 +21,9 @@ var operations = require("./lib/" + (config.CONFIGURED_STORAGE || "fs") + "/disk // base storage object var Inode = require("./lib/inode.js"); +// executable support +var XStream = require("./lib/xstream.js"); + // get this now, rather than at several other points var TOTAL_LOCATIONS = config.STORAGE_LOCATIONS.length; @@ -89,12 +92,25 @@ http.createServer(function(req, res){ return compressed ? zlib.createGunzip() : through(); }; + var create_executor = function create_executor(executable){ + // TODO: We're passing the whole damn request in for now, + // but it might be a good ideal to pair this down at some point. + return executable ? new XStream(req) : through(); + }; + // return status res.statusCode = 200; // return file metadata as HTTP headers - res.setHeader("Content-Type", requested_file.content_type); - res.setHeader("Content-Length", requested_file.file_size); + // TODO: These can change for executable files, + // so for now only set them if we're not executing + // (a better solution would be to count the output + // from the running code, but I don't know how to + // do that yet...) + if(!requested_file.executable){ + res.setHeader("Content-Type", requested_file.content_type); + res.setHeader("Content-Length", requested_file.file_size); + } var total_blocks = requested_file.blocks.length; var idx = 0; @@ -139,6 +155,8 @@ http.createServer(function(req, res){ var read_stream = operations.stream_read(path); var decryptor = create_decryptor({ encrypted : requested_file.encrypted, key : requested_file.access_key}); var unzipper = create_unzipper(try_compressed); + // If access auth is present, don't execute + var executor = create_executor(requested_file.executable && !params.access_key && !params.access_token); var should_end = (idx + 1) === total_blocks; function on_error(){ @@ -166,9 +184,10 @@ http.createServer(function(req, res){ } else { res.setMaxListeners(0); } + read_stream.on("end", on_end); read_stream.on("error", on_error); - read_stream.pipe(unzipper).pipe(decryptor).pipe(res, {end: should_end}); + read_stream.pipe(unzipper).pipe(decryptor).pipe(executor).pipe(res, {end: should_end}); }; var load_from_last_seen = function load_from_last_seen(try_compressed){ @@ -205,6 +224,9 @@ http.createServer(function(req, res){ if (inode){ + // TODO: If the file is executable, and access credentials are not present, execute the file. + // This is really just GET, so maybe there is a way to branch-out to GET and avoid a lot of duplication? + // check authorization if (validate.is_authorized(inode, req.method, params)){ log.message(log.INFO, "File update request authorized"); @@ -235,6 +257,10 @@ http.createServer(function(req, res){ new_file.file_metadata.encrypted = true; } + if(params.executable){ + new_file.file_metadata.executable = true; + } + // if access_key is supplied with update, replace the default one if(params.access_key){ new_file.file_metadata.access_key = params.access_key;