modular server: Adds signup route & implements basePath config

.github/workflows/test-and-lint.yml

@@ -1,9 +1,9 @@
 name: test-and-lint
 on:
   push:
-    branches: [ master ]
+    branches: [ master, modular ]
   pull_request:
-    branches: [ master ]
+    branches: [ master, modular ]
 jobs:
   build:
     name: node.js
@@ -13,9 +13,9 @@ jobs:
         # Support LTS versions based on https://nodejs.org/en/about/releases/
         node-version: ['18', '20', '21']
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node-version }}
       - name: Install dependencies

bin/www

@@ -1,6 +1,5 @@
 #!/usr/bin/env node
 
-const app = require('../lib/app');
 const http = require('http');
 const fs = require("fs");
 const path = require("path");
@@ -29,12 +28,25 @@ try {
 
 configureLogger(conf.logging);
 
+conf.basePath ||= '';
+if (conf.basePath && !conf.basePath.startsWith('/')) { conf.basePath = '/' + conf.basePath; }
+process.env.basePath = conf.basePath;
+
+const app = require('../lib/app');   // process.env.basePath must have been set previously
+const S3 = require("../lib/streaming_stores/S3");
+
 const port = normalizePort( process.env.PORT || conf.http?.port || '8000');
 app.set('port', port);
 
+// If the environment variables aren't set, storage uses a shared public account on play.min.io!
+app.set('streaming store', new S3(process.env.S3_HOSTNAME,
+  process.env.S3_PORT ? parseInt(process.env.S3_PORT) : undefined,
+  process.env.S3_ACCESS_KEY, process.env.S3_SECRET_KEY));
+
+
 app.locals.title = "Modular Armadietto";
-app.locals.basePath = '';
-// rendering should set host: getHost(req)
+app.locals.basePath = conf.basePath;
+// rendering should set locals.host: getHost(req)
 app.locals.host = conf.http?.host + (port ? ':' + port : '');
 app.locals.signup = conf.allow_signup;
 

lib/app.js

@@ -2,10 +2,14 @@ const express = require('express');
 const path = require('path');
 const logger = require('morgan');
 const indexRouter = require('./routes/index');
+const signupRouter = require('./routes/signup');
 const errorPage = require('./util/errorPage');
 const helmet = require('helmet');
 const shorten = require('./util/shorten');
 
+let basePath = process.env.basePath || '';
+if (basePath && !basePath.startsWith('/')) { basePath = '/' + basePath; }
+
 const app = express();
 
 // view engine setup
@@ -38,16 +42,23 @@ app.use(helmet({
 }));
 // app.use(express.json());
 // app.use(express.urlencoded({ extended: false }));
-app.use('/assets', express.static(path.join(__dirname, 'assets')));
+app.use(`${basePath}/assets`, express.static(path.join(__dirname, 'assets')));
+
+app.use(`${basePath}/`, indexRouter);
 
-app.use('/', indexRouter);
+app.use(`${basePath}/signup`, signupRouter);
 
 // catches 404 and forwards to error handler
-app.use(function (req, res, next) {
+app.use(basePath, function (req, res, next) {
   const name = req.path.slice(1);
   errorPage(req, res, 404, { title: 'Not Found', message: `“${name}” doesn't exist` });
 });
 
+// redirect for paths outside the app
+app.use(function (req, res, next) {
+  res.status(308).set('Location', basePath).end();
+});
+
 // error handler
 app.use(function (err, req, res, _next) {
   errorPage(req, res, err.status || 500, {

lib/routes/signup.js

@@ -0,0 +1,52 @@
+const express = require('express');
+const router = express.Router();
+const getHost = require('../util/getHost');
+const errorPage = require('../util/errorPage');
+const { getLogger } = require('../logger');
+
+const DISABLED_LOCALS = { title: 'Forbidden', message: 'Signing up is not allowed currently' };
+const DISABLED_LOG_NOTE = 'signups disabled';
+
+/* initial entry */
+router.get('/', function (req, res) {
+  if (req.app?.locals?.signup) {
+    res.render('signup.html', {
+      title: 'Signup',
+      params: {},
+      error: null,
+      host: getHost(req)
+    });
+  } else {
+    errorPage(req, res, 403, DISABLED_LOCALS, DISABLED_LOG_NOTE);
+  }
+});
+
+/* submission or re-submission */
+router.post('/',
+  express.urlencoded({ extended: false }),
+  async function (req, res) {
+    if (req.app?.locals?.signup) {
+      try {
+        const store = req.app?.get('streaming store');
+        const bucketName = await store.createUser(req.body);
+        getLogger().notice(`created bucket “${bucketName}” for user “${req.body.username}”`);
+        res.status(201).render('signup-success.html', {
+          title: 'Signup Success',
+          params: req.body,
+          host: getHost(req)
+        });
+      } catch (err) {
+        getLogger().error(`while creating user “${req.body?.username}”`, err);
+        res.status(409).render('signup.html', {
+          title: 'Signup Failure',
+          params: req.body,
+          error: err,
+          host: getHost(req)
+        });
+      }
+    } else {
+      errorPage(req, res, 403, DISABLED_LOCALS, DISABLED_LOG_NOTE);
+    }
+  });
+
+module.exports = router;

lib/streaming_stores/S3.js

@@ -0,0 +1,106 @@
+const Minio = require('minio');
+const core = require('../stores/core');
+const { getLogger } = require('../logger');
+
+// const FILE_PREFIX = 'remoteStorageBlob/';
+// const AUTH_PREFIX = 'remoteStorageAuth/';
+
+/** uses the min.io client to connect to any S3-compatible storage that supports versioning */
+class S3 {
+  #minioClient;
+
+  /** Using the default arguments connects you to a public server where anyone can read and delete your data! */
+  constructor (endPoint = 'play.min.io', port = 9000, accessKey = 'Q3AM3UQ867SPQQA43P2F', secretKey = 'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG') {
+    this.#minioClient = new Minio.Client({
+      endPoint,
+      port,
+      accessKey,
+      secretKey,
+      useSSL: !['localhost', '10.0.0.2', '127.0.0.1'].includes(endPoint)
+    });
+  }
+
+  /**
+   * Creates an empty bucket for the new user.
+   * @param {Object} params
+   * @param {string} params.username
+   * @param {string} params.email
+   * @param {string} params.password
+   * @returns {Promise<string>} name of bucket
+   */
+  async createUser (params) {
+    const errors = core.validateUser(params);
+    if (errors.length > 0) {
+      const msg = errors.map(err => err.message).join('|');
+      throw new Error(msg);
+    }
+
+    const bucketName = params.username;
+    const exists = await this.#minioClient.bucketExists(bucketName);
+    if (exists) {
+      throw new Error(`Username “${params.username}” is already taken`);
+    } else {
+      await this.#minioClient.makeBucket(bucketName);
+      await this.#minioClient.setBucketVersioning(bucketName,
+        { Status: 'Enabled', ExcludedPrefixes: [{ Prefix: 'permissions' }, { Prefix: 'meta' }] });
+      getLogger().info(`bucket ${bucketName} created.`);
+      return bucketName;
+    }
+  }
+
+  /**
+   * Deletes all of user's files and the bucket. NOT REVERSIBLE.
+   * @param username
+   * @returns {Promise<number>} number of files deleted
+   */
+  async deleteUser (username) {
+    if (!await this.#minioClient.bucketExists(username)) { return 0; }
+
+    return new Promise((resolve, reject) => {
+      const GROUP_SIZE = 100;
+      const objectVersions = [];
+      let numRequested = 0; let numRemoved = 0; let isReceiveComplete = false;
+
+      const removeObjectVersions = async () => {
+        const group = objectVersions.slice(0);
+        objectVersions.length = 0;
+        numRequested += group.length;
+        await this.#minioClient.removeObjects(username, group);
+        numRemoved += group.length;
+
+        if (isReceiveComplete && numRemoved === numRequested) {
+          await this.#minioClient.removeBucket(username); // will fail if any object versions remain
+          resolve(numRemoved);
+        }
+      };
+
+      const removeObjectVersionsAndBucket = async err => {
+        try {
+          isReceiveComplete = true;
+          await removeObjectVersions();
+          if (err) {
+            reject(err);
+          }
+        } catch (err2) {
+          reject(err || err2);
+        }
+      };
+
+      const objectVersionStream = this.#minioClient.listObjects(username, '', true, { IncludeVersion: true });
+      objectVersionStream.on('data', async item => {
+        try {
+          objectVersions.push(item);
+          if (objectVersions.length >= GROUP_SIZE) {
+            await removeObjectVersions();
+          }
+        } catch (err) { // keeps going
+          getLogger().error(`while deleting user “${username}” object version ${JSON.stringify(item)}:`, err);
+        }
+      });
+      objectVersionStream.on('error', removeObjectVersionsAndBucket);
+      objectVersionStream.on('end', removeObjectVersionsAndBucket);
+    });
+  }
+}
+
+module.exports = S3;

lib/views/signup.html

@@ -1,6 +1,6 @@
 <%- include('header.html'); %>
 
-<h2>Sign up </h2>
+<h2>Sign up</h2>
 
 <form method="post" action="<%= basePath %>/signup">
   <% if (error) { %>

notes/S3 streaming store.md

@@ -0,0 +1,31 @@
+# S3-compatible Streaming Store
+
+Streaming Stores can only be used with the modular server.
+
+You should be able to connect to any S3-compatible service that supports versioning. Tested services include:
+
+* min.io (both self-hosted and cloud)
+
+
+Configure the store by passing to the constructor the endpoint (host name), access key (admin user name) and secret key (password). For non-Amazon providers, you may need to pass in a port number as well.  You can provide these however you like, but typically they are stored in these environment variables:
+
+* S3_HOSTNAME
+* S3_PORT
+* S3_ACCESS_KEY
+* S3_SECRET_KEY
+
+Creating a client then resembles:
+
+```javascript
+const store = new S3(process.env.S3_HOSTNAME,
+    process.env.S3_PORT ? parseInt(process.env.S3_PORT) : undefined,
+    process.env.S3_ACCESS_KEY, process.env.S3_SECRET_KEY);
+```
+
+This one access key is used to create a bucket for each user.
+The bucket name is the username.
+Buckets can be administered using the service's tools, such as a webapp console or command-line tools.
+The bucket can contain non-remoteStorage blobs outside these prefixes:
+
+* remoteStorageBlob/
+* remoteStorageAuth/