From de89258d8f28fee0b5f81cded84de43e92a09128 Mon Sep 17 00:00:00 2001 From: "P. Douglas Reeder" Date: Sun, 10 Mar 2024 16:55:53 -0400 Subject: [PATCH] WIP: Modular: Re-implements streaming store as handler --- lib/routes/S3Handler.js | 209 +++++++++++++ lib/streaming_stores/S3.js | 101 ++----- notes/S3 streaming store.md | 8 +- package-lock.json | 237 ++++++++++++++- package.json | 1 + spec/oauth.spec.js | 2 +- spec/streaming_handler.spec.js | 366 +++++++++++++++++++++++ spec/streaming_handler/S3Handler.spec.js | 19 ++ 8 files changed, 862 insertions(+), 81 deletions(-) create mode 100644 lib/routes/S3Handler.js create mode 100644 spec/streaming_handler.spec.js create mode 100644 spec/streaming_handler/S3Handler.spec.js diff --git a/lib/routes/S3Handler.js b/lib/routes/S3Handler.js new file mode 100644 index 00000000..0dbc0271 --- /dev/null +++ b/lib/routes/S3Handler.js @@ -0,0 +1,209 @@ +/* streaming storage to an S3-compatible service */ + +/* eslint-env node */ +/* eslint-disable camelcase */ +const express = require('express'); +const { posix } = require('node:path'); +const { HeadObjectCommand, S3Client, DeleteObjectCommand, GetObjectCommand, PutObjectCommand } = require('@aws-sdk/client-s3'); +const normalizeETag = require('../util/normalizeETag'); +const ParameterError = require('../util/ParameterError'); +const { dirname, basename } = require('path'); +const { createHash } = require('node:crypto'); +const YAML = require('yaml'); +const TimeoutError = require('../util/timeoutError'); +const { Upload } = require('@aws-sdk/lib-storage'); +const { pipeline } = require('node:stream/promises'); + +const PUT_TIMEOUT = 24 * 60 * 60 * 1000; +// const AUTH_PREFIX = 'remoteStorageAuth'; +// const AUTHENTICATION_LOCAL_PASSWORD = 'authenticationLocalPassword'; +// const USER_METADATA = 'userMetadata'; +const FILE_PREFIX = 'remoteStorageBlob'; +const EMPTY_DIRECTORY = { '@context': 'http://remotestorage.io/spec/folder-description', items: {} }; + +module.exports = function (endPoint = 'play.min.io', accessKey = 'Q3AM3UQ867SPQQA43P2F', secretKey = 'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG', region = 'us-east-1') { + const sslEnabled = !/\blocalhost\b|\b127.0.0.1\b|\b10.0.0.2\b/.test(endPoint); + if (!endPoint.startsWith('http')) { + endPoint = (sslEnabled ? 'https://' : 'http://') + endPoint; + } + // if (!/:\d{1,5}\/?$/.test(endPoint)) { + // endPoint += ':9000'; + // } + + const s3client = new S3Client({ + forcePathStyle: true, + region, + endpoint: endPoint, + sslEnabled, + credentials: { + accessKeyId: accessKey, + secretAccessKey: secretKey, + Version: 1 + } + // logger: getLogger(), + }); + + const router = express.Router(); + router.get('/:username/*', + async function (req, res, next) { + try { + const bucketName = req.params.username.toLowerCase(); + const isDirectoryRequest = req.url.endsWith('/'); + const s3Path = posix.join(FILE_PREFIX, req.url.slice(1 + bucketName.length)); + let getParam; + if (req.get('If-None-Match')) { + getParam = { Bucket: bucketName, Key: s3Path, IfNoneMatch: req.get('If-None-Match') }; + } else if (req.get('If-Match')) { + getParam = { Bucket: bucketName, Key: s3Path, IfMatch: req.get('If-Match') }; + } else { // unconditional + getParam = { Bucket: bucketName, Key: s3Path }; + } + + const { Body, ETag, ContentType, ContentLength } = await s3client.send(new GetObjectCommand(getParam)); + const isDirectory = ContentType === 'application/x.remotestorage-ld+json'; + const contentType = isDirectory ? 'application/ld+json' : ContentType; + if (isDirectoryRequest ^ isDirectory) { + return res.status(409).end(); // Conflict + // return { status: 409, readStream: null, contentType, contentLength: null, ETag: null }; // Conflict + } else { + res.status(200).set('Content-Length', ContentLength).set('Content-Type', contentType).set('ETag', normalizeETag(ETag)); + return pipeline(Body, res); + } + } catch (err) { + if (['NotFound', 'NoSuchKey'].includes(err.name)) { + return res.status(404).end(); // Not Found + // return next(Object.assign(new Error(`No file exists at path “${req.blobPath}”`), { status: 404 })); + } else if (err.name === 'PreconditionFailed') { + return res.status(412).end(); + // return { status: 412 }; + } else if (err.name === 'NotModified' || err.$metadata?.httpStatusCode === 304 || err.name === 304) { + return res.status(304).end(); + } else { + return next(Object.assign(err, { status: 502 })); + } + } + } + ); + + router.delete('/:username/*', + async function (req, res, next) { + try { + const bucketName = req.params.username.toLowerCase(); + const s3Path = posix.join(FILE_PREFIX, req.url.slice(1 + bucketName.length)); + let currentETag; + try { + const headResponse = await s3client.send(new HeadObjectCommand({ Bucket: bucketName, Key: s3Path })); + if (headResponse.ContentType === 'application/x.remotestorage-ld+json') { + return res.status(409).end(); + } + currentETag = normalizeETag(headResponse.ETag); + + if (req.get('If-Match') && req.get('If-Match') !== currentETag) { + return res.status(412).end(); + } else if (req.get('If-None-Match') && req.get('If-None-Match') === currentETag) { + return res.status(412).end(); + } + /* const { DeleteMarker, VersionId } = */ await s3client.send(new DeleteObjectCommand({ Bucket: bucketName, Key: s3Path })); + } catch (err) { + if (['NotFound', 'NoSuchKey'].includes(err.name)) { + return res.status(404).end(); + } else if (err.$metadata?.httpStatusCode === 400 || err.name === '400' || /\bBucket\b/.test(err.message)) { + return next(Object.assign(new ParameterError('A parameter value is bad', { cause: err }), { status: 400 })); + } else { + return next(Object.assign(err, { status: 502 })); + } + } + + // updates all ancestor directories + let itemETag = null; + let itemPath = s3Path; + do { + let directory; + try { + directory = await readJson(bucketName, dirname(itemPath)); + } catch (err) { + if (!['NotFound', 'NoSuchKey'].includes(err.name)) { return next(Object.assign(err, { status: 502 })); } + } + if (!(directory?.items instanceof Object)) { + directory = structuredClone(EMPTY_DIRECTORY); + // TODO: scan for existing blobs + } + if (typeof itemETag === 'string') { // item is folder + if (itemETag.length > 0) { + directory.items[basename(itemPath) + '/'] = { ETag: itemETag }; + } else { + delete directory.items[basename(itemPath) + '/']; + } + } else { + delete directory.items[basename(itemPath)]; + } + if (Array.from(Object.keys(directory.items)).length > 0) { + const dirJSON = JSON.stringify(directory); + await putBlob(bucketName, dirname(itemPath), 'application/x.remotestorage-ld+json', dirJSON.length, dirJSON); + + if (dirname(itemPath) !== FILE_PREFIX) { + // calculates ETag for the folder + const hash = createHash('md5'); + for (const itemMeta of Object.values(directory.items)) { + hash.update(itemMeta?.ETag?.replace(/^W\/"|^"|"$/g, '') || ''); + } + itemETag = '"' + hash.digest('hex') + '"'; + } + } else { // that was the last blob in the folder, so delete the folder + /* const { DeleteMarker, VersionId } = */ await s3client.send(new DeleteObjectCommand({ Bucket: bucketName, Key: dirname(itemPath) })); + itemETag = ''; + } + + itemPath = dirname(itemPath); + } while (itemPath.length > FILE_PREFIX.length); + + if (currentETag) { + res.set('ETag', normalizeETag(currentETag)); + } + res.status(204).end(); + } catch (err) { + if (err.$metadata?.httpStatusCode === 400 || err.name === '400') { + return next(Object.assign(new ParameterError('A parameter value is bad', { cause: err }), { status: 400 })); + } else { + return next(Object.assign(err, { status: 502 })); + } + } + } + ); + + async function putBlob (bucketName, s3Path, contentType, contentLength, contentStream) { + if (contentLength <= 500_000_000) { + const putPrms = s3client.send(new PutObjectCommand( + { Bucket: bucketName, Key: s3Path, Body: contentStream, ContentType: contentType, ContentLength: contentLength })); + const timeoutPrms = new Promise((_resolve, reject) => + setTimeout(reject, PUT_TIMEOUT, new TimeoutError(`PUT of ${contentLength / 1_000_000} MB to ${bucketName} ${s3Path} took more than ${Math.round(PUT_TIMEOUT / 60_000)} minutes`))); + const putResponse = await Promise.race([putPrms, timeoutPrms]); + return normalizeETag(putResponse.ETag); + } else { + const parallelUpload = new Upload({ + client: s3client, + params: { Bucket: bucketName, Key: s3Path, Body: contentStream, ContentType: contentType, ContentLength: contentLength } + }); + + parallelUpload.on('httpUploadProgress', (progress) => { + console.debug(bucketName, s3Path, `part ${progress.part} ${progress.loaded} / ${progress.total} bytes`); + }); + + return normalizeETag((await parallelUpload.done()).ETag); + } + } + + async function readYaml (bucketName, s3Path) { // eslint-disable-line no-unused-vars + const { Body } = await s3client.send(new GetObjectCommand({ Bucket: bucketName, Key: s3Path })); + const string = (await Body.setEncoding('utf-8').toArray())[0]; + return YAML.parse(string); + } + + async function readJson (bucketName, s3Path) { + const { Body } = await s3client.send(new GetObjectCommand({ Bucket: bucketName, Key: s3Path })); + const string = (await Body.setEncoding('utf-8').toArray())[0]; + return JSON.parse(string); + } + + return router; +}; diff --git a/lib/streaming_stores/S3.js b/lib/streaming_stores/S3.js index 7723e049..9ae99d97 100644 --- a/lib/streaming_stores/S3.js +++ b/lib/streaming_stores/S3.js @@ -9,7 +9,7 @@ const { PutObjectCommand, CreateBucketCommand, DeleteBucketCommand, - GetObjectCommand, PutBucketVersioningCommand, DeleteObjectsCommand, ListObjectVersionsCommand, + GetObjectCommand, PutBucketVersioningCommand, ListObjectVersionsCommand, DeleteObjectCommand, HeadObjectCommand } = require('@aws-sdk/client-s3'); const { Upload } = require('@aws-sdk/lib-storage'); @@ -36,9 +36,9 @@ class S3 { if (!endPoint.startsWith('http')) { endPoint = (sslEnabled ? 'https://' : 'http://') + endPoint; } - if (!/:\d{1,5}\/?$/.test(endPoint)) { - endPoint += ':9000'; - } + // if (!/:\d{1,5}\/?$/.test(endPoint)) { + // endPoint += ':9000'; + // } this.#S3Client = new S3Client({ forcePathStyle: true, @@ -107,90 +107,47 @@ class S3 { * @returns {Promise} number of files deleted */ async deleteUser (username) { - return new Promise((resolve, reject) => { - const DELETE_GROUP_SIZE = 100; - const objectVersions = []; - let numRequested = 0; let numResolved = 0; let isReceiveComplete = false; - - const removeObjectVersions = async () => { - let group; - try { - if (objectVersions.length > 0) { - group = objectVersions.slice(0); - objectVersions.length = 0; - numRequested += group.length; - const { Errors } = await this.#S3Client.send(new DeleteObjectsCommand({ Bucket: username, Delete: { Objects: group } })); - numResolved += group.length; - if (Errors?.length > 0) { - getLogger().error('errors deleting object versions:', YAML.stringify(Errors)); - } - } - } catch (err) { - if (err.name === 'NoSuchBucket') { - resolve(numResolved); - } else if (err.name === 'NotImplemented') { // OpenIO - getLogger().warning('while deleting object versions: ' + err); - for (const objectVersion of group) { - const { Errors } = await this.#S3Client.send(new DeleteObjectCommand({ Bucket: username, Key: objectVersion.Key, VersionId: objectVersion.VersionId })); - if (Errors?.length > 0) { - getLogger().error('errors deleting object version:', YAML.stringify(Errors)); - } - ++numResolved; - } - } else { - reject(Object.assign(new Error('while deleting object versions: ' + err), { cause: err })); - } - } - try { - if (isReceiveComplete && numResolved === numRequested) { - // will fail if any object versions remain - await this.#S3Client.send(new DeleteBucketCommand({ Bucket: username })); - resolve(numResolved); - } - } catch (err) { - if (err.name === 'NoSuchBucket') { - resolve(numResolved); - } else { - reject(new Error('while deleting bucket: ' + err)); + await new Promise((resolve, reject) => { + const bucketName = username.toLowerCase(); + + const deleteItems = async items => { + for (const item of items) { + try { + /* const { DeleteMarker } = */ await this.#S3Client.send(new DeleteObjectCommand({ Bucket: bucketName, Key: item.Key, VersionId: item.VersionId })); + // console.log(`deleted ${item.Key} ${DeleteMarker}`); + } catch (err) { + console.warn('while deleting', bucketName, item.Key, item.VersionID); } } }; - const removeObjectVersionsAndBucket = async err => { + const pageObjectVersions = async (KeyMarker) => { try { - isReceiveComplete = true; - await removeObjectVersions(); - if (err) { - reject(err); - } - } catch (err2) { - reject(err || err2); - } - }; + const { Versions, DeleteMarkers, IsTruncated, NextKeyMarker } = await this.#S3Client.send(new ListObjectVersionsCommand({ Bucket: bucketName, ...(KeyMarker ? { KeyMarker } : null) })); - let keyMarker = null; - const pageVersions = async () => { - try { - const { Versions, IsTruncated, NextKeyMarker } = await this.#S3Client.send(new ListObjectVersionsCommand({ Bucket: username, KeyMarker: keyMarker /*, MaxKeys: DELETE_GROUP_SIZE */ })); - keyMarker = NextKeyMarker; - objectVersions.push(...Versions); - isReceiveComplete = !IsTruncated; - if (objectVersions.length >= DELETE_GROUP_SIZE || !IsTruncated) { - await removeObjectVersions(); + if (typeof Versions?.[Symbol.iterator] === 'function') { + await deleteItems(Versions); + } + if (typeof DeleteMarkers?.[Symbol.iterator] === 'function') { + await deleteItems(DeleteMarkers); } + if (IsTruncated) { - return pageVersions(); + return pageObjectVersions(NextKeyMarker).catch(reject); + } else { + await this.#S3Client.send(new DeleteBucketCommand({ Bucket: username })); + resolve(); } } catch (err) { if (err.name === 'NoSuchBucket') { - resolve(numResolved); + resolve(); } else { - return removeObjectVersionsAndBucket(err); + reject(err); } } }; - pageVersions(); + pageObjectVersions(undefined).catch(reject); }); } diff --git a/notes/S3 streaming store.md b/notes/S3 streaming store.md index a91d212a..4e901c49 100644 --- a/notes/S3 streaming store.md +++ b/notes/S3 streaming store.md @@ -4,7 +4,11 @@ Streaming Stores can only be used with the modular server. You should be able to connect to any S3-compatible service that supports versioning. Tested services include: -Tested implementations: +Tested working implementations: + +* AWS S3 + +Tested working with caveats * OpenIO @@ -13,7 +17,7 @@ Incompatible implementations: * min.io (both self-hosted and cloud) -Configure the store by passing to the constructor the endpoint (host name, and port if not 9000), access key (admin user name) and secret key (password). (If you don't pass any arguments, S3 will use the public account on `play.min.io`, where the files can be **read, altered and deleted** by anyone in the world.) If you're using a AWS and a region other than `us-east-1`, include that as a fourth argument. You can provide these however you like, but typically they are stored in these environment variables: +Configure the store by passing to the constructor the endpoint (host name, and port if not 9000), access key (admin user name) and secret key (password). (If you don't pass any arguments, S3 will use the public account on `play.min.io`, where the files can be **read, altered and deleted** by anyone in the world. It's also incompatible.) If you're using a AWS and a region other than `us-east-1`, include that as a fourth argument. You can provide these however you like, but typically they are stored in these environment variables: * S3_ENDPOINT * S3_ACCESS_KEY diff --git a/package-lock.json b/package-lock.json index a112a237..d4adf5d3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,6 +21,7 @@ "lockfile": "^1.0.4", "mkdirp": "^1.0.4", "morgan": "^1.10.0", + "node-mocks-http": "^1.14.1", "pug": "^3.0.2", "winston": "^3.11.0", "yaml": "^2.4.0" @@ -1738,33 +1739,110 @@ "node": ">=14.0.0" } }, + "node_modules/@types/body-parser": { + "version": "1.19.5", + "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.5.tgz", + "integrity": "sha512-fB3Zu92ucau0iQ0JMCFQE7b/dv8Ot07NI3KaZIkIUNXq82k4eBAqUaneXfleGY9JWskeS9y+u0nXMyspcuQrCg==", + "dependencies": { + "@types/connect": "*", + "@types/node": "*" + } + }, "node_modules/@types/chai": { "version": "4.1.6", "resolved": "https://registry.npmjs.org/@types/chai/-/chai-4.1.6.tgz", "integrity": "sha512-CBk7KTZt3FhPsEkYioG6kuCIpWISw+YI8o+3op4+NXwTpvAPxE1ES8+PY8zfaK2L98b1z5oq03UHa4VYpeUxnw==", "dev": true }, + "node_modules/@types/connect": { + "version": "3.4.38", + "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", + "integrity": "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/cookiejar": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@types/cookiejar/-/cookiejar-2.1.5.tgz", "integrity": "sha512-he+DHOWReW0nghN24E1WUqM0efK4kI9oTqDm6XmK8ZPe2djZ90BSNdGnIyCLzCPw7/pogPlGbzI2wHGGmi4O/Q==", "dev": true }, + "node_modules/@types/express": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.21.tgz", + "integrity": "sha512-ejlPM315qwLpaQlQDTjPdsUFSc6ZsP4AN6AlWnogPjQ7CVi7PYF3YVz+CY3jE2pwYf7E/7HlDAN0rV2GxTG0HQ==", + "dependencies": { + "@types/body-parser": "*", + "@types/express-serve-static-core": "^4.17.33", + "@types/qs": "*", + "@types/serve-static": "*" + } + }, + "node_modules/@types/express-serve-static-core": { + "version": "4.17.43", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.43.tgz", + "integrity": "sha512-oaYtiBirUOPQGSWNGPWnzyAFJ0BP3cwvN4oWZQY+zUBwpVIGsKUkpBpSztp74drYcjavs7SKFZ4DX1V2QeN8rg==", + "dependencies": { + "@types/node": "*", + "@types/qs": "*", + "@types/range-parser": "*", + "@types/send": "*" + } + }, + "node_modules/@types/http-errors": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.4.tgz", + "integrity": "sha512-D0CFMMtydbJAegzOyHjtiKPLlvnm3iTZyZRSZoLq2mRhDdmLfIWOCYPfQJ4cu2erKghU++QvjcUjp/5h7hESpA==" + }, "node_modules/@types/json5": { "version": "0.0.29", "resolved": "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz", "integrity": "sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==", "dev": true }, + "node_modules/@types/mime": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", + "integrity": "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==" + }, "node_modules/@types/node": { "version": "20.11.19", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.11.19.tgz", "integrity": "sha512-7xMnVEcZFu0DikYjWOlRq7NTPETrm7teqUT2WkQjrTIkEgUyyGdWsj/Zg8bEJt5TNklzbPD1X3fqfsHw3SpapQ==", - "dev": true, "dependencies": { "undici-types": "~5.26.4" } }, + "node_modules/@types/qs": { + "version": "6.9.12", + "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.12.tgz", + "integrity": "sha512-bZcOkJ6uWrL0Qb2NAWKa7TBU+mJHPzhx9jjLL1KHF+XpzEcR7EXHvjbHlGtR/IsP1vyPrehuS6XqkmaePy//mg==" + }, + "node_modules/@types/range-parser": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.7.tgz", + "integrity": "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==" + }, + "node_modules/@types/send": { + "version": "0.17.4", + "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.4.tgz", + "integrity": "sha512-x2EM6TJOybec7c52BX0ZspPodMsQUd5L6PRwOunVyVUhXiBSKf3AezDL8Dgvgt5o0UfKNfuA0eMLr2wLT4AiBA==", + "dependencies": { + "@types/mime": "^1", + "@types/node": "*" + } + }, + "node_modules/@types/serve-static": { + "version": "1.15.5", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.5.tgz", + "integrity": "sha512-PDRk21MnK70hja/YF8AHfC7yIsiQHn1rcXx7ijCFBX/k+XQJhQT/gw3xekXKJvx+5SXaMMS8oqQy09Mzvz2TuQ==", + "dependencies": { + "@types/http-errors": "*", + "@types/mime": "*", + "@types/node": "*" + } + }, "node_modules/@types/superagent": { "version": "4.1.13", "resolved": "https://registry.npmjs.org/@types/superagent/-/superagent-4.1.13.tgz", @@ -5069,6 +5147,47 @@ "node": ">= 0.6" } }, + "node_modules/node-mocks-http": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/node-mocks-http/-/node-mocks-http-1.14.1.tgz", + "integrity": "sha512-mfXuCGonz0A7uG1FEjnypjm34xegeN5+HI6xeGhYKecfgaZhjsmYoLE9LEFmT+53G1n8IuagPZmVnEL/xNsFaA==", + "dependencies": { + "@types/express": "^4.17.21", + "@types/node": "^20.10.6", + "accepts": "^1.3.7", + "content-disposition": "^0.5.3", + "depd": "^1.1.0", + "fresh": "^0.5.2", + "merge-descriptors": "^1.0.1", + "methods": "^1.1.2", + "mime": "^1.3.4", + "parseurl": "^1.3.3", + "range-parser": "^1.2.0", + "type-is": "^1.6.18" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/node-mocks-http/node_modules/depd": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", + "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/node-mocks-http/node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/nodemon": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-3.0.3.tgz", @@ -6383,8 +6502,7 @@ "node_modules/undici-types": { "version": "5.26.5", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", - "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", - "dev": true + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" }, "node_modules/unpipe": { "version": "1.0.0", @@ -8068,33 +8186,110 @@ "tslib": "^2.5.0" } }, + "@types/body-parser": { + "version": "1.19.5", + "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.5.tgz", + "integrity": "sha512-fB3Zu92ucau0iQ0JMCFQE7b/dv8Ot07NI3KaZIkIUNXq82k4eBAqUaneXfleGY9JWskeS9y+u0nXMyspcuQrCg==", + "requires": { + "@types/connect": "*", + "@types/node": "*" + } + }, "@types/chai": { "version": "4.1.6", "resolved": "https://registry.npmjs.org/@types/chai/-/chai-4.1.6.tgz", "integrity": "sha512-CBk7KTZt3FhPsEkYioG6kuCIpWISw+YI8o+3op4+NXwTpvAPxE1ES8+PY8zfaK2L98b1z5oq03UHa4VYpeUxnw==", "dev": true }, + "@types/connect": { + "version": "3.4.38", + "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", + "integrity": "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==", + "requires": { + "@types/node": "*" + } + }, "@types/cookiejar": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@types/cookiejar/-/cookiejar-2.1.5.tgz", "integrity": "sha512-he+DHOWReW0nghN24E1WUqM0efK4kI9oTqDm6XmK8ZPe2djZ90BSNdGnIyCLzCPw7/pogPlGbzI2wHGGmi4O/Q==", "dev": true }, + "@types/express": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.21.tgz", + "integrity": "sha512-ejlPM315qwLpaQlQDTjPdsUFSc6ZsP4AN6AlWnogPjQ7CVi7PYF3YVz+CY3jE2pwYf7E/7HlDAN0rV2GxTG0HQ==", + "requires": { + "@types/body-parser": "*", + "@types/express-serve-static-core": "^4.17.33", + "@types/qs": "*", + "@types/serve-static": "*" + } + }, + "@types/express-serve-static-core": { + "version": "4.17.43", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.43.tgz", + "integrity": "sha512-oaYtiBirUOPQGSWNGPWnzyAFJ0BP3cwvN4oWZQY+zUBwpVIGsKUkpBpSztp74drYcjavs7SKFZ4DX1V2QeN8rg==", + "requires": { + "@types/node": "*", + "@types/qs": "*", + "@types/range-parser": "*", + "@types/send": "*" + } + }, + "@types/http-errors": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.4.tgz", + "integrity": "sha512-D0CFMMtydbJAegzOyHjtiKPLlvnm3iTZyZRSZoLq2mRhDdmLfIWOCYPfQJ4cu2erKghU++QvjcUjp/5h7hESpA==" + }, "@types/json5": { "version": "0.0.29", "resolved": "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz", "integrity": "sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==", "dev": true }, + "@types/mime": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", + "integrity": "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==" + }, "@types/node": { "version": "20.11.19", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.11.19.tgz", "integrity": "sha512-7xMnVEcZFu0DikYjWOlRq7NTPETrm7teqUT2WkQjrTIkEgUyyGdWsj/Zg8bEJt5TNklzbPD1X3fqfsHw3SpapQ==", - "dev": true, "requires": { "undici-types": "~5.26.4" } }, + "@types/qs": { + "version": "6.9.12", + "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.12.tgz", + "integrity": "sha512-bZcOkJ6uWrL0Qb2NAWKa7TBU+mJHPzhx9jjLL1KHF+XpzEcR7EXHvjbHlGtR/IsP1vyPrehuS6XqkmaePy//mg==" + }, + "@types/range-parser": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.7.tgz", + "integrity": "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==" + }, + "@types/send": { + "version": "0.17.4", + "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.4.tgz", + "integrity": "sha512-x2EM6TJOybec7c52BX0ZspPodMsQUd5L6PRwOunVyVUhXiBSKf3AezDL8Dgvgt5o0UfKNfuA0eMLr2wLT4AiBA==", + "requires": { + "@types/mime": "^1", + "@types/node": "*" + } + }, + "@types/serve-static": { + "version": "1.15.5", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.5.tgz", + "integrity": "sha512-PDRk21MnK70hja/YF8AHfC7yIsiQHn1rcXx7ijCFBX/k+XQJhQT/gw3xekXKJvx+5SXaMMS8oqQy09Mzvz2TuQ==", + "requires": { + "@types/http-errors": "*", + "@types/mime": "*", + "@types/node": "*" + } + }, "@types/superagent": { "version": "4.1.13", "resolved": "https://registry.npmjs.org/@types/superagent/-/superagent-4.1.13.tgz", @@ -10491,6 +10686,37 @@ "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==" }, + "node-mocks-http": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/node-mocks-http/-/node-mocks-http-1.14.1.tgz", + "integrity": "sha512-mfXuCGonz0A7uG1FEjnypjm34xegeN5+HI6xeGhYKecfgaZhjsmYoLE9LEFmT+53G1n8IuagPZmVnEL/xNsFaA==", + "requires": { + "@types/express": "^4.17.21", + "@types/node": "^20.10.6", + "accepts": "^1.3.7", + "content-disposition": "^0.5.3", + "depd": "^1.1.0", + "fresh": "^0.5.2", + "merge-descriptors": "^1.0.1", + "methods": "^1.1.2", + "mime": "^1.3.4", + "parseurl": "^1.3.3", + "range-parser": "^1.2.0", + "type-is": "^1.6.18" + }, + "dependencies": { + "depd": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", + "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==" + }, + "mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==" + } + } + }, "nodemon": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-3.0.3.tgz", @@ -11482,8 +11708,7 @@ "undici-types": { "version": "5.26.5", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", - "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", - "dev": true + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" }, "unpipe": { "version": "1.0.0", diff --git a/package.json b/package.json index 3c1c0826..c7f66253 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "lockfile": "^1.0.4", "mkdirp": "^1.0.4", "morgan": "^1.10.0", + "node-mocks-http": "^1.14.1", "pug": "^3.0.2", "winston": "^3.11.0", "yaml": "^2.4.0" diff --git a/spec/oauth.spec.js b/spec/oauth.spec.js index 06f7ec4a..9400d78f 100644 --- a/spec/oauth.spec.js +++ b/spec/oauth.spec.js @@ -115,7 +115,7 @@ exports.shouldImplementOAuth = function () { it('redirects with an access token', async function () { const res = await post(this.app, '/oauth', this.auth_params); - expect(res).to.redirectTo(/http:\/\/example.com\/cb#access_token=[\w-]+&token_type=bearer&state=the_state/); + expect(res).to.redirectTo(/http:\/\/example\.com\/cb#access_token=[\w-]+&token_type=bearer&state=the_state/); }); }); diff --git a/spec/streaming_handler.spec.js b/spec/streaming_handler.spec.js new file mode 100644 index 00000000..23d6ba2f --- /dev/null +++ b/spec/streaming_handler.spec.js @@ -0,0 +1,366 @@ +/* eslint-env mocha, chai, node */ +/* eslint-disable no-unused-expressions */ + +const chai = require('chai'); +const expect = chai.expect; +chai.use(require('chai-spies')); +chai.use(require('chai-as-promised')); +const httpMocks = require('node-mocks-http'); +const { Readable } = require('node:stream'); + +async function waitForEnd (response) { + return new Promise(resolve => { + setTimeout(checkEnd, 100); + function checkEnd () { + if (response._isEndCalled()) { + resolve(); + } else { + setTimeout(checkEnd, 100); + } + } + }); +} + +module.exports.shouldStoreStream = function () { + before(async function () { + this.timeout(10_000); + this.username = 'automated-test-' + Math.round(Math.random() * Number.MAX_SAFE_INTEGER); + // TODO: replace with user handler + await this.store.createUser({ username: this.username, email: 'l@m.no', password: '12345678' }); + }); + + after(async function () { + this.timeout(10_000); + await this.store.deleteUser(this.username); + }); + + describe('GET', function () { + describe('for files', function () { + describe('unversioned', function () { + it('returns null for a non-existing path', async function () { + const req = httpMocks.createRequest({ + method: 'GET', + url: `/${this.username}/non-existing/non-existing` + }); + const res = httpMocks.createResponse({ req }); + const next = chai.spy(); + + await this.handler(req, res, next); + await waitForEnd(res); + + expect(res.statusCode).to.equal(404); + expect(res._getData()).to.equal(''); + expect(Boolean(res.get('Content-Length'))).to.be.false; + expect(Boolean(res.get('Content-Type'))).to.be.false; + expect(Boolean(res.get('ETag'))).to.be.false; + expect(next).not.to.have.been.called; + }); + + it('returns null for a non-existing path in an existing category', async function () { + const content = 'filename'; + const [putStatus] = await this.store.put(this.username, '/existing/document', 'text/cache-manifest', content.length, Readable.from([content], { objectMode: false }), null); + expect(putStatus).to.equal('CREATED'); + + const req = httpMocks.createRequest({ + method: 'GET', + url: `/${this.username}/existing/not-existing` + }); + const res = httpMocks.createResponse({ req }); + const next = chai.spy(); + + await this.handler(req, res, next); + await waitForEnd(res); + + expect(res.statusCode).to.equal(404); + expect(res._getData()).to.equal(''); + expect(Boolean(res.get('Content-Length'))).to.be.false; + expect(Boolean(res.get('Content-Type'))).to.be.false; + expect(Boolean(res.get('ETag'))).to.be.false; + expect(next).not.to.have.been.called; + }); + }); + + describe('versioned', function () { + it('should return file for If-None-Match with old ETag', async function () { + const content = 'VEVENT'; + const [putStatus, putETag] = await this.store.put(this.username, '/existing/file', 'text/calendar', content.length, Readable.from([content], { objectMode: false }), null); + expect(putStatus).to.equal('CREATED'); + + const req = httpMocks.createRequest({ + method: 'GET', + url: `/${this.username}/existing/file` + }); + const res = httpMocks.createResponse({ req, eventEmitter: require('events').EventEmitter }); + const next = chai.spy(); + + await this.handler(req, res, next); + await waitForEnd(res); + + expect(res.statusCode).to.equal(200); + expect(res._getBuffer().toString()).to.equal(content); + expect(res.get('Content-Length')).to.equal(String(content.length)); + expect(res.get('Content-Type')).to.equal('text/calendar'); + expect(res.get('ETag')).to.equal(putETag); + expect(next).not.to.have.been.called; + }); + + it('should return Not Modified for If-None-Match with matching ETag', async function () { + const content = 'VEVENT'; + const [putStatus, putETag] = await this.store.put(this.username, '/existing/thing', 'text/plain', content.length, Readable.from([content], { objectMode: false }), null); + expect(putStatus).to.equal('CREATED'); + + const req = httpMocks.createRequest({ + method: 'GET', + url: `/${this.username}/existing/thing`, + headers: { 'If-None-Match': putETag } + }); + const res = httpMocks.createResponse({ req }); + const next = chai.spy(); + + await this.handler(req, res, next); + await waitForEnd(res); + + expect(res.statusCode).to.equal(304); + expect(res._getData()).to.equal(''); + expect(Boolean(res.get('Content-Length'))).to.be.false; + expect(Boolean(res.get('Content-Type'))).to.be.false; + expect(Boolean(res.get('ETag'))).to.be.false; + expect(next).not.to.have.been.called; + }); + + it.skip('should return file for If-Match with matching ETag', async function () { + const content = 'VEVENT'; + const [putStatus, putETag] = await this.store.put(this.username, '/existing/novel', 'text/calendar', content.length, Readable.from([content], { objectMode: false }), null); + expect(putStatus).to.equal('CREATED'); + + const { status, readStream, contentType, contentLength, ETag } = + await this.store.get(this.username, '/existing/novel', { name: 'If-Match', ETag: putETag }); + expect(status).to.equal(200); + const retrievedContent = (await readStream.setEncoding('utf-8').toArray())[0]; + expect(retrievedContent).to.be.deep.equal(content); + expect(contentType).to.equal('text/calendar'); + expect(contentLength).to.equal(content.length); + expect(ETag).to.equal(putETag); + }); + + it.skip('should return Precondition Failed for If-Match with mismatched ETag', async function () { + const content = 'VEVENT'; + const [putStatus] = await this.store.put(this.username, '/existing/short-story', 'text/plain', content.length, Readable.from([content], { objectMode: false }), null); + expect(putStatus).to.equal('CREATED'); + + const { status, readStream } = + await this.store.get(this.username, '/existing/short-story', { name: 'If-Match', ETag: '"l6jl546jl453"' }); + expect(status).to.equal(412); + expect(Boolean(readStream)).to.equal(false); + }); + }); + }); + }); + + describe('DELETE', function () { + describe('unversioned', function () { + it('should return Not Found for nonexistent user', async function () { + const req = httpMocks.createRequest({ + method: 'DELETE', + url: '/not-a-user/some-category/some-directory/some-thing' + }); + const res = httpMocks.createResponse({ req }); + const next = chai.spy(); + + await this.handler(req, res, next); + + await waitForEnd(res); + + expect(next).not.to.have.been.called; + expect(res.statusCode).to.equal(404); + expect(res._getData()).to.equal(''); + expect(Boolean(res.get('ETag'))).to.be.false; + }); + + it('should return Not Found for nonexistent path', async function () { + const req = httpMocks.createRequest({ + method: 'DELETE', + url: `/${this.username}/non-existent-category/non-existent-directory/non-existent-thing` + }); + const res = httpMocks.createResponse({ req }); + const next = chai.spy(); + + this.handler(req, res, next); + await waitForEnd(res); + + expect(next).not.to.have.been.called; + expect(res.statusCode).to.equal(404); + expect(res._getData()).to.equal(''); + expect(Boolean(res.get('ETag'))).to.be.false; + }); + + it('should return Conflict when path is a directory', async function () { + this.timeout(10_000); + const content = 'pad thai'; + const [result, ETag] = await this.store.put(this.username, '/consumables/food/thai/noodles/pad-thai', 'text/vnd.q', content.length, Readable.from([content], { objectMode: false }), null); + expect(result).to.equal('CREATED'); + expect(ETag).to.match(/^".{6,128}"$/); + + const req = httpMocks.createRequest({ + method: 'DELETE', + url: `/${this.username}/consumables/food` + }); + const res = httpMocks.createResponse({ req }); + const next = chai.spy(); + + this.handler(req, res, next); + await waitForEnd(res); + + expect(next).not.to.have.been.called; + expect(res.statusCode).to.equal(409); + expect(res._getData()).to.equal(''); + expect(Boolean(res.get('ETag'))).to.be.false; + }); + + it('should remove a file, empty parent directories, and remove directory entries', async function () { + this.timeout(10_000); + const content1 = 'wombat'; + const [result1, ETag1] = await this.store.put(this.username, '/animal/vertebrate/australia/marsupial/wombat', 'text/vnd.latex-z', content1.length, Readable.from([content1], { objectMode: false }), null); + expect(result1).to.equal('CREATED'); + expect(ETag1).to.match(/^".{6,128}"$/); + + const content2 = 'Alpine Ibex'; + const [result2, ETag2] = await this.store.put(this.username, '/animal/vertebrate/europe/Capra ibex', 'text/vnd.abc', content2.length, Readable.from([content2], { objectMode: false }), null); + expect(result2).to.equal('CREATED'); + expect(ETag2).to.match(/^".{6,128}"$/); + + const req = httpMocks.createRequest({ + method: 'DELETE', + url: `/${this.username}/animal/vertebrate/australia/marsupial/wombat` + }); + const res = httpMocks.createResponse({ req }); + const next = chai.spy(); + + this.handler(req, res, next); + await waitForEnd(res); + expect(res.statusCode).to.equal(204); // No Content + expect(res._getData()).to.equal(''); + expect(res.get('ETag')).to.equal(ETag1); + expect(next).not.to.have.been.called; + + const { status: status1, readStream: readStream1 } = await this.store.get(this.username, '/animal/vertebrate/australia/marsupial/wombat', null); + expect(status1).to.equal(404); + expect(Boolean(readStream1)).to.be.false; + + const { status: status2, readStream: readStream2 } = await this.store.get(this.username, '/animal/vertebrate/australia/marsupial/', null); + expect(status2).to.equal(404); + expect(Boolean(readStream2)).to.be.false; + + const { status: status3, readStream: readStream3 } = await this.store.get(this.username, '/animal/vertebrate/australia/', null); + expect(status3).to.equal(404); + expect(Boolean(readStream3)).to.be.false; + + const { status: status4, readStream: readStream4, contentType, ETag: directoryETag } = await this.store.get(this.username, '/animal/vertebrate/', null); + expect(status4).to.equal(200); + expect(contentType).to.equal('application/ld+json'); + expect(directoryETag).to.match(/^".{6,128}"$/); + const directory = JSON.parse((await readStream4.setEncoding('utf-8').toArray())[0]); + expect(directory['@context']).to.equal('http://remotestorage.io/spec/folder-description'); + expect(directory.items['europe/'].ETag).to.match(/^".{6,128}"$/); + }); + }); + + describe('versioned', function () { + it('should delete a blob if the If-Match header is equal', async function () { + this.timeout(5_000); + const content = 'elbow'; + const [result, ETag] = await this.store.put(this.username, '/deleting/if-match/equal', 'text/vnd.r', content.length, Readable.from([content], { objectMode: false }), null); + expect(result).to.equal('CREATED'); + expect(ETag).to.match(/^".{6,128}"$/); + + const req = httpMocks.createRequest({ + method: 'DELETE', + url: `/${this.username}/deleting/if-match/equal`, + headers: { 'If-Match': ETag } + }); + const res = httpMocks.createResponse({ req }); + const next = chai.spy(); + + this.handler(req, res, next); + await waitForEnd(res); + + expect(res.statusCode).to.equal(204); + expect(res._getData()).to.equal(''); + expect(res.get('ETag')).to.equal(ETag); + expect(next).not.to.have.been.called; + }); + + it('should not delete a blob if the If-Match header isn\'t equal', async function () { + this.timeout(5_000); + const content = 'elbow'; + const [result, ETag] = await this.store.put(this.username, '/deleting/if-match/not-equal', 'text/vnd.r', content.length, Readable.from([content], { objectMode: false }), null); + expect(result).to.equal('CREATED'); + expect(ETag).to.match(/^".{6,128}"$/); + + const req = httpMocks.createRequest({ + method: 'DELETE', + url: `/${this.username}/deleting/if-match/not-equal`, + headers: { 'If-Match': '"6a6a6a6a6a6a6"' } + }); + const res = httpMocks.createResponse({ req }); + const next = chai.spy(); + + this.handler(req, res, next); + await waitForEnd(res); + + expect(res.statusCode).to.equal(412); + expect(res._getData()).to.equal(''); + expect(Boolean(res.get('ETag'))).to.be.false; + expect(next).not.to.have.been.called; + }); + + it('should not delete a blob if the If-None-Match header is equal', async function () { + this.timeout(5_000); + const content = 'elbow'; + const [result, ETag] = await this.store.put(this.username, '/deleting/if-none-match/equal', 'text/vnd.r', content.length, Readable.from([content], { objectMode: false }), null); + expect(result).to.equal('CREATED'); + expect(ETag).to.match(/^".{6,128}"$/); + + const req = httpMocks.createRequest({ + method: 'DELETE', + url: `/${this.username}/deleting/if-none-match/equal`, + headers: { 'If-None-Match': ETag } + }); + const res = httpMocks.createResponse({ req }); + const next = chai.spy(); + + this.handler(req, res, next); + await waitForEnd(res); + + expect(res.statusCode).to.equal(412); + expect(res._getData()).to.equal(''); + expect(Boolean(res.get('ETag'))).to.be.false; + expect(next).not.to.have.been.called; + }); + + it('should delete a blob if the If-None-Match header is not equal', async function () { + this.timeout(5_000); + const content = 'elbow'; + const [result, ETag] = await this.store.put(this.username, '/deleting/if-none-match/not-equal', 'text/vnd.r', content.length, Readable.from([content], { objectMode: false }), null); + expect(result).to.equal('CREATED'); + expect(ETag).to.match(/^".{6,128}"$/); + + const req = httpMocks.createRequest({ + method: 'DELETE', + url: `/${this.username}/deleting/if-none-match/not-equal`, + headers: { 'If-None-Match': '"4l54jl5hio452"' } + }); + const res = httpMocks.createResponse({ req }); + const next = chai.spy(); + + this.handler(req, res, next); + await waitForEnd(res); + + expect(res.statusCode).to.equal(204); + expect(res._getData()).to.equal(''); + expect(res.get('ETag')).to.equal(ETag); + expect(next).not.to.have.been.called; + }); + }); + }); +}; diff --git a/spec/streaming_handler/S3Handler.spec.js b/spec/streaming_handler/S3Handler.spec.js new file mode 100644 index 00000000..cb50031b --- /dev/null +++ b/spec/streaming_handler/S3Handler.spec.js @@ -0,0 +1,19 @@ +// If a environment variables aren't set, tests are run using a shared public account on play.min.io +/* eslint-env mocha, chai, node */ +/* eslint-disable no-unused-expressions */ + +const s3handler = require('../../lib/routes/S3Handler'); +const { shouldStoreStream } = require('../streaming_handler.spec'); +const { configureLogger } = require('../../lib/logger'); +const S3 = require('../../lib/streaming_stores/S3'); + +describe('S3 streaming handler', function () { + before(function () { + configureLogger({ stdout: [], log_dir: './test-log', log_files: ['debug'] }); + // If the environment variables aren't set, tests are run using a shared public account on play.min.io + this.store = new S3(process.env.S3_ENDPOINT, process.env.S3_ACCESS_KEY, process.env.S3_SECRET_KEY); + this.handler = s3handler(process.env.S3_ENDPOINT, process.env.S3_ACCESS_KEY, process.env.S3_SECRET_KEY); + }); + + shouldStoreStream(); +});