Skip to content

Commit f44a1ec

Browse files
tmacrotmacro
committed
impr(CLDSRV-766): Add skeleton
1 parent 460a47c commit f44a1ec

File tree

7 files changed

+332
-1
lines changed

7 files changed

+332
-1
lines changed

‎constants.js‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -250,6 +250,9 @@ const constants = {
250250
// if requester is not bucket owner, bucket policy actions should be denied with
251251
// MethodNotAllowed error
252252
onlyOwnerAllowed: ['bucketDeletePolicy', 'bucketGetPolicy', 'bucketPutPolicy'],
253+
// Default value for rate limiting config cache TTL
254+
rateLimitDefaultConfigCacheTTL: 30000,
255+
rateLimitDefaultBurstCapacity: 1,
253256
};
254257

255258
module.exports = constants;

‎lib/Config.js‎

Lines changed: 82 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ const crypto = require('crypto');
88
const { v4: uuidv4 } = require('uuid');
99
const cronParser = require('cron-parser');
1010
const joi = require('@hapi/joi');
11-
const { s3routes, auth: arsenalAuth, s3middleware } = require('arsenal');
11+
const { s3routes, auth: arsenalAuth, s3middleware, errors, ArsenalError } = require('arsenal');
1212
const { isValidBucketName } = s3routes.routesUtils;
1313
const validateAuthConfig = arsenalAuth.inMemory.validateAuthConfig;
1414
const { buildAuthDataAccount } = require('./auth/in_memory/builder');
@@ -32,6 +32,7 @@ const {
3232
isValidType,
3333
isValidProtocol,
3434
} = require('arsenal/build/lib/network/KMSInterface');
35+
const { parseRateLimitConfig } = require('./api/apiUtils/rateLimit/config');
3536

3637
// config paths
3738
const configSearchPaths = [
@@ -1803,6 +1804,86 @@ class Config extends EventEmitter {
18031804
if (config.enableVeeamRoute !== undefined && config.enableVeeamRoute !== null) {
18041805
this.enableVeeamRoute = config.enableVeeamRoute;
18051806
}
1807+
1808+
this.rateLimiting = {
1809+
enabled: false,
1810+
bucket: {
1811+
configCacheTTL: constants.rateLimitDefaultConfigCacheTTL,
1812+
},
1813+
};
1814+
1815+
// {
1816+
// "enabled": true,
1817+
// "bucket": {
1818+
// "defaultConfig": {
1819+
// "requestsPerSecond": {
1820+
// "limit": 1000,
1821+
// "burstCapacity": 2
1822+
// }
1823+
// },
1824+
// "configCacheTTL": 30000,
1825+
// },
1826+
// "account": { // not in first iteration
1827+
// "requestsPerSecond": {
1828+
// "limit": 1000,
1829+
// "burstCapacity": 2
1830+
// }
1831+
// },
1832+
// "configCacheTTL": 30000,
1833+
// },
1834+
// "error": {
1835+
// "code": 429,
1836+
// "description": "Too Many Requests"
1837+
// }
1838+
// }
1839+
1840+
1841+
if (config.rateLimiting?.enabled) {
1842+
this.rateLimiting.enabled = true;
1843+
1844+
assert.strictEqual(typeof config.rateLimiting.serviceUserArn, 'string');
1845+
this.rateLimiting.serviceUserArn = config.rateLimiting.serviceUserArn;
1846+
1847+
// this.rateLimiting.default = {};
1848+
if (config.rateLimiting.bucket) {
1849+
assert.strictEqual(
1850+
typeof config.rateLimiting.bucket, 'object',
1851+
'rateLimiting.bucket must be an object'
1852+
);
1853+
1854+
let defaultConfig = undefined;
1855+
if (config.rateLimiting.bucket.defaultConfig) {
1856+
assert.strictEqual(
1857+
typeof config.rateLimiting.bucket.defaultConfig, 'object',
1858+
'rateLimiting.bucket.defaultConfig must be an object'
1859+
);
1860+
defaultConfig = parseRateLimitConfig(config.rateLimiting.bucket);
1861+
}
1862+
1863+
let configCacheTTL = constants.rateLimitDefaultConfigCacheTTL;
1864+
if (config.rateLimiting.bucket.configCacheTTL) {
1865+
configCacheTTL = config.rateLimiting.bucket.configCacheTTL;
1866+
assert(
1867+
typeof configCacheTTL === 'number' && Number.isInteger(configCacheTTL) && configCacheTTL > 0,
1868+
'ratelimiting.bucket.configCacheTTL must be a postive integer'
1869+
);
1870+
}
1871+
1872+
this.rateLimiting.bucket = {
1873+
defaultConfig,
1874+
configCacheTTL,
1875+
};
1876+
}
1877+
1878+
if (config.error) {
1879+
assert.strictEqual(typeof config.rateLimiting.error, 'object', 'rateLimiting.error must be an object');
1880+
this.rateLimiting.error = new ArsenalError('SlowDown',
1881+
config.throttling.errorResponse.code, config.throttling.errorResponse.message);
1882+
} else {
1883+
this.rateLimiting.error = errors.SlowDown;
1884+
}
1885+
}
1886+
18061887
return config;
18071888
}
18081889

‎lib/api/apiUtils/authorization/serviceUser.js‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
const { timingSafeEqual } = require('crypto');
2+
3+
const { config } = require('../../../Config');
4+
5+
function isRateLimitServiceUser(authInfo) {
6+
try {
7+
return timingSafeEqual(Buffer.from(authInfo.getArn()), Buffer.from(config.rateLimiting.serviceUserArn));
8+
} catch {
9+
return false;
10+
}
11+
}
12+
13+
module.exports = {
14+
isRateLimitServiceUser
15+
};

‎lib/api/apiUtils/rateLimit/config.js‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
const assert = require('assert');
2+
3+
const { rateLimitDefaultBurstCapacity } = require('../../../../constants');
4+
5+
function parseRateLimitConfig(config) {
6+
const limitConfig = {};
7+
8+
if (config.requestsPerSecond) {
9+
assert.strictEqual(typeof config.requestsPerSecond, 'object');
10+
11+
const { limit } = config.requestsPerSecond;
12+
assert(typeof limit === 'number' && Number.isInteger(limit) && limit > 0);
13+
14+
let { burstCapacity } = config.requestsPerSecond;
15+
if (burstCapacity !== undefined) {
16+
assert(typeof burstCapacity === 'number' && Number.isInteger(burstCapacity) && burstCapacity > 0);
17+
} else {
18+
burstCapacity = rateLimitDefaultBurstCapacity;
19+
}
20+
21+
limitConfig.requestsPerSecond = {
22+
interval: Math.ceil((1 / limit) * 1000),
23+
bucketSize: burstCapacity * 1000,
24+
};
25+
}
26+
}
27+
28+
module.exports = {
29+
parseRateLimitConfig,
30+
};

‎lib/api/bucketDeleteRateLimit.js‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
const { errors } = require('arsenal');
2+
3+
const metadata = require('../metadata/wrapper');
4+
const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
5+
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
6+
const { isRateLimitServiceUser } = require('./apiUtils/authorization/serviceUser');
7+
8+
/**
9+
* bucketDeleteRateLimit - Delete the bucket rate limit configuration
10+
* @param {AuthInfo} authInfo - Instance of AuthInfo class with requester's info
11+
* @param {object} request - http request object
12+
* @param {object} log - Werelogs logger
13+
* @param {function} callback - callback to server
14+
* @return {undefined}
15+
*/
16+
function bucketDeleteRateLimit(authInfo, request, log, callback) {
17+
log.debug('processing request', { method: 'bucketDeleteRateLimit' });
18+
19+
if (!isRateLimitServiceUser(authInfo)) {
20+
return callback(errors.AccessDenied);
21+
}
22+
23+
const { bucketName, headers, method } = request;
24+
const metadataValParams = {
25+
authInfo,
26+
bucketName,
27+
requestType: request.apiMethods || 'bucketDeleteRateLimit',
28+
request,
29+
};
30+
return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => {
31+
const corsHeaders = collectCorsHeaders(headers.origin, method, bucket);
32+
if (err) {
33+
log.debug('error processing request', {
34+
error: err,
35+
method: 'bucketDeleteRateLimit',
36+
});
37+
return callback(err, corsHeaders);
38+
}
39+
if (!bucket.getRateLimitConfig()) {
40+
log.trace('no existing bucket rate limit configuration', {
41+
method: 'bucketDeleteRateLimit',
42+
});
43+
// TODO: implement Utapi metric support
44+
return callback(null, corsHeaders);
45+
}
46+
log.trace('deleting bucket rate limit configuration in metadata');
47+
bucket.setRateLimitConfig(null);
48+
return metadata.updateBucket(bucketName, bucket, log, err => {
49+
if (err) {
50+
return callback(err, corsHeaders);
51+
}
52+
// TODO: implement Utapi metric support
53+
return callback(null, corsHeaders);
54+
});
55+
});
56+
}
57+
58+
module.exports = bucketDeleteRateLimit;

‎lib/api/bucketGetRateLimit.js‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
const { errors } = require('arsenal');
2+
3+
const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
4+
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
5+
const { isRateLimitServiceUser } = require('./apiUtils/authorization/serviceUser');
6+
7+
/**
8+
* bucketGetRateLimit - Get the bucket rate limit config
9+
* @param {AuthInfo} authInfo - Instance of AuthInfo class with requester's info
10+
* @param {object} request - http request object
11+
* @param {object} log - Werelogs logger
12+
* @param {function} callback - callback to server
13+
* @return {undefined}
14+
*/
15+
function bucketGetRateLimit(authInfo, request, log, callback) {
16+
log.debug('processing request', { method: 'bucketGetRateLimit' });
17+
18+
if (!isRateLimitServiceUser(authInfo)) {
19+
return callback(errors.AccessDenied);
20+
}
21+
22+
const { bucketName, headers, method } = request;
23+
const metadataValParams = {
24+
authInfo,
25+
bucketName,
26+
requestType: request.apiMethods || 'bucketGetRateLimit',
27+
request,
28+
};
29+
30+
return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => {
31+
const corsHeaders = collectCorsHeaders(headers.origin, method, bucket);
32+
if (err) {
33+
log.debug('error processing request', {
34+
error: err,
35+
method: 'bucketGetRateLimit',
36+
});
37+
return callback(err, null, corsHeaders);
38+
}
39+
40+
const rateLimitConfig = bucket.getRateLimitConfig();
41+
if (!rateLimitConfig) {
42+
log.debug('error processing request', {
43+
error: errors.NoSuchBucketRateLimit,
44+
method: 'bucketGetRateLimit',
45+
});
46+
return callback(errors.NoSuchBucketRateLimit, null,
47+
corsHeaders);
48+
}
49+
50+
return callback(null, JSON.stringify(rateLimitConfig), corsHeaders);
51+
});
52+
}
53+
54+
module.exports = bucketGetRateLimit;

‎lib/api/bucketPutRateLimit.js‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
const async = require('async');
2+
const { parseString } = require('xml2js');
3+
const { errorInstances, errors } = require('arsenal');
4+
5+
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
6+
const metadata = require('../metadata/wrapper');
7+
const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
8+
const { isRateLimitServiceUser } = require('./apiUtils/authorization/serviceUser');
9+
10+
function parseRequestBody(requestBody, callback) {
11+
try {
12+
const jsonData = JSON.parse(requestBody);
13+
if (typeof jsonData !== 'object') {
14+
throw new Error('Invalid JSON');
15+
}
16+
return callback(null, jsonData);
17+
} catch {
18+
return parseString(requestBody, (xmlError, xmlData) => {
19+
if (xmlError) {
20+
return callback(errorInstances.InvalidArgument
21+
.customizeDescription('Request body must be a JSON object'));
22+
}
23+
return callback(null, xmlData);
24+
});
25+
}
26+
}
27+
28+
function validateRateLimitConfig(config, callback) {
29+
const limit = parseInt(config.RequestsPerSecond, 10);
30+
if (Number.isNaN(limit) || !Number.isInteger(limit) || limit <= 0) {
31+
return callback(errorInstances.InvalidArgument
32+
.customizeDescription('RequestsPerSecond must be a positive integer'));
33+
}
34+
return callback(null, {
35+
RequestsPerSecond: limit,
36+
});
37+
}
38+
39+
/**
40+
* bucketPutRateLimit - create or update a bucket policy
41+
* @param {AuthInfo} authInfo - Instance of AuthInfo class with requester's info
42+
* @param {object} request - http request object
43+
* @param {object} log - Werelogs logger
44+
* @param {function} callback - callback to server
45+
* @return {undefined}
46+
*/
47+
function bucketPutRateLimit(authInfo, request, log, callback) {
48+
log.debug('processing request', { method: 'bucketPutRateLimit' });
49+
50+
if (!isRateLimitServiceUser(authInfo)) {
51+
return callback(errors.AccessDenied);
52+
}
53+
54+
const { bucketName } = request;
55+
const metadataValParams = {
56+
authInfo,
57+
bucketName,
58+
requestType: request.apiMethods || 'bucketPutRateLimit',
59+
request,
60+
};
61+
62+
return async.waterfall([
63+
next => parseRequestBody(request.post, next),
64+
(requestBody, next) => validateRateLimitConfig(requestBody, next),
65+
(limitConfig, next) => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log,
66+
(err, bucket) => {
67+
if (err) {
68+
return next(err, bucket);
69+
}
70+
return next(null, bucket, limitConfig);
71+
}),
72+
(bucket, limitConfig, next) => {
73+
bucket.setRateLimitConfig(limitConfig);
74+
metadata.updateBucket(bucket.getName(), bucket, log,
75+
err => next(err, bucket));
76+
},
77+
], (err, bucket) => {
78+
const corsHeaders = collectCorsHeaders(request.headers.origin,
79+
request.method, bucket);
80+
if (err) {
81+
log.trace('error processing request',
82+
{ error: err, method: 'bucketPutRateLimit' });
83+
return callback(err, corsHeaders);
84+
}
85+
// TODO: implement Utapi metric support
86+
return callback(null, corsHeaders);
87+
});
88+
}
89+
90+
module.exports = bucketPutRateLimit;

0 commit comments

Comments
 (0)