From 71a634497952b87fe4556cf141999aba9ff07f62 Mon Sep 17 00:00:00 2001 From: Dina Yakovlev Date: Thu, 5 Jul 2018 11:18:48 +0300 Subject: [PATCH] Add the option to create request that run once before test starts --- core/lib/engine_http.js | 9 +- core/lib/runner.js | 280 +++++++++++--------- core/lib/schemas/artillery_test_script.json | 12 + test/core/scripts/before_test.json | 49 ++++ test/core/test_capture.js | 30 +++ test/core/test_reuse.js | 10 +- 6 files changed, 263 insertions(+), 127 deletions(-) create mode 100644 test/core/scripts/before_test.json diff --git a/core/lib/engine_http.js b/core/lib/engine_http.js index b52a989010..f99c455d36 100644 --- a/core/lib/engine_http.js +++ b/core/lib/engine_http.js @@ -436,17 +436,18 @@ HttpEngine.prototype.step = function step(requestSpec, ee, opts) { request(requestParams, maybeCallback) .on('request', function(req) { - debugRequests("request start: %s", req.path); + debugRequests('request start: %s', req.path); ee.emit('request'); const startedAt = process.hrtime(); req.on('response', function updateLatency(res) { let code = res.statusCode; + let path = res.req.method + ' ' + res.req.path; const endedAt = process.hrtime(startedAt); let delta = (endedAt[0] * 1e9) + endedAt[1]; - debugRequests("request end: %s", req.path); - ee.emit('response', delta, code, context._uid); + debugRequests('request end: %s', req.path); + ee.emit('response', delta, code, context._uid, path); }); }).on('end', function() { context._successCount++; @@ -458,7 +459,7 @@ HttpEngine.prototype.step = function step(requestSpec, ee, opts) { debug(err); // Run onError hooks and end the scenario - runOnErrorHooks(onErrorHandlers, config.processor, err, requestParams, context, ee, function(asyncErr) { + runOnErrorHooks(onErrorHandlers, config.processor, err, requestParams, context, ee, function() { let errCode = err.code || err.message; ee.emit('error', errCode); return callback(err, context); diff --git a/core/lib/runner.js b/core/lib/runner.js index 3a961d0359..ece1b6eaa2 100644 --- a/core/lib/runner.js +++ b/core/lib/runner.js @@ -1,6 +1,6 @@ /* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +* License, v. 2.0. If a copy of the MPL was not distributed with this +* file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 'use strict'; @@ -20,24 +20,26 @@ const engineUtil = require('./engine_util'); const wl = require('./weighted-pick'); const Engines = { - http: {}, - ws: {}, - socketio: {} + http: {}, + ws: {}, + socketio: {} }; +let contextVars; + JSCK.Draft4 = JSCK.draft4; const schema = new JSCK.Draft4(require('./schemas/artillery_test_script.json')); module.exports = { - runner: runner, - validate: validate, - stats: Stats + runner: runner, + validate: validate, + stats: Stats }; function validate(script) { - let validation = schema.validate(script); - return validation; + let validation = schema.validate(script); + return validation; } function runner(script, payload, options, callback) { @@ -120,6 +122,10 @@ function runner(script, payload, options, callback) { } ); + if (script.before) { + handleBeforeRequests(script, runnableScript, runnerEngines, ee); + } + // // load plugins: // @@ -244,72 +250,76 @@ function runner(script, payload, options, callback) { } function run(script, ee, options, runState) { - let intermediate = Stats.create(); - let aggregate = []; - - let phaser = createPhaser(script.config.phases); - phaser.on('arrival', function() { - runScenario(script, intermediate, runState); - }); - phaser.on('phaseStarted', function(spec) { - ee.emit('phaseStarted', spec); - }); - phaser.on('phaseCompleted', function(spec) { - ee.emit('phaseCompleted', spec); - }); - phaser.on('done', function() { - debug('All phases launched'); - - const doneYet = setInterval(function checkIfDone() { - if (runState.pendingScenarios === 0) { - if (runState.pendingRequests !== 0) { - debug('DONE. Pending requests: %s', runState.pendingRequests); + let intermediate = Stats.create(); + let aggregate = []; + + let phaser = createPhaser(script.config.phases); + phaser.on('arrival', function() { + if (runState.pendingRequests >= (process.env.CONCURRENCY_LIMIT || 250)) { + intermediate._scenariosAvoided++; + } else { + runScenario(script, intermediate, runState); } + }); + phaser.on('phaseStarted', function(spec) { + ee.emit('phaseStarted', spec); + }); + phaser.on('phaseCompleted', function(spec) { + ee.emit('phaseCompleted', spec); + }); + phaser.on('done', function() { + debug('All phases launched'); - clearInterval(doneYet); - clearInterval(periodicStatsTimer); + const doneYet = setInterval(function checkIfDone() { + if (runState.pendingScenarios === 0) { + if (runState.pendingRequests !== 0) { + debug('DONE. Pending requests: %s', runState.pendingRequests); + } - sendStats(); + clearInterval(doneYet); + clearInterval(periodicStatsTimer); - intermediate.free(); + sendStats(); - let aggregateReport = Stats.combine(aggregate).report(); - return ee.emit('done', aggregateReport); - } else { - debug('Pending requests: %s', runState.pendingRequests); - debug('Pending scenarios: %s', runState.pendingScenarios); - } - }, 500); - }); + intermediate.free(); + + let aggregateReport = Stats.combine(aggregate).report(); + return ee.emit('done', aggregateReport); + } else { + debug('Pending requests: %s', runState.pendingRequests); + debug('Pending scenarios: %s', runState.pendingScenarios); + } + }, 500); + }); - const periodicStatsTimer = setInterval(sendStats, options.periodicStats * 1000); + const periodicStatsTimer = setInterval(sendStats, options.periodicStats * 1000); - function sendStats() { - intermediate._concurrency = runState.pendingScenarios; - intermediate._pendingRequests = runState.pendingRequests; - ee.emit('stats', intermediate.clone()); - delete intermediate._entries; - aggregate.push(intermediate.clone()); - intermediate.reset(); - } + function sendStats() { + intermediate._concurrency = runState.pendingScenarios; + intermediate._pendingRequests = runState.pendingRequests; + ee.emit('stats', intermediate.clone()); + delete intermediate._entries; + aggregate.push(intermediate.clone()); + intermediate.reset(); + } - phaser.run(); + phaser.run(); } function runScenario(script, intermediate, runState) { - const start = process.hrtime(); - - // - // Compile scenarios if needed - // - if (!runState.compiledScenarios) { - _.each(script.scenarios, function(scenario) { - if (!scenario.weight) { - scenario.weight = 1; - } - }); + const start = process.hrtime(); + + // + // Compile scenarios if needed + // + if (!runState.compiledScenarios) { + _.each(script.scenarios, function(scenario) { + if (!scenario.weight) { + scenario.weight = 1; + } + }); - runState.picker = wl(script.scenarios); + runState.picker = wl(script.scenarios); runState.scenarioEvents = new EventEmitter(); runState.scenarioEvents.on('counter', function(name, value) { @@ -331,74 +341,83 @@ function runScenario(script, intermediate, runState) { runState.scenarioEvents.on('request', function() { intermediate.newRequest(); - runState.pendingRequests++; - }); - runState.scenarioEvents.on('match', function() { - intermediate.addMatch(); - }); - runState.scenarioEvents.on('response', function(delta, code, uid) { - intermediate.completedRequest(); - intermediate.addLatency(delta); - intermediate.addCode(code); + runState.pendingRequests++; + }); + runState.scenarioEvents.on('match', function() { + intermediate.addMatch(); + }); + runState.scenarioEvents.on('response', function(delta, code, uid) { + intermediate.completedRequest(); + intermediate.addLatency(delta); + intermediate.addCode(code); - let entry = [Date.now(), uid, delta, code]; - intermediate.addEntry(entry); + let entry = [Date.now(), uid, delta, code]; - runState.pendingRequests--; - }); + intermediate.addEntry(entry); - runState.compiledScenarios = _.map( - script.scenarios, - function(scenarioSpec) { - const name = scenarioSpec.engine || 'http'; - const engine = runState.engines.find((e) => e.__name === name); - return engine.createScenario(scenarioSpec, runState.scenarioEvents); - } - ); - } + runState.pendingRequests--; + }); + + runState.compiledScenarios = _.map( + script.scenarios, + function(scenarioSpec) { + const name = scenarioSpec.engine || 'http'; + const engine = runState.engines.find((e) => e.__name === name); + return engine.createScenario(scenarioSpec, runState.scenarioEvents); + } + ); + } - let i = runState.picker()[0]; + let i = runState.picker()[0]; - debug('picking scenario %s (%s) weight = %s', + debug('picking scenario %s (%s) weight = %s', i, script.scenarios[i].name, script.scenarios[i].weight); - intermediate.newScenario(script.scenarios[i].name || i); - - const scenarioStartedAt = process.hrtime(); - const scenarioContext = createContext(script); - const finish = process.hrtime(start); - const runScenarioDelta = (finish[0] * 1e9) + finish[1]; - debugPerf('runScenarioDelta: %s', Math.round(runScenarioDelta / 1e6 * 100) / 100); - runState.compiledScenarios[i](scenarioContext, function(err, context) { - runState.pendingScenarios--; - if (err) { - debug(err); - } else { - const scenarioFinishedAt = process.hrtime(scenarioStartedAt); - const delta = (scenarioFinishedAt[0] * 1e9) + scenarioFinishedAt[1]; - intermediate.addScenarioLatency(delta); - intermediate.completedScenario(); - } - }); + intermediate.newScenario(script.scenarios[i].name || i); + + const scenarioStartedAt = process.hrtime(); + const scenarioContext = createContext(script); + const finish = process.hrtime(start); + const runScenarioDelta = (finish[0] * 1e9) + finish[1]; + debugPerf('runScenarioDelta: %s', Math.round(runScenarioDelta / 1e6 * 100) / 100); + runState.compiledScenarios[i](scenarioContext, function(err, context) { + runState.pendingScenarios--; + if (err) { + debug(err); + } else { + const scenarioFinishedAt = process.hrtime(scenarioStartedAt); + const delta = (scenarioFinishedAt[0] * 1e9) + scenarioFinishedAt[1]; + intermediate.addScenarioLatency(delta); + intermediate.completedScenario(); + } + }); } /** - * Create initial context for a scenario. - */ +* Create initial context for a scenario. +*/ function createContext(script) { - const INITIAL_CONTEXT = { - vars: { - target: script.config.target, - $environment: script._environment - }, - funcs: { - $randomNumber: $randomNumber, - $randomString: $randomString - } + let initialContext = {}; + if (contextVars) { + initialContext = { + vars: contextVars + }; + } else { + initialContext = { + vars: { + target: script.config.target, + $environment: script._environment + } + }; + } + initialContext.funcs = { + $randomNumber: $randomNumber, + $randomString: $randomString }; - let result = _.cloneDeep(INITIAL_CONTEXT); + + let result = _.cloneDeep(initialContext); // // variables from payloads @@ -435,9 +454,30 @@ function createContext(script) { // Generator functions for template strings: // function $randomNumber(min, max) { - return _.random(min, max); + return _.random(min, max); } function $randomString(length) { - return Math.random().toString(36).substr(2, length); + return Math.random().toString(36).substr(2, length); +} + +function handleBeforeRequests(script, runnableScript, runnerEngines, testEvents) { + let ee = new EventEmitter(); + ee.on('request', function() { + testEvents.emit('beforeTestRequest'); + }); + ee.on('error', function(error) { + testEvents.emit('beforeTestError', error); + }); + let name = runnableScript.before.engine || 'http'; + let engine = runnerEngines.find((e) => e.__name === name); + let beforeTestScenario = engine.createScenario(runnableScript.before, ee); + let beforeTestContext = createContext(script); + beforeTestScenario(beforeTestContext, function(err, context) { + if (err) { + debug(err); + } else { + contextVars = context.vars; + } + }); } diff --git a/core/lib/schemas/artillery_test_script.json b/core/lib/schemas/artillery_test_script.json index ef0264f881..85a98c1472 100644 --- a/core/lib/schemas/artillery_test_script.json +++ b/core/lib/schemas/artillery_test_script.json @@ -31,6 +31,18 @@ } } }, + "before": { + "type": "object", + "properties": { + "flow": { + "type": "array", + "items": { + "type": "object" + } + } + }, + "required": ["flow"] + }, "scenarios": { "type": "array" } diff --git a/test/core/scripts/before_test.json b/test/core/scripts/before_test.json new file mode 100644 index 0000000000..40b9ce1b46 --- /dev/null +++ b/test/core/scripts/before_test.json @@ -0,0 +1,49 @@ +{ + "config": { + "target": "http://127.0.0.1:3003", + "phases": [ + { "duration": 10, "arrivalRate": 1 } + ], + "payload": { + "fields": ["species", "name"] + }, + "ensure": { + "p95": 300 + }, + "variables": { + "jsonPathExpr": ["$.id"] + } + }, + "before": { + "flow": [ + {"post": + { + "url": "/pets", + "json": {"name": "Guiness", "species": "Dog"}, + "capture": [{ + "json": "{{{ jsonPathExpr }}}", + "as": "id" + }, { + "json": "$.doesnotexist", + "transform": "this.doesnotexist.toUpperCase()", + "as": "doesnotexist" + }, { + "regexp": ".+", + "as": "id2" + }] + } + } + ] + }, + "scenarios": [ + { + "name": "Get the same pet in every scenario, as the pet was created once before the test started. ", + "flow": [ + {"get": { + "url": "/pets/{{ id }}", + "match": {"json": "$.name", "value": "Guiness"} + }} + ] + } + ] +} diff --git a/test/core/test_capture.js b/test/core/test_capture.js index be2b13897b..a11e6d030a 100644 --- a/test/core/test_capture.js +++ b/test/core/test_capture.js @@ -60,6 +60,36 @@ test('Capture - JSON', (t) => { }); }); +test('Capture before test - JSON', (t) => { + const fn = path.resolve(__dirname, './scripts/before_test.json'); + const script = require(fn); + const data = fs.readFileSync(path.join(__dirname, 'pets.csv')); + csv(data, function(err, parsedData) { + if (err) { + t.fail(err); + } + let beforeRequest = 0; + + runner(script, parsedData, {}).then(function(ee) { + ee.on('beforeTestRequest', function(){ + beforeRequest++; + }); + ee.on('done', function(report) { + let c200 = report.codes[200]; + let expectedAmountRequests = script.config.phases[0].duration * script.config.phases[0].arrivalRate; + t.assert(c200 === expectedAmountRequests, + 'There should be ' + expectedAmountRequests + ' requests'); + t.assert(report.matches === expectedAmountRequests, 'All requests should have the same match'); + t.assert(beforeRequest === 1, + 'There should be only one request before test starts'); + t.end(); + }); + + ee.run(); + }); + }); +}); + test('Capture - XML', (t) => { if (!xmlCapture) { console.log('artillery-xml-capture does not seem to be installed, skipping XML capture test.'); diff --git a/test/core/test_reuse.js b/test/core/test_reuse.js index 5f246cd219..3e4a276564 100644 --- a/test/core/test_reuse.js +++ b/test/core/test_reuse.js @@ -26,6 +26,9 @@ test('reuse', function(t) { } } expected *= weightedFlowLengths; + ee.on('beforeTestRequest', function(){ + t.assert('should preform before requests in the \'before requests\' test', script === 'before requests'); + }); ee.on('stats', function(stats) { intermediate.push(stats.report()); }); @@ -47,15 +50,16 @@ test('reuse', function(t) { report.latencies.length === expected ); if (first) { - let last = report.latencies.length - 1; + let lastIntermediate = intermediate.length - 1; + let last = intermediate[lastIntermediate].latencies.length - 1; first = false; - lastLatency = report.latencies[last][0]; + lastLatency = intermediate[lastIntermediate].latencies[last]; ee.run(); } else { t.assert( 'first latency of second aggregate should be after ' + 'the last latency of the first aggregate', - lastLatency <= report.latencies[0][0] + lastLatency <= intermediate[0].latencies[0] ); t.end(); }