Add scenario benchmarks and fix ops/sec benchmarks on v1

.gitignore

@@ -16,3 +16,5 @@ isolate-*
 v8*.log
 *.cpuprofile
 .cache.json
+benchmarks/scenarios/results
+benchmarks/scenarios/projects.local.json

benchmarks/helpers.ts

@@ -0,0 +1,48 @@
+import type { BsConfig, ProgramBuilder } from '../src';
+import type { TargetOptions } from './target-runner';
+
+/**
+ * Create a ProgramBuilder that doesn't print diagnostics (they still get collected).
+ *
+ * This used to be done with `diagnosticFilters: ['**\/*']`, but in v1 a string filter is a diagnostic
+ * code, not a glob, so it filtered nothing. A `{ files: '**\/*' }` filter would be worse - v1 skips
+ * scope validation for completely filtered files, so the validate benchmarks would measure nothing
+ */
+export function createBuilder(options: TargetOptions): ProgramBuilder {
+    const builder = new options.brighterscript.ProgramBuilder();
+    (builder as any).printDiagnostics = () => { };
+    return builder;
+}
+
+/**
+ * The standard config for loading the benchmark project without emitting anything
+ */
+export function getConfig(options: TargetOptions, projectPath = options.projectPath): BsConfig {
+    return {
+        cwd: projectPath,
+        createPackage: false,
+        copyToStaging: false,
+        noEmit: true,
+        logLevel: 'error',
+        ...options.additionalConfig
+    } as BsConfig & Record<string, any>;
+}
+
+/**
+ * Get the files that can be transpiled, and a function that transpiles one of them.
+ * v1 moved BrsFile transpiling out of the file and into the program (`getTranspiledFileContents`),
+ * older versions only have `file.transpile()`
+ */
+export function getTranspiler(builder: ProgramBuilder) {
+    const program = builder.program as any;
+    const useProgram = typeof program.getTranspiledFileContents === 'function';
+    return {
+        canTranspile: (file: any) => useProgram || typeof file.transpile === 'function',
+        transpile: (file: any) => {
+            if (useProgram) {
+                return program.getTranspiledFileContents(file.srcPath);
+            }
+            return file.transpile();
+        }
+    };
+}

benchmarks/index.ts

@@ -7,12 +7,13 @@ const tempDir = path.join(cwd, '.tmp');
 let MAX_OLD_SPACE = 8192;
 
 clean();
-execSync('npm install');
+//installing here would prune the brighterscriptN packages from the last run, which is exactly what --noprepare wants to reuse
+if (!(process.argv.includes('--noprepare') || process.argv.includes('--noinstall')) || !fs.existsSync(path.join(cwd, 'node_modules'))) {
+    execSync('npm install');
+}
 
 import * as fsExtra from 'fs-extra';
 import * as yargs from 'yargs';
-import * as rimraf from 'rimraf';
-import * as fastGlob from 'fast-glob';
 
 interface RunnerOptions {
     versions: string[];
@@ -131,10 +132,6 @@ class Runner {
             return curr.length > acc ? curr.length : acc;
         }, 0);
 
-        if (this.options.profile) {
-            console.log('Deleting previous profile runs\n');
-            rimraf.sync(path.join(__dirname, 'isolate-*'), { glob: true });
-        }
 
         //run one target at a time
         for (const target of this.options.targets) {
@@ -144,26 +141,19 @@ class Runner {
                 process.stdout.write(`Benchmarking ${target}@${version}`);
                 const alias = `brighterscript${versionIndex + 1}`;
 
-                //get the list of current profiler logs
-                const beforeLogs = fastGlob.sync('isolate-*.log', {
-                    cwd: cwd
-                });
-
                 execSync(`npx ts-node target-runner.ts "${version}" "${maxVersionLength}" "${target}" "${maxTargetLength}" "${alias}" "${this.options.project}" "${this.options.quick}" "${this.options.profile}" "${(this.options.config ?? '{}').replaceAll('\"', '\\"')}"`, {
                     env: {
                         ...process.env,
                         'NODE_OPTIONS': `--max-old-space-size=${MAX_OLD_SPACE}`
                     }
                 });
-                if (this.options.profile) {
-                    const logFile = fastGlob.sync('isolate-*.log', {
-                        cwd: cwd
-                    }).filter(x => !beforeLogs.includes(x))[0];
-                }
             }
             //print a newline to separate the targets
             console.log('');
         }
+        if (this.options.profile) {
+            console.log(`benchmark: cpu profiles written to ${path.join(tempDir, 'profiles')}`);
+        }
     }
 }