feat(4): add test runner

.freeCodeCamp/plugin/index.js

@@ -34,14 +34,14 @@ import { logover } from '../tooling/logger.js';
  * @typedef {Object} Lesson
  * @property {{watch?: string[]; ignore?: string[]} | undefined} meta
  * @property {string} description
- * @property {[[string, string]]} tests
+ * @property {Array<{ text: string; runner: string; code: string; }>} tests
  * @property {string[]} hints
  * @property {[{filePath: string; fileSeed: string} | string]} seed
  * @property {boolean?} isForce
- * @property {string?} beforeAll
- * @property {string?} afterAll
- * @property {string?} beforeEach
- * @property {string?} afterEach
+ * @property {{ runner: string; code: string; } | null} beforeAll
+ * @property {{ runner: string; code: string; } | null} afterAll
+ * @property {{ runner: string; code: string; } | null} beforeEach
+ * @property {{ runner: string; code: string; } | null} afterEach
  */
 
 export const pluginEvents = {
@@ -141,10 +141,9 @@ export const pluginEvents = {
     const { afterAll, afterEach, beforeAll, beforeEach, isForce, meta } =
       lesson;
     const description = parseMarkdown(lesson.description).trim();
-    const tests = lesson.tests.map(([testText, test]) => [
-      parseMarkdown(testText).trim(),
-      test
-    ]);
+    const tests = lesson.tests.map(t => {
+      return { ...t, text: parseMarkdown(t.text).trim() };
+    });
     const hints = lesson.hints.map(h => parseMarkdown(h).trim());
     return {
       meta,

.freeCodeCamp/tooling/parser.js

@@ -183,6 +183,7 @@ export class CoffeeDown {
    * Get first code block text from tokens
    *
    * Meant to be used with `getBeforeAll`, `getAfterAll`, `getBeforeEach`, and `getAfterEach`
+   * @returns {{ runner: string; code: string; } | null}
    */
   get code() {
     const callers = [
@@ -198,7 +199,33 @@ export class CoffeeDown {
         }`
       );
     }
-    return this.tokens.find(t => t.type === 'code')?.text;
+
+    for (const token of this.tokens) {
+      if (token.type === 'code') {
+        let runner = 'node';
+        switch (token.lang) {
+          case 'js':
+          case 'javascript':
+            runner = 'Node';
+            break;
+          case 'py':
+          case 'python':
+            runner = 'Python';
+            break;
+          default:
+            break;
+        }
+
+        const code = token.text;
+        const test = {
+          runner,
+          code
+        };
+        return test;
+      }
+    }
+
+    return null;
   }
 
   get seed() {
@@ -210,25 +237,46 @@ export class CoffeeDown {
     return seedToIterator(this.tokens);
   }
 
+  /**
+   * @returns {Array<{ text: string; runner: string; code: string; }>}
+   */
   get tests() {
     if (this.caller !== 'getTests') {
       throw new Error(
         `textsAndTests must be called on getTests. Called on ${this.caller}`
       );
     }
     const textTokens = [];
-    const testTokens = [];
+    const tests = [];
     for (const token of this.tokens) {
       if (token.type === 'paragraph') {
         textTokens.push(token);
       }
       if (token.type === 'code') {
-        testTokens.push(token);
+        let runner = 'node';
+        switch (token.lang) {
+          case 'js':
+          case 'javascript':
+            runner = 'Node';
+            break;
+          case 'py':
+          case 'python':
+            runner = 'Python';
+            break;
+          default:
+            break;
+        }
+
+        const code = token.text;
+        const test = {
+          runner,
+          code
+        };
+        tests.push(test);
       }
     }
     const texts = textTokens.map(t => t.text);
-    const tests = testTokens.map(t => t.text);
-    return texts.map((text, i) => [text, tests[i]]);
+    return texts.map((text, i) => ({ text, ...tests[i] }));
   }
 
   get hints() {
@@ -332,7 +380,7 @@ marked.use(
 );
 
 export function parseMarkdown(markdown) {
-  return marked.parse(markdown, { gfm: true });
+  return marked.parse(markdown, { gfm: true, async: false });
 }
 
 const TOKENS = [

.freeCodeCamp/tooling/tests/Python.js

@@ -0,0 +1,20 @@
+import { parentPort, workerData } from 'node:worker_threads';
+import { runPython } from './utils.js';
+
+const { beforeEach = '' } = workerData;
+
+parentPort.on('message', async ({ testCode, testId }) => {
+  let passed = false;
+  let error = null;
+  try {
+    const _out = await runPython(`
+${beforeEach?.code ?? ''}
+
+${testCode}
+`);
+    passed = true;
+  } catch (e) {
+    error = e;
+  }
+  parentPort.postMessage({ passed, testId, error });
+});

.freeCodeCamp/tooling/tests/main.js

@@ -17,6 +17,7 @@ import { join } from 'node:path';
 import { Worker } from 'node:worker_threads';
 import { pluginEvents } from '../../plugin/index.js';
 import { t } from '../t.js';
+import { runPython } from './utils.js';
 
 try {
   const plugins = freeCodeCampConfig.tooling?.plugins;
@@ -51,10 +52,35 @@ export async function runTests(ws, projectDashedName) {
     );
     const { beforeAll, beforeEach, afterAll, afterEach, hints, tests } = lesson;
 
+    const testers = [beforeAll, beforeEach, afterAll, afterEach, ...tests];
+    let firstRunner = testers.filter(t => t !== null).at(0).runner;
+
+    for (const tester of testers) {
+      if (!tester) {
+        continue;
+      }
+      const runner = tester.runner;
+
+      if (runner !== firstRunner) {
+        throw new Error(
+          `All tests and hooks must use the same runner. Found: ${runner}, expected: ${firstRunner}`
+        );
+      }
+    }
+
     if (beforeAll) {
       try {
         logover.debug('Starting: --before-all-- hook');
-        await eval(`(async () => {${beforeAll}})()`);
+        switch (beforeAll.runner) {
+          case 'Node':
+            await eval(`(async () => {${beforeAll.code}})()`);
+            break;
+          case 'Python':
+            await runPython(beforeAll.code);
+            break;
+          default:
+            throw new Error(`Unsupported runner: ${beforeAll.runner}`);
+        }
         logover.debug('Finished: --before-all-- hook');
       } catch (e) {
         logover.error('--before-all-- hook failed to run:');
@@ -63,10 +89,10 @@ export async function runTests(ws, projectDashedName) {
     }
     // toggleLoaderAnimation(ws);
 
-    testsState = tests.map((text, i) => {
+    testsState = tests.map((t, i) => {
       return {
         passed: false,
-        testText: text[0],
+        testText: t.text,
         testId: i,
         isLoading: !project.blockingTests
       };
@@ -80,7 +106,10 @@ export async function runTests(ws, projectDashedName) {
     // Create one worker for each test if non-blocking.
     // TODO: See if holding pool of workers is better.
     if (project.blockingTests) {
-      const worker = createWorker('blocking-worker', { beforeEach, project });
+      const worker = createWorker('blocking-worker', firstRunner, {
+        beforeEach,
+        project
+      });
       WORKER_POOL.push(worker);
 
       // When result is received back from worker, update the client state
@@ -104,20 +133,23 @@ export async function runTests(ws, projectDashedName) {
       });
 
       for (let i = 0; i < tests.length; i++) {
-        const [_text, testCode] = tests[i];
+        const { code } = tests[i];
         testsState[i].isLoading = true;
         updateTest(ws, testsState[i]);
 
-        worker.postMessage({ testCode, testId: i });
+        worker.postMessage({ testCode: code, testId: i });
       }
     } else {
       // Run tests in parallel, and in own worker threads
       for (let i = 0; i < tests.length; i++) {
-        const [_text, testCode] = tests[i];
+        const { code, runner } = tests[i];
         testsState[i].isLoading = true;
         updateTest(ws, testsState[i]);
 
-        const worker = createWorker(`worker-${i}`, { beforeEach, project });
+        const worker = createWorker(`worker-${i}`, runner, {
+          beforeEach,
+          project
+        });
         WORKER_POOL.push(worker);
 
         // When result is received back from worker, update the client state
@@ -141,7 +173,7 @@ export async function runTests(ws, projectDashedName) {
           });
         });
 
-        worker.postMessage({ testCode, testId: i });
+        worker.postMessage({ testCode: code, testId: i });
       }
     }
 
@@ -150,9 +182,7 @@ export async function runTests(ws, projectDashedName) {
       testsState[testId].isLoading = false;
       testsState[testId].passed = passed;
       if (error) {
-        if (error.type !== 'AssertionError') {
-          logover.error(`Test #${testId}:`, error);
-        }
+        logover.error(`Test #${testId}:`, error);
 
         if (error.message) {
           const assertionTranslation = await t(error.message, {});
@@ -217,7 +247,16 @@ async function checkTestsCallback({
     if (afterAll) {
       try {
         logover.debug('Starting: --after-all-- hook');
-        await eval(`(async () => {${afterAll}})()`);
+        switch (afterAll.runner) {
+          case 'Node':
+            await eval(`(async () => {${afterAll.code}})()`);
+            break;
+          case 'Python':
+            await runPython(afterAll.code);
+            break;
+          default:
+            throw new Error(`Unsupported runner: ${afterAll.runner}`);
+        }
         logover.debug('Finished: --after-all-- hook');
       } catch (e) {
         logover.error('--after-all-- hook failed to run:');
@@ -236,11 +275,11 @@ async function checkTestsCallback({
  * @param {number} param0.exitCode
  * @param {Array} param0.testsState
  * @param {number} param0.i
- * @param {string} param0.afterEach
+ * @param {{ runner: string; code: string;} | null} param0.afterEach
  * @param {object} param0.error
  * @param {object} param0.project
  * @param {Array} param0.hints
- * @param {string} param0.afterAll
+ * @param {{ runner: string; code: string;} | null} param0.afterAll
  * @param {number} param0.lessonNumber
  */
 async function handleWorkerExit({
@@ -280,8 +319,22 @@ async function handleWorkerExit({
     updateTests(ws, testsState);
   }
   // Run afterEach even if tests are cancelled
+  // TODO: Double-check this is not run twice
   try {
-    const _afterEachOut = await eval(`(async () => { ${afterEach} })();`);
+    if (afterEach) {
+      logover.debug('Starting: --after-each-- hook');
+      switch (afterEach.runner) {
+        case 'Node':
+          await eval(`(async () => {${afterEach.code}})()`);
+          break;
+        case 'Python':
+          await runPython(afterEach.code);
+          break;
+        default:
+          throw new Error(`Unsupported runner: ${afterEach.runner}`);
+      }
+      logover.debug('Finished: --after-each-- hook');
+    }
   } catch (e) {
     logover.error('--after-each-- hook failed to run:');
     logover.error(e);
@@ -297,13 +350,20 @@ async function handleWorkerExit({
   });
 }
 
-function createWorker(name, workerData) {
+/**
+ *
+ * @param {string} name unique name for the worker
+ * @param {string} runner runner to use
+ * @param {*} workerData
+ * @returns {Worker}
+ */
+function createWorker(name, runner, workerData) {
   return new Worker(
     join(
       ROOT,
       'node_modules/@freecodecamp/freecodecamp-os',
       '.freeCodeCamp/tooling/tests',
-      'test-worker.js'
+      `${runner}.js`
     ),
     {
       name,

.freeCodeCamp/tooling/tests/utils.js

@@ -0,0 +1,34 @@
+import { spawn } from 'node:child_process';
+
+/**
+ * Runs the provided Python code.
+ *
+ * If the python code raises an error, this function rejects with that error as a string.
+ * Otherwise, resolves with the python process stdout.
+ *
+ * @param {string} code
+ */
+export async function runPython(code) {
+  const child = spawn('python3', ['-c', code]);
+
+  let stdout = '';
+  let stderr = '';
+
+  child.stdout.on('data', data => {
+    process.stdout.write(data);
+    stdout += data;
+  });
+
+  child.stderr.on('data', data => {
+    stderr += data;
+  });
+
+  return new Promise((resolve, reject) => {
+    child.on('close', code => {
+      if (stderr) {
+        reject(stderr);
+      }
+      resolve(stdout);
+    });
+  });
+}