Merge pull request #209 from data-douser/cds-compile-retry

CDS extractor : Implement retry for CDS compilation tasks

Author: data-douser

.github/workflows/run-codeql-unit-tests-javascript.yml

@@ -77,48 +77,29 @@ jobs:
         run: |
           qlt query run install-packs
 
-      - name: Ensure presence of cds shell command
-        run: |
-          if ! command -v cds &> /dev/null
-          then
-            ## Workaround for https://github.tools.sap/cap/issues/issues/17840
-            npm install -g @sap/[email protected]
-          fi
+      - name: Setup Node.js for CDS compilation
+        uses: actions/setup-node@v4
+        with:
+          node-version: '18'
+          cache: 'npm'
+          cache-dependency-path: 'extractors/cds/tools/package-lock.json'
 
-      # Compile .cds files to .cds.json files.
+      - name: Verify Node.js and npm tools
+        run: |
+          echo "Node.js version: $(node --version)"
+          echo "npm version: $(npm --version)"
+          echo "npx version: $(npx --version)"
+          # Verify npx can access @sap/cds-dk without installing globally
+          echo "Testing npx access to @sap/cds-dk..."
+          npx --yes --package @sap/cds-dk@latest cds --version || echo "CDS will be installed per-project as needed"
+
+      # Compile .cds files to .cds.json files using the dedicated test script
       - name: Compile CAP CDS files
         run: |
-          for test_dir in $(find . -type f -name '*.expected' -exec dirname {} \;);
-          do
-            # The CDS compiler produces locations relative to the working directory
-            # so we switch to the test directory before running the compiler.
-            pushd $test_dir
-            for cds_file in $(find . -type f \( -iname '*.cds' \) -print)
-            do
-              echo "I am compiling $cds_file"
-              _out_path="${cds_file}.json"
-              cds compile $cds_file \
-                --locations \
-                --to json \
-                --dest "$_out_path" \
-                2> "$cds_file.err"
-              # Check if the output is a regular file or a (sub)directory, where
-              # files generated in an output directory will need to have the file
-              # extension changed from '.json' to '.cds.json', but we don't need
-              # to rename anything if the cds compiler just generated a single
-              # '.cds.json' file.
-              if [ -d "$_out_path" ]
-              then
-                for json_file in $(find "$_out_path" -type f \( -iname '*.json' \) -print)
-                do
-                  _new_path="${json_file%.json}.cds.json"
-                  echo "Renaming CDS compiler generated JSON file $json_file to $_new_path"
-                  mv "$json_file" "$_new_path"
-                done
-              fi
-            done
-            popd
-          done
+          # Use the dedicated CDS compilation script that includes proper version resolution
+          # This script follows the same logic as the CDS extractor's resolveCdsVersions function
+          chmod +x extractors/cds/tools/test/cds-compilation-for-actions.test.sh
+          ./extractors/cds/tools/test/cds-compilation-for-actions.test.sh
 
       - name: Run test suites
         id: run-test-suites

extractors/cds/tools/README.md

@@ -63,6 +63,8 @@ node dist/cds-extractor.js /path/to/source/root
 
 ## Development
 
+> **⚠️ IMPORTANT NOTE**: Any changes to the CDS extractor's compilation task behavior (including how and where `cds compile` commands are executed, project detection logic, or output file generation patterns) **MUST** be reflected in the `extractors/cds/tools/test/cds-compilation-for-actions.test.sh` script. The `.github/workflows/run-codeql-unit-tests-javascript.yml` workflow executes this script during the "Compile CAP CDS files" step to simulate the CDS extractor's compilation process for unit tests. If the script and extractor implementations diverge, the `CodeQL - Run Unit Tests (javascript)` workflow will fail on PRs, causing status check failures. Always review and update the test script when modifying compilation behavior to maintain consistency between local testing and CI/CD environments.
+
 ### Project Structure
 
 ```text

extractors/cds/tools/cds-extractor.ts

@@ -17,7 +17,7 @@ import {
   logPerformanceTrackingStop,
   setSourceRootDirectory,
 } from './src/logging';
-import { installDependencies } from './src/packageManager';
+import { cacheInstallDependencies } from './src/packageManager';
 import { validateArguments } from './src/utils';
 
 // Validate the script arguments.
@@ -99,7 +99,7 @@ try {
     for (const [projectDir, project] of dependencyGraph.projects.entries()) {
       cdsExtractorLog(
         'info',
-        `Project: ${projectDir}, Status: ${project.status}, CDS files: ${project.cdsFiles.length}, Compilations to run: ${project.cdsFilesToCompile.length}`,
+        `Project: ${projectDir}, Status: ${project.status}, CDS files: ${project.cdsFiles.length}, Compilation targets: ${project.compilationTargets.length}`,
       );
     }
   } else {
@@ -151,7 +151,7 @@ try {
 }
 
 logPerformanceTrackingStart('Dependency Installation');
-const projectCacheDirMap = installDependencies(dependencyGraph, sourceRoot, codeqlExePath);
+const projectCacheDirMap = cacheInstallDependencies(dependencyGraph, sourceRoot, codeqlExePath);
 logPerformanceTrackingStop('Dependency Installation');
 
 // Check if dependency installation resulted in any usable project mappings