Add CritPt benchmark #4557
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CPU tests | |
| on: | |
| pull_request: | |
| branches: [ "main" ] | |
| # Allows you to run this workflow manually from the Actions tab | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| unit-tests: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v5 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to GHCR | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GHCR_PAT }} | |
| - name: Free up disk space on Ubuntu | |
| run: | | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /opt/ghc | |
| sudo rm -rf /usr/local/share/boost | |
| sudo rm -rf "$AGENT_TOOLSDIRECTORY" | |
| - name: Set up Python 3.10 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.10" | |
| cache: pip | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -e .[dev] --extra-index-url https://download.pytorch.org/whl/cpu | |
| # Clear pip cache | |
| pip cache purge || true | |
| - name: Build Images | |
| run: | | |
| # Calculate image tags that match what Python code expects | |
| # Python generates: locally-built-{sanitized_path}:{sha256_hash[:12]} | |
| REPO_LOWER=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]') | |
| # Build nemo-skills-image with expected tag | |
| NEMO_SKILLS_HASH=$(sha256sum dockerfiles/Dockerfile.nemo-skills | cut -d' ' -f1 | cut -c1-12) | |
| NEMO_SKILLS_TAG="locally-built-dockerfiles-dockerfile-nemo-skills:${NEMO_SKILLS_HASH}" | |
| docker buildx build \ | |
| --tag nemo-skills-image \ | |
| --tag ${NEMO_SKILLS_TAG} \ | |
| --file dockerfiles/Dockerfile.nemo-skills \ | |
| --cache-from type=registry,ref=ghcr.io/${REPO_LOWER}/nemo-skills-image:cache \ | |
| --cache-to type=registry,ref=ghcr.io/${REPO_LOWER}/nemo-skills-image:cache,mode=min \ | |
| --load \ | |
| . | |
| # Free up build cache before building the next image. | |
| # buildx --load exports a tarball then imports layers, so both | |
| # exist on disk at once. Pruning the builder cache between builds | |
| # reclaims enough space for the sandbox image to load. | |
| docker builder prune -f | |
| # Build sandbox-image with expected tag | |
| SANDBOX_HASH=$(sha256sum dockerfiles/Dockerfile.sandbox | cut -d' ' -f1 | cut -c1-12) | |
| SANDBOX_TAG="locally-built-dockerfiles-dockerfile-sandbox:${SANDBOX_HASH}" | |
| docker buildx build \ | |
| --tag nemo-skills-sandbox-image \ | |
| --tag ${SANDBOX_TAG} \ | |
| --file dockerfiles/Dockerfile.sandbox \ | |
| --build-arg GITHUB_CI=1 \ | |
| --cache-from type=registry,ref=ghcr.io/${REPO_LOWER}/nemo-skills-sandbox-image:cache \ | |
| --cache-to type=registry,ref=ghcr.io/${REPO_LOWER}/nemo-skills-sandbox-image:cache,mode=min \ | |
| --load \ | |
| . | |
| - name: Run all tests | |
| env: | |
| NV_INFERENCE_API_KEY: ${{ secrets.NV_INFERENCE_API_KEY }} | |
| NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| run: | | |
| # Default shared runtime directory | |
| sudo mkdir -p /nemo_run | |
| sudo chmod 777 /nemo_run | |
| docker run --rm --network=host -v /nemo_run:/nemo_run nemo-skills-sandbox-image & | |
| sleep 10 | |
| set -o pipefail # this will make sure next line returns non-0 exit code if tests fail | |
| ns prepare_data gsm8k math-500 hle | |
| python -m pytest tests/ -m "not gpu" --junitxml=pytest.xml --cov-report=term-missing:skip-covered --cov=nemo_skills --cov=pipeline --durations=30 -rs -s -vvv |