Skip to content
CI

Modify CI to build when version is incremented; increment to v1.0.1 #517

Sign in to view logs

Modify CI to build when version is incremented; increment to v1.0.1

Modify CI to build when version is incremented; increment to v1.0.1 #517

Workflow file for this run

.github/workflows/CI.yml at 898498e
name: CI
on:
push:
branches:
- main
- master
tags:
- '*'
pull_request:
branches:
- main
- master
workflow_dispatch:
permissions:
contents: read
env:
DOLMA_TESTS_SKIP_AWS: ${{ secrets.AWS_ACCESS_KEY_ID == '' && 'true' || 'false' }}
DOLMA_TEST_S3_PREFIX: s3://dolma-tests
RUST_CHANNEL: stable
jobs:
info:
name: Run info
runs-on: ubuntu-latest
steps:
- name: Echo environment variables
run: |
echo "reference: ${{ github.ref }}"
echo "event name: ${{ github.event_name }}"
echo "run tests: ${{ github.event_name == 'pull_request' || github.event_name == 'push' }}"
echo "is main: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master' }}"
echo "is release: ${{ startsWith(github.ref, 'refs/tags/') }}"
echo "skip AWS: ${{ env.DOLMA_TESTS_SKIP_AWS }}"
echo "commit: ${{ github.sha }}"
echo "PR base repo: ${{ github.event.pull_request.base.repo.full_name }}/tree/${{ github.event.pull_request.base.ref }}"
echo "PR head repo: ${{ github.event.pull_request.head.repo.full_name }}/tree/${{ github.event.pull_request.head.ref }}"
new_version:
name: "New Version"
runs-on: ubuntu-latest
outputs:
should_run: ${{ steps.check.outputs.should_run }}
steps:
- name: checkout
uses: actions/checkout@v3
- name: List branches and tags
run: |
git branch -a
git tag -l
git log | head -n 1000
- id: check_version
run: |
has_updated=$(git diff --name-only '${{ github.event.pull_request.base.sha }}' | grep -E 'pyproject.toml|Cargo.toml')
if [[ -n $has_updated ]]; then
echo "should_run=true" >> $GITHUB_ENV
else
echo "should_run=false" >> $GITHUB_ENV
fi
shell: bash
temp:
name: temp
needs: new_version
runs-on: ubuntu-latest
steps:
- name: echo
run: |
echo "should_run: ${{ needs.new_version.outputs.should_run }}"
echo "should_run: ${{ steps.new_version.outputs.should_run }}"
# prepare-venv:
# runs-on: ubuntu-latest
# name: "Prepare Virtual Env"
# steps:
# - name: Checkout repository
# uses: actions/checkout@v3
# - name: Cache Virtual Env
# uses: actions/cache@v3
# # name for referring later
# id: cache-venv
# with:
# # what we cache: the virtualenv
# path: ./.venv/
# # The cache key depends on pyproject.toml and Cargo.toml
# key: ${{ runner.os }}-venv-${{ hashFiles('**/pyproject.toml', '**/Cargo.toml, **/Cargo.lock') }}--${{ hashFiles('python/**', 'src/**') }}
# - name: Setup system libraries
# if: steps.cache-venv.outputs.cache-hit != 'true'
# run: |
# sudo apt-get update
# sudo apt-get install --yes --upgrade build-essential cmake protobuf-compiler libssl-dev glibc-source musl-tools
# - name: Install Rust toolchain
# if: steps.cache-venv.outputs.cache-hit != 'true'
# run: |
# rustup update ${{ env.RUST_CHANNEL }}
# rustup component add --toolchain ${{ env.RUST_CHANNEL }} rustfmt rust-src
# rustup default ${{ env.RUST_CHANNEL }}
# - name: Install Python
# if: steps.cache-venv.outputs.cache-hit != 'true'
# uses: actions/setup-python@v4
# with:
# python-version: '3.8'
# architecture: "x64"
# - name: Create a new Python environment & install maturin
# if: steps.cache-venv.outputs.cache-hit != 'true'
# run: |
# python -m venv .venv
# source .venv/bin/activate
# pip install -U pip
# pip install maturin
# - name: Install dolma wheels
# if: steps.cache-venv.outputs.cache-hit != 'true'
# run: |
# source .venv/bin/activate
# maturin build --release -i $(which python) --out dist
# wheel_path=$(ls dist/*.whl)
# pip install "${wheel_path}[all]"
# tests:
# runs-on: ubuntu-latest
# needs: prepare-venv
# env:
# AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
# AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
# if: ${{ github.event_name == 'pull_request' || github.event_name == 'push' }}
# name: "${{ matrix.task.name }}"
# strategy:
# fail-fast: true
# matrix:
# task:
# - name: Check Python style
# run: |
# set -e
# isort --check --verbose .
# black --check --verbose .
# - name: Check Rust style
# run: |
# rustfmt --edition 2021 src/*.rs --check
# - name: Lint Python
# run: |
# flake8 tests/python/ && flake8 python/
# - name: Types Python
# run: |
# mypy tests/python/ && mypy python/
# - name: Run Python tests
# run: |
# pytest -vs --color=yes tests/python/
# steps:
# - name: Checkout repository
# uses: actions/checkout@v3
# - name: Cache Virtual Env
# uses: actions/cache@v3
# # name for referring later
# id: cache-venv
# with:
# # what we cache: the virtualenv
# path: ./.venv/
# # The cache key depends on pyproject.toml and Cargo.toml
# key: ${{ runner.os }}-venv-${{ hashFiles('**/pyproject.toml', '**/Cargo.toml, **/Cargo.lock') }}--${{ hashFiles('python/**', 'src/**') }}
# - name: ${{ matrix.task.name }}
# run: |
# source .venv/bin/activate
# ${{ matrix.task.run }}
# build-linux:
# if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/') || contains(github.event.modified_files, 'pyproject.toml') || contains(github.event.modified_files, 'Cargo.toml') }}
# runs-on: ubuntu-latest
# strategy:
# matrix:
# # fails to build on x86, so removing for now; low use anyway.
# # target: [x86_64, x86, aarch64, armv7]
# target: [x86_64, aarch64, armv7]
# name: "Build Linux (${{ matrix.target }})"
# steps:
# - uses: actions/checkout@v3
# - uses: actions/setup-python@v4
# with:
# python-version: '3.10'
# - name: Install 32bit version of libc
# if: ${{ matrix.target == 'x86' || contains(matrix.target, 'i686') }}
# run: |
# sudo apt-get update
# sudo apt-get install --yes --upgrade libc6-dev-i386
# - name: Build wheels
# uses: PyO3/maturin-action@v1
# with:
# target: ${{ matrix.target }}
# args: --release --out dist --find-interpreter
# manylinux: manylinux_2_28
# container: "ghcr.io/rust-cross/manylinux_2_28-cross:${{ matrix.target }}"
# before-script-linux: |
# sudo apt-get update
# sudo apt-get install --yes --upgrade build-essential cmake protobuf-compiler libssl-dev glibc-source musl-tools
# - name: Upload wheels
# uses: actions/upload-artifact@v3
# with:
# name: wheels
# path: dist
# build-windows:
# if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/') || contains(github.event.modified_files, 'pyproject.toml') || contains(github.event.modified_files, 'Cargo.toml') }}
# runs-on: windows-latest
# name: "Build Windows (${{ matrix.target }})"
# strategy:
# matrix:
# target: [x64, x86]
# steps:
# - uses: actions/checkout@v3
# - uses: actions/setup-python@v4
# with:
# python-version: '3.10'
# architecture: ${{ matrix.target }}
# - name: Build wheels
# uses: PyO3/maturin-action@v1
# with:
# target: ${{ matrix.target }}
# args: --release --out dist --find-interpreter
# - name: Upload wheels
# uses: actions/upload-artifact@v3
# with:
# name: wheels
# path: dist
# build-macos:
# if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/') || contains(github.event.modified_files, 'pyproject.toml') || contains(github.event.modified_files, 'Cargo.toml') }}
# runs-on: macos-latest
# name: "Build macOS (${{ matrix.target }})"
# strategy:
# matrix:
# target: [x86_64, aarch64]
# steps:
# - uses: actions/checkout@v3
# - uses: actions/setup-python@v4
# with:
# python-version: '3.10'
# - name: Build wheels
# uses: PyO3/maturin-action@v1
# with:
# target: ${{ matrix.target }}
# args: --release --out dist --find-interpreter
# - name: Upload wheels
# uses: actions/upload-artifact@v3
# with:
# name: wheels
# path: dist
# sdist:
# runs-on: ubuntu-latest
# if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/') || contains(github.event.modified_files, 'pyproject.toml') || contains(github.event.modified_files, 'Cargo.toml') }}
# steps:
# - uses: actions/checkout@v3
# - name: Build sdist
# uses: PyO3/maturin-action@v1
# with:
# command: sdist
# args: --out dist
# - name: Upload sdist
# uses: actions/upload-artifact@v3
# with:
# name: wheels
# path: dist
# release:
# name: Release
# runs-on: ubuntu-latest
# if: "startsWith(github.ref, 'refs/tags/')"
# needs: [build-linux, build-windows, build-macos, sdist]
# steps:
# - uses: actions/download-artifact@v3
# with:
# name: wheels
# - name: Publish to PyPI
# uses: PyO3/maturin-action@v1
# env:
# MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
# with:
# command: upload
# args: --skip-existing *