diff --git a/.github/workflows/ci_examples_python.yml b/.github/workflows/ci_examples_python.yml new file mode 100644 index 000000000..6eb659ca0 --- /dev/null +++ b/.github/workflows/ci_examples_python.yml @@ -0,0 +1,98 @@ +# This workflow performs tests in Python. +name: Python Examples + +on: + workflow_call: + inputs: + dafny: + description: "The Dafny version to run" + required: true + type: string + regenerate-code: + description: "Regenerate code using smithy-dafny" + required: false + default: false + type: boolean + mpl-version: + description: "MPL version to use" + required: false + type: string + mpl-head: + description: "Running on MPL HEAD" + required: false + default: false + type: boolean + +jobs: + testPython: + strategy: + max-parallel: 1 + matrix: + python-version: [3.11, 3.12, 3.13] + os: [macos-13] + runs-on: ${{ matrix.os }} + permissions: + id-token: write + contents: read + steps: + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-west-2 + role-to-assume: arn:aws:iam::370957321024:role/GitHub-CI-DDBEC-Dafny-Role-us-west-2 + role-session-name: DDBEC-Dafny-Python-Tests + + - uses: actions/checkout@v3 + with: + submodules: recursive + + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Setup Python ${{ matrix.python-version }} for running tests + run: | + python -m pip install --upgrade pip + pip install --upgrade tox + pip install poetry + + - name: Setup Dafny + uses: ./submodules/MaterialProviders/.github/actions/setup_dafny/ + with: + dafny-version: ${{ inputs.dafny }} + + - name: Update MPL submodule if using MPL HEAD + if: ${{ inputs.mpl-head == true }} + working-directory: submodules/MaterialProviders + run: | + git checkout main + git pull + git submodule update --init --recursive + git rev-parse HEAD + + - name: Install Smithy-Dafny codegen dependencies + uses: ./.github/actions/install_smithy_dafny_codegen_dependencies + + - name: Regenerate code using smithy-dafny if necessary + if: ${{ inputs.regenerate-code }} + uses: ./.github/actions/polymorph_codegen + with: + dafny: ${{ env.DAFNY_VERSION }} + library: DynamoDbEncryption + diff-generated-code: false + update-and-regenerate-mpl: true + + - name: Build and locally deploy dependencies for examples + shell: bash + working-directory: ./DynamoDbEncryption + run: | + make transpile_python + + - name: Test DynamoDbEncryption Examples + working-directory: ./Examples/runtimes/python + run: | + # Run simple examples + tox -e dynamodbencryption + # Run migration examples + # tox -e migration diff --git a/.github/workflows/ci_static_analysis_python.yml b/.github/workflows/ci_static_analysis_python.yml new file mode 100644 index 000000000..2559227a6 --- /dev/null +++ b/.github/workflows/ci_static_analysis_python.yml @@ -0,0 +1,79 @@ +# This workflow performs static analysis in Python. +name: Python Static Analysis + +on: + workflow_call: + inputs: + regenerate-code: + description: "Regenerate code using smithy-dafny" + required: false + default: false + type: boolean + mpl-version: + description: "MPL version to use" + required: false + type: string + mpl-head: + description: "Running on MPL HEAD" + required: false + default: false + type: boolean + +jobs: + testPython: + strategy: + matrix: + python-version: [3.11] + os: [ubuntu-latest] + runs-on: ${{ matrix.os }} + permissions: + id-token: write + contents: read + steps: + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-west-2 + role-to-assume: arn:aws:iam::370957321024:role/GitHub-CI-DDBEC-Dafny-Role-us-west-2 + role-session-name: DDBEC-Dafny-Python-Tests + + - uses: actions/checkout@v3 + with: + submodules: recursive + + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Setup Python ${{ matrix.python-version }} for running tests + run: | + python -m pip install --upgrade pip + pip install --upgrade tox + pip install poetry + + - name: Update MPL submodule if using MPL HEAD + if: ${{ inputs.mpl-head == true }} + working-directory: submodules/MaterialProviders + run: | + git checkout main + git pull + git submodule update --init --recursive + git rev-parse HEAD + + - name: Install Smithy-Dafny codegen dependencies + uses: ./.github/actions/install_smithy_dafny_codegen_dependencies + + - name: Regenerate code using smithy-dafny if necessary + if: ${{ inputs.regenerate-code }} + uses: ./.github/actions/polymorph_codegen + with: + dafny: ${{ env.DAFNY_VERSION }} + library: DynamoDbEncryption + diff-generated-code: false + update-and-regenerate-mpl: true + + - name: Run static analysis + working-directory: ./DynamoDbEncryption/runtimes/python + run: | + tox -e lint-check diff --git a/.github/workflows/ci_test_python.yml b/.github/workflows/ci_test_python.yml new file mode 100644 index 000000000..51be31292 --- /dev/null +++ b/.github/workflows/ci_test_python.yml @@ -0,0 +1,131 @@ +# This workflow runs only Dafny-transpiled Python tests. +name: test python + +on: + workflow_call: + inputs: + dafny: + description: "The Dafny version to run" + required: true + type: string + regenerate-code: + description: "Regenerate code using smithy-dafny" + required: false + default: false + type: boolean + mpl-head: + description: "Running on MPL HEAD" + required: false + default: false + type: boolean + +jobs: + testPython: + strategy: + fail-fast: false + matrix: + library: [DynamoDbEncryption] + python-version: ["3.11", "3.12", "3.13"] + os: [ + macos-13, + ubuntu-22.04, + # Dafny-transpiled Python tests use a PYTHONPATH hack that doesn't work on Windows. + # Windows is tested with non-Dafny-transpiled Python tests. + # windows-latest + ] + runs-on: ${{ matrix.os }} + permissions: + id-token: write + contents: read + steps: + - name: Support longpaths on Git checkout + run: | + git config --global core.longpaths true + - uses: actions/checkout@v3 + with: + submodules: recursive + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Setup Python ${{ matrix.python-version }} for running tests + run: | + python -m pip install --upgrade pip + pip install --upgrade tox + pip install poetry + + - name: Setup Dafny + uses: ./submodules/MaterialProviders/.github/actions/setup_dafny/ + with: + dafny-version: ${{ inputs.dafny }} + + - name: Update MPL submodule if using MPL HEAD + if: ${{ inputs.mpl-head == true }} + working-directory: submodules/MaterialProviders + run: | + git checkout main + git pull + git submodule update --init --recursive + git rev-parse HEAD + + - name: Install Smithy-Dafny codegen dependencies + uses: ./.github/actions/install_smithy_dafny_codegen_dependencies + + - name: Regenerate code using smithy-dafny if necessary + if: ${{ inputs.regenerate-code }} + uses: ./.github/actions/polymorph_codegen + with: + dafny: ${{ env.DAFNY_VERSION }} + library: ${{ matrix.library }} + diff-generated-code: false + update-and-regenerate-mpl: true + + - name: Download Dependencies + working-directory: ./${{ matrix.library }} + run: make setup_python + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-west-2 + role-to-assume: arn:aws:iam::370957321024:role/GitHub-CI-DDBEC-Dafny-Role-us-west-2 + role-session-name: DDBEC-Dafny-Net-Tests + + - name: Compile ${{ matrix.library }} implementation + shell: bash + working-directory: ./${{ matrix.library }} + run: | + # This works because `node` is installed by default on GHA runners + CORES=$(node -e 'console.log(os.cpus().length)') + make transpile_python CORES=$CORES + + - name: Test ${{ matrix.library }} Dafny-transpiled Python tests + # Dafny-transpiled Python tests use a PYTHONPATH hack that doesn't work on Windows. + # Windows is tested with non-Dafny-transpiled Python tests. + if: ${{ matrix.os != 'windows-latest' }} + working-directory: ./${{ matrix.library }}/runtimes/python + shell: bash + run: | + tox -e dafnytests + + - name: Test ${{ matrix.library }} Python unit tests + working-directory: ./${{ matrix.library }}/runtimes/python + shell: bash + run: | + tox -e unit + + - name: Test ${{ matrix.library }} Python integration tests + working-directory: ./${{ matrix.library }}/runtimes/python + shell: bash + run: | + tox -e integ + + - name: Test ${{ matrix.library }} Python coverage + working-directory: ./${{ matrix.library }}/runtimes/python + shell: bash + run: | + tox -e encrypted-interface-coverage + tox -e client-to-resource-conversions-coverage + tox -e resource-to-client-conversions-coverage diff --git a/.github/workflows/ci_test_vector_python.yml b/.github/workflows/ci_test_vector_python.yml new file mode 100644 index 000000000..f525fdaec --- /dev/null +++ b/.github/workflows/ci_test_vector_python.yml @@ -0,0 +1,111 @@ +# This workflow performs test vectors in Python. +name: Library Python Test Vectors + +on: + workflow_call: + inputs: + dafny: + description: "The Dafny version to run" + required: true + type: string + regenerate-code: + description: "Regenerate code using smithy-dafny" + required: false + default: false + type: boolean + mpl-version: + description: "MPL version to use" + required: false + type: string + mpl-head: + description: "Running on MPL HEAD" + required: false + default: false + type: boolean + +jobs: + testPython: + strategy: + fail-fast: false + matrix: + library: [TestVectors] + python-version: [3.11, 3.12, 3.13] + # Only ubuntu for now; + # As of 4.10.1, Dafny's Python JSON processing is still very slow (1 hour to run test vectors on ubuntu) + # and Github's macOS runners are also very slow (~2x slower than ubuntu). + # If Dafny's JSON processing speed is improved, we can add macOS back. + os: [ubuntu-22.04] + interface: [client, resource, table] + runs-on: ${{ matrix.os }} + permissions: + id-token: write + contents: read + steps: + - name: Setup DynamoDB Local + uses: rrainn/dynamodb-action@v4.0.0 + with: + port: 8000 + cors: "*" + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-west-2 + role-to-assume: arn:aws:iam::370957321024:role/GitHub-CI-DDBEC-Dafny-Role-us-west-2 + role-session-name: DDBEC-Dafny-Python-Tests + role-duration-seconds: 7200 + + - uses: actions/checkout@v3 + with: + submodules: recursive + + - name: Setup Dafny + uses: ./submodules/MaterialProviders/.github/actions/setup_dafny/ + with: + dafny-version: ${{ inputs.dafny }} + + - name: Update MPL submodule if using MPL HEAD + if: ${{ inputs.mpl-head == true }} + working-directory: submodules/MaterialProviders + run: | + git checkout main + git pull + git submodule update --init --recursive + git rev-parse HEAD + + - name: Install Smithy-Dafny codegen dependencies + uses: ./.github/actions/install_smithy_dafny_codegen_dependencies + + - name: Regenerate code using smithy-dafny if necessary + if: ${{ inputs.regenerate-code }} + uses: ./.github/actions/polymorph_codegen + with: + dafny: ${{ env.DAFNY_VERSION }} + library: ${{ matrix.library }} + diff-generated-code: false + update-and-regenerate-mpl: true + + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Setup Python ${{ matrix.python-version }} for running tests + run: | + python -m pip install --upgrade pip + pip install --upgrade tox + pip install poetry + + - name: Build Python TestVectors implementation + shell: bash + working-directory: ${{matrix.library}} + run: | + # This works because `node` is installed by default on GHA runners + CORES=$(node -e 'console.log(os.cpus().length)') + make transpile_python CORES=$CORES + + - name: Test Python TestVectors with ${{matrix.interface}} interface + working-directory: ${{matrix.library}} + run: | + cp runtimes/java/*.json runtimes/python + make test_python_${{matrix.interface}}_interface diff --git a/.github/workflows/daily_ci.yml b/.github/workflows/daily_ci.yml index ed5cf7e78..a00038ac5 100644 --- a/.github/workflows/daily_ci.yml +++ b/.github/workflows/daily_ci.yml @@ -61,6 +61,26 @@ jobs: uses: ./.github/workflows/library_rust_tests.yml with: dafny: ${{needs.getVersion.outputs.version}} + daily-ci-python: + needs: getVersion + uses: ./.github/workflows/ci_test_python.yml + with: + dafny: ${{needs.getVersion.outputs.version}} + daily-ci-python-examples: + needs: getVersion + uses: ./.github/workflows/ci_examples_python.yml + with: + dafny: ${{needs.getVersion.outputs.version}} + daily-ci-python-test-vectors: + needs: getVersion + uses: ./.github/workflows/ci_test_vector_python.yml + with: + dafny: ${{needs.getVersion.outputs.version}} + daily-ci-python-static-analysis: + needs: getVersion + uses: ./.github/workflows/ci_static_analysis_python.yml + with: + dafny: ${{needs.getVersion.outputs.version}} daily-ci-net-test-vectors: needs: getVersion uses: ./.github/workflows/ci_test_vector_net.yml diff --git a/.github/workflows/manual.yml b/.github/workflows/manual.yml index 8679b4f30..ca703d850 100644 --- a/.github/workflows/manual.yml +++ b/.github/workflows/manual.yml @@ -57,6 +57,26 @@ jobs: with: dafny: ${{ inputs.dafny }} regenerate-code: ${{ inputs.regenerate-code }} + manual-ci-python: + uses: ./.github/workflows/ci_test_python.yml + with: + dafny: ${{ inputs.dafny }} + regenerate-code: ${{ inputs.regenerate-code }} + manual-ci-python-examples: + uses: ./.github/workflows/ci_examples_python.yml + with: + dafny: ${{ inputs.dafny }} + regenerate-code: ${{ inputs.regenerate-code }} + manual-ci-python-test-vectors: + uses: ./.github/workflows/ci_test_vector_python.yml + with: + dafny: ${{ inputs.dafny }} + regenerate-code: ${{ inputs.regenerate-code }} + manual-ci-python-static-analysis: + uses: ./.github/workflows/ci_static_analysis_python.yml + with: + dafny: ${{ inputs.dafny }} + regenerate-code: ${{ inputs.regenerate-code }} manual-ci-net-test-vectors: uses: ./.github/workflows/ci_test_vector_net.yml with: diff --git a/.github/workflows/mpl-head.yml b/.github/workflows/mpl-head.yml index 6e2e06234..fffa31e0d 100644 --- a/.github/workflows/mpl-head.yml +++ b/.github/workflows/mpl-head.yml @@ -73,6 +73,18 @@ jobs: with: dafny: ${{needs.getVersion.outputs.version}} mpl-head: true + mpl-head-ci-python: + needs: getVersion + uses: ./.github/workflows/ci_test_python.yml + with: + dafny: ${{needs.getVersion.outputs.version}} + mpl-head: true + mpl-head-ci-python-examples: + needs: getVersion + uses: ./.github/workflows/ci_examples_python.yml + with: + dafny: ${{needs.getVersion.outputs.version}} + mpl-head: true mpl-head-ci-net-test-vectors: needs: getVersion uses: ./.github/workflows/ci_test_vector_net.yml diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index eb4bc6e20..ba72181f4 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -58,6 +58,18 @@ jobs: with: dafny: "nightly-latest" regenerate-code: true + dafny-nightly-python: + if: github.event_name != 'schedule' || github.repository_owner == 'aws' + uses: ./.github/workflows/ci_test_python.yml + with: + dafny: "nightly-latest" + regenerate-code: true + dafny-nightly-python-test-vectors: + if: github.event_name != 'schedule' || github.repository_owner == 'aws' + uses: ./.github/workflows/ci_test_vector_python.yml + with: + dafny: "nightly-latest" + regenerate-code: true dafny-nightly-test-vectors-net: if: github.event_name != 'schedule' || github.repository_owner == 'aws' uses: ./.github/workflows/ci_test_vector_net.yml diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index 3f237ee0d..e9e237653 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -54,6 +54,24 @@ jobs: uses: ./.github/workflows/library_rust_tests.yml with: dafny: ${{needs.getVersion.outputs.version}} + pr-ci-python: + needs: getVersion + uses: ./.github/workflows/ci_test_python.yml + with: + dafny: ${{needs.getVersion.outputs.version}} + pr-ci-python-testvectors: + needs: getVersion + uses: ./.github/workflows/ci_test_vector_python.yml + with: + dafny: ${{needs.getVersion.outputs.version}} + pr-ci-python-examples: + needs: getVersion + uses: ./.github/workflows/ci_examples_python.yml + with: + dafny: ${{needs.getVersion.outputs.version}} + pr-ci-python-static-analysis: + needs: getVersion + uses: ./.github/workflows/ci_static_analysis_python.yml pr-ci-net-test-vectors: needs: getVersion uses: ./.github/workflows/ci_test_vector_net.yml @@ -80,6 +98,10 @@ jobs: - pr-ci-rust - pr-ci-net-test-vectors - pr-ci-net-examples + - pr-ci-python + - pr-ci-python-testvectors + - pr-ci-python-examples + - pr-ci-python-static-analysis runs-on: ubuntu-22.04 steps: - name: Verify all required jobs passed diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 9e49cf133..b3f66e2d2 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -56,6 +56,24 @@ jobs: uses: ./.github/workflows/library_rust_tests.yml with: dafny: ${{needs.getVersion.outputs.version}} + pr-ci-python: + needs: getVersion + uses: ./.github/workflows/ci_test_python.yml + with: + dafny: ${{needs.getVersion.outputs.version}} + pr-ci-python-examples: + needs: getVersion + uses: ./.github/workflows/ci_examples_python.yml + with: + dafny: ${{needs.getVersion.outputs.version}} + pr-ci-python-static-analysis: + needs: getVersion + uses: ./.github/workflows/ci_static_analysis_python.yml + pr-ci-python-test-vectors: + needs: getVersion + uses: ./.github/workflows/ci_test_vector_python.yml + with: + dafny: ${{needs.getVersion.outputs.version}} pr-ci-net-test-vectors: needs: getVersion uses: ./.github/workflows/ci_test_vector_net.yml diff --git a/DynamoDbEncryption/runtimes/python/tox.ini b/DynamoDbEncryption/runtimes/python/tox.ini new file mode 100644 index 000000000..a4edd1639 --- /dev/null +++ b/DynamoDbEncryption/runtimes/python/tox.ini @@ -0,0 +1,108 @@ +[tox] +isolated_build = True +envlist = + py{311,312,313}-{dafnytests,unit,integ}, + encrypted-interface-coverage, + client-to-resource-conversions-coverage, + resource-to-client-conversions-coverage, + docs, + isort-check, + black-check + +[testenv:base-command] +commands = poetry run pytest -s -v -l {posargs} + +[testenv] +skip_install = true +allowlist_externals = poetry,ruff,black +passenv = AWS_* +commands_pre = + poetry lock + poetry install --with test +commands = + dafnytests: {[testenv:base-command]commands} test/internaldafny/ + unit: {[testenv:base-command]commands} test/unit/ + integ: {[testenv:base-command]commands} test/integ/ + +[testenv:encrypted-interface-coverage] +description = Run integ + unit tests for encrypted interfaces with coverage +commands = + python -m pytest -s -vv \ + test/integ/encrypted \ + test/unit/encrypted \ + --cov aws_dbesdk_dynamodb.encrypted \ + --cov-report=term-missing \ + --cov-fail-under=100 + +[testenv:client-to-resource-conversions-coverage] +description = Run boto3 conversion tests with coverage +commands = + python -m pytest -s -vv \ + test/unit/internal/test_client_to_resource.py \ + --cov aws_dbesdk_dynamodb.internal.client_to_resource \ + --cov-report=term-missing \ + --cov-fail-under=100 + +[testenv:resource-to-client-conversions-coverage] +description = Run boto3 conversion tests with coverage +commands = + python -m pytest -s -vv \ + test/unit/internal/test_resource_to_client.py \ + --cov aws_dbesdk_dynamodb.internal.resource_to_client \ + --cov-report=term-missing \ + --cov-fail-under=100 + +# Linters +[testenv:ruff] +commands_pre = + poetry install --with linting +deps = + ruff +commands = + ruff check \ + src/aws_dbesdk_dynamodb/ \ + ../../../Examples/runtimes/python/DynamoDBEncryption/ \ + test/ \ + {posargs} + +[testenv:blacken] +commands_pre = + poetry install --with linting +deps = + black +basepython = python3 +commands = + black --line-length 120 \ + src/aws_dbesdk_dynamodb/ \ + ../../../Examples/runtimes/python/DynamoDBEncryption/ \ + test/ \ + {posargs} + +[testenv:black-check] +commands_pre = + {[testenv:blacken]commands_pre} +basepython = python3 +deps = + {[testenv:blacken]deps} +commands = + {[testenv:blacken]commands} --diff --check + +[testenv:lint] +commands_pre = + poetry install --with linting +deps = + black +basepython = python3 +commands = + {[testenv:blacken]commands} + {[testenv:ruff]commands} --fix + +[testenv:lint-check] +commands_pre = + poetry install --with linting +deps = + black +basepython = python3 +commands = + {[testenv:black-check]commands} + {[testenv:ruff]commands} diff --git a/Examples/runtimes/python/tox.ini b/Examples/runtimes/python/tox.ini new file mode 100644 index 000000000..b3edda147 --- /dev/null +++ b/Examples/runtimes/python/tox.ini @@ -0,0 +1,18 @@ +[tox] +isolated_build = True +envlist = + py{311,312,313}-{dynamodbencryption,migration} + +[testenv:base-command] +commands = poetry run pytest -s -v -l {posargs} + +[testenv] +skip_install = true +allowlist_externals = poetry +passenv = AWS_* +commands_pre = + poetry lock + poetry install --with test --no-root +commands = + dynamodbencryption: {[testenv:base-command]commands} DynamoDBEncryption/test/ + migration: {[testenv:base-command]commands} Migration/PlaintextToAWSDBE/test/ \ No newline at end of file diff --git a/TestVectors/runtimes/python/tox.ini b/TestVectors/runtimes/python/tox.ini new file mode 100644 index 000000000..4f64adc59 --- /dev/null +++ b/TestVectors/runtimes/python/tox.ini @@ -0,0 +1,26 @@ +[tox] +isolated_build = True +envlist = + py{311,312,313} + +[testenv] +skip_install = true +allowlist_externals = poetry +passenv = AWS_* +commands_pre = + poetry lock + poetry install +commands = + poetry run pytest test/ -s -v + +[testenv:client] +commands = + poetry run pytest test/client -s -v + +[testenv:resource] +commands = + poetry run pytest test/resource -s -v + +[testenv:table] +commands = + poetry run pytest test/table -s -v