feat!: prepare 2.0.0 application packages and release pipeline

.github/workflows/build.yaml

@@ -1,146 +1,217 @@
-name: build
+name: Release application packages
 on:
   push:
-    branches:
-    - master
-    - main
-
+    branches: [master, main]
     paths:
-    # Only rebuild website when apps have changed
-    - 'water-bodies/**'
-    - .github/**
-    - docs/**
-    - cwl-workflow/*.cwl
-    - codemeta.json
+      - 'water-bodies/**'
+      - '.github/**'
+      - 'cwl-cli/**'
+      - 'cwl-workflow/**'
+      - 'scripts/**'
+      - 'tests/**'
+      - 'codemeta.json'
+      - 'CHANGELOG.md'
+  workflow_dispatch:
+permissions:
+  contents: read
+concurrency:
+  group: application-release
+  cancel-in-progress: false
 jobs:
-  validate:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - uses: actions/setup-python@v2
-      with:
-        python-version: 3.x
-    - run: pip install cwltool
-    - run: cwltool --validate cwl-workflow/app-water-bodies-cloud-native.cwl
-    - run: cwltool --validate cwl-workflow/app-water-body-cloud-native.cwl
-    - run: cwltool --validate cwl-workflow/app-water-body.cwl
+  tests:
+    uses: ./.github/workflows/processing-tests.yaml
 
   version:
-    needs: validate
-
+    needs: tests
     runs-on: ubuntu-latest
-
     outputs:
-      app-version: ${{ steps.set-version.outputs.version }}
-
+      version: ${{ steps.metadata.outputs.version }}
+      registry: ${{ steps.metadata.outputs.registry }}
     steps:
-    - uses: actions/checkout@v2
-    - run: echo "APP_VERSION=$(cat codemeta.json | jq -r .version )" >> $GITHUB_ENV
-    - run: echo app version is $APP_VERSION
-    - id: set-version
-      run: echo "::set-output name=version::$APP_VERSION"
+      - uses: actions/checkout@v4
+      - id: metadata
+        name: Check version and publication targets
+        env:
+          GH_TOKEN: ${{ github.token }}
+        shell: bash
+        run: |
+          version=$(jq -er '.version' codemeta.json)
+          [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9.-]+)?$ ]]
+          grep -Fq "## [$version]" CHANGELOG.md || {
+            echo "::error::CHANGELOG.md must describe release $version."
+            exit 1
+          }
+          registry="ghcr.io/${GITHUB_REPOSITORY,,}"
+          echo "version=$version" >> "$GITHUB_OUTPUT"
+          echo "registry=$registry" >> "$GITHUB_OUTPUT"
+          gh api "repos/$GITHUB_REPOSITORY/releases" --paginate --jq '.[].tag_name' > /tmp/release-tags
+          gh api "repos/$GITHUB_REPOSITORY/git/matching-refs/tags/$version" --jq '.[].ref' > /tmp/tag-refs
+          if grep -Fxq "$version" /tmp/release-tags || grep -Fxq "refs/tags/$version" /tmp/tag-refs; then
+            echo "::error::Version $version already exists. Bump codemeta.json before releasing."
+            exit 1
+          fi
 
   container-build:
-
     needs: version
-
     runs-on: ubuntu-latest
-
+    permissions:
+      contents: read
+      packages: write
     strategy:
+      fail-fast: false
       matrix:
-        step: [crop, norm_diff, otsu, stac, stage]
-
+        step: [crop, norm_diff, otsu, stac, stage, stage-in, stage-out]
+    env:
+      IMAGE: ${{ needs.version.outputs.registry }}/${{ matrix.step }}
+      VERSION: ${{ needs.version.outputs.version }}
     steps:
-    - uses: actions/checkout@v2
-    - run: echo version ${{needs.version.outputs.app-version}}
-    - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
-    - name: build & push image
-      run: |
-        IMAGE_ID=ghcr.io/eoap/mastering-app-package/${{ matrix.step }}
-        docker build water-bodies/command-line-tools/${{ matrix.step }} --file water-bodies/command-line-tools/${{ matrix.step }}/Dockerfile --tag ${{ matrix.step }}
-        docker tag ${{ matrix.step }} $IMAGE_ID:${{needs.version.outputs.app-version}}
-        docker push $IMAGE_ID:${{needs.version.outputs.app-version}}
-
-  create-release:
-
-    needs:
-    - container-build
-    - version
+      - uses: actions/checkout@v4
+      - uses: oras-project/setup-oras@v1
+        with:
+          version: '1.3.0'
+      - name: Authenticate
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          echo "$GH_TOKEN" | docker login ghcr.io -u "$GITHUB_ACTOR" --password-stdin
+          echo "$GH_TOKEN" | oras login ghcr.io -u "$GITHUB_ACTOR" --password-stdin
+      - name: Refuse to replace an existing image tag
+        run: |
+          oras repo tags "$IMAGE" > /tmp/image-tags 2>/tmp/registry-error || {
+            # A repository that has never been published has no tags.
+            grep -Eq 'NAME_UNKNOWN|name unknown|404' /tmp/registry-error || { cat /tmp/registry-error; exit 1; }
+          }
+          if grep -Fxq "$VERSION" /tmp/image-tags; then
+            echo "::error::Image $IMAGE:$VERSION already exists. Use a new release version."
+            exit 1
+          fi
+      - name: Build image
+        run: docker build -t "$IMAGE:$VERSION" "water-bodies/command-line-tools/${{ matrix.step }}"
+      - name: Scan image (block fixable HIGH and CRITICAL vulnerabilities)
+        uses: aquasecurity/trivy-action@0.35.0
+        with:
+          version: 'v0.69.3'
+          image-ref: ${{ env.IMAGE }}:${{ env.VERSION }}
+          scanners: vuln
+          format: json
+          output: scan-${{ matrix.step }}.json
+          severity: HIGH,CRITICAL
+          ignore-unfixed: true
+          exit-code: '1'
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: scan-${{ matrix.step }}
+          path: scan-${{ matrix.step }}.json
+          if-no-files-found: ignore
+      - name: Push and record immutable image reference
+        run: |
+          docker push "$IMAGE:$VERSION"
+          mkdir -p digests
+          digest=$(oras resolve "$IMAGE:$VERSION")
+          [[ "$digest" =~ ^sha256:[a-f0-9]{64}$ ]]
+          printf '%s@%s\n' "$IMAGE" "$digest" > "digests/${{ matrix.step }}.txt"
+      - uses: actions/upload-artifact@v4
+        with:
+          name: digest-${{ matrix.step }}
+          path: digests/*.txt
+          if-no-files-found: error
 
+  publish-artifacts:
+    needs: [version, container-build]
     runs-on: ubuntu-latest
-
-    outputs:
-
-      upload_url: ${{ steps.set-upload-url.outputs.upload_url }}
-
+    permissions:
+      contents: read
+      packages: write
+    env:
+      VERSION: ${{ needs.version.outputs.version }}
+      REGISTRY: ${{ needs.version.outputs.registry }}
     steps:
-    - name: release
-      uses: actions/create-release@v1
-      id: create_release
-      with:
-        draft: false
-        prerelease: false
-        release_name: ${{needs.version.outputs.app-version}}
-        tag_name: ${{needs.version.outputs.app-version}}
-      env:
-        GITHUB_TOKEN: ${{ github.token }}
-    - id: set-upload-url
-      run: echo "::set-output name=upload_url::${{ steps.create_release.outputs.upload_url }}"
-
-  publish-artifacts:
-
-    needs:
-    - create-release
-    - version
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - run: python -m pip install -r scripts/requirements-cwl-release.txt
+      - uses: oras-project/setup-oras@v1
+        with:
+          version: '1.3.0'
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: digest-*
+          merge-multiple: true
+          path: digests
+      - name: Prepare and validate packed release CWL
+        run: python scripts/prepare-cwl-release.py --digests digests --output downloads --version "$VERSION"
+      - name: Authenticate to GHCR
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: echo "$GH_TOKEN" | oras login ghcr.io -u "$GITHUB_ACTOR" --password-stdin
+      - name: Annotate, publish, and verify CWL artifacts
+        shell: bash
+        run: |
+          : > downloads/oci-artifacts.txt
+          for name in app-water-bodies-cloud-native app-water-body-cloud-native app-water-body; do
+            file="$name.$VERSION.cwl"
+            annotations="$name.$VERSION.annotations.json"
+            transpiler-mate cwl2oci \
+              --image-source "https://github.com/$GITHUB_REPOSITORY" \
+              --image-revision "$GITHUB_SHA" \
+              --output "downloads/$annotations" "downloads/$file#main"
+            python scripts/check-oci-annotations.py "downloads/$annotations" "$file" "$VERSION"
+            target="$REGISTRY/cwl/$name:$VERSION"
+            oras repo tags "$REGISTRY/cwl/$name" > /tmp/cwl-tags 2>/tmp/registry-error || {
+              grep -Eq 'NAME_UNKNOWN|name unknown|404' /tmp/registry-error || { cat /tmp/registry-error; exit 1; }
+            }
+            if grep -Fxq "$VERSION" /tmp/cwl-tags; then
+              echo "::error::Artifact $target already exists. Use a new release version."
+              exit 1
+            fi
+            (cd downloads && oras push --artifact-type application/cwl \
+              --annotation-file "$annotations" "$target" "$file:application/cwl")
+            digest=$(oras resolve "$target")
+            [[ "$digest" =~ ^sha256:[a-f0-9]{64}$ ]]
+            reference="$REGISTRY/cwl/$name@$digest"
+            oras manifest fetch "$reference" > "downloads/$name.$VERSION.manifest.json"
+            jq -e --slurpfile expected "downloads/$annotations" \
+              '.annotations as $actual | $expected[0]."$manifest" | to_entries | all(.[]; $actual[.key] == .value)' \
+              "downloads/$name.$VERSION.manifest.json"
+            mkdir -p "verified/$name"
+            oras pull "$reference" -o "verified/$name"
+            cmp "downloads/$file" "verified/$name/$file"
+            cwltool --validate "verified/$name/$file#main"
+            printf '%s\n' "$reference" >> downloads/oci-artifacts.txt
+          done
+      - uses: actions/upload-artifact@v4
+        with:
+          name: application-packages
+          path: downloads/*
+          if-no-files-found: error
 
+  create-release:
+    needs: [version, publish-artifacts]
     runs-on: ubuntu-latest
-
-    strategy:
-      matrix:
-        step: ["app-water-bodies-cloud-native", "app-water-body-cloud-native", "app-water-body"]
-
+    permissions:
+      contents: write
+    env:
+      VERSION: ${{ needs.version.outputs.version }}
+      GH_TOKEN: ${{ github.token }}
     steps:
-    - uses: actions/checkout@v2
-    - run: |
-        for step in crop norm_diff otsu stac
-        do
-          tag="ghcr.io/eoap/mastering-app-package/${step}:${{needs.version.outputs.app-version}}"
-          docker pull ${tag}
-          shatag=$( docker inspect ${tag} | yq -r '.[0]["RepoDigests"][0]' )
-          for cwl in $(ls cwl-workflow/*.cwl)
-          do
-            s="${step}" t="${shatag}" yq -i eval '(.$graph[] | select (.id == env(s)) ).hints.DockerRequirement.dockerPull = env(t)' $cwl
-          done
-        done
-    - run: |
-        for cwl in $(ls cwl-workflow/*.cwl)
-        do
-          r=$( cat codemeta.json | jq -r ".codeRepository" ) yq -i eval '."s:codeRepository" = {"URL" : env(r)}' $cwl
-          v="${{needs.version.outputs.app-version}}" yq -i eval '."s:softwareVersion" = env(v)' $cwl
-          n=$(cat codemeta.json | jq -r '(.author[0].givenName + " " + .author[0].familyName)') \
-          e=$(cat codemeta.json | jq -r '.author[0].email') \
-          a=$(cat codemeta.json | jq -r '.author[0].affiliation["name"]') \
-          yq eval -i  '."s:author" += [{"class": "s:Person", "s.name": env(n), "s.email": env(e), "s.affiliation": env(a)}]' $cwl
-        done
-    - name: Cleanup downloads folder
-      run: rm -rf downloads/*
-    - run: |
-        mkdir downloads
-        for cwl in "app-water-bodies-cloud-native" "app-water-body-cloud-native" "app-water-body"
-        do
-          cp cwl-workflow/${cwl}.cwl downloads/${cwl}.${{needs.version.outputs.app-version}}.cwl
-        done
-    - uses: actions/upload-artifact@v4
-      with:
-        name: application-package-${{matrix.step}}
-        path: downloads
-        overwrite: true
-    - name: upload linux artifact 1
-      uses: actions/upload-release-asset@v1
-      env:
-        GITHUB_TOKEN: ${{ github.token }}
-      with:
-        upload_url: ${{needs.create-release.outputs.upload_url}}
-        asset_path: downloads/${{matrix.step}}.${{needs.version.outputs.app-version}}.cwl
-        asset_name: ${{matrix.step}}.${{needs.version.outputs.app-version}}.cwl
-        asset_content_type: text/yaml
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with:
+          name: application-packages
+          path: downloads
+      - name: Create draft, attach verified assets, and publish
+        run: |
+          awk -v version="$VERSION" '
+            index($0, "## [" version "]") == 1 { found=1; active=1 }
+            active && /^## / && index($0, "## [" version "]") != 1 { active=0 }
+            active { print }
+            END { if (!found) exit 1 }
+          ' CHANGELOG.md > /tmp/release-notes.md
+          printf '\n### OCI artifacts\n\n' >> /tmp/release-notes.md
+          cat downloads/oci-artifacts.txt >> /tmp/release-notes.md
+          gh release create "$VERSION" --target "$GITHUB_SHA" --title "$VERSION" \
+            --draft --notes-file /tmp/release-notes.md
+          gh release upload "$VERSION" downloads/*
+          gh release edit "$VERSION" --draft=false

.github/workflows/docs.yaml

@@ -10,6 +10,12 @@ on:
       - 'README.md'
       - 'docs/**'
       - 'mkdocs.yml'
+      - 'scripts/requirements-docs.txt'
+      - 'water-bodies/command-line-tools/**'
+      - 'cwl-cli/**'
+      - 'cwl-workflow/**'
+      - 'practice-labs/**'
+      - 'scripts/**'
       - .github/**
 
 jobs:
@@ -23,5 +29,5 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: 3.x
-      - run: pip install mkdocs-material mkdocs-mermaid2-plugin
+      - run: python -m pip install -r scripts/requirements-docs.txt
       - run: mkdocs gh-deploy --force