Targets-based workflows leveraging slurm for dist. computing - workflow 1a and POC

.github/workflows/apptainer-build-image.yml

@@ -0,0 +1,163 @@
+name: build-image
+
+on:
+  workflow_call:
+    inputs:
+      image-name:
+        required: true
+        type: string
+      build-context:
+        required: true
+        type: string
+      dockerfile:
+        required: true
+        type: string
+      r-version:
+        required: true
+        type: string
+      parent-image:
+        required: false
+        default: ''
+        type: string
+      model-version:
+        required: false
+        default: ''
+        type: string
+      image-version:
+        required: false
+        default: "latest"
+        type: string
+      dockerhub-repo:
+        required: false
+        default: "hdpriest0uiuc"
+        type: string
+      platforms:
+        required: false
+        default: "linux/amd64"
+        type: string
+    secrets:
+      DOCKERHUB_USERNAME:
+        description: 'DockerHub username used to push images'
+        required: false
+      DOCKERHUB_PASSWORD:
+        description: 'DockerHub password used to push images'
+        required: false
+
+env:
+  DEFAULT_R_VERSION: "4.4"
+  GITHUB_PAT: ${{ secrets.GH_TOKEN }}
+
+jobs:
+  build:
+    runs-on: ubuntu-24.04
+    permissions:
+      packages: write
+
+    steps:
+
+      - name: lowercase image name
+        id: name
+        run: |
+          echo "image_name=$(echo ${{ inputs.image-name }} | tr '[:upper:]' '[:lower:]')" >> $GITHUB_OUTPUT
+          echo "repository=$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]')" >> $GITHUB_OUTPUT
+
+      - name: set PARENT_IMAGE only if specified
+        id: parent
+        shell: bash
+        run: |
+          echo "PARENT_IMAGE_IF_SET=$(
+            [[ -n '${{ inputs.parent-image }}' ]] &&
+            echo "PARENT_IMAGE=ghcr.io/${{ steps.name.outputs.repository }}/"'${{ inputs.parent-image }}'
+            )" >> $GITHUB_OUTPUT
+
+      - name: set MODEL_VERSION only if specified
+        id: modelver
+        shell: bash
+        run: |
+          echo "MODEL_VERSION_IF_SET=$(
+            [[ -n '${{ inputs.model-version }}' ]] &&
+            echo 'MODEL_VERSION=${{ inputs.model-version }}'
+            )" >> $GITHUB_OUTPUT
+
+      - uses: actions/checkout@v4
+
+      # create metadata for image
+      - name: Docker meta
+        env:
+          check_var: ${{ secrets.DOCKERHUB_USERNAME }}
+          is_default_R: ${{ inputs.r-version == env.DEFAULT_R_VERSION }}
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          # list of Docker images to use as base name for tags
+          images: |
+            name=ghcr.io/${{ steps.name.outputs.repository }}/${{ steps.name.outputs.image_name }}
+            name=${{ inputs.dockerhub-repo }}/${{ steps.name.outputs.image_name }},enable=${{ env.check_var != null }}
+          # generate Docker tags based on the following events/attributes
+          tags: |
+            type=raw,value=${{ inputs.image-version }}
+          #  type=schedule
+          # type=ref,event=branch,enable=${{ env.is_default_R }}
+          # type=ref,event=branch,suffix=-R${{ inputs.r-version }}
+          # type=ref,event=pr
+          # type=semver,pattern={{version}},enable=${{ env.is_default_R }}
+          # type=semver,pattern={{major}}.{{minor}},enable=${{ env.is_default_R }}
+          # type=semver,pattern={{major}},enable=${{ env.is_default_R }}
+          # type=semver,pattern={{version}},suffix=-R${{ inputs.r-version }}
+
+      # setup docker build
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Inspect Builder
+        run: |
+          echo "Name:      ${{ steps.buildx.outputs.name }}"
+          echo "Endpoint:  ${{ steps.buildx.outputs.endpoint }}"
+          echo "Status:    ${{ steps.buildx.outputs.status }}"
+          echo "Flags:     ${{ steps.buildx.outputs.flags }}"
+          echo "Platforms: ${{ steps.buildx.outputs.platforms }}"
+
+      # login to registries
+      - name: Login to DockerHub
+        env: 
+          check_var: ${{ secrets.DOCKERHUB_USERNAME }}
+        if: env.check_var != null
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # build the docker images
+      - name: Build and push ${{ steps.name.outputs.image_name }}
+        uses: docker/build-push-action@v6
+        with:
+          context: ${{ inputs.build-context }}
+          file: ${{ inputs.dockerfile }}
+          push: true
+          platforms: ${{ inputs.platforms }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          build-args: |
+            VERSION=${{ steps.meta.outputs.version }}
+            IMAGE_VERSION=${{ steps.meta.outputs.version }}
+            PECAN_VERSION=${{ steps.meta.outputs.version }}
+            R_VERSION=${{ inputs.r-version }}
+            ${{ steps.parent.outputs.PARENT_IMAGE_IF_SET }}
+            ${{ steps.modelver.outputs.MODEL_VERSION_IF_SET }}
+            GITHUB_PAT=${{ secrets.GITHUB_TOKEN }}
+            PECAN_GIT_BRANCH=${{ github.head_ref || github.ref_name }}
+            PECAN_GIT_CHECKSUM=${{ github.sha }}
+            PECAN_GIT_DATE=${{ github.event.repository.updated_at }}

.github/workflows/apptainer-sipnet-carb.yml

@@ -0,0 +1,73 @@
+name: Apptainer GHA CARB
+
+env:
+  DEFAULT_R_VERSION: 4.4
+  R_VERSION: 4.4
+  GITHUB_PAT: ${{ secrets.GH_TOKEN }}
+
+on:
+  push:
+    branches:
+      - main
+      - develop
+
+  pull_request:
+  merge_group:
+  workflow_dispatch:
+    inputs:
+      r_version:
+        description: 'R version to use'
+        required: true
+        type: choice
+        default: "$DEFAULT_R_VERSION"
+        options:
+          - 4.1
+          - 4.2
+          - 4.3
+          - 4.4
+          - devel
+      image_version:
+        description: 'version of sipnet container to use'
+        required: true
+        type: choice
+        default: "latest"
+        options:
+          - develop
+          - latest
+
+jobs:
+  # ----------------------------------------------------------------------
+  # Set R version.
+  # This is a hack: We really just want a global env var here, but it seems
+  # `env:` values can't be passed into a `jobs.<jobid>.with` context
+  # (see https://github.com/actions/runner/issues/2372).
+  # As an ugly workaround, we assign it to a job output instead.
+  # ----------------------------------------------------------------------
+  rversion:
+    runs-on: ubuntu-latest
+    steps:
+      - id: default
+        if: github.event_name != 'schedule'
+        run: echo "R_VERSION=4.4" >> "$GITHUB_OUTPUT"
+    outputs:
+      # Note: "steps.*" seems to mean "all step ids", not "all steps"
+      # If seeing weird results here, check that all steps above have an id set.
+      R_VERSION: 4.4
+
+# ----------------------------------------------------------------------
+# Next are images that have specific layers added 
+# ----------------------------------------------------------------------
+  sipnet-carb:
+      needs: [rversion]
+      uses: ./.github/workflows/apptainer-build-image.yml
+      with:
+        image-name: sipnet-carb
+        build-context: tools/apptainer-sipnet-carb
+        dockerfile: tools/apptainer-sipnet-carb/Dockerfile
+        r-version: ${{ needs.rversion.outputs.R_VERSION }}
+        parent-image: "pecan/model-sipnet-git"
+        image-version: ${{ inputs.image_version }}
+      secrets:
+        DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+        DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} 
+

.github/workflows/run-workflow-examples.yml

@@ -0,0 +1,103 @@
+name: Run Workflow Examples
+
+env:
+  GITHUB_PAT: ${{ secrets.GH_TOKEN }}
+
+on:
+  push:
+    branches:
+      - main
+      - develop
+    paths:
+      - 'workflow_examples/**'
+      - '.github/workflows/run-workflow-examples.yml'
+  pull_request:
+    paths:
+      - 'workflow_examples/**'
+      - '.github/workflows/run-workflow-examples.yml'
+  workflow_dispatch:
+    inputs:
+      orchestration_version:
+        description: 'Orchestration XML version to use (devel or latest)'
+        required: true
+        type: choice
+        default: 'devel'
+        options:
+          - devel
+          - latest
+
+jobs:
+  # ----------------------------------------------------------------------
+  # Workflow 01: Data Prep Workflow
+  # This is the first workflow that prepares the base data
+  # ----------------------------------------------------------------------
+  workflow_01_data_prep:
+    runs-on: ['self-hosted', 'Linux', 'X64']
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set orchestration XML version
+        id: orchestration
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "version=${{ github.event.inputs.orchestration_version }}" >> $GITHUB_OUTPUT
+          else
+            echo "version=devel" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Run 01_data_prep_workflow
+        working-directory: workflow_examples/01_simple_data_workflow
+        run: |
+          Rscript 01_data_prep_workflow.R -s ../../.github/workflows_resources/01_orchestration_${{ steps.orchestration.outputs.version }}.xml
+
+  # ----------------------------------------------------------------------
+  # Workflow 02: Data Reference Workflow
+  # This workflow references data from workflow 01
+  # ----------------------------------------------------------------------
+  workflow_02_data_reference:
+    needs: [workflow_01_data_prep]
+    runs-on: ['self-hosted', 'Linux', 'X64']
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set orchestration XML version
+        id: orchestration
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "version=${{ github.event.inputs.orchestration_version }}" >> $GITHUB_OUTPUT
+          else
+            echo "version=devel" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Run 02_run_data_reference_workflow
+        working-directory: workflow_examples/02_referencing_data_workflow
+        run: |
+          Rscript 02_run_data_reference_workflow.R -s ../../.github/workflows_resources/02_orchestration_${{ steps.orchestration.outputs.version }}.xml
+
+  # ----------------------------------------------------------------------
+  # Workflow 03: Distributed Workflow
+  # This workflow runs the distributed analysis workflow
+  # ----------------------------------------------------------------------
+  workflow_03_distributed:
+    needs: [workflow_02_data_reference]
+    runs-on: ['self-hosted', 'Linux', 'X64']
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set orchestration XML version
+        id: orchestration
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "version=${{ github.event.inputs.orchestration_version }}" >> $GITHUB_OUTPUT
+          else
+            echo "version=devel" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Run 03_run_distributed_workflow
+        working-directory: workflow_examples/03_distributed_workflow
+        run: |
+          Rscript 03_run_distributed_workflow.R -s ../../.github/workflows_resources/03_orchestration_${{ steps.orchestration.outputs.version }}.xml
+

.github/workflows_resources/01_orchestration_devel.xml

@@ -0,0 +1,19 @@
+<?xml version="1.0"?>
+<pecan>
+ <orchestration>
+    <workflow.base.run.directory>/project/60007/hpriest/data/workflow_runs_ci</workflow.base.run.directory>
+    <functions.source>../../tools/workflow_functions.R</functions.source>
+    <distributed.compute.adapter>
+        <name>localhost</name>
+        <qsub>sbatch -J @NAME@ -o @STDOUT@ -e @STDERR@ apptainer run ./sipnet-carb_develop.sif </qsub>
+        <qsub.jobid>Submitted batch job ([0-9]+)</qsub.jobid>
+        <qstat>if test -z "$(squeue -h -j @JOBID@)"; then echo "DONE"; fi</qstat>
+    </distributed.compute.adapter>
+    <workflow.data.prep.1>
+        <run.identifier>data_prep_run_01</run.identifier>
+        <pecan.xml.path>./01_pecan_config_devel.xml</pecan.xml.path>
+        <ccmmf.data.s3.url>s3://carb/data/workflows/phase_1a</ccmmf.data.s3.url>
+        <ccmmf.data.tarball.filename>00_cccmmf_phase_1a_input_artifacts.tgz</ccmmf.data.tarball.filename>
+    </workflow.data.prep.1>
+</orchestration>
+</pecan>