CI: enable IOC tests

.ci/bl-specific.sh

@@ -1,3 +1,20 @@
 #!/bin/bash
 
-# cp -v <...> ~/.ipython/profile_${TEST_PROFILE}/
+set -veo pipefail
+
+# Per https://stackoverflow.com/a/66982842:
+sudo echo "127.0.0.1 info.cms.nsls2.bnl.gov" | sudo tee -a /etc/hosts
+cat /etc/hosts
+
+ping -c 5 info.cms.nsls2.bnl.gov
+echo "" | telnet info.cms.nsls2.bnl.gov 6379
+
+# Copy config files into the dummy IPython profile:
+mkdir -v -p ~/.ipython/profile_test/startup/
+cp -v startup/.cms_config ~/.ipython/profile_test/startup/
+
+# Create pilatus and /nsls2 dir trees:
+sudo mkdir -v -p /nsls2/
+sudo chown -R -v $USER: /nsls2/
+mkdir -v -p $HOME/cms-epics-containers/pilatus-data/data/
+ln -sv $HOME/cms-epics-containers/pilatus-data/data/ /nsls2/

.ci/drop-in.py

@@ -1,8 +1,16 @@
+import numpy as np
+
 caput(beam.mono_bragg_pv, 1.03953)
 pilatus2M.tiff.create_directory.set(-20)
 sam = SampleGISAXS('test')
 detselect(pilatus2M)
 
 pilatus2M.cam.num_images.put(1)
 
-RE(bp.count([pilatus2M], num=3))
+uid, = RE(bp.count([pilatus2M], num=3))
+
+hdr = db[uid]
+print(hdr.table(fill=True))
+
+data = np.array(list(hdr.data("pilatus2M_image")))
+print(f"{data = }\n{data.shape = }")

.ci/kafka-config.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -veo pipefail
+
+# Kafka config:
+cat << EOF > kafka.yml
+---
+abort_run_on_kafka_exception: false
+bootstrap_servers:
+    - localhost:9092
+runengine_producer_config:
+    security.protocol: PLAINTEXT
+EOF
+
+mkdir -v -p $HOME/.config/bluesky/
+mv -v kafka.yml $HOME/.config/bluesky/kafka.yml
+cat $HOME/.config/bluesky/kafka.yml

.ci/tiled-config.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -veo pipefail
+
 # Tiled profile config:
 
 tiled_profiles_dir="$HOME/.config/tiled/profiles/"

.github/workflows/testing.yml

@@ -14,81 +14,134 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        conda-env-version: ["2024-3.0"]
-        zenodo-id: ["14019710"]
-        md5-checksum: ["0ec1899a4f5989292e07380f96b97498"]
-        python-version: ["3.10"]  # , "3.11", "3.12"]
+        zenodo-deposition:
+          # https://zenodo.org/records/14862443
+          - env-version: "2025-1.0"
+            zenodo-id: "14862443"
+        artifact:
+          - python-version: "3.10"
+            md5-checksum: "d031ed24ad7beebb836f93712d8def2e"
+          - python-version: "3.11"
+            md5-checksum: "3b1074c0827ee3baf0cdcd64917b9afa"
+          - python-version: "3.12"
+            md5-checksum: "624239ed0846ef95d8334ea100581b83"
       fail-fast: false
     env:
       TZ: America/New_York
 
     defaults:
       run:
-        shell: bash -l {0}
+        shell: bash -leo pipefail {0}
 
     steps:
       - name: Set env vars
         run: |
           export REPOSITORY_NAME=${GITHUB_REPOSITORY#*/}  # just the repo, as opposed to org/repo
           echo "REPOSITORY_NAME=${REPOSITORY_NAME}" >> $GITHUB_ENV
 
-          export PYTHONVER=$(echo ${{ matrix.python-version }} | sed 's/\.//g')
+          export PYTHONVER=$(echo ${{ matrix.artifact.python-version }} | sed 's/\.//g')
           echo "PYTHONVER=${PYTHONVER}" >> $GITHUB_ENV
 
-          export CONDA_ENV_NAME="${{ matrix.conda-env-version }}-py${PYTHONVER}-tiled"
+          export CONDA_ENV_NAME="${{ matrix.zenodo-deposition.env-version }}-py${PYTHONVER}-tiled"
           echo "CONDA_ENV_NAME=${CONDA_ENV_NAME}" >> $GITHUB_ENV
 
+          export BLUESKY_KAFKA_CONFIG_PATH="$HOME/.config/bluesky/kafka.yml"
+          echo "BLUESKY_KAFKA_CONFIG_PATH=${BLUESKY_KAFKA_CONFIG_PATH}" >> $GITHUB_ENV
+
+      - name: Install system packages
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y xorg openbox
+
+      - name: Check Docker version
+        run: |
+          docker version
+          docker compose --version
+
+      - name: Start Redis
+        uses: supercharge/redis-github-action@1.0.0
+
+      - name: Start MongoDB
+        uses: supercharge/mongodb-github-action@1.6.0
+
       - name: Checkout the code
         uses: actions/checkout@v4
 
-      - name: Setup umamba
-        uses: mamba-org/setup-micromamba@v1
+      - name: Configure epics-containers
+        run: |
+          # git clone --depth 1 --branch CMS-IOCs https://github.com/NSLS2/cms-epics-containers.git $HOME/cms-epics-containers
+          git clone --depth 1 --branch ci-gha-profile https://github.com/mrakitin/epics-containers-example-services.git $HOME/cms-epics-containers
+
+          # source $HOME/cms-epics-containers/environment.sh
+          # echo "EPICS_CA_ADDR_LIST=${EPICS_CA_ADDR_LIST}" >> $GITHUB_ENV
+
+      - name: Prepare Tiled and Kafka configs
+        run: |
+          .ci/tiled-config.sh
+          .ci/kafka-config.sh
+
+      - name: Run beamline-specific setup
+        run: |
+          .ci/bl-specific.sh
+
+      - name: Start epics-containers
+        run: |
+          cd $HOME/cms-epics-containers
+          source $HOME/cms-epics-containers/environment.sh
+          docker compose up -d
+          sleep 20
 
-      - name: Setup umamba
+      - name: Check Docker state
         run: |
-          set -vxeo pipefail
-          url="https://zenodo.org/record/${{ matrix.zenodo-id }}/files/${CONDA_ENV_NAME}.tar.gz?download=1"
+          docker images
+          docker ps -a
+          docker network ls
+          docker volume ls
+
+      - name: Check Docker logs
+        run: |
+          cd $HOME/cms-epics-containers
+          docker compose logs
+
+      - uses: conda-incubator/setup-miniconda@v3
+        with:
+          activate-environment: ${{ env.REPOSITORY_NAME }}
+          auto-update-conda: true
+          miniforge-version: latest
+          python-version: "3.10"
+          conda-remove-defaults: true
+
+      - name: Create a conda env from Zenodo
+        run: |
+          url="https://zenodo.org/record/${{ matrix.zenodo-deposition.zenodo-id }}/files/${CONDA_ENV_NAME}.tar.gz?download=1"
           wget --progress=dot:giga ${url} -O ${CONDA_ENV_NAME}.tar.gz
           status=$?
           if [ $status -gt 0 ]; then
               echo "Cannot download from ${url}. Exit code: ${status}"
               exit $status
           fi
-          echo "${{ matrix.md5-checksum }} ${CONDA_ENV_NAME}.tar.gz" > checksum.txt
+          echo "${{ matrix.artifact.md5-checksum }} ${CONDA_ENV_NAME}.tar.gz" > checksum.txt
           md5sum --check checksum.txt
           mkdir -p $HOME/${CONDA_ENV_NAME}
+          echo "Unarchiving the tarball..."
           tar -xf ${CONDA_ENV_NAME}.tar.gz -C $HOME/${CONDA_ENV_NAME}
           conda activate $HOME/${CONDA_ENV_NAME}
-          conda unpack
-
-      - name: Start MongoDB
-        uses: supercharge/mongodb-github-action@1.6.0
-
-      - name: Prepare databroker config
-        run: |
-          set -vxeuo pipefail
-          mkdir -v -p $HOME/.config/databroker/
-          cp -v configs/databroker/local.yml $HOME/.config/databroker/
-
-      - name: Start epics-containers
-        run: |
-          set -vxeuo pipefail
-          git clone --depth 1 --branch CMS-IOCs https://github.com/NSLS2/cms-epics-containers.git ~/cms-epics-containers
-          source ~/cms-epics-containers/environment.sh
-          cd ~/cms-epics-containers
-          docker-compose up -d
-          sleep 20
+          conda-unpack
 
       - name: Start caproto IOC
         run: |
-          set -vxeuo pipefail
-          source ~/cms-epics-containers/environment.sh
-          echo "\n" | python iocs/spoof_beamline.py --list-pvs
+          source $HOME/cms-epics-containers/environment.sh
+          conda activate $HOME/${CONDA_ENV_NAME}
+          echo "\n" | python iocs/spoof_beamline.py --list-pvs &
           sleep 20
 
       - name: Test the code
         run: |
-          set -vxeuo pipefail
+          # Start Xvfb
+          # (from https://developercommunity.visualstudio.com/content/problem/336288/headless-testing-using-xvfb-on-hosted-ubuntu-1604.html)
+          /sbin/start-stop-daemon --start --quiet --pidfile /tmp/custom_xvfb_99.pid --make-pidfile --background --exec /usr/bin/Xvfb -- :99 -ac -screen 0 1280x1024x16
+          export DISPLAY=:99
+
           # This is what IPython does internally to load the startup files:
           command="
           import os
@@ -105,4 +158,20 @@ jobs:
               print(f'Executing {f} in CI')
               ip.parent._exec_file(f)"
 
+          conda activate $HOME/${CONDA_ENV_NAME}
+
+          source $HOME/cms-epics-containers/environment.sh
+          env | grep EPICS
           ipython --profile=test -c "$command"
+
+      - name: Show bluesky logs
+        if: always()
+        run: |
+          cat $HOME/.cache/bluesky/log/bluesky.log
+
+      - name: Check dir tree
+        if: always()
+        run: |
+          ls -laF /nsls2/
+          tree -a $HOME/cms-epics-containers/pilatus-data/
+          tree -a /nsls2/data/