diff --git a/.ci/bl-specific.sh b/.ci/bl-specific.sh old mode 100644 new mode 100755 index f00b1ca..71fe435 --- a/.ci/bl-specific.sh +++ b/.ci/bl-specific.sh @@ -1,3 +1,20 @@ #!/bin/bash -# cp -v <...> ~/.ipython/profile_${TEST_PROFILE}/ +set -veo pipefail + +# Per https://stackoverflow.com/a/66982842: +sudo echo "127.0.0.1 info.cms.nsls2.bnl.gov" | sudo tee -a /etc/hosts +cat /etc/hosts + +ping -c 5 info.cms.nsls2.bnl.gov +echo "" | telnet info.cms.nsls2.bnl.gov 6379 + +# Copy config files into the dummy IPython profile: +mkdir -v -p ~/.ipython/profile_test/startup/ +cp -v startup/.cms_config ~/.ipython/profile_test/startup/ + +# Create pilatus and /nsls2 dir trees: +sudo mkdir -v -p /nsls2/ +sudo chown -R -v $USER: /nsls2/ +mkdir -v -p $HOME/cms-epics-containers/pilatus-data/data/ +ln -sv $HOME/cms-epics-containers/pilatus-data/data/ /nsls2/ diff --git a/.ci/drop-in.py b/.ci/drop-in.py index d26eb14..06f6a2d 100644 --- a/.ci/drop-in.py +++ b/.ci/drop-in.py @@ -1,3 +1,5 @@ +import numpy as np + caput(beam.mono_bragg_pv, 1.03953) pilatus2M.tiff.create_directory.set(-20) sam = SampleGISAXS('test') @@ -5,4 +7,10 @@ pilatus2M.cam.num_images.put(1) -RE(bp.count([pilatus2M], num=3)) +uid, = RE(bp.count([pilatus2M], num=3)) + +hdr = db[uid] +print(hdr.table(fill=True)) + +data = np.array(list(hdr.data("pilatus2M_image"))) +print(f"{data = }\n{data.shape = }") diff --git a/.ci/kafka-config.sh b/.ci/kafka-config.sh new file mode 100755 index 0000000..baf11fb --- /dev/null +++ b/.ci/kafka-config.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -veo pipefail + +# Kafka config: +cat << EOF > kafka.yml +--- +abort_run_on_kafka_exception: false +bootstrap_servers: + - localhost:9092 +runengine_producer_config: + security.protocol: PLAINTEXT +EOF + +mkdir -v -p $HOME/.config/bluesky/ +mv -v kafka.yml $HOME/.config/bluesky/kafka.yml +cat $HOME/.config/bluesky/kafka.yml diff --git a/.ci/tiled-config.sh b/.ci/tiled-config.sh index 9277949..21f27bf 100755 --- a/.ci/tiled-config.sh +++ b/.ci/tiled-config.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -veo pipefail + # Tiled profile config: tiled_profiles_dir="$HOME/.config/tiled/profiles/" diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index a41e6b3..06f83de 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -14,17 +14,24 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - conda-env-version: ["2024-3.0"] - zenodo-id: ["14019710"] - md5-checksum: ["0ec1899a4f5989292e07380f96b97498"] - python-version: ["3.10"] # , "3.11", "3.12"] + zenodo-deposition: + # https://zenodo.org/records/14862443 + - env-version: "2025-1.0" + zenodo-id: "14862443" + artifact: + - python-version: "3.10" + md5-checksum: "d031ed24ad7beebb836f93712d8def2e" + - python-version: "3.11" + md5-checksum: "3b1074c0827ee3baf0cdcd64917b9afa" + - python-version: "3.12" + md5-checksum: "624239ed0846ef95d8334ea100581b83" fail-fast: false env: TZ: America/New_York defaults: run: - shell: bash -l {0} + shell: bash -leo pipefail {0} steps: - name: Set env vars @@ -32,63 +39,109 @@ jobs: export REPOSITORY_NAME=${GITHUB_REPOSITORY#*/} # just the repo, as opposed to org/repo echo "REPOSITORY_NAME=${REPOSITORY_NAME}" >> $GITHUB_ENV - export PYTHONVER=$(echo ${{ matrix.python-version }} | sed 's/\.//g') + export PYTHONVER=$(echo ${{ matrix.artifact.python-version }} | sed 's/\.//g') echo "PYTHONVER=${PYTHONVER}" >> $GITHUB_ENV - export CONDA_ENV_NAME="${{ matrix.conda-env-version }}-py${PYTHONVER}-tiled" + export CONDA_ENV_NAME="${{ matrix.zenodo-deposition.env-version }}-py${PYTHONVER}-tiled" echo "CONDA_ENV_NAME=${CONDA_ENV_NAME}" >> $GITHUB_ENV + export BLUESKY_KAFKA_CONFIG_PATH="$HOME/.config/bluesky/kafka.yml" + echo "BLUESKY_KAFKA_CONFIG_PATH=${BLUESKY_KAFKA_CONFIG_PATH}" >> $GITHUB_ENV + + - name: Install system packages + run: | + sudo apt-get update + sudo apt-get install -y xorg openbox + + - name: Check Docker version + run: | + docker version + docker compose --version + + - name: Start Redis + uses: supercharge/redis-github-action@1.0.0 + + - name: Start MongoDB + uses: supercharge/mongodb-github-action@1.6.0 + - name: Checkout the code uses: actions/checkout@v4 - - name: Setup umamba - uses: mamba-org/setup-micromamba@v1 + - name: Configure epics-containers + run: | + # git clone --depth 1 --branch CMS-IOCs https://github.com/NSLS2/cms-epics-containers.git $HOME/cms-epics-containers + git clone --depth 1 --branch ci-gha-profile https://github.com/mrakitin/epics-containers-example-services.git $HOME/cms-epics-containers + + # source $HOME/cms-epics-containers/environment.sh + # echo "EPICS_CA_ADDR_LIST=${EPICS_CA_ADDR_LIST}" >> $GITHUB_ENV + + - name: Prepare Tiled and Kafka configs + run: | + .ci/tiled-config.sh + .ci/kafka-config.sh + + - name: Run beamline-specific setup + run: | + .ci/bl-specific.sh + + - name: Start epics-containers + run: | + cd $HOME/cms-epics-containers + source $HOME/cms-epics-containers/environment.sh + docker compose up -d + sleep 20 - - name: Setup umamba + - name: Check Docker state run: | - set -vxeo pipefail - url="https://zenodo.org/record/${{ matrix.zenodo-id }}/files/${CONDA_ENV_NAME}.tar.gz?download=1" + docker images + docker ps -a + docker network ls + docker volume ls + + - name: Check Docker logs + run: | + cd $HOME/cms-epics-containers + docker compose logs + + - uses: conda-incubator/setup-miniconda@v3 + with: + activate-environment: ${{ env.REPOSITORY_NAME }} + auto-update-conda: true + miniforge-version: latest + python-version: "3.10" + conda-remove-defaults: true + + - name: Create a conda env from Zenodo + run: | + url="https://zenodo.org/record/${{ matrix.zenodo-deposition.zenodo-id }}/files/${CONDA_ENV_NAME}.tar.gz?download=1" wget --progress=dot:giga ${url} -O ${CONDA_ENV_NAME}.tar.gz status=$? if [ $status -gt 0 ]; then echo "Cannot download from ${url}. Exit code: ${status}" exit $status fi - echo "${{ matrix.md5-checksum }} ${CONDA_ENV_NAME}.tar.gz" > checksum.txt + echo "${{ matrix.artifact.md5-checksum }} ${CONDA_ENV_NAME}.tar.gz" > checksum.txt md5sum --check checksum.txt mkdir -p $HOME/${CONDA_ENV_NAME} + echo "Unarchiving the tarball..." tar -xf ${CONDA_ENV_NAME}.tar.gz -C $HOME/${CONDA_ENV_NAME} conda activate $HOME/${CONDA_ENV_NAME} - conda unpack - - - name: Start MongoDB - uses: supercharge/mongodb-github-action@1.6.0 - - - name: Prepare databroker config - run: | - set -vxeuo pipefail - mkdir -v -p $HOME/.config/databroker/ - cp -v configs/databroker/local.yml $HOME/.config/databroker/ - - - name: Start epics-containers - run: | - set -vxeuo pipefail - git clone --depth 1 --branch CMS-IOCs https://github.com/NSLS2/cms-epics-containers.git ~/cms-epics-containers - source ~/cms-epics-containers/environment.sh - cd ~/cms-epics-containers - docker-compose up -d - sleep 20 + conda-unpack - name: Start caproto IOC run: | - set -vxeuo pipefail - source ~/cms-epics-containers/environment.sh - echo "\n" | python iocs/spoof_beamline.py --list-pvs + source $HOME/cms-epics-containers/environment.sh + conda activate $HOME/${CONDA_ENV_NAME} + echo "\n" | python iocs/spoof_beamline.py --list-pvs & sleep 20 - name: Test the code run: | - set -vxeuo pipefail + # Start Xvfb + # (from https://developercommunity.visualstudio.com/content/problem/336288/headless-testing-using-xvfb-on-hosted-ubuntu-1604.html) + /sbin/start-stop-daemon --start --quiet --pidfile /tmp/custom_xvfb_99.pid --make-pidfile --background --exec /usr/bin/Xvfb -- :99 -ac -screen 0 1280x1024x16 + export DISPLAY=:99 + # This is what IPython does internally to load the startup files: command=" import os @@ -105,4 +158,20 @@ jobs: print(f'Executing {f} in CI') ip.parent._exec_file(f)" + conda activate $HOME/${CONDA_ENV_NAME} + + source $HOME/cms-epics-containers/environment.sh + env | grep EPICS ipython --profile=test -c "$command" + + - name: Show bluesky logs + if: always() + run: | + cat $HOME/.cache/bluesky/log/bluesky.log + + - name: Check dir tree + if: always() + run: | + ls -laF /nsls2/ + tree -a $HOME/cms-epics-containers/pilatus-data/ + tree -a /nsls2/data/