Skip to content

Commit 2d15a1e

Browse files
author
jakubk
committed
update circleci config.yml
- install dependency codecov - update nipype/info.py if CIRCLE_TAG is set - retry docker image builds - download test data - fix workdir permission denied issue - save all docker images to single tarball - pipe dockerhub password to `docker login` - push all docker images to docker hub on success
1 parent 86d70f5 commit 2d15a1e

File tree

1 file changed

+99
-58
lines changed

1 file changed

+99
-58
lines changed

‎.circleci/config.yml

Lines changed: 99 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -1,49 +1,34 @@
1-
# Examples:
2-
# https://github.com/circleci/frontend/blob/master/.circleci/config.yml
3-
#
4-
# Questions
5-
# ---------
6-
# 1. Regarding the cache: what if the base Dockerfile is reverted to a previous
7-
# version? The cache for that Dockerfile will exist, so it will pull the
8-
# image, which is incorrect. Include a note in generate_dockerfiles.sh to
9-
# increase the version of the cache.
10-
111
version: 2
122
jobs:
133

144
compare_base_dockerfiles:
155
docker:
16-
- image: docker:17.06.2-ce-git # shell is /bin/ash (bash not available)
6+
- image: docker:17.09.0-ce-git
177
steps:
188
- checkout:
199
path: /home/circleci/nipype
2010
- run:
2111
name: Prune base Dockerfile in preparation for cache check
12+
working_directory: /home/circleci/nipype/docker
2213
command: |
2314
mkdir -p /tmp/docker
24-
25-
# Remove empty lines, comments, and the timestamp from the base
26-
# Dockerfile. Use the sha256 sum of this pruned Dockerfile as the
27-
# cache key.
28-
sed -e '/\s*#.*$/d' \
29-
-e '/^\s*$/d' \
30-
-e '/generation_timestamp/d' \
31-
/home/circleci/nipype/docker/Dockerfile.base \
32-
> /tmp/docker/Dockerfile.base-pruned
15+
# Use the sha256 sum of the pruned Dockerfile as the cache key.
16+
ash prune_dockerfile.sh Dockerfile.base > /tmp/docker/Dockerfile.base-pruned
3317
- restore_cache:
34-
key: dftest-v4-master-{{ checksum "/tmp/docker/Dockerfile.base-pruned" }}
18+
# TODO: change this to 'master' after we are sure this works.
19+
key: dftest-v5-enh/circleci-neurodocker-{{ checksum "/tmp/docker/Dockerfile.base-pruned" }}
3520
- run:
3621
name: Determine how to get base image
3722
command: |
3823
GET_BASE="/tmp/docker/get_base_image.sh"
3924
4025
# This directory comes from the cache.
4126
if [ -d /cache/base-dockerfile ]; then
42-
echo 'echo Pulling base image ...' > "$GET_BASE"
43-
echo 'docker pull kaczmarj/nipype:base' >> "$GET_BASE"
27+
echo "echo Pulling base image ..." > "$GET_BASE"
28+
echo "docker pull kaczmarj/nipype:base" >> "$GET_BASE"
4429
else
45-
echo 'echo Building base image ...' > "$GET_BASE"
46-
echo 'docker build -t kaczmarj/nipype:base - < /home/circleci/nipype/docker/Dockerfile.base' >> "$GET_BASE"
30+
echo "echo Building base image ..." > "$GET_BASE"
31+
echo "docker build -t kaczmarj/nipype:base - < /home/circleci/nipype/docker/Dockerfile.base" >> "$GET_BASE"
4732
fi
4833
- persist_to_workspace:
4934
root: /tmp
@@ -52,8 +37,7 @@ jobs:
5237

5338

5439
build_and_test:
55-
parallelism: 1
56-
# Ideally, we could test inside the main docker image.
40+
parallelism: 4
5741
machine:
5842
# Ubuntu 14.04 with Docker 17.03.0-ce
5943
image: circleci/classic:201703-01
@@ -62,48 +46,92 @@ jobs:
6246
path: /home/circleci/nipype
6347
- attach_workspace:
6448
at: /tmp
49+
- run:
50+
name: Get test dependencies
51+
command: |
52+
pip install --no-cache-dir codecov
53+
- run:
54+
name: Modify Nipype version if necessary
55+
working_directory: /home/circleci/nipype
56+
command: |
57+
if [ "$CIRCLE_TAG" != "" ]; then
58+
sed -i -E "s/(__version__ = )'[A-Za-z0-9.-]+'/\1'$CIRCLE_TAG'/" nipype/info.py
59+
fi
6560
- run:
6661
name: Get base image (pull or build)
6762
no_output_timeout: 60m
63+
# TODO: remove `docker pull` once once caching works.
6864
command: |
69-
bash /tmp/docker/get_base_image.sh
65+
# bash /tmp/docker/get_base_image.sh
66+
docker pull kaczmarj/nipype:base
7067
- run:
71-
name: Build main image (latest & py36)
68+
name: Build main image (py36)
7269
no_output_timeout: 60m
70+
working_directory: /home/circleci/nipype
7371
command: |
74-
cd /home/circleci/nipype
75-
76-
docker build --rm=false \
77-
--tag kaczmarj/nipype:latest \
78-
--tag kaczmarj/nipype:py36 \
79-
--build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` \
80-
--build-arg VCS_REF=`git rev-parse --short HEAD` \
81-
--build-arg VERSION=$CIRCLE_TAG .
72+
e=1 && for i in {1..5}; do
73+
docker build \
74+
--rm=false \
75+
--tag kaczmarj/nipype:latest \
76+
--tag kaczmarj/nipype:py36 \
77+
--build-arg BUILD_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
78+
--build-arg VCS_REF="$(git rev-parse --short HEAD)" \
79+
--build-arg VERSION="${CIRCLE_TAG}" /home/circleci/nipype \
80+
&& e=0 && break || sleep 15
81+
done && [ "$e" -eq "0" ]
8282
- run:
8383
name: Build main image (py27)
8484
no_output_timeout: 60m
85+
working_directory: /home/circleci/nipype
8586
command: |
86-
cd /home/circleci/nipype
87+
e=1 && for i in {1..5}; do
88+
docker build \
89+
--rm=false \
90+
--tag kaczmarj/nipype:py27 \
91+
--build-arg PYTHON_VERSION_MAJOR=2 \
92+
--build-arg PYTHON_VERSION_MINOR=7 \
93+
--build-arg BUILD_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
94+
--build-arg VCS_REF="$(git rev-parse --short HEAD)" \
95+
--build-arg VERSION="${CIRCLE_TAG}-py27" /home/circleci/nipype \
96+
&& e=0 && break || sleep 15
97+
done && [ "$e" -eq "0" ]
98+
- run:
99+
name: Download test data
100+
no_output_timeout: 20m
101+
working_directory: /home/circleci/examples
102+
environment:
103+
OSF_NIPYPE_URL: "https://files.osf.io/v1/resources/nefdp/providers/osfstorage"
104+
command: |
105+
export DATA_NIPYPE_TUTORIAL_URL="${OSF_NIPYPE_URL}/57f4739cb83f6901ed94bf21"
106+
curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_TUTORIAL_URL" | tar xj
107+
108+
export DATA_NIPYPE_FSL_COURSE="${OSF_NIPYPE_URL}/57f472cf9ad5a101f977ecfe"
109+
curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_COURSE" | tar xz
87110
88-
docker build --rm=false \
89-
--tag kaczmarj/nipype:py27 \
90-
--build-arg PYTHON_VERSION_MAJOR=2 \
91-
--build-arg PYTHON_VERSION_MINOR=7 \
92-
--build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` \
93-
--build-arg VCS_REF=`git rev-parse --short HEAD` \
94-
--build-arg VERSION=$CIRCLE_TAG-py27 /home/circleci/nipype
111+
export DATA_NIPYPE_FSL_FEEDS="${OSF_NIPYPE_URL}/57f473066c613b01f113e7af"
112+
curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_FEEDS" | tar xz
95113
- run:
96114
name: Run tests
115+
no_output_timeout: 4h
116+
environment:
117+
WORKDIR: /home/circleci/work
97118
command: |
98-
echo "This is node $CIRCLE_NODE_INDEX"
99-
echo "No tests to run yet."
119+
mkdir -p "$WORKDIR"
120+
chmod -R 777 "$WORKDIR"
121+
bash /home/circleci/nipype/.circleci/tests.sh
122+
- store_artifacts:
123+
path: /home/circleci/work/tests
100124
- run:
101125
name: Save Docker images to workspace
102126
no_output_timeout: 60m
103127
command: |
104128
if [ "$CIRCLE_NODE_INDEX" -eq "0" ]; then
105129
echo "Saving Docker images to tar.gz files ..."
106-
docker save kaczmarj/nipype:latest kaczmarj/nipype:py36 | gzip > /tmp/docker/nipype-latest-py36.tar.gz
130+
docker save kaczmarj/nipype:base \
131+
kaczmarj/nipype:latest \
132+
kaczmarj/nipype:py36 \
133+
kaczmarj/nipype:py27 > /tmp/docker/nipype-base-latest-py36-py27.tar
134+
echo "$(du -h /tmp/docker/nipype-base-latest-py36-py27.tar)"
107135
fi
108136
- persist_to_workspace:
109137
root: /tmp
@@ -113,7 +141,7 @@ jobs:
113141

114142
deploy:
115143
docker:
116-
- image: docker:17.06.2-ce-git
144+
- image: docker:17.09.0-ce-git
117145
steps:
118146
- checkout
119147
- setup_remote_docker
@@ -123,18 +151,27 @@ jobs:
123151
name: Load saved Docker images.
124152
no_output_timeout: 60m
125153
command: |
126-
docker load < /tmp/docker/nipype-latest-py36.tar.gz
154+
docker load < /tmp/docker/nipype-base-latest-py36-py27.tar
127155
- run:
128156
name: Push to DockerHub
129-
no_output_timeout: 60m
157+
no_output_timeout: 120m
130158
command: |
131-
if [ "${CIRCLE_BRANCH}" == "enh/circleci-neurodocker" ]; then
132-
docker login -u $DOCKER_USER -p $DOCKER_PASS
133-
docker push kaczmarj/nipype:latest
134-
docker push kaczmarj/nipype:py36
135-
fi
136-
# TODO: write pruned Dockerfile to cache here. Make a shell script that will
137-
# prune Dockerfiles
159+
echo "$DOCKER_PASS" | docker login -u "$DOCKER_USER" --password-stdin
160+
docker push kaczmarj/nipype:base
161+
docker push kaczmarj/nipype:latest
162+
docker push kaczmarj/nipype:py36
163+
docker push kaczmarj/nipype:py27
164+
- run:
165+
name: Prune base Dockerfile to update cache
166+
command: |
167+
cd /home/circleci/nipype/docker
168+
# Use the sha256 sum of the pruned Dockerfile as the cache key.
169+
ash prune_dockerfile.sh Dockerfile.base > /tmp/docker/Dockerfile.base-pruned
170+
- save_cache:
171+
paths:
172+
- /tmp/docker/Dockerfile.base-pruned
173+
key: dftest-v5-{{ .Branch }}-{{ checksum "/tmp/docker/Dockerfile.base-pruned" }}
174+
138175

139176
workflows:
140177
version: 2
@@ -145,5 +182,9 @@ workflows:
145182
requires:
146183
- compare_base_dockerfiles
147184
- deploy:
185+
filters:
186+
branches:
187+
# TODO: change this to master after we are sure this works.
188+
only: enh/circleci-neurodocker
148189
requires:
149190
- build_and_test

0 commit comments

Comments
 (0)