Skip to content

Commit 473096f

Browse files
pditommasopditommaso
committed
Added AWS batch CI tests
1 parent ba5ad7a commit 473096f

File tree

6 files changed

+127
-76
lines changed

6 files changed

+127
-76
lines changed

‎tests

Submodule tests updated from bddf366 to 1989dfe

‎validation/await.sh

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
#!/bin/bash
2+
set -eu
3+
4+
TIMEOUT=3600 # terminate after one hour
5+
KEEPALIVE=480 # time after which print an output to keep alive travis
6+
7+
# launch main command to await
8+
( "$@" | tee stdout.log ) &
9+
pid=$!
10+
11+
# make sure to terminate target command on exit
12+
trap "kill $pid" TERM INT USR1 USR2
13+
14+
# monitor task execution
15+
begin=$(date +%s)
16+
size=0
17+
while kill -0 $pid > /dev/null 2>&1; do
18+
sleep 1
19+
20+
# continue to await if there's a change
21+
# in the stdout file
22+
### BSD current=$(stat -f%z stdout.log)
23+
current=$(stat -c '%s' stdout.log)
24+
if [[ $current != $size ]]; then
25+
size=$current
26+
begin=$(date +%s)
27+
continue
28+
fi
29+
30+
# kill the execution if it's taking too
31+
# much time without producing any output
32+
now=$(date +%s)
33+
delta=$((now-begin))
34+
if ((delta>=$TIMEOUT)); then
35+
echo Taking too long ... killing it!
36+
kill $pid
37+
exit 1
38+
elif ((delta>=$KEEPALIVE)); then
39+
echo "[keep alive]"
40+
fi
41+
done

‎validation/awsbatch.config

Lines changed: 2 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,33 +1,5 @@
1+
workDir = 's3://cbcrg-eu/work'
12
process.executor = 'awsbatch'
2-
process.queue = 'circleci'
3+
process.queue = 'demo'
34
executor.awscli = '/home/ec2-user/miniconda/bin/aws'
4-
workDir = 's3://cbcrg-eu/work'
55
aws.region = 'eu-west-1'
6-
7-
profiles {
8-
hello {
9-
process.container = 'debian'
10-
}
11-
12-
rnatoy {
13-
params.reads = "s3://cbcrg-eu/ggal/*_{1,2}.fq"
14-
params.annot = "s3://cbcrg-eu/ggal/ggal_1_48850000_49020000.bed.gff"
15-
params.genome = "s3://cbcrg-eu/ggal/ggal_1_48850000_49020000.Ggal71.500bpflank.fa"
16-
process.container = 'nextflow/rnatoy@sha256:9ac0345b5851b2b20913cb4e6d469df77cf1232bafcadf8fd929535614a85c75'
17-
}
18-
19-
rnaseq {
20-
params.reads = "s3://cbcrg-eu/ggal/*_{1,2}.fq"
21-
params.transcriptome = "s3://cbcrg-eu/ggal/ggal_1_48850000_49020000.Ggal71.500bpflank.fa"
22-
params.multiqc = "s3://cbcrg-eu/ggal/multiqc"
23-
process.container = 'nextflow/rnaseq-nf:latest'
24-
}
25-
26-
callings {
27-
docker.enabled = true
28-
docker.runOptions = ""
29-
process.memory = 4.GB
30-
process.container = "885800555707.dkr.ecr.us-east-1.amazonaws.com/cbcrg/callings-with-gatk:latest"
31-
params.gatk = '/opt/broad/GenomeAnalysisTK.jar'
32-
}
33-
}

‎validation/awsbatch.nf

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
process foo {
2+
publishDir 'foo'
3+
container 'debian:latest'
4+
output:
5+
file '*.fa' into ch1
6+
file 'hello.txt' into ch2
7+
file '*.{zip,html}' into ch3
8+
file '01_A(R{1,2}).fastq' into ch4
9+
file 'sample_(1 2).vcf' into ch5
10+
file '.alpha' into ch6
11+
12+
script:
13+
$/
14+
echo A > hello.txt
15+
echo B > sample.zip
16+
echo C > sample.html
17+
echo D > 01_A\(R1\).fastq
18+
echo E > 01_A\(R2\).fastq
19+
echo F > sample_\(1\ 2\).vcf
20+
echo 1 > f1.fa
21+
echo 2 > f2.fa
22+
echo 3 > f3.fa
23+
mkdir .alpha
24+
echo "Hello world!" > .alpha/hello.txt
25+
/$
26+
}
27+
28+
process bar {
29+
echo true
30+
container 'debian:latest'
31+
input:
32+
file '*' from ch1.mix(ch2,ch3,ch4,ch5,ch6).collect()
33+
34+
script:
35+
$/
36+
cat .alpha/hello.txt
37+
[ `cat * | grep -c ''` == 9 ] || false
38+
/$
39+
}

‎validation/awsbatch.sh

Lines changed: 31 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,32 @@
1+
get_abs_filename() {
2+
echo "$(cd "$(dirname "$1")" && pwd)/$(basename "$1")"
3+
}
14

2-
#
3-
# AWS Batch tests
4-
#
5-
set -e
6-
$NXF_CMD run hello -c awsbatch.config -profile hello
7-
$NXF_CMD run rnatoy -c awsbatch.config -profile rnatoy
8-
$NXF_CMD run https://github.com/CRG-CNAG/CalliNGS-NF -c awsbatch.config -profile callings
5+
export NXF_CMD=${NXF_CMD:-$(get_abs_filename ../launch.sh)}
6+
7+
$NXF_CMD run awsbatch.nf -c awsbatch.config
8+
[[ -d foo ]] || false
9+
[[ -e 'foo/.alpha' ]] || false
10+
[[ -e 'foo/01_A(R1).fastq' ]] || false
11+
[[ -e 'foo/01_A(R2).fastq' ]] || false
12+
[[ -e 'foo/f1.fa' ]] || false
13+
[[ -e 'foo/f2.fa' ]] || false
14+
[[ -e 'foo/f3.fa' ]] || false
15+
[[ -e 'foo/hello.txt' ]] || false
16+
[[ -e 'foo/sample.html' ]] || false
17+
[[ -e 'foo/sample.zip' ]] || false
18+
[[ -e 'foo/sample_(1 2).vcf' ]] || false
19+
20+
rm -rf foo
21+
$NXF_CMD run awsbatch.nf -resume -c awsbatch.config
22+
[[ -d foo ]] || false
23+
[[ -e 'foo/.alpha' ]] || false
24+
[[ -e 'foo/01_A(R1).fastq' ]] || false
25+
[[ -e 'foo/01_A(R2).fastq' ]] || false
26+
[[ -e 'foo/f1.fa' ]] || false
27+
[[ -e 'foo/f2.fa' ]] || false
28+
[[ -e 'foo/f3.fa' ]] || false
29+
[[ -e 'foo/hello.txt' ]] || false
30+
[[ -e 'foo/sample.html' ]] || false
31+
[[ -e 'foo/sample.zip' ]] || false
32+
[[ -e 'foo/sample_(1 2).vcf' ]] || false

‎validation/test.sh

Lines changed: 13 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ get_abs_filename() {
77
}
88

99
export NXF_CMD=${NXF_CMD:-$(get_abs_filename ../launch.sh)}
10+
export TRAVIS_PULL_REQUEST=${TRAVIS_PULL_REQUEST:=false}
1011

1112
#
1213
# Tests
@@ -26,16 +27,6 @@ git clone https://github.com/nextflow-io/hello
2627
$NXF_CMD run . -resume
2728
)
2829

29-
#
30-
# Rna-Toy
31-
#
32-
git clone https://github.com/nextflow-io/rnatoy
33-
(
34-
cd rnatoy;
35-
$NXF_CMD run . -with-docker
36-
$NXF_CMD run . -with-docker -resume
37-
)
38-
3930
#
4031
# AMPA-NF
4132
#
@@ -48,37 +39,21 @@ docker pull cbcrg/ampa-nf
4839
)
4940

5041
#
51-
# MTA-NF
42+
# RNASEQ-NF
5243
#
53-
git clone https://github.com/cbcrg/mta-nf
54-
docker pull cbcrg/mta-nf
55-
(
56-
cd mta-nf;
57-
$NXF_CMD run . -with-docker --seq tutorial/small.fa --ntree 5 --msa clustalw
58-
$NXF_CMD run . -with-docker --seq tutorial/small.fa --ntree 5 --msa clustalw -resume
59-
)
44+
echo nextflow-io/rnaseq-nf
45+
$NXF_CMD run nextflow-io/rnaseq-nf -with-docker
46+
$NXF_CMD run nextflow-io/rnaseq-nf -with-docker -resume
6047

61-
#
62-
# GRAPE-NF
63-
#
64-
git clone https://github.com/cbcrg/grape-nf
65-
docker pull cbcrg/grape-nf
66-
(
67-
cd grape-nf;
68-
$NXF_CMD run . -with-docker
69-
$NXF_CMD run . -with-docker -resume
70-
)
48+
49+
if [[ $TRAVIS_PULL_REQUEST == false ]]; then
7150

7251
#
73-
# PIPER-NF
52+
# AWS Batch tests
7453
#
75-
git clone https://github.com/cbcrg/piper-nf
76-
docker pull cbcrg/piper-nf
77-
(
78-
cd piper-nf;
79-
$NXF_CMD run . -with-docker
80-
$NXF_CMD run . -with-docker -resume
81-
)
82-
83-
54+
echo aws batch tests
55+
./await.sh bash awsbatch.sh
8456

57+
else
58+
echo Skipping tests requiring AWS credentials
59+
fi

0 commit comments

Comments
 (0)