Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Multiple databases lock #1165

Draft
wants to merge 4 commits into
base: development
Choose a base branch
from
Draft

Multiple databases lock #1165

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
db47b1d
Allow multiple test databases to be locked
arteymix Jun 26, 2024
0777e65
Comment uniwanted parts of the build (to be reverted)
arteymix Jun 26, 2024
4f8520f
Override gemma.testdb.name in pom.xml
arteymix Jun 26, 2024
dd93060
Ignore testComputeDevRankForExpressionExperimentB for now (see #1158)
arteymix Jun 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
336 changes: 174 additions & 162 deletions .jenkins/Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
//file:noinspection GrUnresolvedAccess

def testdbUrl
def gemmaVersion
def baseCodeVersion
def dataDir
Expand Down Expand Up @@ -41,7 +40,6 @@ pipeline {
steps {
checkout scm
script {
testdbUrl = sh script: 'mvn help:evaluate -Dexpression=gemma.testdb.build.url -q -DforceStdout', returnStdout: true
gemmaVersion = sh script: 'mvn help:evaluate -Dexpression=project.version -q -DforceStdout', returnStdout: true
baseCodeVersion = sh script: 'mvn help:evaluate -Dartifact=baseCode:baseCode -Dexpression=project.version -q -DforceStdout', returnStdout: true
if (env.BRANCH_NAME == "master" && gemmaVersion.endsWith("-SNAPSHOT")) {
Expand Down Expand Up @@ -78,103 +76,117 @@ pipeline {
sh 'mvn -B clean'
}
}
stage('Build') {
steps {
setBuildStatus "Build", "Build #${env.BUILD_NUMBER} has started...", 'PENDING'
sh 'mvn -B compile'
}
post {
success {
setBuildStatus "Build", "Build #${env.BUILD_NUMBER} succeeded.", 'SUCCESS'
}
unsuccessful {
setBuildStatus "Build", "Build #${env.BUILD_NUMBER} failed.", 'FAILURE'
}
}
}
stage('Run quick unit tests') {
steps {
setBuildStatus "Tests", "Tests for build #${env.BUILD_NUMBER} have started...", 'PENDING'
sh 'mvn -B test -DexcludedGroups=SlowTest'
}
post {
always {
junit testResults: '**/target/surefire-reports/*.xml'
}
success {
setBuildStatus "Tests", "Tests for build #${env.BUILD_NUMBER} have passed.", 'SUCCESS'
}
unsuccessful {
setBuildStatus "Tests", "Tests for build #${env.BUILD_NUMBER} failed.", 'FAILURE'
}
}
}
stage('Package') {
steps {
setBuildStatus "Package", "Package for build #${env.BUILD_NUMBER} have started...", 'PENDING'
sh 'mvn -B package -DskipTests'
}
post {
success {
archiveArtifacts artifacts: '**/target/*.jar, **/target/*.war', fingerprint: true
setBuildStatus "Package", "Package for build #${env.BUILD_NUMBER} have passed.", 'SUCCESS'
}
unsuccessful {
setBuildStatus "Package", "Package for build #${env.BUILD_NUMBER} failed.", 'FAILURE'
}
}
}
stage('Run integration tests and perform deployment in parallel') {
when {
anyOf {
branch 'master'
branch 'hotfix-*'
branch 'release-*'
branch 'development'
expression { params.FORCE_INTEGRATION_TESTS }
}
}
parallel {
stage('Run slow unit tests') {
when {
anyOf {
// we can safely ignore integration tests on the master branch, those were run in hotfix/release
branch 'hotfix-*'
branch 'release-*'
branch 'development'
expression { params.FORCE_SLOW_TESTS }
}
}
steps {
setBuildStatus "Slow tests", "Slow tests #${env.BUILD_NUMBER} have started...", 'PENDING'
sh 'mvn -B test -Dgroups=SlowTest'
}
post {
always {
junit testResults: '**/target/surefire-reports/*.xml'
}
success {
setBuildStatus "Slow tests", "Slow tests for build #${env.BUILD_NUMBER} have passed.", 'SUCCESS'
}
unsuccessful {
setBuildStatus "Slow tests", "Slow tests for build #${env.BUILD_NUMBER} failed.", 'FAILURE'
}
}
}
// stage('Build') {
// steps {
// setBuildStatus "Build", "Build #${env.BUILD_NUMBER} has started...", 'PENDING'
// sh 'mvn -B compile'
// }
// post {
// success {
// setBuildStatus "Build", "Build #${env.BUILD_NUMBER} succeeded.", 'SUCCESS'
// }
// unsuccessful {
// setBuildStatus "Build", "Build #${env.BUILD_NUMBER} failed.", 'FAILURE'
// }
// }
// }
// stage('Run quick unit tests') {
// steps {
// setBuildStatus "Tests", "Tests for build #${env.BUILD_NUMBER} have started...", 'PENDING'
// sh 'mvn -B test -DexcludedGroups=SlowTest'
// }
// post {
// always {
// junit testResults: '**/target/surefire-reports/*.xml'
// }
// success {
// setBuildStatus "Tests", "Tests for build #${env.BUILD_NUMBER} have passed.", 'SUCCESS'
// }
// unsuccessful {
// setBuildStatus "Tests", "Tests for build #${env.BUILD_NUMBER} failed.", 'FAILURE'
// }
// }
// }
// stage('Package') {
// steps {
// setBuildStatus "Package", "Package for build #${env.BUILD_NUMBER} have started...", 'PENDING'
// sh 'mvn -B package -DskipTests'
// }
// post {
// success {
// archiveArtifacts artifacts: '**/target/*.jar, **/target/*.war', fingerprint: true
// setBuildStatus "Package", "Package for build #${env.BUILD_NUMBER} have passed.", 'SUCCESS'
// }
// unsuccessful {
// setBuildStatus "Package", "Package for build #${env.BUILD_NUMBER} failed.", 'FAILURE'
// }
// }
// }
//stage('Run integration tests and perform deployment in parallel') {
// when {
// anyOf {
// branch 'master'
// branch 'hotfix-*'
// branch 'release-*'
// branch 'development'
// expression { params.FORCE_INTEGRATION_TESTS }
// }
// }
// parallel {
// stage('Run slow unit tests') {
// when {
// anyOf {
// // we can safely ignore integration tests on the master branch, those were run in hotfix/release
// branch 'hotfix-*'
// branch 'release-*'
// branch 'development'
// expression { params.FORCE_SLOW_TESTS }
// }
// }
// steps {
// setBuildStatus "Slow tests", "Slow tests #${env.BUILD_NUMBER} have started...", 'PENDING'
// sh 'mvn -B test -Dgroups=SlowTest'
// }
// post {
// always {
// junit testResults: '**/target/surefire-reports/*.xml'
// }
// success {
// setBuildStatus "Slow tests", "Slow tests for build #${env.BUILD_NUMBER} have passed.", 'SUCCESS'
// }
// unsuccessful {
// setBuildStatus "Slow tests", "Slow tests for build #${env.BUILD_NUMBER} failed.", 'FAILURE'
// }
// }
// }
stage('Run integration tests') {
when {
anyOf {
// we can safely ignore integration tests on the master branch, those were run in hotfix/release
branch 'hotfix-*'
branch 'release-*'
branch 'development'
expression { params.FORCE_INTEGRATION_TESTS }
}
}
// when {
// anyOf {
// // we can safely ignore integration tests on the master branch, those were run in hotfix/release
// branch 'hotfix-*'
// branch 'release-*'
// branch 'development'
// expression { params.FORCE_INTEGRATION_TESTS }
// }
// }
steps {
setBuildStatus "Integration tests", "Integration tests #${env.BUILD_NUMBER} have started...", 'PENDING'
lock('database/' + testdbUrl) {
sh 'mvn -B verify -DskipUnitTests -Dmaven.javadoc.skip=true'
// lock the first available test database
script {
def availableTestdbs = ['gemdtest', 'gemdtest2', 'gemdtest3', 'gemdtest4']
def successfullyLocked = false
for (def testdbName in availableTestdbs) {
def testdbUrl = sh script: "mvn help:evaluate -Dgemma.testdb.name=${testdbName} -Dexpression=gemma.testdb.build.url -q -DforceStdout", returnStdout: true
lock(resource: 'database/' + testdbUrl, skipIfLocked: availableTestdbs[-1] != testdbName) {
successfullyLocked = true
sh "mvn -B verify -DskipUnitTests -Dmaven.javadoc.skip=true -Dgemma.testdb.name=${testdbName}"
}
if (successfullyLocked) {
break;
} else {
echo "${testdbUrl} is already in use, trying the next available test database..."
}
}
}
}
post {
Expand All @@ -189,72 +201,72 @@ pipeline {
}
}
}
stage('Deploy artifacts') {
when {
anyOf {
branch 'master'
branch 'hotfix-*'
branch 'release-*'
branch 'development'
}
}
steps {
sh 'mvn -B deploy -DskipTests'
}
}
stage('Deploy Maven website') {
when {
anyOf {
branch 'master'
branch 'hotfix-*'
branch 'release-*'
branch 'development'
}
}
steps {
sh 'mvn -B site-deploy'
script {
if (dataDir != null) {
sh "ln -Tsf ${params.MAVEN_SITES_DIR}/gemma/gemma-${gemmaVersion} ${dataDir}/gemma-devsite"
sh "ln -Tsf ${params.MAVEN_SITES_DIR}/baseCode/baseCode-${baseCodeVersion} ${dataDir}/baseCode-site"
}
}
}
}
stage('Deploy Gemma Web') {
when {
anyOf {
branch 'master'
// we only support one staging server
branch params.STAGING_BRANCH + '-*'
branch 'development'
}
}
steps {
sh "rsync gemma-web/src/main/config/log4j.properties ${deployDir}/lib/log4j.properties"
sh "rsync gemma-web/target/Gemma.war ${deployDir}/Gemma.war"
}
}
stage('Deploy Gemma CLI') {
when {
anyOf {
branch 'master'
branch 'hotfix-*'
branch 'release-*'
// never deploy the development CLI, it's too risky
}
}
steps {
sh "rsync -av --delete gemma-cli/target/appassembler/ ${cliDir}/"
script {
if (cliDirSymlink != null) {
sh "ln -Tsf ${cliDir} ${cliDirSymlink}"
}
}
}
}
}
}
// stage('Deploy artifacts') {
// when {
// anyOf {
// branch 'master'
// branch 'hotfix-*'
// branch 'release-*'
// branch 'development'
// }
// }
// steps {
// sh 'mvn -B deploy -DskipTests'
// }
// }
// stage('Deploy Maven website') {
// when {
// anyOf {
// branch 'master'
// branch 'hotfix-*'
// branch 'release-*'
// branch 'development'
// }
// }
// steps {
// sh 'mvn -B site-deploy'
// script {
// if (dataDir != null) {
// sh "ln -Tsf ${params.MAVEN_SITES_DIR}/gemma/gemma-${gemmaVersion} ${dataDir}/gemma-devsite"
// sh "ln -Tsf ${params.MAVEN_SITES_DIR}/baseCode/baseCode-${baseCodeVersion} ${dataDir}/baseCode-site"
// }
// }
// }
// }
// stage('Deploy Gemma Web') {
// when {
// anyOf {
// branch 'master'
// // we only support one staging server
// branch params.STAGING_BRANCH + '-*'
// branch 'development'
// }
// }
// steps {
// sh "rsync gemma-web/src/main/config/log4j.properties ${deployDir}/lib/log4j.properties"
// sh "rsync gemma-web/target/Gemma.war ${deployDir}/Gemma.war"
// }
// }
// stage('Deploy Gemma CLI') {
// when {
// anyOf {
// branch 'master'
// branch 'hotfix-*'
// branch 'release-*'
// // never deploy the development CLI, it's too risky
// }
// }
// steps {
// sh "rsync -av --delete gemma-cli/target/appassembler/ ${cliDir}/"
// script {
// if (cliDirSymlink != null) {
// sh "ln -Tsf ${cliDir} ${cliDirSymlink}"
// }
// }
// }
// }
// }
//}
}
post {
fixed {
Expand Down
2 changes: 2 additions & 0 deletions ...st/java/ubic/gemma/core/analysis/preprocess/ProcessedExpressionDataCreateServiceTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.RandomStringUtils;
import org.junit.After;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.springframework.beans.factory.annotation.Autowired;
Expand Down Expand Up @@ -112,6 +113,7 @@ public void tearDown() {

@Test
@Category(SlowTest.class)
@Ignore("This test randomly fails, see https://github.com/PavlidisLab/Gemma/issues/1158")
public void testComputeDevRankForExpressionExperimentB() throws Exception {

try {
Expand Down
Loading