Skip to content

Commit d74fea4

Browse files
dliappisdliappis
authored
Fix IT tests after version bumps (#15827)
This commit fixes IT failures that frequently occur after version bumps due to missing unified release snapshot builds for the new version. This commit uses project specific DRA snapshot URLs for ES and Filebeat in all cases apart from release builds. Closes #2825
1 parent 15e19a9 commit d74fea4

File tree

2 files changed

+83
-26
lines changed

2 files changed

+83
-26
lines changed

‎build.gradle

+55-26
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ import org.logstash.gradle.tooling.ListProjectDependencies
5151
import org.logstash.gradle.tooling.ExtractBundledJdkVersion
5252
import org.logstash.gradle.tooling.SignAliasDefinitions
5353
import org.logstash.gradle.tooling.ToolingUtils
54+
import org.logstash.gradle.tooling.SnapshotArtifactURLs
5455

5556
allprojects {
5657
group = 'org.logstash'
@@ -174,12 +175,17 @@ tasks.register("configureArtifactInfo") {
174175
version = "$version-$versionQualifier"
175176
}
176177

177-
def isReleaseBuild = System.getenv('RELEASE') == "1" || versionQualifier
178+
boolean isReleaseBuild = System.getenv('RELEASE') == "1" || versionQualifier
178179
String apiResponse = artifactVersionsApi.toURL().text
179180

180181
def dlVersions = new JsonSlurper().parseText(apiResponse)
181182
String qualifiedVersion = dlVersions['versions'].grep(isReleaseBuild ? ~/^${version}$/ : ~/^${version}-SNAPSHOT/)[0]
182183
if (qualifiedVersion == null) {
184+
if (!isReleaseBuild) {
185+
project.ext.set("useProjectSpecificArtifactSnapshotUrl", true)
186+
project.ext.set("stackArtifactSuffix", "${version}-SNAPSHOT")
187+
return
188+
}
183189
throw new GradleException("could not find the current artifact from the artifact-api ${artifactVersionsApi} for ${version}")
184190
}
185191
// find latest reference to last build
@@ -190,6 +196,7 @@ tasks.register("configureArtifactInfo") {
190196

191197
project.ext.set("artifactApiVersionedBuildUrl", "${artifactVersionsApi}/${qualifiedVersion}/builds/${stackBuildVersion}")
192198
project.ext.set("stackArtifactSuffix", qualifiedVersion)
199+
project.ext.set("useProjectSpecificArtifactSnapshotUrl", false)
193200
}
194201
}
195202

@@ -432,16 +439,23 @@ tasks.register("downloadFilebeat") {
432439

433440
doLast {
434441
download {
435-
String downloadedFilebeatName = "filebeat-${project.ext.get("stackArtifactSuffix")}-${project.ext.get("beatsArchitecture")}"
442+
String beatVersion = project.ext.get("stackArtifactSuffix")
443+
String downloadedFilebeatName = "filebeat-${beatVersion}-${project.ext.get("beatsArchitecture")}"
436444
project.ext.set("unpackedFilebeatName", downloadedFilebeatName)
437445

438-
// find url of build artifact
439-
String artifactApiUrl = "${project.ext.get("artifactApiVersionedBuildUrl")}/projects/beats/packages/${downloadedFilebeatName}.tar.gz"
440-
String apiResponse = artifactApiUrl.toURL().text
441-
def buildUrls = new JsonSlurper().parseText(apiResponse)
442-
443-
project.ext.set("filebeatSnapshotUrl", System.getenv("FILEBEAT_SNAPSHOT_URL") ?: buildUrls["package"]["url"])
444-
project.ext.set("filebeatDownloadLocation", "${projectDir}/build/${downloadedFilebeatName}.tar.gz")
446+
if (project.ext.get("useProjectSpecificArtifactSnapshotUrl")) {
447+
def res = SnapshotArtifactURLs.packageUrls("beats", beatVersion, downloadedFilebeatName)
448+
project.ext.set("filebeatSnapshotUrl", System.getenv("FILEBEAT_SNAPSHOT_URL") ?: res.packageUrl)
449+
project.ext.set("filebeatDownloadLocation", "${projectDir}/build/${downloadedFilebeatName}.tar.gz")
450+
} else {
451+
// find url of build artifact
452+
String artifactApiUrl = "${project.ext.get("artifactApiVersionedBuildUrl")}/projects/beats/packages/${downloadedFilebeatName}.tar.gz"
453+
String apiResponse = artifactApiUrl.toURL().text
454+
def buildUrls = new JsonSlurper().parseText(apiResponse)
455+
456+
project.ext.set("filebeatSnapshotUrl", System.getenv("FILEBEAT_SNAPSHOT_URL") ?: buildUrls["package"]["url"])
457+
project.ext.set("filebeatDownloadLocation", "${projectDir}/build/${downloadedFilebeatName}.tar.gz")
458+
}
445459

446460
src project.ext.filebeatSnapshotUrl
447461
onlyIfNewer true
@@ -477,14 +491,20 @@ tasks.register("checkEsSHA") {
477491
description "Download ES version remote's fingerprint file"
478492

479493
doLast {
480-
String downloadedElasticsearchName = "elasticsearch-${project.ext.get("stackArtifactSuffix")}-${project.ext.get("esArchitecture")}"
481-
482-
483-
// find url of build artifact
484-
String artifactApiUrl = "${project.ext.get("artifactApiVersionedBuildUrl")}/projects/elasticsearch/packages/${downloadedElasticsearchName}.tar.gz"
485-
String apiResponse = artifactApiUrl.toURL().text
486-
def buildUrls = new JsonSlurper().parseText(apiResponse)
487-
String remoteSHA = buildUrls.package.sha_url.toURL().text
494+
String esVersion = project.ext.get("stackArtifactSuffix")
495+
String downloadedElasticsearchName = "elasticsearch-${esVersion}-${project.ext.get("esArchitecture")}"
496+
String remoteSHA
497+
498+
if (project.ext.get("useProjectSpecificArtifactSnapshotUrl")) {
499+
def res = SnapshotArtifactURLs.packageUrls("elasticsearch", esVersion, downloadedElasticsearchName)
500+
remoteSHA = res.packageShaUrl
501+
} else {
502+
// find url of build artifact
503+
String artifactApiUrl = "${project.ext.get("artifactApiVersionedBuildUrl")}/projects/elasticsearch/packages/${downloadedElasticsearchName}.tar.gz"
504+
String apiResponse = artifactApiUrl.toURL().text
505+
def buildUrls = new JsonSlurper().parseText(apiResponse)
506+
remoteSHA = buildUrls.package.sha_url.toURL().text
507+
}
488508

489509
def localESArchive = new File("${projectDir}/build/${downloadedElasticsearchName}.tar.gz")
490510
if (localESArchive.exists()) {
@@ -512,22 +532,31 @@ tasks.register("checkEsSHA") {
512532

513533
tasks.register("downloadEs") {
514534
dependsOn = [configureArtifactInfo, checkEsSHA]
515-
description "Download ES Snapshot for current branch version: ${version}"
516535

536+
description "Download ES Snapshot for current branch version: ${version}"
517537
inputs.file("${projectDir}/versions.yml")
518538

519539
doLast {
520540
download {
521-
String downloadedElasticsearchName = "elasticsearch-${project.ext.get("stackArtifactSuffix")}-${project.ext.get("esArchitecture")}"
522-
project.ext.set("unpackedElasticsearchName", "elasticsearch-${project.ext.get("stackArtifactSuffix")}")
541+
String esVersion = project.ext.get("stackArtifactSuffix")
542+
String downloadedElasticsearchName = "elasticsearch-${esVersion}-${project.ext.get("esArchitecture")}"
523543

524-
// find url of build artifact
525-
String artifactApiUrl = "${project.ext.get("artifactApiVersionedBuildUrl")}/projects/elasticsearch/packages/${downloadedElasticsearchName}.tar.gz"
526-
String apiResponse = artifactApiUrl.toURL().text
527-
def buildUrls = new JsonSlurper().parseText(apiResponse)
544+
project.ext.set("unpackedElasticsearchName", "elasticsearch-${esVersion}")
545+
546+
if (project.ext.get("useProjectSpecificArtifactSnapshotUrl")) {
547+
def res = SnapshotArtifactURLs.packageUrls("elasticsearch", esVersion, downloadedElasticsearchName)
548+
project.ext.set("elasticsearchSnapshotURL", System.getenv("ELASTICSEARCH_SNAPSHOT_URL") ?: res.packageUrl)
549+
project.ext.set("elasticsearchDownloadLocation", "${projectDir}/build/${downloadedElasticsearchName}.tar.gz")
550+
} else {
551+
// find url of build artifact
552+
String artifactApiUrl = "${project.ext.get("artifactApiVersionedBuildUrl")}/projects/elasticsearch/packages/${downloadedElasticsearchName}.tar.gz"
553+
String apiResponse = artifactApiUrl.toURL().text
528554

529-
project.ext.set("elasticsearchSnapshotURL", System.getenv("ELASTICSEARCH_SNAPSHOT_URL") ?: buildUrls["package"]["url"])
530-
project.ext.set("elasticsearchDownloadLocation", "${projectDir}/build/${downloadedElasticsearchName}.tar.gz")
555+
def buildUrls = new JsonSlurper().parseText(apiResponse)
556+
557+
project.ext.set("elasticsearchSnapshotURL", System.getenv("ELASTICSEARCH_SNAPSHOT_URL") ?: buildUrls["package"]["url"])
558+
project.ext.set("elasticsearchDownloadLocation", "${projectDir}/build/${downloadedElasticsearchName}.tar.gz")
559+
}
531560

532561
src project.ext.elasticsearchSnapshotURL
533562
onlyIfNewer true

‎buildSrc/src/main/groovy/org/logstash/gradle/tooling/SnapshotArtifactURLs.groovy

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
package org.logstash.gradle.tooling
2+
3+
import groovy.json.JsonSlurper
4+
5+
/**
6+
* Helper class to obtain project specific (e.g. Elasticsearch or beats/filebeat) snapshot-DRA artifacts.
7+
* We use it in all cases apart from release builds.
8+
* */
9+
class SnapshotArtifactURLs {
10+
/**
11+
* Returns a list of the package and package SHA(512) URLs for a given project / version / downloadedPackageName
12+
* */
13+
static def packageUrls(String project, String projectVersion, String downloadedPackageName) {
14+
String artifactSnapshotVersionsApiPrefix = "https://artifacts-snapshot.elastic.co"
15+
16+
// e.g. https://artifacts-snapshot.elastic.co/elasticsearch/latest/8.11.5-SNAPSHOT.json
17+
String apiResponse = "${artifactSnapshotVersionsApiPrefix}/${project}/latest/${projectVersion}.json".toURL().text
18+
def artifactUrls = new JsonSlurper().parseText(apiResponse)
19+
String manifestUrl = artifactUrls["manifest_url"]
20+
// e.g. https://artifacts-snapshot.elastic.co/elasticsearch/8.11.5-12345678/manifest-8.11.5-SNAPSHOT.json
21+
apiResponse = manifestUrl.toURL().text
22+
def packageArtifactUrls = new JsonSlurper().parseText(apiResponse)
23+
String packageUrl = packageArtifactUrls["projects"]["${project}"]["packages"]["${downloadedPackageName}.tar.gz"]["url"]
24+
String packageShaUrl = packageArtifactUrls["projects"]["${project}"]["packages"]["${downloadedPackageName}.tar.gz"]["sha_url"]
25+
26+
return ["packageUrl": packageUrl, "packageShaUrl": packageShaUrl]
27+
}
28+
}

0 commit comments

Comments
 (0)