From fa45ede3a37acee3cbdc1b613715efed90f43865 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Tue, 25 Mar 2025 21:00:47 +0100 Subject: [PATCH 1/6] Reference cleanup scripts in generateCleanupCommand.sh Signed-off-by: Dennis Behm --- .../generateCleanupCommands.sh | 11 +++++----- .../pipelineBackend.config | 20 +++++++++++++++++++ 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/Templates/Common-Backend-Scripts/generateCleanupCommands.sh b/Templates/Common-Backend-Scripts/generateCleanupCommands.sh index 0daedaa6..d430ca95 100755 --- a/Templates/Common-Backend-Scripts/generateCleanupCommands.sh +++ b/Templates/Common-Backend-Scripts/generateCleanupCommands.sh @@ -72,13 +72,12 @@ Help() { } # -# Customization # Configuration file leveraged by the backend scripts # Either an absolute path or a relative path to the current working directory SCRIPT_HOME="$(dirname "$0")" pipelineConfiguration="${SCRIPT_HOME}/pipelineBackend.config" buildUtilities="${SCRIPT_HOME}/utilities/dbbBuildUtils.sh" -# Customization - End +deleteScript="${SCRIPT_HOME}/../..//Utilities/DeletePDS/DeletePDS.groovy" # # Internal Variables @@ -319,7 +318,7 @@ genDeleteStatementsCollections() { for applicationCollection in ${collectionsToBeDeleted[@]}; do - echo "dbb collection delete $applicationCollection ${dbbMetadataStoreOptions}" >>${cmdFileDeleteCollections} + echo "dbb collection delete $applicationCollection ${cleanupDbbMetadataStoreOptions}" >>${cmdFileDeleteCollections} done @@ -341,7 +340,7 @@ genDeleteStatementsBuildGroups() { for buildGroup in ${buildgroupsToBeDeleted[@]}; do - echo "dbb build-group delete $buildGroup ${dbbMetadataStoreOptions}" >>${cmdFileDeleteBuildGroups} + echo "dbb build-group delete $buildGroup ${cleanupDbbMetadataStoreOptions}" >>${cmdFileDeleteBuildGroups} done @@ -381,7 +380,7 @@ if [ $rc -eq 0 ]; then echo $PGM": [INFO] ** Cmd obsolete collections:" ${cmdFileDeleteCollections} echo $PGM": [INFO] ** Cmd obsolete build groups:" ${cmdFileDeleteBuildGroups} echo $PGM": [INFO] ** Cmd obsolete build datasets:" ${cmdFileDeleteBuildDatasets} - echo $PGM": [INFO] ** DBB Metadastore Config:" ${dbbMetadataStoreOptions} + echo $PGM": [INFO] ** DBB Metadastore Config:" ${cleanupDbbMetadataStoreOptions} echo $PGM": [INFO] ** Process Cleanup Scripts:" ${executeCleanupCommandScripts} echo $PGM": [INFO] **************************************************************" echo "" @@ -390,7 +389,7 @@ fi if [ $rc -eq 0 ]; then # Retrieve existing DBB Collections echo $PGM": [STAGE] Retrieve all collections with application qualifier $App" - applicationCollections=$(dbb collection list $dbbMetadataStoreOptions | grep $App) + applicationCollections=$(dbb collection list $cleanupDbbMetadataStoreOptions | grep $App) rc=$? if [ ! $rc -eq 0 ]; then ERRMSG=$PGM": [ERROR] Retrieving Collections failed. Check Log. rc="$rc diff --git a/Templates/Common-Backend-Scripts/pipelineBackend.config b/Templates/Common-Backend-Scripts/pipelineBackend.config index a7b77fc0..9c7c182b 100644 --- a/Templates/Common-Backend-Scripts/pipelineBackend.config +++ b/Templates/Common-Backend-Scripts/pipelineBackend.config @@ -81,6 +81,10 @@ dbbMetadataStoreJdbcId="" # Default = None, Required. dbbMetadataStoreJdbcPwdFile="" +# Absolute path to Db2 conf file containing db2 schema etc +# Default = None, Required. +dbbMetadataStoreJdbcConfigFile="" + # JDBC connection server url # sample jdbc:db2://10.3.20.201:4740/MOPDBC0 # Optional if zAppBuild is configured to use the @@ -326,6 +330,22 @@ wdDeployPackageDir() { ## End of wazideploy-generate.sh, wazideploy-deploy.sh and wazideploy-evidences.sh parameters #### ##################################################################################################### +##################################################################################################### +## genCleanupInstructions. parameters ############################################################### +##################################################################################################### + +# DBB Metadatastore Options String to configure Db2 or File Metadatastore in cleanup commands +# e.g. +# for file metadatastore +# cleanupDbbMetadataStoreOptions="--type file --location /u/github/" +# for db2 metadatastore, reusing build settings +# cleanupDbbMetadataStoreOptions="--type db2 --user ${dbbMetadataStoreJdbcId} --password-file ${dbbMetadataStoreJdbcPwdFile} --db2-config ${dbbMetadataStoreJdbcConfigFile} --url ${dbbMetadataStoreJdbcUrl}" +cleanupDbbMetadataStoreOptions="" + +##################################################################################################### +## End of genCleanupInstructions. parameters #### +##################################################################################################### + ##################################################################################################### ## Central functions shared across scripts ####################################################### ##################################################################################################### From ee7aa4ef2a83ab6569bbc7f4c28e1055172bd071 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Wed, 26 Mar 2025 08:34:17 +0100 Subject: [PATCH 2/6] Apply suggestions from code review Co-authored-by: Lauren Li <45975633+lauren-li@users.noreply.github.com> --- Templates/Common-Backend-Scripts/generateCleanupCommands.sh | 2 +- Templates/Common-Backend-Scripts/pipelineBackend.config | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Templates/Common-Backend-Scripts/generateCleanupCommands.sh b/Templates/Common-Backend-Scripts/generateCleanupCommands.sh index d430ca95..d79edea2 100755 --- a/Templates/Common-Backend-Scripts/generateCleanupCommands.sh +++ b/Templates/Common-Backend-Scripts/generateCleanupCommands.sh @@ -77,7 +77,7 @@ Help() { SCRIPT_HOME="$(dirname "$0")" pipelineConfiguration="${SCRIPT_HOME}/pipelineBackend.config" buildUtilities="${SCRIPT_HOME}/utilities/dbbBuildUtils.sh" -deleteScript="${SCRIPT_HOME}/../..//Utilities/DeletePDS/DeletePDS.groovy" +deleteScript="${SCRIPT_HOME}/../../Utilities/DeletePDS/DeletePDS.groovy" # # Internal Variables diff --git a/Templates/Common-Backend-Scripts/pipelineBackend.config b/Templates/Common-Backend-Scripts/pipelineBackend.config index 9c7c182b..5bc74adc 100644 --- a/Templates/Common-Backend-Scripts/pipelineBackend.config +++ b/Templates/Common-Backend-Scripts/pipelineBackend.config @@ -331,7 +331,7 @@ wdDeployPackageDir() { ##################################################################################################### ##################################################################################################### -## genCleanupInstructions. parameters ############################################################### +## generateCleanupCommands.sh parameters ############################################################### ##################################################################################################### # DBB Metadatastore Options String to configure Db2 or File Metadatastore in cleanup commands @@ -343,7 +343,7 @@ wdDeployPackageDir() { cleanupDbbMetadataStoreOptions="" ##################################################################################################### -## End of genCleanupInstructions. parameters #### +## End of generateCleanupCommands.sh parameters #### ##################################################################################################### ##################################################################################################### From 0dbcf119e73363b51d8ccf87a8e5bbeb73c3020c Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Wed, 26 Mar 2025 08:45:13 +0100 Subject: [PATCH 3/6] Minor formatting updates Signed-off-by: Dennis Behm --- Templates/Common-Backend-Scripts/pipelineBackend.config | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Templates/Common-Backend-Scripts/pipelineBackend.config b/Templates/Common-Backend-Scripts/pipelineBackend.config index 5bc74adc..a2e8d526 100644 --- a/Templates/Common-Backend-Scripts/pipelineBackend.config +++ b/Templates/Common-Backend-Scripts/pipelineBackend.config @@ -82,7 +82,7 @@ dbbMetadataStoreJdbcId="" dbbMetadataStoreJdbcPwdFile="" # Absolute path to Db2 conf file containing db2 schema etc -# Default = None, Required. +# Default = None, currently only referenced in generateCleanupCommands.sh dbbMetadataStoreJdbcConfigFile="" # JDBC connection server url @@ -331,7 +331,7 @@ wdDeployPackageDir() { ##################################################################################################### ##################################################################################################### -## generateCleanupCommands.sh parameters ############################################################### +## generateCleanupCommands.sh parameters ############################################################ ##################################################################################################### # DBB Metadatastore Options String to configure Db2 or File Metadatastore in cleanup commands @@ -343,7 +343,7 @@ wdDeployPackageDir() { cleanupDbbMetadataStoreOptions="" ##################################################################################################### -## End of generateCleanupCommands.sh parameters #### +## End of generateCleanupCommands.sh parameters #### ##################################################################################################### ##################################################################################################### From 5532c1b7e447a6d3d6b3f1d98b78d2f6de5aa6d8 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Mon, 7 Apr 2025 17:29:22 +0200 Subject: [PATCH 4/6] CBS script enhancements to manage packages lifecycle and ability to fetch external dependencies (#299) * (CBS) packageBuildOutputs.sh to follow implemented naming conventions how archives are published to Artifact repository * (CBS) dbbBuild.sh and zBuilder.sh to process build dependency configuration defined in the Application Descriptor * (CBS) Download external dependencies based on naming conventions from the artifact repository (Nexus, Artifactory) using the ArtifactRepositoryHelpers * (CBS) Expanding the external dependencies in the build workspace for consumption by the build framework * (CBS) Central cache of external archives * (CBS) Enable wazideploy-generate to automatically download the tar from the Artifact repository * (CBS) Scripts no longer carry configuration parameters to point to existing scripts. It references the scripts directly within this repo. --------- Signed-off-by: Dennis Behm Co-authored-by: Mathieu Dalbin Co-authored-by: Lauren Li <45975633+lauren-li@users.noreply.github.com> --- .../ArtifactRepositoryHelpers.groovy | 78 ++- .../PackageBuildOutputs.groovy | 289 +++++----- .../packageBuildOutputs.properties | 2 +- .../WaziDeployManifestGenerator.groovy | 2 +- .../applicationDescriptorUtils.groovy | 368 +++++++------ Templates/Common-Backend-Scripts/README.md | 521 ++++++++++-------- Templates/Common-Backend-Scripts/dbbBuild.sh | 28 +- .../packageBuildOutputs.sh | 299 +++++++--- .../pipelineBackend.config | 106 ++-- Templates/Common-Backend-Scripts/ucdDeploy.sh | 31 +- .../Common-Backend-Scripts/ucdPackaging.sh | 9 +- .../utilities/README.md | 3 + .../utilities/fetchBuildDependencies.groovy | 461 ++++++++++++++++ .../utilities/fetchBuildDependenciesUtils.sh | 104 ++++ .../utilities/packagingUtilities.sh | 152 +++++ .../wazideploy-deploy.sh | 17 +- .../wazideploy-generate.sh | 168 +++++- Templates/Common-Backend-Scripts/zBuilder.sh | 24 +- Templates/GitlabCIPipeline/.gitlab-ci.yml | 2 +- 19 files changed, 1922 insertions(+), 742 deletions(-) create mode 100644 Templates/Common-Backend-Scripts/utilities/README.md create mode 100644 Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy create mode 100644 Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh create mode 100755 Templates/Common-Backend-Scripts/utilities/packagingUtilities.sh mode change 100644 => 100755 Templates/Common-Backend-Scripts/zBuilder.sh diff --git a/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy b/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy index 96606766..6473ad15 100644 --- a/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy +++ b/Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy @@ -44,6 +44,7 @@ def CompletableFuture> run(args) +// public methods def upload(String url, String fileName, String user, String password, boolean verbose, String httpClientVersion) throws IOException { System.setProperty("jdk.httpclient.allowRestrictedHeaders", "Connection") println( "** ArtifactRepositoryHelper started for upload of $fileName to $url" ); @@ -66,7 +67,7 @@ def upload(String url, String fileName, String user, String password, boolean ve // } HttpClient httpClient = httpClientBuilder.build(); - + // build http request HttpRequest.Builder httpRequestBuilder = HttpRequest.newBuilder() .uri(URI.create("$url")) @@ -74,7 +75,7 @@ def upload(String url, String fileName, String user, String password, boolean ve .header("Connection","Keep-Alive") .PUT(BodyPublishers.ofFile(Paths.get(fileName))); - // set http client version if set + // set http client version if set if (httpClientVersion) { def httpVer = HttpClient.Version.valueOf(httpClientVersion) if (httpVer) { @@ -90,9 +91,10 @@ def upload(String url, String fileName, String user, String password, boolean ve HttpResponse.BodyHandler handler = HttpResponse.BodyHandlers.ofString(); // submit request + CompletableFuture> response = httpClient.sendAsync(request, handler).thenComposeAsync(r -> tryResend(httpClient, request, handler, 1, r)); HttpResponse finalResponse = response.get() - + if (verbose) println("** Response: " + finalResponse); @@ -104,6 +106,7 @@ def upload(String url, String fileName, String user, String password, boolean ve else { println("*! Upload failed."); } + return rc } def download(String url, String fileName, String user, String password, boolean verbose) throws IOException { @@ -154,8 +157,18 @@ def download(String url, String fileName, String user, String password, boolean } else { println("*! Download failed."); } + + return rc +} + +// Method directly accessed by PackageBuildOutputs and Common Backend script functionality +def computeArchiveUrl(Properties props) { + def String remotePath = (props.versionName) ? (props.versionName + "/" + props.tarFileName) : (props.tarFileLabel + "/" + props.tarFileName) + def url = new URI(props.get('artifactRepository.url') + "/" + props.get('artifactRepository.repo') + "/" + (props.get('artifactRepository.directory') ? "${props.get('artifactRepository.directory')}/" : "") + remotePath).normalize().toString() // Normalized URL + return url } +// private methods def evaluateHttpResponse (HttpResponse response, String action, boolean verbose) { int rc = 0 def statusCode = response.statusCode() @@ -173,20 +186,41 @@ def evaluateHttpResponse (HttpResponse response, String action, boolean verbose) //Parsing the command line def run(String[] cliArgs) { def cli = new CliBuilder(usage: "ArtifactRepositoryHelpers.groovy [options]", header: '', stopAtNonOption: false) + def Properties props = new Properties() cli.h(longOpt:'help', 'Prints this message') - cli.u(longOpt:'url', args:1, required:true, 'Artifactory file uri location') + cli.u(longOpt:'url', args:1,'Absolute artifact repository url location to store package') cli.fU(longOpt:'fileToUpload', args:1, 'The full path of the file to upload') cli.fD(longOpt:'fileToDownload', args:1, 'The full path of the file to download') - cli.U(longOpt:'user', args:1, required:true, 'Artifactory user id') - cli.P(longOpt:'password', args:1, required:true, 'Artifactory password') + cli.U(longOpt:'user', args:1,'Artifact repository user id or token') + cli.P(longOpt:'password', args:1, 'Artifact repository password') cli.ht(longOpt:'httpClientVersion', args:1, 'HTTP Client protocol version') cli.v(longOpt:'verbose', 'Flag to turn on script trace') + + // recompute options + // Compute Flag to recompute url + + cli.c(longOpt:'computeArchiveUrl', 'Action Flag to identify to recompute the uri of a given package') + cli.t(longOpt:'tarFileName', args:1, argName:'filename', 'Name of the package tar file. (Optional unless using --buildReportOrder or --buildReportOrderFile)') + cli.aRU(longOpt:'artifactRepositoryUrl', args:1, 'Artifact repository url') + cli.aRN(longOpt:'artifactRepositoryName', args:1, 'Artifact repository name') + cli.aRD(longOpt:'artifactRepositoryDirectory', args:1, 'Artifact repository directory') + cli.aVN(longOpt:'versionName', args:1, argName:'versionName', 'Name of the version/package folder on the Artifact repository server.') + def opts = cli.parse(cliArgs) // if opt parsing fails, exit if (opts == null || !opts) { System.exit(1) } + + if (opts.c) props.computeArchiveUrl = true + if (opts.t) props.tarFileName = opts.t + if (opts.aRU) props.put('artifactRepository.url', opts.aRU) + if (opts.aRN) props.put('artifactRepository.repo', opts.aRN) + if (opts.aRD) props.put('artifactRepository.directory', opts.aRD) + if (opts.aVN) props.versionName = opts.aVN + + if (opts.h) { cli.usage() @@ -194,8 +228,34 @@ def run(String[] cliArgs) { } if ( opts.fU) { + // assert required CLI options for upload + assert opts.u : "Missing option: Absolute artifact repository url location to store package" + assert opts.U : "Missing option: Artifact repository user id or token" + assert opts.P : "Missing option: Artifactory password" upload(opts.u, opts.fU, opts.U, opts.P, opts.v) - } else { + } else if (opts.fD) { + // assert required CLI options for download + assert opts.u : "Missing option: Absolute artifact repository url location to store package" + assert opts.U : "Missing option: Artifact repository user id or token" + assert opts.P : "Missing option: Artifactory password" download(opts.u, opts.fD, opts.U, opts.P, opts.v) - } -} \ No newline at end of file + } else if (props.computeArchiveUrl){ + + // invoke processing + if (props.computeArchiveUrl && props.computeArchiveUrl.toBoolean()) { + // check requires cli arguments for this operation + assert props.tarFileName : "Missing option tarFileName (--tarFileName)" + assert props.versionName : "Missing option versionName (--versionName)" + assert props.get('artifactRepository.url') : "Missing option artifactRepository.url (--artifactRepositoryUrl)" + assert props.get('artifactRepository.repo'): "Missing option artifactRepository.repo (--artifactRepositoryName)" + assert props.get('artifactRepository.directory'): "Missing option artifactRepository.directory (--artifactRepositoryDirectory)" + + // load script + archiveUrl = computeArchiveUrl(props) + // the println is used in a script by the CBS to grep the url + println "url=$archiveUrl" + } else + println("** No action has been specified for the ArtifactoryHelpers (available action triggers 'fileToUpload' or 'fileToDownload' or 'computeArchiveUrl') "); + } +} + diff --git a/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy b/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy index 1e87657b..f9e72be6 100644 --- a/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy +++ b/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy @@ -66,9 +66,11 @@ import com.ibm.jzos.ZFile; ************************************************************************************/ // start create & publish package -@Field Properties props = null +@Field Properties props = new Properties() def scriptDir = new File(getClass().protectionDomain.codeSource.location.path).parent @Field def wdManifestGeneratorUtilities = loadScript(new File("${scriptDir}/utilities/WaziDeployManifestGenerator.groovy")) +@Field def artifactRepositoryHelpers = loadScript(new File("${scriptDir}/ArtifactRepositoryHelpers.groovy")) + @Field def applicationDescriptorUtils @Field def applicationDescriptor @Field def sbomUtilities @@ -81,11 +83,14 @@ def scriptDir = new File(getClass().protectionDomain.codeSource.location.path).p @Field String libSubfolder = "lib" @Field String binSubfolder = "bin" @Field def tempLoadDir +@Field def String tarFileLabel = "Default" +@Field def String tarFileName = "" + -def startTime = new Date() +parseInput(args) -props = parseInput(args) +startTime = new Date() props.startTime = startTime.format("yyyyMMdd.HHmmss.SSS") println("** PackageBuildOutputs start at $props.startTime") println("** Properties at startup:") @@ -112,9 +117,8 @@ def copyModeMap = parseCopyModeMap(props.copyModeMap) Map buildOutputsMap = new HashMap() // Field to store default tarFileLabel (buildInfo.label) when cli argument tarFileName is not passed. -@Field def String tarFileLabel = "Default" -@Field def String tarFileName = "" -// Field to store build number, set to default when its not decipherable from build report + +// Field to store build number, set to default when its not decipherable from build report def String buildNumber = "UNKNOWN" // Object to store scm information for Wazi Deploy Application Manifest file @@ -144,7 +148,7 @@ props.buildReportOrder.each { buildReportFile -> rc = 1 } else { def buildReport= BuildReport.parse(new FileInputStream(jsonOutputFile)) - + // Read buildInfo to obtain build information def buildInfo = buildReport.getRecords().findAll{ try { @@ -155,14 +159,14 @@ props.buildReportOrder.each { buildReportFile -> tarFileLabel = buildInfo[0].label buildNumber = buildInfo[0].label } - + // retrieve the buildResultPropertiesRecord def buildResultPropertiesRecord = buildReport.getRecords().find { try { it.getType()==DefaultRecordFactory.TYPE_PROPERTIES && it.getId()=="DBB.BuildResultProperties" } catch (Exception e){} } - + // finds all the build outputs with a deployType def buildRecords = buildReport.getRecords().findAll{ try { @@ -170,7 +174,7 @@ props.buildReportOrder.each { buildReportFile -> !it.getOutputs().isEmpty() } catch (Exception e){} } - + // finds all the build outputs with a deployType // Today the USS_RECORD type is built using an AnyTypeRecord record // An Idea is currently opened to have an official USS_RECORD: https://ideas.ibm.com/ideas/DBB-I-43 @@ -182,14 +186,14 @@ props.buildReportOrder.each { buildReportFile -> // find all deletions using the DELETE_RECORD of zAppBuild def deletionRecords = buildReport.getRecords().findAll { - try { - // Obtain delete records, which got added by zAppBuild - it.getType() == "DELETE_RECORD" - } catch (Exception e) { - println e - } + try { + // Obtain delete records, which got added by zAppBuild + it.getType() == "DELETE_RECORD" + } catch (Exception e) { + println e + } } - + if (props.deployTypeFilter) { println("** Filter Output Records on following deployTypes: ${props.deployTypeFilter}...") buildRecords.each { @@ -206,7 +210,7 @@ props.buildReportOrder.each { buildReportFile -> it.getAttribute("outputs").split(';').collectEntries { entry -> outputs += entry.replaceAll('\\[|\\]', '').split(',') } - + ArrayList filteredOutputs = [] outputs.each { output -> rootDir = output[0].trim() @@ -216,11 +220,10 @@ props.buildReportOrder.each { buildReportFile -> filteredOutputs += output.toString() } } - + def filteredOutputsStrings = String.join(";", filteredOutputs) it.setAttribute("outputs", filteredOutputsStrings) } - } else { // Remove outputs without deployType + ZUNIT-TESTCASEs println("** Remove output records without deployType or with deployType=ZUNIT-TESTCASE") @@ -231,13 +234,13 @@ props.buildReportOrder.each { buildReportFile -> it.getOutputs().removeAll(unwantedOutputs) } } - + buildRecords += ussBuildRecords // append USS records - + def datasetMembersCount = 0 def zFSFilesCount = 0 def deletionCount = 0 - + // adding files and executes with outputs to Hashmap to remove redundant data buildRecords.each{ buildRecord -> if (buildRecord.getType()=="USS_RECORD") { @@ -254,7 +257,7 @@ props.buildReportOrder.each { buildReportFile -> def dependencySetRecord = buildReport.getRecords().find { it.getType()==DefaultRecordFactory.TYPE_DEPENDENCY_SET && it.getFile().equals(file) } - + temporaryBuildOutputsMap.put(new DeployableArtifact(file, deployType, "zFSFile"), [ container: rootDir, owningApplication: props.application, @@ -273,7 +276,7 @@ props.buildReportOrder.each { buildReportFile -> fileUsage = applicationDescriptorUtils.getFileUsageByType(applicationDescriptor, "Program", member) } // If the artifact is not an Object Deck or has no usage or its usage is not main - if ((output.deployType.equals("OBJ") && fileUsage && (fileUsage.equals("internal submodule") || fileUsage.equals("service submodule"))) || !output.deployType.equals("OBJ")) { + if ((output.deployType.equals("OBJ") && fileUsage && (fileUsage.equals("internal submodule") || fileUsage.equals("service submodule"))) || !output.deployType.equals("OBJ")) { datasetMembersCount++ String file = buildRecord.getFile() def dependencySetRecord = buildReport.getRecords().find { @@ -296,20 +299,20 @@ props.buildReportOrder.each { buildReportFile -> deletionRecords.each { deleteRecord -> deletionCount += deleteRecord.getAttributeAsList("deletedBuildOutputs").size() - deleteRecord.getAttributeAsList("deletedBuildOutputs").each{ deletedFile -> - + deleteRecord.getAttributeAsList("deletedBuildOutputs").each{ deletedFile -> + String cleansedDeletedFile = ((String) deletedFile).replace('"', ''); def (dataset, member) = getDatasetName(cleansedDeletedFile) - + // search for an existing deployableArtifacts record ArrayList filteredDeployableArtifacts = new ArrayList() - + temporaryBuildOutputsMap.each { DeployableArtifact deployableArtifact, Map info -> if (deployableArtifact.file == deleteRecord.getAttribute("file")) { filteredDeployableArtifacts.add(deployableArtifact, info) } } - + if (filteredDeployableArtifacts){ filteredDeployableArtifacts.each {deployableArtifact, info -> String container = info.get("container") @@ -324,7 +327,7 @@ props.buildReportOrder.each { buildReportFile -> record: buildRecord, propertiesRecord: buildResultPropertiesRecord, dependencySetRecord: dependencySetRecord - ]) + ]) } } } else { @@ -334,12 +337,12 @@ props.buildReportOrder.each { buildReportFile -> owningApplication: props.application, record: deleteRecord, propertiesRecord: buildResultPropertiesRecord - ]) + ]) } - } + } } - + // Print summary of BuildReport if ( datasetMembersCount + zFSFilesCount == 0 ) { println("** No items to package in '$buildReportFile'.") @@ -354,19 +357,19 @@ props.buildReportOrder.each { buildReportFile -> println("\t'${deployableArtifact.file}' from '${container}' with Deploy Type '${deployableArtifact.deployType}'") } } - + // Log detected deleted files if (deletionCount != 0) { println("** Deleted files detected in '$buildReportFile':") deletionRecords.each { it.getAttributeAsList("deletedBuildOutputs").each { println(" ${it}")}} } - + buildOutputsMap.putAll(temporaryBuildOutputsMap) - + // generate scmInfo for Wazi Deploy Application Manifest file - if ((props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean()) || - (props.generateConcertBuildManifest && props.generateConcertBuildManifest.toBoolean() )) { - + if ((props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean()) || + (props.generateConcertBuildManifest && props.generateConcertBuildManifest.toBoolean() )) { + if (props.buildReportOrder.size() == 1) { scmInfo.put("type", "git") gitUrl = retrieveBuildResultProperty (buildResultPropertiesRecord, "giturl") @@ -399,7 +402,7 @@ if (rc == 0) { } else { println("** External build dependencies file not found (${props.externalDependenciesEvidences}). Exiting.") rc=4 - } + } } } @@ -407,30 +410,29 @@ if (rc == 0) { if (props.generateSBOM && props.generateSBOM.toBoolean()) { sbomUtilities = loadScript(new File("${scriptDir}/utilities/sbomGenerator.groovy")) sbomSerialNumber = "url:uuid:" + UUID.randomUUID().toString() - sbomFileName = "${buildNumber}_sbom.json" + sbomFileName = "${buildNumber}_sbom.json" sbomUtilities.initializeSBOM(props.sbomAuthor, sbomSerialNumber) - } // Initialize Concert Build Manifest Generator if (props.generateConcertBuildManifest && props.generateConcertBuildManifest.toBoolean()) { - // Concert Build Manifest - + // Concert Build Manifest + concertManifestGeneratorUtilities = loadScript(new File("${scriptDir}/utilities/concertBuildManifestGenerator.groovy")) concertManifestGeneratorUtilities.initConcertBuildManifestGenerator() concertBuild = concertManifestGeneratorUtilities.addBuild(props.application, props.versionName, buildNumber) concertManifestGeneratorUtilities.addRepositoryToBuild(concertBuild, scmInfo.uri, scmInfo.branch, scmInfo.shortCommit) } - + // Local variables tarFileName = (props.tarFileName) ? props.tarFileName : "${tarFileLabel}.tar" def tarFile = "$props.workDir/${tarFileName}" - + //Create a temporary directory on zFS to copy the load modules from data sets to tempLoadDir = new File("$props.workDir/tempPackageDir") !tempLoadDir.exists() ?: tempLoadDir.deleteDir() tempLoadDir.mkdirs() - + // A baseline Package has been specified, we then extract it in the $tempLoadDir folder if (props.baselinePackageFilePath) { File baselinePackageFile = new File(props.baselinePackageFilePath) @@ -441,7 +443,7 @@ if (rc == 0) { "-c", "tar -xUXf ${props.baselinePackageFilePath}" ] - + def processRC = runProcess(processCmd, tempLoadDir) rc = Math.max(rc, processRC) if (rc == 0) { @@ -464,11 +466,11 @@ if (rc == 0) { wdManifestGeneratorUtilities.setScmInfo(scmInfo) } } - + // Search in all subfolders of the archive except the folders // that contains includes "$includeSubfolder" and binaries "$binSubfolder" // Copy the artifacts found to comply with the right structure - // All the artifact that don't comply will end up in the binSubfolder + // All the artifact that don't comply will end up in the binSubfolder tempLoadDir.eachDir() { subfolder -> if (!subfolder.getName().equals(includeSubfolder) && !subfolder.getName().equals(libSubfolder)) { subfolder.eachFileRecurse(FileType.FILES) { file -> @@ -480,7 +482,7 @@ if (rc == 0) { if (props.fullPackage && props.fullPackage.toBoolean()) { String expectedFilePath = "$tempLoadDir/$binSubfolder/$fileDeployType/$fileName" try { - Path destinationPath = Paths.get("$tempLoadDir/$binSubfolder/$fileDeployType/${fileName}.${fileDeployType}") + Path destinationPath = Paths.get("$tempLoadDir/$binSubfolder/$fileDeployType/${fileName}.${fileDeployType}") Path destinationDirPath = destinationPath.getParent() destinationDirPath.toFile().mkdirs() Path sourcePath = file.toPath() @@ -492,13 +494,13 @@ if (rc == 0) { } } catch (IOException e) { println("!* [ERROR] Error when moving file '${sourcePath}' to '${destinationPath}' during baseline package extraction.") - rc = 1 + rc = 1 } } else { file.delete() if (props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean()) { wdManifestGeneratorUtilities.removeArtifactFromManifest(fileName, fileDeployType) - } + } } } } @@ -506,20 +508,20 @@ if (rc == 0) { } if (subfolder.getName().equals("tmp")) { subfolder.deleteDir() - } + } } } else { println("*! [ERROR] Error when extracting baseline package '${created}' with rc=$rc.") rc = 1 - } + } } else { println("*! [ERROR] The Baseline Package '${props.baselinePackageFilePath}' was not found.") rc = 1 - } + } } - - if (rc == 0) { - + + if (rc == 0) { + println("** Total number of build outputs to package: ${buildOutputsMap.size()}") def publicInterfacesDeployTypes @@ -583,7 +585,7 @@ if (rc == 0) { processedArtifacts += privateInterfaces.size() } if ((publicInterfaces && !publicInterfaces.isEmpty()) || - (privateInterfaces && !privateInterfaces.isEmpty())) { + (privateInterfaces && !privateInterfaces.isEmpty())) { // Checks if all binary interfaces (submodules) are in the archive or not println("** Validate if all interfaces known in Application Descriptor are packaged.") checkBinaryInterfaces(tempLoadDir, applicationDescriptor) @@ -614,7 +616,6 @@ if (rc == 0) { } println("\tCopy '${includeFilePath}' file to '${targetIncludeFilePath}'") copyFiles(includeFilePath.toString(), targetIncludeFilePath.toString()) - } catch (IOException exception) { println "!* [ERROR] Copy failed: an error occurred when copying '${includeFilePath}' to '${targetIncludeFilePath}'" rc = Math.max(rc, 1) @@ -627,17 +628,17 @@ if (rc == 0) { println("*! [WARNING] The number of copied artifacts ($processedArtifacts) doesn't match the number of identified build outputs (${buildOutputsMap.size()}). Some files might have an incorrect 'usage' in the Application Descriptor.") } } - + if (props.generateSBOM && props.generateSBOM.toBoolean() && rc == 0) { - sbomUtilities.writeSBOM("$tempLoadDir/$sbomFileName", props.fileEncoding) + sbomUtilities.writeSBOM("$tempLoadDir/$sbomFileName", props.fileEncoding) } if (wdManifestGeneratorUtilities && props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean() && rc == 0) { if (props.publish && props.publish.toBoolean()) { HashMap packageInfo = new HashMap() packageInfo.put("type", "artifactRepository") - packageInfo.put("name", props.versionName) - packageUrl = computeAbsoluteRepositoryUrl() + packageInfo.put("name", props.buildIdentifier) + packageUrl = artifactRepositoryHelpers.computeArchiveUrl(props) if (packageUrl) packageInfo.put("uri", packageUrl) wdManifestGeneratorUtilities.setPackageInfo(packageInfo) } @@ -645,9 +646,9 @@ if (rc == 0) { // wazideploy_manifest.yml is the default name of the manifest file wdManifestGeneratorUtilities.writeApplicationManifest(new File("$tempLoadDir/wazideploy_manifest.yml"), props.fileEncoding, props.verbose) } - + if (rc == 0) { - + // log buildReportOrder file and add build reports to tar file File buildReportOrder = new File("$tempLoadDir/buildReportOrder.txt") ArrayList buildReportOrderLines = new ArrayList() @@ -660,16 +661,16 @@ if (rc == 0) { } } } - + println("** Generate package build report order file to '$buildReportOrder'") - + props.buildReportOrder.each { buildReportFile -> Path buildReportFilePath = Paths.get(buildReportFile) // Always prefix the buildreport with sequence number int nextIndex = buildReportOrderLines.size() + 1 Path copiedBuildReportFilePath = Paths.get(tempLoadDir.getPath() + "/" + "$nextIndex".padLeft(3, "0") + "_" + buildReportFilePath.getFileName().toString()) - + copyFiles(buildReportFilePath.toString(), copiedBuildReportFilePath.toString()) buildReportOrderLines.add("${copiedBuildReportFilePath.getFileName().toString()}\n") } @@ -677,17 +678,17 @@ if (rc == 0) { buildReportOrderLines.each { line -> if (!line.isEmpty()) { File buildReportFilePath = new File(line) - String buildReportFileName = buildReportFilePath.getName() + String buildReportFileName = buildReportFilePath.getName() writer.write("$buildReportFileName\n") } } } - + Path packagingPropertiesFilePath = Paths.get(props.packagingPropertiesFile) Path copiedPackagingPropertiesFilePath = Paths.get(tempLoadDir.getPath() + "/" + packagingPropertiesFilePath.getFileName().toString()) if (props.verbose) println("** Copy packaging properties config file to '$copiedPackagingPropertiesFilePath'") copyFiles(packagingPropertiesFilePath.toString(), copiedPackagingPropertiesFilePath.toString()) - + if (props.owner) { def processCmd = [ "sh", @@ -702,92 +703,86 @@ if (rc == 0) { println("*! [ERROR] Error when changing ownership to '${props.owner}' with rc=$rc.") } } - - if (rc == 0) { - println("** Create tar file at ${tarFile}") - // Note: https://www.ibm.com/docs/en/zos/2.4.0?topic=scd-tar-manipulate-tar-archive-files-copy-back-up-file - // To save all attributes to be restored on z/OS and non-z/OS systems : tar -UX - def processCmd = [ - "sh", - "-c", - "tar cUXf $tarFile *" - ] - - def processRC = runProcess(processCmd, tempLoadDir) - rc = Math.max(rc, processRC) + if (rc == 0) { - println("** Package '${tarFile}' successfully created.") - } else { - println("*! [ERROR] Error when creating Package '${tarFile}' with rc=$rc.") + println("** Create tar file at ${tarFile}") + // Note: https://www.ibm.com/docs/en/zos/2.4.0?topic=scd-tar-manipulate-tar-archive-files-copy-back-up-file + // To save all attributes to be restored on z/OS and non-z/OS systems : tar -UX + def processCmd = [ + "sh", + "-c", + "tar cUXf $tarFile *" + ] + + def processRC = runProcess(processCmd, tempLoadDir) + rc = Math.max(rc, processRC) + if (rc == 0) { + println("** Package '${tarFile}' successfully created.") + } else { + println("*! [ERROR] Error when creating Package '${tarFile}' with rc=$rc.") } } } - + //Package additional outputs to tar file. if (props.includeLogs && rc == 0) { (props.includeLogs).split(",").each { logPattern -> - println("** Add files with file pattern '$logPattern' from '${props.workDir}' to '${tarFile}'") - processCmd = [ - "sh", - "-c", - "tar rUXf $tarFile $logPattern" - ] - - processRC = runProcess(processCmd, new File(props.workDir)) - rc = Math.max(rc, processRC) - if (rc != 0) { - println("*! [ERROR] Error when appending '$logPattern' files to Package '${tarFile}' with rc=$rc.") + println("** Add files with file pattern '$logPattern' from '${props.workDir}' to '${tarFile}'") + processCmd = [ + "sh", + "-c", + "tar rUXf $tarFile $logPattern" + ] + + processRC = runProcess(processCmd, new File(props.workDir)) + rc = Math.max(rc, processRC) + if (rc != 0) { + println("*! [ERROR] Error when appending '$logPattern' files to Package '${tarFile}' with rc=$rc.") } } } - + if (props.verbose && props.verbose.toBoolean() && rc == 0) { println ("** List package contents.") - + processCmd = [ "sh", "-c", "tar tvf $tarFile" ] - + processRC = runProcess(processCmd, new File(props.workDir)) rc = Math.max(rc, processRC) if (rc != 0) { println("*! [ERROR] Error when listing contents of Package '${tarFile}' with rc=$rc.") } } - + //Set up the artifact repository information to publish the tar file if (props.publish && props.publish.toBoolean() && rc == 0){ // Configuring artifact repositoryHelper parms - def url = computeAbsoluteRepositoryUrl() - + def url = artifactRepositoryHelpers.computeArchiveUrl(props) + def apiKey = props.'artifactRepository.user' def user = props.'artifactRepository.user' def password = props.'artifactRepository.password' def httpClientVersion = props.'artifactRepository.httpClientVersion' def repo = props.get('artifactRepository.repo') as String - - //Call the artifactRepositoryHelpers to publish the tar file - File artifactRepoHelpersFile = new File("$scriptDir/ArtifactRepositoryHelpers.groovy") - Class artifactRepositoryHelpersClass = new GroovyClassLoader(getClass().getClassLoader()).parseClass(artifactRepoHelpersFile) - GroovyObject artifactRepositoryHelpers = (GroovyObject) artifactRepositoryHelpersClass.newInstance() - println ("** Upload package to Artifact Repository '$url'.") - artifactRepositoryHelpers.upload(url, tarFile as String, user, password, props.verbose.toBoolean(), httpClientVersion) + rc = artifactRepositoryHelpers.upload(url, tarFile as String, user, password, props.verbose.toBoolean(), httpClientVersion) - // generate PackageInfo for Concert Manifest file - if (props.generateConcertBuildManifest && props.generateConcertBuildManifest.toBoolean()) { + // generate PackageInfo for Concert Manifest file + if (props.generateConcertBuildManifest && props.generateConcertBuildManifest.toBoolean()) { concertManifestGeneratorUtilities.addLibraryInfoTobuild(concertBuild, tarFileName, url) } } - + if (concertManifestGeneratorUtilities && props.generateConcertBuildManifest && props.generateConcertBuildManifest.toBoolean() && rc == 0) { // concert_build_manifest.yaml is the default name of the manifest file - + if (props.generateSBOM && props.generateSBOM.toBoolean() && rc == 0) { concertManifestGeneratorUtilities.addSBOMInfoToBuild(concertBuild, sbomFileName, sbomSerialNumber) - } + } concertManifestGeneratorUtilities.writeBuildManifest(new File("$tempLoadDir/concert_build_manifest.yaml"), props.fileEncoding, props.verbose) println("** Add concert build config yaml to tar file at ${tarFile}") // Note: https://www.ibm.com/docs/en/zos/2.4.0?topic=scd-tar-manipulate-tar-archive-files-copy-back-up-file @@ -797,7 +792,7 @@ if (rc == 0) { "-c", "tar rUXf $tarFile concert_build_manifest.yaml" ] - + def processRC = runProcess(processCmd, tempLoadDir) rc = Math.max(rc, processRC) if (rc == 0) { @@ -823,7 +818,7 @@ def copyArtifactsToUSS(Map buildOutputsMap, String tarS Record record = info.get("record") PropertiesRecord propertiesRecord = info.get("propertiesRecord") DependencySetRecord dependencySetRecord = info.get("dependencySetRecord") - + def relativeFilePath = "" if (deployableArtifact.artifactType.equals("zFSFile")) { relativeFilePath = "$binSubfolder/uss" @@ -886,7 +881,6 @@ def copyArtifactsToUSS(Map buildOutputsMap, String tarS if (wdManifestGeneratorUtilities && props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean()) { wdManifestGeneratorUtilities.appendArtifactToManifest(deployableArtifact, "$relativeFilePath/$fileName", record, dependencySetRecord, propertiesRecord) } - } else { println "*! [ERROR] Copy failed: The file '$container(${deployableArtifact.file})' doesn't exist." rc = Math.max(rc, 1) @@ -896,10 +890,10 @@ def copyArtifactsToUSS(Map buildOutputsMap, String tarS rc = Math.max(rc, 1) } } else if (deployableArtifact.artifactType.equals("DatasetMemberDelete")) { - // generate delete instruction for Wazi Deploy - if (wdManifestGeneratorUtilities && props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean()) { - wdManifestGeneratorUtilities.appendArtifactDeletionToManifest(deployableArtifact, "$relativeFilePath/$fileName", record, propertiesRecord) - } + // generate delete instruction for Wazi Deploy + if (wdManifestGeneratorUtilities && props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean()) { + wdManifestGeneratorUtilities.appendArtifactDeletionToManifest(deployableArtifact, "$relativeFilePath/$fileName", record, propertiesRecord) + } } if (props.generateSBOM && props.generateSBOM.toBoolean() && rc == 0) { @@ -950,8 +944,10 @@ def parseInput(String[] cliArgs){ // Wazi Deploy Application Manifest generation cli.wd(longOpt:'generateWaziDeployAppManifest', 'Flag indicating to generate and add the Wazi Deploy Application Manifest file.') + cli.bi(longOpt:'buildIdentifier', args:1, argName:'buildIdentifier', 'Unique build identifier stored in Wazi Deploy Application Manifest file.') cli.ed(longOpt:'externalDependenciesEvidences', args:1, argName:'externalDependenciesEvidences', 'File documenting the external dependencies that were provided to the build phase.') + // Concert Build Manifest generation cli.ic(longOpt:'generateConcertBuildManifest', 'Flag indicating to generate and add the IBM Concert Build Manifest file.') @@ -965,7 +961,7 @@ def parseInput(String[] cliArgs){ // Artifact repository options :: cli.p(longOpt:'publish', 'Flag to indicate package upload to the provided Artifact Repository server. (Optional)') - cli.v(longOpt:'versionName', args:1, argName:'versionName', 'Name of the version/package on the Artifact repository server. (Optional)') + cli.v(longOpt:'versionName', args:1, argName:'versionName', 'Name of the version/package folder on the Artifact repository server. (Optional)') // Artifact repository info cli.au(longOpt:'artifactRepositoryUrl', args:1, argName:'url', 'URL to the Artifact repository server. (Optional)') @@ -992,13 +988,12 @@ def parseInput(String[] cliArgs){ cli.h(longOpt:'help', 'Prints this message') def opts = cli.parse(cliArgs) - if (opts.h) { // if help option used, print usage and exit + if (opts.h) { + // if help option used, print usage and exit cli.usage() - System.exit(0) + System.exit(0) } - def props = new Properties() - // read properties file if (opts.properties) { def propertiesFile = new File(opts.properties) @@ -1006,7 +1001,8 @@ def parseInput(String[] cliArgs){ props.packagingPropertiesFile = opts.properties propertiesFile.withInputStream { props.load(it) } } - } else { // read default sample properties file shipped with the script + } else { + // read default sample properties file shipped with the script def scriptDir = new File(getClass().protectionDomain.codeSource.location.path).parent def defaultPackagePropFile = new File("$scriptDir/packageBuildOutputs.properties") if (defaultPackagePropFile.exists()) { @@ -1022,6 +1018,7 @@ def parseInput(String[] cliArgs){ if (opts.il) props.includeLogs = opts.il if (opts.a) props.application = opts.a if (opts.b) props.branch = opts.b + if (opts.bi) props.buildIdentifier = opts.bi // cli overrides defaults set in 'packageBuildOutputs.properties' props.generateWaziDeployAppManifest = (opts.wd) ? 'true' : (props.generateWaziDeployAppManifest ? props.generateWaziDeployAppManifest : 'false') @@ -1034,7 +1031,7 @@ def parseInput(String[] cliArgs){ if (opts.o) props.owner = opts.o props.verbose = (opts.verb) ? 'true' : 'false' - + if (opts.af) { props.applicationFolderPath = opts.af if (opts.bp) props.baselinePackageFilePath = opts.bp @@ -1079,7 +1076,6 @@ def parseInput(String[] cliArgs){ println("*! [ERROR] Missing required property 'tarFilename'. 'tarFilename' is only optional when no build report order is specified.") rc = 2 } - } if (opts.bO) { opts.bO.split(',').each{ @@ -1098,7 +1094,7 @@ def parseInput(String[] cliArgs){ if (opts.sbomAuthor) { props.sbomAuthor = opts.sbomAuthor - } + } if (!props.workDir) { println("*! [ERROR] Missing required Working Directory parameter ('--workDir'). ") @@ -1115,7 +1111,7 @@ def parseInput(String[] cliArgs){ println("*! [ERROR] Missing Application ('-a') property required when generating SBOM.") rc = 2 } - + // validate publishing options if (props.publish && props.publish.toBoolean()){ if (!props.'artifactRepository.url') { @@ -1148,7 +1144,7 @@ def parseInput(String[] cliArgs){ } } - // assess required options to generate Concert Build manifest + // assess required options to generate Concert Build manifest if (props.generateConcertBuildManifest && props.generateConcertBuildManifest.toBoolean()) { if (!props.branch) { println("*! [ERROR] Missing Branch parameter ('--branch'). It is required for generating the Concert Build Manifest file.") @@ -1163,7 +1159,7 @@ def parseInput(String[] cliArgs){ rc = 2 } } - + if (props.publishInterfaces && props.publishInterfaces.toBoolean()) { if (!props.applicationFolderPath) { println("*! [ERROR] Missing Application Folder Path parameter ('--applicationFolderPath'). It is required for publishing intefaces in the archive.") @@ -1171,7 +1167,6 @@ def parseInput(String[] cliArgs){ } } - return props } /* @@ -1200,14 +1195,6 @@ def parseCopyModeMap(String copyModeMapString) { return copyModeMap } -/* - * build package url - */ -def computeAbsoluteRepositoryUrl() { - def String remotePath = (props.versionName) ? (props.versionName + "/" + tarFileName) : (tarFileLabel + "/" + tarFileName) - def url = new URI(props.get('artifactRepository.url') + "/" + props.get('artifactRepository.repo') + "/" + (props.get('artifactRepository.directory') ? "${props.get('artifactRepository.directory')}/" : "") + remotePath).normalize().toString() // Normalized URL - return url -} /* * checksBinaryInterfaces - Checks if all interfaces @@ -1215,14 +1202,14 @@ def computeAbsoluteRepositoryUrl() { * are present in the archive. * If not, issue a warning message */ - + def checkBinaryInterfaces(File tempLoadDir, applicationDescriptor) { ArrayList binaryPublicInterfaces = applicationDescriptorUtils.getFilesByTypeAndUsage(applicationDescriptor, "Program", "service submodule") ArrayList binaryPrivateInterfaces = applicationDescriptorUtils.getFilesByTypeAndUsage(applicationDescriptor, "Program", "internal submodule") binaryPublicInterfaces.each { binaryInterface -> String sourceFileName = Paths.get(binaryInterface).getFileName().toString() String member = sourceFileName.split("\\.")[0].toUpperCase() - + String expectedInterfaceFileName = "$includeSubfolder/bin/" + member + ".OBJ" File expectedInterfaceFile = new File("${tempLoadDir.getAbsolutePath()}/$expectedInterfaceFileName") if (!expectedInterfaceFile.exists()) { @@ -1232,7 +1219,7 @@ def checkBinaryInterfaces(File tempLoadDir, applicationDescriptor) { binaryPrivateInterfaces.each { binaryInterface -> String sourceFileName = Paths.get(binaryInterface).getFileName().toString() String member = sourceFileName.split("\\.")[0].toUpperCase() - + String expectedInterfaceFileName = "$libSubfolder/bin/" + member + ".OBJ" File expectedInterfaceFile = new File("${tempLoadDir.getAbsolutePath()}/$expectedInterfaceFileName") if (!expectedInterfaceFile.exists()) { diff --git a/Pipeline/PackageBuildOutputs/packageBuildOutputs.properties b/Pipeline/PackageBuildOutputs/packageBuildOutputs.properties index 9ba3781e..eb65ca11 100644 --- a/Pipeline/PackageBuildOutputs/packageBuildOutputs.properties +++ b/Pipeline/PackageBuildOutputs/packageBuildOutputs.properties @@ -71,7 +71,7 @@ addExtension=true # Boolean setting to define if the Wazi Deploy Application Manifest file should be generated # Please note that the cli option `generateWaziDeployAppManifest` can override this setting and activate it. # Default: false -generateWaziDeployAppManifest=false +generateWaziDeployAppManifest=true # Boolean setting to define if the IBM Concert Build Manifest file should be generated # Please note that the cli option `generateConcertBuildManifest` can override this setting and activate it. diff --git a/Pipeline/PackageBuildOutputs/utilities/WaziDeployManifestGenerator.groovy b/Pipeline/PackageBuildOutputs/utilities/WaziDeployManifestGenerator.groovy index a4a98efd..a762c47f 100644 --- a/Pipeline/PackageBuildOutputs/utilities/WaziDeployManifestGenerator.groovy +++ b/Pipeline/PackageBuildOutputs/utilities/WaziDeployManifestGenerator.groovy @@ -29,7 +29,7 @@ def initWaziDeployManifestGenerator(Properties props) { } // Metadata information - wdManifest.metadata.version = (props.versionName) ? props.versionName : props.startTime + wdManifest.metadata.version = (props.buildIdentifier) ? props.buildIdentifier : props.startTime if (props.application) wdManifest.metadata.name = props.application // Annotations diff --git a/Pipeline/PackageBuildOutputs/utilities/applicationDescriptorUtils.groovy b/Pipeline/PackageBuildOutputs/utilities/applicationDescriptorUtils.groovy index 53aab543..3290cbe3 100644 --- a/Pipeline/PackageBuildOutputs/utilities/applicationDescriptorUtils.groovy +++ b/Pipeline/PackageBuildOutputs/utilities/applicationDescriptorUtils.groovy @@ -12,158 +12,170 @@ import groovy.util.* import java.nio.file.* /** - * Utilities to read, update or export existing ApplicationDescriptor from/to YAML + * Utilities to read, update or export existing ApplicationDescriptor from/to YAML */ class ApplicationDescriptor { - String application - String description - String owner - ArrayList sources - ArrayList baselines - ArrayList dependencies - ArrayList consumers + String application + String schemaVersion = "applicationDescriptor/0.11.0" + String description + String owner + ArrayList sources + ArrayList baselines + ArrayList dependencies + ArrayList consumers } class Source { - String name - String repositoryPath - String language - String languageProcessor - String fileExtension - String artifactsType - ArrayList files + String name + String repositoryPath + String language + String languageProcessor + String fileExtension + String artifactsType + ArrayList files } class FileDef { - String name - String type - String usage + String name + String type + String usage } class Baseline { - String branch - String baseline + String branch + String type + String reference + String buildid } class DependencyDescriptor { - String name - String version - String type + String name + String type + String reference + String buildid +} + +class Consumer { + String name } /** - * + * * Reads an existing application descriptor YAML * returns an ApplicationDescriptor Object - * + * */ def readApplicationDescriptor(File yamlFile) { - // Internal objects - def yamlSlurper = new groovy.yaml.YamlSlurper() - ApplicationDescriptor applicationDescriptor = yamlSlurper.parse(yamlFile) - return applicationDescriptor + // Internal objects + def yamlSlurper = new groovy.yaml.YamlSlurper() + ApplicationDescriptor applicationDescriptor = yamlSlurper.parse(yamlFile) + return applicationDescriptor } /** * Write an ApplicationDescriptor Object into a YAML file */ def writeApplicationDescriptor(File yamlFile, ApplicationDescriptor applicationDescriptor) { - // Sort source groups and files by name before writing to YAML file - if (applicationDescriptor.sources) { - applicationDescriptor.sources.sort { - it.name - } - applicationDescriptor.sources.each() { source -> - source.files.sort { - it.name - } - } - } - - def yamlBuilder = new YamlBuilder() - // build updated application descriptor - - yamlBuilder { - application applicationDescriptor.application - description applicationDescriptor.description - owner applicationDescriptor.owner - sources (applicationDescriptor.sources) - baselines (applicationDescriptor.baselines) - if (applicationDescriptor.dependencies) { - dependencies applicationDescriptor.dependencies - } - if (applicationDescriptor.consumers) { - consumers applicationDescriptor.consumers - } - } - - // write file - yamlFile.withWriter("IBM-1047") { writer -> - writer.write(yamlBuilder.toString()) - } + // Sort source groups and files by name before writing to YAML file + if (applicationDescriptor.sources) { + applicationDescriptor.sources.sort { + it.name + } + applicationDescriptor.sources.each() { source -> + source.files.sort { + it.name + } + } + } + + def yamlBuilder = new YamlBuilder() + // build updated application descriptor + + yamlBuilder { + application applicationDescriptor.application + schemaVersion applicationDescriptor.schemaVersion + description applicationDescriptor.description + owner applicationDescriptor.owner + sources (applicationDescriptor.sources) + baselines (applicationDescriptor.baselines) + if (applicationDescriptor.dependencies) { + dependencies applicationDescriptor.dependencies + } + if (applicationDescriptor.consumers) { + consumers applicationDescriptor.consumers + } + } + + // write file + yamlFile.withWriter("IBM-1047") { writer -> + writer.write(yamlBuilder.toString()) + } Process process = "chtag -tc IBM-1047 ${yamlFile.getAbsolutePath()}".execute() - process.waitFor() + process.waitFor() } /** * Method to update the Application Descriptor - * + * * Appends to an existing source sourceGroupName, if it exists. * If the sourceGroupName cannot be found, it creates a new sourceGroup - * + * */ def appendFileDefinition(ApplicationDescriptor applicationDescriptor, String sourceGroupName, - String language, String languageProcessor, String artifactsType, - String fileExtension, String repositoryPath, String name, - String type, String usage) { - - def sourceGroupRecord - - def fileRecord = new FileDef() - fileRecord.name = name - fileRecord.type = type - fileRecord.usage = usage - - if (!applicationDescriptor.sources) { - applicationDescriptor.sources = new ArrayList() - } - - existingSourceGroup = applicationDescriptor.sources.find(){ source -> - source.name == sourceGroupName - } - - if (existingSourceGroup) { // append file record definition to existing sourceGroup - sourceGroupRecord = existingSourceGroup - - // check if the fileRecord already exists, and this is an update - - existingFileRecord = sourceGroupRecord.files.find(){ file -> - file.name == fileRecord.name - } - - if (existingFileRecord) { // update existing file record - existingFileRecord.type = type - existingFileRecord.usage = usage - } else { // add a new record - sourceGroupRecord.files.add(fileRecord) - } + String language, String languageProcessor, String artifactsType, + String fileExtension, String repositoryPath, String name, + String type, String usage) { + + def sourceGroupRecord + + def fileRecord = new FileDef() + fileRecord.name = name + fileRecord.type = type + fileRecord.usage = usage + + if (!applicationDescriptor.sources) { + applicationDescriptor.sources = new ArrayList() + } + + existingSourceGroup = applicationDescriptor.sources.find(){ source -> + source.name == sourceGroupName + } + + if (existingSourceGroup) { + // append file record definition to existing sourceGroup + sourceGroupRecord = existingSourceGroup + + // check if the fileRecord already exists, and this is an update + + existingFileRecord = sourceGroupRecord.files.find(){ file -> + file.name == fileRecord.name + } + + if (existingFileRecord) { + // update existing file record + existingFileRecord.type = type + existingFileRecord.usage = usage + } else { + // add a new record + sourceGroupRecord.files.add(fileRecord) + } } else { - // create a new source group entry - sourceGroupRecord = new Source() - sourceGroupRecord.name = sourceGroupName - sourceGroupRecord.language = language - sourceGroupRecord.languageProcessor = languageProcessor - sourceGroupRecord.fileExtension = fileExtension - sourceGroupRecord.artifactsType = artifactsType - sourceGroupRecord.repositoryPath = repositoryPath - - sourceGroupRecord.files = new ArrayList() - // append file record - sourceGroupRecord.files.add(fileRecord) - applicationDescriptor.sources.add(sourceGroupRecord) - } + // create a new source group entry + sourceGroupRecord = new Source() + sourceGroupRecord.name = sourceGroupName + sourceGroupRecord.language = language + sourceGroupRecord.languageProcessor = languageProcessor + sourceGroupRecord.fileExtension = fileExtension + sourceGroupRecord.artifactsType = artifactsType + sourceGroupRecord.repositoryPath = repositoryPath + + sourceGroupRecord.files = new ArrayList() + // append file record + sourceGroupRecord.files.add(fileRecord) + applicationDescriptor.sources.add(sourceGroupRecord) + } } @@ -173,43 +185,46 @@ def appendFileDefinition(ApplicationDescriptor applicationDescriptor, String sou def removeFileDefinition(ApplicationDescriptor applicationDescriptor, String sourceGroupName, String name) { - if (applicationDescriptor.sources) { - def existingSourceGroup = applicationDescriptor.sources.find() { source -> - source.name == sourceGroupName - } - if (existingSourceGroup) { // Found an existing Source Group that matches - def existingFileDef = existingSourceGroup.files.find { file -> - file.name.equals(name) - } - if (existingFileDef) { -// println "Found matching file ${existingFileDef.name}" - existingSourceGroup.files.remove(existingFileDef) - } - } - } + if (applicationDescriptor.sources) { + def existingSourceGroup = applicationDescriptor.sources.find() { source -> + source.name == sourceGroupName + } + if (existingSourceGroup) { + // Found an existing Source Group that matches + def existingFileDef = existingSourceGroup.files.find { file -> + file.name.equals(name) + } + if (existingFileDef) { + // println "Found matching file ${existingFileDef.name}" + existingSourceGroup.files.remove(existingFileDef) + } + } + } } /** - * Method to add an application dependency + * Method to add an application dependency */ -def addApplicationDependency(ApplicationDescriptor applicationDescriptor, String applicationDependency, String version, String type) { - if (!applicationDescriptor.dependencies) { - applicationDescriptor.dependencies = new ArrayList() - } - def existingDependencies = applicationDescriptor.dependencies.findAll() { - it.name.equals(applicationDependency) & it.type.equals(type) - } - if (!existingDependencies) { - def dependency = new DependencyDescriptor() - dependency.name = applicationDependency - dependency.version = version - dependency.type = type - applicationDescriptor.dependencies.add(dependency) - applicationDescriptor.dependencies.sort { - it.name - } - } +def addApplicationDependency(ApplicationDescriptor applicationDescriptor, String applicationDependency, String type, String reference, String buildid) { + if (!applicationDescriptor.dependencies) { + applicationDescriptor.dependencies = new ArrayList() + } + // skip readding same/similar entries + def existingDependencies = applicationDescriptor.dependencies.findAll() { + it.name.equals(applicationDependency) + } + if (!existingDependencies) { + def dependency = new DependencyDescriptor() + dependency.name = applicationDependency + dependency.type = type + dependency.reference = reference + dependency.buildid = buildid + applicationDescriptor.dependencies.add(dependency) + applicationDescriptor.dependencies.sort { + it.name + } + } } /** @@ -217,20 +232,22 @@ def addApplicationDependency(ApplicationDescriptor applicationDescriptor, String */ def addApplicationConsumer(ApplicationDescriptor applicationDescriptor, String consumingApplication) { - - if (!applicationDescriptor.consumers) { - applicationDescriptor.consumers = new ArrayList() - } - // don't add the "owning" application - if (applicationDescriptor.application != consumingApplication) { - def existingConsumers = applicationDescriptor.consumers.findAll() { - it.equals(consumingApplication) - } - if (!existingConsumers) { - applicationDescriptor.consumers.add(consumingApplication) - applicationDescriptor.consumers.sort() - } - } + + if (!applicationDescriptor.consumers) { + applicationDescriptor.consumers = new ArrayList() + } + // don't add the "owning" application + if (applicationDescriptor.application != consumingApplication) { + def existingConsumers = applicationDescriptor.consumers.findAll() { + it.name.equals(consumingApplication) + } + if (!existingConsumers) { + Consumer consumer = new Consumer() + consumer.name = consumingApplication + applicationDescriptor.consumers.add(consumer) + applicationDescriptor.consumers.sort() + } + } } /** @@ -238,7 +255,7 @@ def addApplicationConsumer(ApplicationDescriptor applicationDescriptor, String c */ def resetAllSourceGroups(ApplicationDescriptor applicationDescriptor) { - applicationDescriptor.sources = new ArrayList() + applicationDescriptor.sources = new ArrayList() } /** @@ -246,18 +263,18 @@ def resetAllSourceGroups(ApplicationDescriptor applicationDescriptor) { */ def resetConsumersAndDependencies(ApplicationDescriptor applicationDescriptor) { - applicationDescriptor.consumers = new ArrayList() - applicationDescriptor.dependencies = new ArrayList() + applicationDescriptor.consumers = new ArrayList() + applicationDescriptor.dependencies = new ArrayList() } /** - * Method to create an empty application descriptor object + * Method to create an empty application descriptor object */ def createEmptyApplicationDescriptor() { - ApplicationDescriptor applicationDescriptor = new ApplicationDescriptor() - applicationDescriptor.sources = new ArrayList() - applicationDescriptor.baselines = new ArrayList() - return applicationDescriptor + ApplicationDescriptor applicationDescriptor = new ApplicationDescriptor() + applicationDescriptor.sources = new ArrayList() + applicationDescriptor.baselines = new ArrayList() + return applicationDescriptor } /** @@ -285,7 +302,7 @@ def getFileUsage(ApplicationDescriptor applicationDescriptor, String sourceGroup } else { println("*! [WARNING] No file found matching '${name}'. Skipping search.") return null - } + } } else { return null } @@ -323,12 +340,12 @@ def getFileUsageByType(ApplicationDescriptor applicationDescriptor, String artif if (allMatchingFiles.size() == 1) { return allMatchingFiles[0].usage } else if (allMatchingFiles.size() > 1) { - println("*! [WARNING] Multiple files found matching '${name}'. Skipping search.") + println("*! [WARNING] Multiple files found matching '${name}'. Skipping search.") return null } else { - println("*! [WARNING] No file found matching '${name}'. Skipping search.") + println("*! [WARNING] No file found matching '${name}'. Skipping search.") return null - } + } } else { return null } @@ -369,7 +386,7 @@ def getFilesByTypeAndUsage(ApplicationDescriptor applicationDescriptor, String a } /** - * Method to add a baseline + * Method to add a baseline * If an existing baseline for a given branch already exists, the method replaces it */ @@ -380,16 +397,13 @@ def addBaseline(ApplicationDescriptor applicationDescriptor, String branch, Stri } existingBaselines.forEach() { existingBaseline -> applicationDescriptor.baselines.remove(existingBaseline) - } + } } else { applicationDescriptor.baselines = new ArrayList() } - if (applicationDescriptor.baselines) { - applicationDescriptor.sources = new ArrayList() - } Baseline newBaseline = new Baseline() newBaseline.branch = branch newBaseline.baseline = baseline applicationDescriptor.baselines.add(newBaseline) -} +} \ No newline at end of file diff --git a/Templates/Common-Backend-Scripts/README.md b/Templates/Common-Backend-Scripts/README.md index e24ea8c4..a029a43d 100644 --- a/Templates/Common-Backend-Scripts/README.md +++ b/Templates/Common-Backend-Scripts/README.md @@ -2,7 +2,7 @@ ## Overview -The Common Backend Scripts for Pipeline Implementations is a collection of scripts that deliver central "services" and a simplified interface for pipeline configurations that implement a Git/DBB-based pipeline for Mainframe applications. +The Common Backend Scripts for Pipeline Implementations is a collection of scripts that deliver central "services" and a simplified interface for pipeline configurations that implement a Git/DBB-based pipeline for mainframe applications. They use and simplify the parameterization of existing scripts in this repository to perform build, packaging, and deployment steps. Implementing a pipeline configuration, such as an Azure pipeline, a JenkinsFile, or the .gitlab-ci.yml file, requires accommodation of the selected development workflow with Git. To achieve consistency across various applications, rules must be implemented in pipeline code or configurations to address: * naming conventions of build datasets, @@ -23,22 +23,23 @@ This asset implements the rules and conventions of the Git-based workflow outlin ## Setup -The provided scripts of this asset are implemented as bash scripts and need to be installed on UNIX System Services of the z/OS system that is used to execute the pipeline's tasks. +The provided scripts of this asset are implemented as bash scripts. The entire repository needs to be installed on UNIX System Services of the z/OS system that is used to execute the pipeline's tasks, as the scripts invoke individual pipeline scripts from the [Pipeline](../../Pipeline/) directory. ### Pre-requisites The following are required to use these scripts: -* DBB v2.x toolkit is installed. -* zAppBuild is set up on Unix Systems Services. +* DBB 2.x or DBB 3.x toolkit is installed. +* zAppBuild or zBuilder is set up on Unix Systems Services. * Git repository which follows the Git-based workflow outlined in IBM's documentation `The Git-based workflow for Mainframe development`. -* Build dependency information is available before performing the build run. - ### Installation -* Copy/clone the Common Backend Scripts into z/OS UNIX System Services under a protected directory, e.g. `/usr/dbb/pipelineBackend`. - * Update the permission of these scripts to allow for `read/execute` to only the users who will invoke the scripts. This is typically the technical user defined for the pipeline orchestrator. +1. Make this Git repository available in your enterprise Git provider. Perform necessary site-specific [script configurations](#script-configuration). + +2. Clone the repository customized in the previous step (including the Common Backend Scripts) into z/OS UNIX System Services under a protected directory. + + * Verify that permission of these scripts to allow for `read/execute` to only the users who will invoke the scripts. This is typically the technical user defined for the pipeline orchestrator. -* The following environment variables need to be defined (for instance within the `.profile`) for the mainframe users who will execute the scripts on UNIX System Services: +3. The following environment variables need to be defined (for instance within the `.profile`) for the mainframe users who will execute the scripts on UNIX System Services: * `PIPELINE_SCRIPTS` - Environment variable to define the path to the Common Backend Scripts. @@ -135,10 +136,10 @@ Artifact Name | Description | Script details [zBuilder.sh](zBuilder.sh) | Pipeline Shell script to invoke the zBuilder framework [zBuilder](https://www.ibm.com/docs/en/dbb/3.0?topic=building-zos-applications-zbuilder) | [script details](#zbuildersh-for-dbb-zbuilder) [packageBuildOutputs.sh](packageBuildOutputs.sh) | Pipeline Shell Script to create a Package using the [PackageBuildOutputs groovy script](https://github.com/IBM/dbb/tree/main/Pipeline/PackageBuildOutputs) | [script details](#packagebuildoutputssh) [ucdPackage.sh](ucdPackaging.sh) | Pipeline Shell Script to publish to UCD Code Station binary repository using the [CreateUCDComponentVersion groovy script](https://github.com/IBM/dbb/tree/main/Pipeline/CreateUCDComponentVersion) | [script details](#ucdpackagingsh) -[ucdDeploy.sh](ucdDeploy.sh) | Pipeline Shell Script to trigger a UCD Deployment via its REST interface using the [DeployUCDComponentVersion groovy script](https://github.com/IBM/dbb/tree/main/Pipeline/DeployUCDComponentVersion) | [script details](#ucddeploysh) [wazideploy-generate.sh](wazideploy-generate.sh) | Pipeline Shell Script to generate a Deployment Plan to be used with Wazi Deploy | [script details](#wazideploy-generatesh) [wazideploy-deploy.sh](wazideploy-deploy.sh) | Pipeline Shell Script to trigger a deployment of a package based on Deployment Plan with Wazi Deploy | [script details](#wazideploy-deploysh) [wazideploy-evidence.sh](wazideploy-evidence.sh) | Pipeline Shell Script to query the Wazi Deploy Evidence YAML file and create a deployment report | [script details](#wazideploy-generatesh) +[ucdDeploy.sh](ucdDeploy.sh) | Pipeline Shell Script to trigger a UCD Deployment via its REST interface using the [DeployUCDComponentVersion groovy script](https://github.com/IBM/dbb/tree/main/Pipeline/DeployUCDComponentVersion) | [script details](#ucddeploysh) [prepareLogs.sh](prepareLogs.sh) | Pipeline Shell Script to prepare a TAR file containing log files that can then be retrieved. | [script details](#preparelogssh) [generateCleanupCommands.sh](generateCleanupCommands.sh) | Pipeline Shell Script to generate necessary DBB Metadatastore cleanup tasks including the deletion of the build datasets. | [script details](#generatecleanupcommandssh) [deleteWorkspace.sh](deleteWorkspace.sh) | Pipeline Shell Script to delete the working directory on Unix System Services. | [script details](#deleteworkspacesh) @@ -209,7 +210,7 @@ gitClone.sh: [INFO] Clone Repository Complete. rc=0 ## Build stage -### Conventions +### Conventions and capabilities #### Git branch naming conventions @@ -236,6 +237,41 @@ Note that the location of the baselineReferences.config file can be customized i [baselineReference.config](samples/baselineReference.config) is a sample, that indicates the baseline for the `main` and `release maintenance` branches. +#### Fetching build dependencies + +The build stage can be enabled to pull external build dependencies into the build workspace based on the dependencies definition specified in the Application Descriptor file. + +Each application version, represented by an archive, can export shared components such as public or shared include files, and build outputs such as object decks or NCAL load modules. The application archive needs to be created with the [packageBuildOutputs.sh](#packagebuildoutputssh) script and be uploaded to the artifact repository based on the implemented conventions. + +The Application Descriptor contains metadata about the application itself, but can contain the dependency configuration to other applications versions managed in an artifact repository, which are necessary inputs to the build process. Additional information about the Application Descriptor can be found at the [dbb-git-migration-modeler](https://github.com/IBM/dbb-git-migration-modeler) project, which documents cross-application dependencies and generates Application Descriptor files. + +In the `dependencies` section in the Application Descriptor file, users can configure which application versions should be fetched into the build workspace. The below snippet references the release build of the Cards application with the reference to `rel-1.2.0` and the concrete buildid `build-20241112.1` + +```yaml +dependencies: +- name: ”Cards" + type: "release" + reference: "rel-1.2.0" + buildid: "build-20241112.1" +``` + +The Application Descriptor file, called `applicationDescriptor.yml`, is expected to be on the root level of the application's Git repository. + +To fetch the dependencies, the subscript [fetchBuildDependenciesUtils.sh](utilities/fetchBuildDependenciesUtils.sh) is used. Under the covers, it uses the [fetchBuildDependencies.groovy](utilities/fetchBuildDependencies.groovy) and the [ArtifactoryHelpers](../../Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy) script to download the external dependencies into the working directory. The downloaded archives can be stored at a cache location to improve performance. Fetched archives are expanded in the `imports` subfolder of the pipeline's working directory. + +**Fetch baseline package** + +Along with the fetching of external build dependencies, the fetch phase can retrieve the application's baseline package from the Artifact repository. This is configured through the `baselines` section of the Application Descriptor. Use the baseline if your application architecture uses static calls or requires derived build outputs that is an mandatory input to subsequent builds. A good sample for derived build outputs are bms copybooks, that are inputs to CICS programs. Instead of including the generated bms copybooks via concatenation of pds libraries, it is made available through the baseline package. + +Baseline archives are defined similarly like external dependencies. Under the `baselines` section, the application team manages the references for the corresponding build branch: + +``` +baselines: +- branch: ”main" + type: "release" + reference: "rel-1.2.0" + buildid: "build-7656" +``` ### dbbBuild.sh for zAppBuild framework @@ -452,11 +488,35 @@ The [dbbzBuilderUtils](utilities/dbbzBuilderUtils.sh) script is a core utility s * the `mainBuildBranch` to configure feature branch pipelines to clone the corresponding DBB dependency metadata collections by generating a config.yaml that is passed into zBuilder. -## packageBuildOutputs.sh +## Packaging stage + +Depending on the Deployment Manager tool you are using, you can choose from either creating a package with the [PackageBuildOutputs](#packagebuildoutputssh) script that can be used with IBM Wazi Deploy, or the [UCD packaging](#ucdpackagingsh) script that creates the UCD shiplist and UCD component version. + +### packageBuildOutputs.sh This script is to execute the `PackageBuildOutputs.groovy` that packages up the build outputs and optionally uploads it to an artifact repository to publish the artifacts created by a DBB build in the pipeline. -### Invocation +When uploading the archive to an artifact repository, this script implements naming conventions for the repository layout. The conventions are implemented in [utilities/packageUtils.sh](utilities/packageUtils.sh). +The rules for the naming conventions are detailed hereafter. + +For any preliminary build (that uses the `pipelineType=build`), the outputs are uploaded into the directory + +`build//-`: + +* **build** is defined for any builds, that are considered to be preliminary. They cannot be deployed to production. +* **reference** is the name of the branch which the build originates from: for instance, `feature/123-update-mortgage-computation`, `main` or any hotfix and epic branches. +* The archive's file name is computed using the application's name and a unique build identifier (`-i` argument). This parameter is typically the pipeline build number that is passed by the pipeline orchestrator. If a build identifier is not provided, the current timestamp is used. + +For release builds (that use the `pipelineType=release`), the archive is uploaded to the directory + +`release//-`: + +* **release** is defined for release builds. +* **reference** is the release name: for instance, `rel-1.2.3` (provided through the mandatory `-r` argument). +The archive's file name is computed using the application's name, the release name (`-r` argument) and a unique build identifier (`-i` argument). This parameter is typically the pipeline build number that is passed by the pipeline orchestrator. If a build identifier is not provided, the current timestamp is used. + + +#### Invocation The `packageBuildOutputs.sh` script can be invoked as follows: @@ -467,7 +527,7 @@ packageBuildOutputs.sh -w MortApp/main/build-1 -t rel-1.0.0.tar ``` - Package and Upload ``` -packageBuildOutputs.sh -w MortApp/main/build-1 -t rel-1.0.0.tar -a MortgageApplication -b main -u -p release -v rel-1.0.0.2023-09-22-08.55.20 +packageBuildOutputs.sh -w MortApp/main/build-1 -t rel-1.0.0.tar -a MortgageApplication -b main -u -p release -r rel-1.0.0 -i 4657 ``` CLI parameter | Description @@ -476,30 +536,24 @@ CLI parameter | Description -w `` | **Workspace directory**, an absolute or relative path that represents unique directory for this pipeline definition, that needs to be consistent through multiple steps. The `packageBuildOutputs.sh` script is evaluating the logs directory. -t `` | (Optional) Name of the **tar file** to create. **Artifact Upload options** --u | Flag to enable upload of outputs to the configured artifact repository. --a `` | **Application name** leveraged to define the artifact repository name. See function `computeArtifactRepositoryName()` in the pipelineBackend.config file. Ex.: `MortgageApplication-repo-local`. --b ``| Name of the **git branch** turning into a segment of the directory path in the artifact repository. See function `computeArtifactRepositoryDirectory()` in the pipelineBackend.config file. +-u | Flag to enable upload of outputs to the configured artifact repository. Also available as a general setting in `pipelineBackend.config`. +-a `` | **Application name** leveraged to define the artifact repository name. +-b ``| Name of the **git branch** turning into a segment of the directory path in the artifact repository. Naming convention rules are implemented in `utilities/packageUtils.sh`. -p `` | **Pipeline type** to indicate a `build` pipeline (build only with test/debug options) or a `release` pipeline (build for optimized load modules) to determine the directory in the artifact repository for development and pipeline builds. --v `` | Label of the **version** in the artifact repository turning into a segment of the directory path in the artifact repo. +-r `` | **Release identifier** to indicate the next planned release name. This is a computed value based on the pipeline templates. +-i `` | **Build identifier** a unique value, typically the build number provided by the pipeline orchestrator or a timestamp. Used to help qualifying the archive file. This is a computed value provided by the pipeline templates. +-v `` **deprecated** | Label of the **version** in the artifact repository turning into a segment of the directory path in the artifact repo. Deprecated - switch to `-r ` and `-i `. -s `""` | (Optional) Name and email of the SBOM author enclosed with double quotes. Ex: "Build Engineer \" -### Script conventions - -**Directory Path within the artifact repo** +Check out the pipelineBackend.config to define the `artifactRepositoryNameSuffix` that is appended to the application name to set the repository name in the artifact repository. -For uploading, the backend script computes the directory path within the artifact repository : -* **Branch**/**artifactVersion** - -If it is the `main` branch, the pipeline type (-p) is evaluated to -* **Branch**/**pipelineType **/**artifactVersion** - -while **artifactVersion** is appended by the `PackageBuildOutputs.groovy` script. +#### Script conventions **SBOM Generation** The generation of an SBOM is controlled by the `generateSBOM` property defined in the [pipelineBackend.config](pipelineBackend.config) file. The default SBOM Author is also specified in the [pipelineBackend.config](pipelineBackend.config) file in the `sbomAuthor` property, but this property can be overridden with the `-s` parameter of this script. When the SBOM Author is provided as a parameter, it automatically enables the SBOM generation, even if set to `false` in the [pipelineBackend.config](pipelineBackend.config) file. -### Output +#### Output The section below contains the output that is produced by the `packageBuildOutputs.sh` script. @@ -530,7 +584,7 @@ packageBuildOutputs.sh: [INFO] ** ArtifactRepo Repo Dir: main/build packageBuildOutputs.sh: [INFO] ** DBB_HOME: /usr/lpp/dbb/v2r0 packageBuildOutputs.sh: [INFO] ************************************************************** -packageBuildOutputs.sh: [INFO] Invoking the Package Build Outputs script. +packageBuildOutputs.sh: [INFO] Invoking the ArtifactRepositoryHelper groovy script. packageBuildOutputs.sh: [INFO] groovyz /var/dbb/extensions/dbb20/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy --workDir /var/dbb/pipelineBackend/workspace/MortApp/main/build-1/logs --tarFileName package.tar --packagingPropertiesFile /var/dbb/extensions/dbb20/Pipeline/PackageBuildOutputs/packageBuildOutputs.properties --addExtension --publish --artifactRepositoryUrl "http://10.3.20.231:8081/artifactory" --versionName MortgageApplication.2023-09-22_13-55-20 --artifactRepositoryUser admin --artifactRepositoryPassword artifactoryadmin --artifactRepositoryName MortgageApplication-repo-local --artifactRepositoryDirectory main/build ** PackageBuildOutputs start at 20230922.125616.056 ** Properties at startup: @@ -577,11 +631,11 @@ rc=0 -## ucdPackaging.sh +### ucdPackaging.sh This script is to execute the `dbb-ucd-packaging.groovy` that invokes the Urban Code Deploy (UCD) buztool utility, to publish the artifacts created by the DBB Build from a pipeline. -### Invocation +#### Invocation The `ucdPackaging.sh` script can be invoked as follows: @@ -600,7 +654,7 @@ CLI parameter | Description -b `` | (Optional) Name of the **git branch**. -p `` | (Optional) URL to the pull request. -### Output +#### Output The section below contains the output that is produced by the `ucdPackaging.sh` script. @@ -632,80 +686,18 @@ ucdPackaging.sh: [INFO] groovyz /var/dbb/extensions/dbb20/Pipeline/CreateUCDCom -## ucdDeploy.sh - -This script is implementing the invocation of the `ucd-deploy.groovy` script to perform Urban Code Deploy (UCD) deployments. - - -### Invocation - -The `ucdDeploy.sh` script can be invoked as follows: - -``` -ucdDeploy.sh -a ucdApplicationName -p ucdApplicationProcessName -e ucdEnvironmentName -d ucdComponentName:ucdDeployVersion -``` - -CLI parameter | Description ----------- | ---------------------------------------------------------------------------------------- --a `` | **Application** name defined in UCD containing the component version to be deployed. --p `` | **Process** name in UCD associated with the application being deployed. --e `` | **Environment** name in UCD that the component version will be deployed to. --d `` | **Component name and version** to be deployed to the UCD environment. --t `` | (Optional) **Deployment timeout** value in seconds. Defaults to 300 seconds. --s ``| (Optional) **SSL protocol** to use. Default is TLSv1.2. --k | (Optional) Disable SSL verification flag. --v | (Optional) Verbose tracing flag. Used to produce additional tracing in the groovy script. - -### Output - -The section below contains the output that is produced by the `ucdDeploy.sh` script. - -
- Script Output - -``` -ucdDeploy.sh -a rbs-retirementCalculator -p rbs-retirementCalculator-process -e rbs-IntegrationTest -d rbs-retirementCalculator:latest -bash: [INFO] Deploy UCD Component. Version=1.00 -bash: [INFO] ************************************************************** -bash: [INFO] ** Start UCD Component Deploy on HOST/USER: z/OS ZT01 04.00 02 8561/BPXROOT -bash: [INFO] ** Location of ucd-deploy.groovy: /u/brice/groovyScripts -bash: [INFO] ** UCD Application Name: rbs-retirementCalculator -bash: [INFO] ** UCD Environment Name: rbs-IntegrationTest -bash: [INFO] ** UCD User Name: admin -bash: [INFO] ** UCD Server URL: http://10.3.20.231:8080/ -bash: [INFO] ** UCD Component Version: rbs-retirementCalculator:latest -bash: [INFO] ** UCD Application Process Name: rbs-retirementCalculator-process -bash: [INFO] ** Verbose: No -bash: [INFO] ** SSL Verification: Yes -bash: [INFO] ************************************************************** +## Deployment Stage -/usr/lpp/dbb/v2r0/bin/groovyz /u/brice/groovyScripts/ucd-deploy.groovy -a "rbs-retirementCalculator" -e "rbs-IntegrationTest" -U admin -P ******** -u http://10.3.20.231:8080/ -d "rbs-retirementCalculator:latest" -p rbs-retirementCalculator-process -** Request UCD Deployment start at 20230830.064205.042 -** Properties at startup: - application -> rbs-retirementCalculator - environment -> rbs-IntegrationTest - user -> admin - password -> xxxxxx - url -> http://10.3.20.231:8080/ - deployVersions -> rbs-retirementCalculator:latest - applicationProcess -> rbs-retirementCalculator-process -** Deploying component versions: rbs-retirementCalculator:latest -*** Starting deployment process 'rbs-retirementCalculator-process' of application 'rbs-retirementCalculator' in environment 'rbs-IntegrationTest' -*** SSL Verification disabled -*** Follow Process Request: https://ucd.server.com:8443/#applicationProcessRequest/184c812f-605f-5040-ad31-d3a31f87bb3c -Executing ...... -*** The deployment result is SUCCEEDED. See the UrbanCode Deploy deployment logs for details. -** Build finished -``` +Depending on the selected Deployment tool, select either from the scripts for IBM Wazi Deploy ([wazideploy-generate.sh](#wazideploy-generatesh), [wazideploy-deploy.sh](#wazideploy-deploysh) and [wazideploy-generate.sh](#wazideploy-evidencesh)) or [UCD deployment](#ucddeploysh) to submit a deployment request in IBM UrbanCode Deploy. -
-## wazideploy-generate.sh +### wazideploy-generate.sh This script invokes the Wazi Deploy Generate command to generate a Deployment Plan based on the content of a package. The package should be created with the `PackageBuildOutputs.groovy` script or through the `packageBuildOutputs.sh` script. +This script assesses the configuration option `publish` from the `pipelineBackend.config` file. In case the configuration has enabled the upload to the Artifact repository, the script computes the URL where the package is expected to be found, and passes the URL into the wazideploy-generate command. This means that wazideloy-generate will download the package from the Artifact repository and allows to restore the package on a different system. It requires to pass in the additional arguments `-P`, `-R`, `-B` -### Invocation +#### Invocation The `wazideploy-generate.sh` script can be invoked as follows: @@ -717,18 +709,29 @@ Or by fully specifying the settings wazideploy-generate.sh -m deploymentMethod -p deploymentPlan -r deploymentPlanReport -i packageInputFile ``` +To enable the download based on build and release identifier +``` +wazideploy-generate.sh -w MortgageApplication/feature/15-fetch-application-dependencies/dbb-zappbuild.build_1234 -a MortgageApplication -P build -b feature/15-fetch-application-dependencies -I 1234 +``` + CLI parameter | Description ---------- | ---------------------------------------------------------------------------------------- -w `` | **Workspace directory**, an absolute or relative path that represents unique directory for this pipeline definition, that needs to be consistent through multiple steps. Optional, if `deploymentPlan`, `deploymentPlanReport` and `packageOutputFile` are fully referenced. --i `` | **Package Input File** to be used for the generation phase with Wazi Deploy. This is likely the package to be deployed. If providing a relative path, the file is assumed to be located in the directory `/`. This parameter can either be path to a TAR file on UNIX System Services, or the URL of the TAR file to retrieve (only Artifactory is supported). +-i `` | **Package Input File** to be used for the generation phase with Wazi Deploy. This is likely the package to be deployed. If providing a relative path, the file is assumed to be located in the directory `/`. This parameter can either be path to an archive (TAR file) on UNIX System Services, or the URL of the archive (TAR file) to retrieve (only Artifactory is supported).

If the Common Backend Scripts are configured to perform the upload, the scripts computes the location in the artifact repo and overrides the `-i` and `-o` argument and in that case the this arument is not required. -m `` | (Optional) Absolute path to the Wazi Deploy **Deployment Method** stored on UNIX System Services. If not specified, the deployment method file location is obtained from the `pipelineBackend.config`. -p `` | (Optional) Absolute or relative path to the **Deployment Plan** file, generated based on the content of the input package. If providing a relative path, the file path is prefixed with Wazi Deploy Packaging directory `` configured in `pipelineBackend.config`. If not specified, the deployment plan location is obtained from the `pipelineBackend.config`. -r `` | (Optional) Absolute or relative path to the **Deployment Plan Report**. If providing a relative path, the file path is prefixed with Wazi Deploy Packaging directory `` configured in `pipelineBackend.config`. If not specified, the deployment plan report location is obtained from the `pipelineBackend.config`. -o `` | (Optional) Absolute or relative path to the **Package Output File** that specifies the location where to store the downloaded tar file. If providing a relative path, the file path is prefixed with Wazi Deploy Packaging directory `` configured in `pipelineBackend.config`. Only required when wazideploy-generate is used to download the package. This is indicated when a URL is specified for the **Package Input File**. --c `` | (Optional) Absolute path to the **Configuration File** that contains information to connect to Artifactory. Only required when wazideploy-generate is used to download the package. This is indicated when a URL is specified for the **Package Input File**. + -d | (Optional) Debug tracing flag. Used to produce additional tracing with Wazi Deploy. +-- | - when retrieving the tar file from Artifact repo the below options are mandatory - +-b ``| Name of the **git branch** turning into a segment of the directory path for the location within the artifact repository. +-p `` | **Pipeline Type** to indicate a `build` pipeline (build only with test/debug options) or a `release` pipeline (build for optimized load modules for release candidates). +-R `` | **Release identifier** to indicate the next planned release name. This is a computed value based on the pipeline templates. +-I `` | **Build identifier** a unique value to identify the tar file. This is a computed value provided by the pipeline templates. Typically the build number of the pipeline run. +-c `` | Absolute path to the Wazi Deploy **Configuration File** that contains information to connect to the artifact repository. See [IBM Wazi Deploy documentation](https://www.ibm.com/docs/en/developer-for-zos/17.0?topic=files-configuration-file) -### Output +#### Output The section below contains the output that is produced by the `wazideploy-generate.sh` script. @@ -774,11 +777,11 @@ wazideploy-generate.sh: [INFO] ************************************************* -## wazideploy-deploy.sh +### wazideploy-deploy.sh This script invokes the Wazi Deploy Deploy (with the Python Translator) command to deploy the content of a provided package with a Deployment Plan. -### Invocation +#### Invocation The `wazideploy-deploy.sh` script can be invoked as follows: @@ -864,11 +867,11 @@ wazideploy-deploy -wf /u/ado/workspace/MortgageApplication/main/build-20231019.1 -## wazideploy-evidence.sh +### wazideploy-evidence.sh This script invokes the Wazi Deploy Evidence command to generate a Deployment report from the Wazi Deploy Evidence YAML file created by the Wazi Deploy Deploy command. -### Invocation +#### Invocation The `wazideploy-evidence.sh` script can be invoked as follows: @@ -887,7 +890,7 @@ CLI parameter | Description -l `` | (Optional) Absolute or relative path to the **Evidence File** that contains the logs of all Wazi Deploy tasks. If not specified, evidence file location will be obtained from the `pipelineBackend.config`. -o `` | (Optional) Absolute or relative path to the **Output File** that will contain the Deployment Report. If not specified, evidence file location will be obtained from the `pipelineBackend.config`. -### Output +#### Output The section below contains the output that is produced by the `wazideploy-evidence.sh` script. @@ -921,11 +924,85 @@ wazideploy-evidence --index /u/ado/workspace/MortgageApplication/main/build-2023 -## prepareLogs.sh +### ucdDeploy.sh -Script to obtain the logs that were produced as part of the pipeline steps in the *logs* directory. +This script is implementing the invocation of the `ucd-deploy.groovy` script to perform Urban Code Deploy (UCD) deployments. -### Invocation + +#### Invocation + +The `ucdDeploy.sh` script can be invoked as follows: + +``` +ucdDeploy.sh -a ucdApplicationName -p ucdApplicationProcessName -e ucdEnvironmentName -d ucdComponentName:ucdDeployVersion +``` + +CLI parameter | Description +---------- | ---------------------------------------------------------------------------------------- +-a `` | **Application** name defined in UCD containing the component version to be deployed. +-p `` | **Process** name in UCD associated with the application being deployed. +-e `` | **Environment** name in UCD that the component version will be deployed to. +-d `` | **Component name and version** to be deployed to the UCD environment. +-t `` | (Optional) **Deployment timeout** value in seconds. Defaults to 300 seconds. +-s ``| (Optional) **SSL protocol** to use. Default is TLSv1.2. +-k | (Optional) Disable SSL verification flag. +-v | (Optional) Verbose tracing flag. Used to produce additional tracing in the groovy script. + +#### Output + +The section below contains the output that is produced by the `ucdDeploy.sh` script. + +
+ Script Output + +``` +ucdDeploy.sh -a rbs-retirementCalculator -p rbs-retirementCalculator-process -e rbs-IntegrationTest -d rbs-retirementCalculator:latest +bash: [INFO] Deploy UCD Component. Version=1.00 +bash: [INFO] ************************************************************** +bash: [INFO] ** Start UCD Component Deploy on HOST/USER: z/OS ZT01 04.00 02 8561/BPXROOT +bash: [INFO] ** Location of ucd-deploy.groovy: /u/brice/groovyScripts +bash: [INFO] ** UCD Application Name: rbs-retirementCalculator +bash: [INFO] ** UCD Environment Name: rbs-IntegrationTest +bash: [INFO] ** UCD User Name: admin +bash: [INFO] ** UCD Server URL: http://10.3.20.231:8080/ +bash: [INFO] ** UCD Component Version: rbs-retirementCalculator:latest +bash: [INFO] ** UCD Application Process Name: rbs-retirementCalculator-process +bash: [INFO] ** Verbose: No +bash: [INFO] ** SSL Verification: Yes +bash: [INFO] ************************************************************** + +/usr/lpp/dbb/v2r0/bin/groovyz /u/brice/groovyScripts/ucd-deploy.groovy -a "rbs-retirementCalculator" -e "rbs-IntegrationTest" -U admin -P ******** -u http://10.3.20.231:8080/ -d "rbs-retirementCalculator:latest" -p rbs-retirementCalculator-process +** Request UCD Deployment start at 20230830.064205.042 +** Properties at startup: + application -> rbs-retirementCalculator + environment -> rbs-IntegrationTest + user -> admin + password -> xxxxxx + url -> http://10.3.20.231:8080/ + deployVersions -> rbs-retirementCalculator:latest + applicationProcess -> rbs-retirementCalculator-process +** Deploying component versions: rbs-retirementCalculator:latest +*** Starting deployment process 'rbs-retirementCalculator-process' of application 'rbs-retirementCalculator' in environment 'rbs-IntegrationTest' +*** SSL Verification disabled +*** Follow Process Request: https://ucd.server.com:8443/#applicationProcessRequest/184c812f-605f-5040-ad31-d3a31f87bb3c +Executing ...... +*** The deployment result is SUCCEEDED. See the UrbanCode Deploy deployment logs for details. +** Build finished +``` + +
+ +## Generic stages + +When your pipeline setup uses a remote agents that runs outside of the z/OS build machine (like with an agent on an x86 environment), you can benefit from the [prepareLogs.sh](#preparelogssh) script to create an archive of the logs, before transferring it to the remote agent for attaching them to the pipeline run. + +Once the pipeline has completed all its tasks, you can use the [deleteWorkspace.sh](#deleteworkspacesh) to cleanup the created workspace on z/OS Unix Services and optionally the [generateCleanupCommands.sh](#generatecleanupcommandssh) for housekeeping activities of the DBB metadataststore. + +### prepareLogs.sh + +Script to obtain the logs that were produced as part of the pipeline step in the *logs* directory. + +#### Invocation The `prepareLogs.sh` script can be invoked as follows: @@ -942,7 +1019,7 @@ On successful completion, the script writes a message to indicate the output dir Logs successfully stored at /var/dbb/pipelineBackend/workspace/MortApp/feature/setmainbuildbranch/build-1/logs.tar ``` -### Script output +#### Script output The section below contains the output that is produced by the `prepareLogs.sh` script. @@ -976,7 +1053,7 @@ prepareLogs.sh: [INFO] Logs successfully stored at /var/dbb/pipelineBackend/work -### Download logs to non-z/OS runner/agent environment +#### Download logs to non-z/OS runner/agent environment While the script `prepareLogs.sh` only creates the TAR file on the workspace directory, the next step is to download the TAR file to the non-z/OS runner/agent environment, in order to attach it to the pipeline results. @@ -1001,116 +1078,11 @@ It _greps_ the information and invokes a download action. fi ``` -## generateCleanupCommands.sh - -Script to generate and run the necessary cleanup steps of DBB Metadatastore collections and build groups (build results), and the deletion of the build datasets using the [DeletePDS.groovy](../../Utilities/DeletePDS/README.md) utility. - -The script lists all the existing DBB collections obtained by applying a filter based on the zAppBuild naming conventions. It checks if Git branches corresponding to the provided application name exist in the Git repository. If one or more branches are found, it generates the necessary command files that contain the removal statements. The generated scripts can be can automatically executed, if the `-p` flag is passed to the script. - -### Invocation - -The `generateCleanupCommands.sh` script can be invoked as follows: - -``` -generateCleanupCommands.sh -w MortApp/main/build-1 -a MortApp -p -``` - -CLI parameter | Description ----------- | ---------------------------------------------------------------------------------------- --w `` | **Workspace directory** - an absolute or relative path that represents unique directory for this pipeline definition, that needs to be consistent through multiple steps. --a `` | **Application name** to be analyzed for stale DBB Metadatastore objects and build datasets. --p | Flag to control if the generated commands files should be executed by the pipeline. If the commands are not executed by the script, it is recommended to publish the generated files to the pipeline orchestrator, where an administrator can review and eventually execute them manually. - -### Additional notes - -This script can be embedded into a pipeline execution, but can also be used in a standalone setup. For a pipeline implementation, this task can be included in the release process to facilitate the cleanup of stale DBB collections and DBB build groups, and to delete the build datasets as well. - -For the standalone implementation, use the following process: -1. Have the Common Backend Scripts installed to z/OS Unix System Services and have them configured. -2. Clone the application repository including all remote references. -3. Execute the `generateCleanupCommands.sh` script like in the above sample. The user executing the script needs proper permissions on the DBB Metadatastore. - -Please note that the script leverages the [utilities/dbbBuildUtils.sh](utilities/dbbBuildUtils.sh) to compute the build high-level qualifier (HLQ). - -### Script output - -The section below contains the output that is produced by the `generateCleanupCommands.sh` script. - -
- Script Output - -``` -". ./.profile && generateCleanupCommands.sh -w /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104 -a MortgageApplication -p" -generateCleanupCommands.sh: [INFO] Generate Cleanup Command File. Version=1.0.0 -generateCleanupCommands.sh: [INFO] Creating output directory. /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104/cleanupCmds -generateCleanupCommands.sh: [INFO] ************************************************************** -generateCleanupCommands.sh: [INFO] ** Start Gen Cleanup Cmds on HOST/USER: z/OS ZT01 05.00 02 8561/GITHUB -generateCleanupCommands.sh: [INFO] ** Workspace: /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104 -generateCleanupCommands.sh: [INFO] ** Application: MortgageApplication -generateCleanupCommands.sh: [INFO] ** AppDir: /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104/MortgageApplication -generateCleanupCommands.sh: [INFO] ** Cmd obsolete collections: /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104/cleanupCmds/deleteStaleCollections.cmd -generateCleanupCommands.sh: [INFO] ** Cmd obsolete build groups: /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104/cleanupCmds/deleteStaleBuildGroups.cmd -generateCleanupCommands.sh: [INFO] ** Cmd obsolete build datasets: /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104/cleanupCmds/deleteStaleBuildDatasets.cmd -generateCleanupCommands.sh: [INFO] ** DBB Metadastore Config: --type file --location /u/github/ -generateCleanupCommands.sh: [INFO] ** Process Cleanup Scripts: true -generateCleanupCommands.sh: [INFO] ************************************************************** - -generateCleanupCommands.sh: [STAGE] Retrieve all collections with application qualifier MortgageApplication -generateCleanupCommands.sh: [STAGE] Verifying Git references -generateCleanupCommands.sh: [INFO] Check if MortgageApplication-main has a corresponding active Git branch -generateCleanupCommands.sh: For the collection MortgageApplication-main a corresponding branch (main) was detected. -generateCleanupCommands.sh: [INFO] Check if MortgageApplication-main-outputs has a corresponding active Git branch -generateCleanupCommands.sh: For the collection MortgageApplication-main-outputs a corresponding branch (main) was detected. -generateCleanupCommands.sh: [INFO] Check if MortgageApplication-feature/johnpipelinetesting has a corresponding active Git branch -generateCleanupCommands.sh: For the collection MortgageApplication-feature/johnpipelinetesting a corresponding branch (feature/johnpipelinetesting) was detected. -generateCleanupCommands.sh: [INFO] Check if MortgageApplication-feature/johnpipelinetesting-outputs has a corresponding active Git branch -generateCleanupCommands.sh: For the collection MortgageApplication-feature/johnpipelinetesting-outputs a corresponding branch (feature/johnpipelinetesting) was detected. -generateCleanupCommands.sh: [INFO] Check if MortgageApplication-feature/20-some-more-testing has a corresponding active Git branch -generateCleanupCommands.sh: DBB Collection MortgageApplication-feature/20-some-more-testing does not have a corresponding branch (feature/20-some-more-testing). It can be deleted. -generateCleanupCommands.sh: [INFO] Check if MortgageApplication-feature/20-some-more-testing-outputs has a corresponding active Git branch -generateCleanupCommands.sh: DBB Collection MortgageApplication-feature/20-some-more-testing-outputs does not have a corresponding branch (feature/20-some-more-testing). It can be deleted. -generateCleanupCommands.sh: [INFO] Check if MortgageApplication-feature/pipeline-trigger-testing has a corresponding active Git branch -generateCleanupCommands.sh: For the collection MortgageApplication-feature/pipeline-trigger-testing a corresponding branch (feature/pipeline-trigger-testing) was detected. -generateCleanupCommands.sh: [INFO] Check if MortgageApplication-feature/pipeline-trigger-testing-outputs has a corresponding active Git branch -generateCleanupCommands.sh: For the collection MortgageApplication-feature/pipeline-trigger-testing-outputs a corresponding branch (feature/pipeline-trigger-testing) was detected. -generateCleanupCommands.sh: [INFO] Check if MortgageApplication-feature/18-add-generate-cleanup-instructions has a corresponding active Git branch -generateCleanupCommands.sh: For the collection MortgageApplication-feature/18-add-generate-cleanup-instructions a corresponding branch (feature/18-add-generate-cleanup-instructions) was detected. -generateCleanupCommands.sh: [INFO] Check if MortgageApplication-feature/18-add-generate-cleanup-instructions-outputs has a corresponding active Git branch -generateCleanupCommands.sh: For the collection MortgageApplication-feature/18-add-generate-cleanup-instructions-outputs a corresponding branch (feature/18-add-generate-cleanup-instructions) was detected. -generateCleanupCommands.sh: [STAGE] Generate Cmd File with Delete Statements for stale collections for application MortgageApplication -generateCleanupCommands.sh: [INFO] Cmd File /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104/cleanupCmds/deleteStaleCollections.cmd created. -generateCleanupCommands.sh: [STAGE] Generate Cmd File with Delete Statements for stale build groups for application MortgageApplication -generateCleanupCommands.sh: [INFO] Cmd File /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104/cleanupCmds/deleteStaleBuildGroups.cmd created. -generateCleanupCommands.sh: [STAGE] Generate Cmd File with Delete Statements for stale build datasets for application MortgageApplication -generateCleanupCommands.sh: [INFO] Cmd File /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104/cleanupCmds/deleteStaleBuildDatasets.cmd created. -generateCleanupCommands.sh: [STAGE] Executing Cleanup of DBB Metadatastore Objects -generateCleanupCommands.sh: [INFO] Executing cleanup script /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104/cleanupCmds/deleteStaleCollections.cmd -BGZTK0195I Successfully deleted collection "MortgageApplication-feature/20-some-more-testing" -BGZTK0195I Successfully deleted collection "MortgageApplication-feature/20-some-more-testing-outputs" -generateCleanupCommands.sh: [INFO] Executing cleanup script /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104/cleanupCmds/deleteStaleBuildGroups.cmd -BGZTK0195I Successfully deleted group "MortgageApplication-feature/20-some-more-testing" -generateCleanupCommands.sh: [INFO] Executing cleanup script /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104/cleanupCmds/deleteStaleBuildDatasets.cmd -** Deleting all datasets filtered with HLQ 'GITHUB.MORTGAGE.F20' -*** Deleting 'GITHUB.MORTGAGE.F20.COBOL' -*** Deleting 'GITHUB.MORTGAGE.F20.COPY' -*** Deleting 'GITHUB.MORTGAGE.F20.DBRM' -*** Deleting 'GITHUB.MORTGAGE.F20.LOAD' -*** Deleting 'GITHUB.MORTGAGE.F20.OBJ' -** Deleted 5 entries. -** Build finished -generateCleanupCommands.sh: [INFO] Generate Cleanup Cmds Complete. rc=0 - -``` - -
- - - -## deleteWorkspace.sh +### deleteWorkspace.sh Script delete the workspace and all empty directories in the working tree. -### Invocation +#### Invocation The `deleteWorkspace.sh` script can be invoked as follows: @@ -1125,7 +1097,7 @@ CLI parameter | Description Note that the script deletes all empty folders in the working tree. It supresses the message `EDC5136I Directory not empty.` and handles that as a INFO message. -### Script output +#### Script output The section below contains the output that is produced by the `deleteWorkspace.sh` script. @@ -1220,6 +1192,113 @@ deleteWorkspace.sh: [INFO] Workspace directory successfully deleted. +### generateCleanupCommands.sh + +Script to generate and run the necessary cleanup steps of DBB Metadatastore collections and build groups (build results), and the deletion of the build datasets using the [DeletePDS.groovy](../../Utilities/DeletePDS/README.md) utility. + +The script lists all the existing DBB collections obtained by applying a filter based on the zAppBuild naming conventions. It checks if Git branches corresponding to the provided application name exist in the Git repository. If one or more branches are found, it generates the necessary command files that contain the removal statements. The generated scripts can be can automatically executed, if the `-p` flag is passed to the script. + +#### Invocation + +The `generateCleanupCommands.sh` script can be invoked as follows: + +``` +generateCleanupCommands.sh -w MortApp/main/build-1 -a MortApp -p +``` + +CLI parameter | Description +---------- | ---------------------------------------------------------------------------------------- +-w `` | **Workspace directory** - an absolute or relative path that represents unique directory for this pipeline definition, that needs to be consistent through multiple steps. +-a `` | **Application name** to be analyzed for stale DBB Metadatastore objects and build datasets. +-p | Flag to control if the generated commands files should be executed by the pipeline. If the commands are not executed by the script, it is recommended to publish the generated files to the pipeline orchestrator, where an administrator can review and eventually execute them manually. + +#### Additional notes + +This script can be embedded into a pipeline execution, but can also be used in a standalone setup. For a pipeline implementation, this task can be included in the release process to facilitate the cleanup of stale DBB collections and DBB build groups, and to delete the build datasets as well. + +For the standalone implementation, use the following process: +1. Have the Common Backend Scripts installed to z/OS Unix System Services and have them configured. +2. Clone the application repository including all remote references. +3. Execute the `generateCleanupCommands.sh` script like in the above sample. The user executing the script needs proper permissions on the DBB Metadatastore. + +Please note that the script leverages the [utilities/dbbBuildUtils.sh](utilities/dbbBuildUtils.sh) to compute the build high-level qualifier (HLQ). + +### Script output + +The section below contains the output that is produced by the `generateCleanupCommands.sh` script. + +
+ Script Output + +``` +". ./.profile && generateCleanupCommands.sh -w /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104 -a MortgageApplication -p" +generateCleanupCommands.sh: [INFO] Generate Cleanup Command File. Version=1.0.0 +generateCleanupCommands.sh: [INFO] Creating output directory. /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104/cleanupCmds +generateCleanupCommands.sh: [INFO] ************************************************************** +generateCleanupCommands.sh: [INFO] ** Start Gen Cleanup Cmds on HOST/USER: z/OS ZT01 05.00 02 8561/GITHUB +generateCleanupCommands.sh: [INFO] ** Workspace: /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104 +generateCleanupCommands.sh: [INFO] ** Application: MortgageApplication +generateCleanupCommands.sh: [INFO] ** AppDir: /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104/MortgageApplication +generateCleanupCommands.sh: [INFO] ** Cmd obsolete collections: /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104/cleanupCmds/deleteStaleCollections.cmd +generateCleanupCommands.sh: [INFO] ** Cmd obsolete build groups: /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104/cleanupCmds/deleteStaleBuildGroups.cmd +generateCleanupCommands.sh: [INFO] ** Cmd obsolete build datasets: /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104/cleanupCmds/deleteStaleBuildDatasets.cmd +generateCleanupCommands.sh: [INFO] ** DBB Metadastore Config: --type file --location /u/github/ +generateCleanupCommands.sh: [INFO] ** Process Cleanup Scripts: true +generateCleanupCommands.sh: [INFO] ************************************************************** + +generateCleanupCommands.sh: [STAGE] Retrieve all collections with application qualifier MortgageApplication +generateCleanupCommands.sh: [STAGE] Verifying Git references +generateCleanupCommands.sh: [INFO] Check if MortgageApplication-main has a corresponding active Git branch +generateCleanupCommands.sh: For the collection MortgageApplication-main a corresponding branch (main) was detected. +generateCleanupCommands.sh: [INFO] Check if MortgageApplication-main-outputs has a corresponding active Git branch +generateCleanupCommands.sh: For the collection MortgageApplication-main-outputs a corresponding branch (main) was detected. +generateCleanupCommands.sh: [INFO] Check if MortgageApplication-feature/johnpipelinetesting has a corresponding active Git branch +generateCleanupCommands.sh: For the collection MortgageApplication-feature/johnpipelinetesting a corresponding branch (feature/johnpipelinetesting) was detected. +generateCleanupCommands.sh: [INFO] Check if MortgageApplication-feature/johnpipelinetesting-outputs has a corresponding active Git branch +generateCleanupCommands.sh: For the collection MortgageApplication-feature/johnpipelinetesting-outputs a corresponding branch (feature/johnpipelinetesting) was detected. +generateCleanupCommands.sh: [INFO] Check if MortgageApplication-feature/20-some-more-testing has a corresponding active Git branch +generateCleanupCommands.sh: DBB Collection MortgageApplication-feature/20-some-more-testing does not have a corresponding branch (feature/20-some-more-testing). It can be deleted. +generateCleanupCommands.sh: [INFO] Check if MortgageApplication-feature/20-some-more-testing-outputs has a corresponding active Git branch +generateCleanupCommands.sh: DBB Collection MortgageApplication-feature/20-some-more-testing-outputs does not have a corresponding branch (feature/20-some-more-testing). It can be deleted. +generateCleanupCommands.sh: [INFO] Check if MortgageApplication-feature/pipeline-trigger-testing has a corresponding active Git branch +generateCleanupCommands.sh: For the collection MortgageApplication-feature/pipeline-trigger-testing a corresponding branch (feature/pipeline-trigger-testing) was detected. +generateCleanupCommands.sh: [INFO] Check if MortgageApplication-feature/pipeline-trigger-testing-outputs has a corresponding active Git branch +generateCleanupCommands.sh: For the collection MortgageApplication-feature/pipeline-trigger-testing-outputs a corresponding branch (feature/pipeline-trigger-testing) was detected. +generateCleanupCommands.sh: [INFO] Check if MortgageApplication-feature/18-add-generate-cleanup-instructions has a corresponding active Git branch +generateCleanupCommands.sh: For the collection MortgageApplication-feature/18-add-generate-cleanup-instructions a corresponding branch (feature/18-add-generate-cleanup-instructions) was detected. +generateCleanupCommands.sh: [INFO] Check if MortgageApplication-feature/18-add-generate-cleanup-instructions-outputs has a corresponding active Git branch +generateCleanupCommands.sh: For the collection MortgageApplication-feature/18-add-generate-cleanup-instructions-outputs a corresponding branch (feature/18-add-generate-cleanup-instructions) was detected. +generateCleanupCommands.sh: [STAGE] Generate Cmd File with Delete Statements for stale collections for application MortgageApplication +generateCleanupCommands.sh: [INFO] Cmd File /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104/cleanupCmds/deleteStaleCollections.cmd created. +generateCleanupCommands.sh: [STAGE] Generate Cmd File with Delete Statements for stale build groups for application MortgageApplication +generateCleanupCommands.sh: [INFO] Cmd File /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104/cleanupCmds/deleteStaleBuildGroups.cmd created. +generateCleanupCommands.sh: [STAGE] Generate Cmd File with Delete Statements for stale build datasets for application MortgageApplication +generateCleanupCommands.sh: [INFO] Cmd File /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104/cleanupCmds/deleteStaleBuildDatasets.cmd created. +generateCleanupCommands.sh: [STAGE] Executing Cleanup of DBB Metadatastore Objects +generateCleanupCommands.sh: [INFO] Executing cleanup script /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104/cleanupCmds/deleteStaleCollections.cmd +BGZTK0195I Successfully deleted collection "MortgageApplication-feature/20-some-more-testing" +BGZTK0195I Successfully deleted collection "MortgageApplication-feature/20-some-more-testing-outputs" +generateCleanupCommands.sh: [INFO] Executing cleanup script /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104/cleanupCmds/deleteStaleBuildGroups.cmd +BGZTK0195I Successfully deleted group "MortgageApplication-feature/20-some-more-testing" +generateCleanupCommands.sh: [INFO] Executing cleanup script /u/github/workspace/IBM-DAT/MortgageApplication/feature/18-add-generate-cleanup-instructions/build_f104/cleanupCmds/deleteStaleBuildDatasets.cmd +** Deleting all datasets filtered with HLQ 'GITHUB.MORTGAGE.F20' +*** Deleting 'GITHUB.MORTGAGE.F20.COBOL' +*** Deleting 'GITHUB.MORTGAGE.F20.COPY' +*** Deleting 'GITHUB.MORTGAGE.F20.DBRM' +*** Deleting 'GITHUB.MORTGAGE.F20.LOAD' +*** Deleting 'GITHUB.MORTGAGE.F20.OBJ' +** Deleted 5 entries. +** Build finished +generateCleanupCommands.sh: [INFO] Generate Cleanup Cmds Complete. rc=0 + +``` + +
+ + + + + # Disclaimer THIS SAMPLE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/Templates/Common-Backend-Scripts/dbbBuild.sh b/Templates/Common-Backend-Scripts/dbbBuild.sh index 86445150..c66a7a59 100755 --- a/Templates/Common-Backend-Scripts/dbbBuild.sh +++ b/Templates/Common-Backend-Scripts/dbbBuild.sh @@ -125,13 +125,13 @@ Help() { } # -# Build Type Customization # Configuration file leveraged by the backend scripts # Either an absolute path or a relative path to the current working directory SCRIPT_HOME="`dirname "$0"`" pipelineConfiguration="${SCRIPT_HOME}/pipelineBackend.config" +# Utility scripts buildUtilities="${SCRIPT_HOME}/utilities/dbbBuildUtils.sh" -# Customization - End +fetchBuildDependenciesUtilities="${SCRIPT_HOME}/utilities/fetchBuildDependenciesUtils.sh" # # Internal Variables @@ -139,7 +139,7 @@ buildUtilities="${SCRIPT_HOME}/utilities/dbbBuildUtils.sh" #export BASH_XTRACEFD=1 # Write set -x trace to file descriptor PGM=$(basename "$0") -PGMVERS="1.10" +PGMVERS="1.20" USER=$USER SYS=$(uname -Ia) @@ -154,7 +154,6 @@ PipelineType="" HELP=$1 # Local Variables -# TLD: Always a good idea to initialize any local varables AppDir="" # Derived Application Directory HLQ="" # Derived High Level Qualifier HLQPrefix="" # Prefix of HLQ, either specified via the cli option -q or via configuration file @@ -211,6 +210,13 @@ if [ $rc -eq 0 ]; then source $buildUtilities fi + # Read and import utilities + if [ ! -f "${fetchBuildDependenciesUtilities}" ]; then + rc=8 + ERRMSG=$PGM": [ERROR] DBB-Build internal utilities (${fetchBuildDependenciesUtilities}) was not found. rc="$rc + echo $ERRMSG + fi + # # Get Options if [ $rc -eq 0 ]; then @@ -338,7 +344,7 @@ validateOptions() { # Check if application directory contains if [ -d "${AppDir}/${App}" ]; then - echo $PGM": [INFO] Detected the application respository (${App}) within the git repository layout structure." + echo $PGM": [INFO] Detected the application repository (${App}) within the git repository layout structure." echo $PGM": [INFO] Assuming this as the new application location." AppDir="${AppDir}/${App}" nestedApplicationFolder="true" @@ -430,7 +436,15 @@ if [ $rc -eq 0 ]; then fi fi -# Ready to go TLD: Suggest in the section to echo as much as possible + +# Setup build environment and pull external dependencies if an Application Descriptor is found +if [ $rc -eq 0 ] && [ "$fetchBuildDependencies" == "true" ]; then + # call utilities script + . ${fetchBuildDependenciesUtilities} +fi + +# +# Echo build configuration if [ $rc -eq 0 ]; then echo $PGM": [INFO] **************************************************************" echo $PGM": [INFO] ** Started - DBB Build on HOST/USER: ${SYS}/${USER}" @@ -495,7 +509,7 @@ if [ $rc -eq 0 ]; then CMD="${CMD} ${Type}" # Append zAppBuild Build Type echo $PGM": [INFO] ${CMD}" - ${CMD} #TLD: I commented this out for testing purposed + ${CMD} rc=$? #exit 0 diff --git a/Templates/Common-Backend-Scripts/packageBuildOutputs.sh b/Templates/Common-Backend-Scripts/packageBuildOutputs.sh index 30bca6d8..de4b78b9 100755 --- a/Templates/Common-Backend-Scripts/packageBuildOutputs.sh +++ b/Templates/Common-Backend-Scripts/packageBuildOutputs.sh @@ -23,9 +23,6 @@ # 1. Review and update the Customization Section to reference the # central configuration file pipelineBackend.config # -# 2. Review the Customization Section in the pipelineBackend.config file : -# -# PackagingScript - Location of the PackageBuildOutputs.groovy # #=================================================================================== Help() { @@ -57,11 +54,6 @@ Help() { echo " " echo " Optional parameters " echo " " - echo " -t - Name of the package tar file " - echo " (Optional) " - echo " " - echo " Ex: package.tar " - echo " " echo " -u - flag to enable uploading " echo " outputs to configured " echo " artifact repository " @@ -72,20 +64,10 @@ Help() { echo " " echo " Ex: MortgageApplication " echo " " - echo " -p - Type of the pipeline to " - echo " control in which directory builds " - echo " are stored in the artifact repo " - echo " Accepted values: " - echo " build - " - echo " development builds " - echo " release - " - echo " builds with options for " - echo " performance optimized " - echo " executables for production env " - echo " " - echo " -b - Name of the git branch. " + echo " -t - Name of the package tar file " + echo " (Optional) " echo " " - echo " Ex: main " + echo " Ex: package.tar " echo " " echo " -v " echo " - Name of the artifactVersion " @@ -95,13 +77,39 @@ Help() { echo " " echo " Ex: Pipeline Build ID (Build.buildid.tar) " echo " " - echo " -s "" - Name and email of " + echo " -s " " - Name and email of " echo " the SBOM author " echo " enclosed with double quotes " echo " (Optional) " echo " " - echo " Ex: \"Build Engineer \" " + echo " Ex: \"Build Engineer \" " + echo " " echo " " + echo " Mandatory arguments when publishing to artifact repo " + echo " " + echo " -I - A unique build identifier " + echo " typically the buildID of the " + echo " pipeline " + echo " Ex: 6756 " + echo " " + echo " -R - The release identifier for " + echo " release pipeline builds " + echo " Ex: rel-1.2.3 " + echo " " + echo " -p - Type of the pipeline to " + echo " control in which directory builds " + echo " are stored in the artifact repo " + echo " Accepted values: " + echo " build - " + echo " development builds " + echo " release - " + echo " builds with options for " + echo " performance optimized " + echo " executables for production env " + echo " " + echo " -b - Name of the git branch. " + echo " " + echo " Ex: main " echo " -h - Display this Help. " echo " " exit 0 @@ -111,6 +119,9 @@ Help() { # Central configuration file leveraged by the backend scripts SCRIPT_HOME="$(dirname "$0")" pipelineConfiguration="${SCRIPT_HOME}/pipelineBackend.config" +packagingScript="${SCRIPT_HOME}/../../Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy" +packagingUtilities="${SCRIPT_HOME}/utilities/packagingUtilities.sh" + # Path and File Name to the advanced debug options. #log4j2="-Dlog4j.configurationFile=file:/../log4j2.properties" @@ -120,7 +131,7 @@ pipelineConfiguration="${SCRIPT_HOME}/pipelineBackend.config" #export BASH_XTRACEFD=1 # Write set -x trace to file descriptor PGM=$(basename "$0") -PGMVERS="1.00" +PGMVERS="1.10" USER=$(whoami) SYS=$(uname -Ia) @@ -131,6 +142,7 @@ rc=0 ERRMSG="" Workspace="" App="" +AppDir="" tarFileName="" PkgPropFile="" PipelineType="" @@ -142,6 +154,8 @@ generateSBOM="" sbomAuthor="" publish="" +buildIdentifier="" +releaseIdentifier="" artifactVersionName="" # required for publishing to artifact repo artifactRepositoryUrl="" # required if artifactRepositoryPropertyFile not specified artifactRepositoryUser="" # required if artifactRepositoryPropertyFile not specified @@ -149,6 +163,12 @@ artifactRepositoryPassword="" # required if artifactRepositoryPropertyFile n artifactRepositoryName="" # required if artifactRepositoryPropertyFile not specified artifactRepositoryDirectory="" # required if artifactRepositoryPropertyFile not specified artifactRepositoryPropertyFile="" # alternative to above cli parms +externalDependenciesLogFile="" # document fetched build dependencies + +# managing baseline package +baselineTarFile="" # computed if a baseline package has been retrieved during the fetch phase +baselineFolder="baseline" # per convention this is the subdir into which the fetch script loads the baseline + HELP=$1 if [ "$HELP" = "?" ]; then @@ -180,10 +200,21 @@ if [ $rc -eq 0 ]; then fi fi +# Source packaging helper +if [ $rc -eq 0 ]; then + if [ ! -f "${packagingUtilities}" ]; then + rc=8 + ERRMSG=$PGM": [ERROR] Packaging Utils (${packagingUtilities}) was not found. rc="$rc + echo $ERRMSG + else + source $packagingUtilities + fi +fi + # # Get Options if [ $rc -eq 0 ]; then - while getopts ":h:w:a:t:b:v:p:us:" opt; do + while getopts ":h:w:a:b:t:i:r:v:p:us:" opt; do case $opt in h) Help @@ -221,6 +252,17 @@ if [ $rc -eq 0 ]; then fi tarFileName="$argument" ;; + i) + argument="$OPTARG" + nextchar="$(expr substr $argument 1 1)" + if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then + rc=4 + ERRMSG=$PGM": [WARNING] The build identifier is required. rc="$rc + echo $ERRMSG + break + fi + buildIdentifier="$argument" + ;; b) argument="$OPTARG" nextchar="$(expr substr $argument 1 1)" @@ -258,15 +300,28 @@ if [ $rc -eq 0 ]; then fi PipelineType="$argument" ;; + r) + argument="$OPTARG" + nextchar="$(expr substr $argument 1 1)" + if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then + rc=4 + ERRMSG=$PGM": [WARNING] The release identifier is required. rc="$rc + echo $ERRMSG + break + fi + releaseIdentifier="$argument" + ;; v) argument="$OPTARG" nextchar="$(expr substr $argument 1 1)" if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then rc=4 - ERRMSG=$PGM": [WARNING] The name of the artifact version in Artifact repository is required. rc="$rc + ERRMSG=$PGM": [WARNING] The artifact version is required. rc="$rc echo $ERRMSG break fi + ERRMSG=$PGM": [WARNING] The argument (-v) for naming the version is deprecated. Please switch to the new options and supply the build identifier argument (-i) and for release pipelines the release identifier argument (-r)." + echo $ERRMSG artifactVersionName="$argument" ;; \?) @@ -284,7 +339,7 @@ if [ $rc -eq 0 ]; then done fi # -# Validate options + validateOptions() { if [ -z "${Workspace}" ]; then rc=8 @@ -303,9 +358,9 @@ validateOptions() { fi # Validate Packaging script - if [ ! -f "${PackagingScript}" ]; then + if [ ! -f "${packagingScript}" ]; then rc=8 - ERRMSG=$PGM": [ERR] Unable to locate ${PackagingScript}. rc="$rc + ERRMSG=$PGM": [ERROR] Unable to locate ${packagingScript}. rc="$rc echo $ERRMSG fi @@ -313,11 +368,66 @@ validateOptions() { if [ ! -z "${PkgPropFile}" ]; then if [ ! -f "${PkgPropFile}" ]; then rc=8 - ERRMSG=$PGM": [ERR] Unable to locate ${PkgPropFile}. rc="$rc + ERRMSG=$PGM": [ERROR] Unable to locate ${PkgPropFile}. rc="$rc + echo $ERRMSG + fi + fi + + if [ -z "${buildIdentifier}" ] && [ "$publish" == "true" ]; then + ERRMSG=$PGM": [INFO] No buildIdentifier (option -i) was supplied. Using timestamp." + echo $ERRMSG + buildIdentifier=$(date +%Y%m%d_%H%M%S) + fi + + if [ -z "${App}" ]; then + rc=8 + ERRMSG=$PGM": [ERROR] Application parameter (-a) is required. rc="$rc + echo $ERRMSG + else + + AppDir=$(getApplicationDir) + + # Check if application directory contains + if [ -d "${AppDir}/${App}" ]; then + echo $PGM": [INFO] Detected the application repository (${App}) within the git repository layout structure." + echo $PGM": [INFO] Assuming this as the new application location." + AppDir="${AppDir}/${App}" + fi + + if [ ! -d "${AppDir}" ]; then + rc=8 + ERRMSG=$PGM": [ERROR] Application Directory (${AppDir}) was not found. rc="$rc echo $ERRMSG fi fi + # validate if external dependency log exists + if [ "${fetchBuildDependencies}" == "true" ] && [ ! -z "${externalDependenciesLogName}" ]; then + externalDependenciesLogFile="$(getLogDir)/${externalDependenciesLogName}" + # Validate Properties file + if [ ! -f "${externalDependenciesLogFile}" ]; then + rc=8 + ERRMSG=$PGM": [ERROR] Unable to locate ${externalDependenciesLogFile}. rc="$rc + echo $ERRMSG + fi + fi + + # validate baseline package + if [ "${fetchBuildDependencies}" == "true" ]; then + + # validate baseline directory and baseline package + baselineDirectory="$(getWorkDirectory)/${baselineFolder}" + if [ -d "${baselineDirectory}" ]; then + baselineTarName=$(ls "${baselineDirectory}") + baselineTarFile="${baselineDirectory}/${baselineTarName}" + if [ ! -f "$baselineTarFile" ]; then + echo $PGM": [INFO] The baseline package $baselineTarFile was not found." + fi + else + echo $PGM": [INFO] The directory for the baseline package $baselineDirectory was not found." + fi + fi + } # function to validate publishing options @@ -338,17 +448,11 @@ validatePublishingOptions() { if [ ! -z "${artifactRepositoryPropertyFile}" ]; then if [ ! -f "${artifactRepositoryPropertyFile}" ]; then rc=8 - ERRMSG=$PGM": [ERR] Unable to locate ${artifactRepositoryPropertyFile}. rc="$rc + ERRMSG=$PGM": [ERROR] Unable to locate ${artifactRepositoryPropertyFile}. rc="$rc echo $ERRMSG fi else - if [ -z "${artifactVersionName}" ]; then - rc=8 - ERRMSG=$PGM": [ERROR] Name of the artifact version (artifactVersionName) is required. rc="$rc - echo $ERRMSG - fi - if [ -z "${artifactRepositoryUrl}" ]; then rc=8 ERRMSG=$PGM": [ERROR] URL to artifact repository (artifactRepositoryUrl) is required. rc="$rc @@ -392,12 +496,12 @@ validatePublishingOptions() { ;; "preview") rc=4 - ERRMSG=$PGM": [WARN] Default Pipeline Type : ${PipelineType} not supported for packaging." + ERRMSG=$PGM": [WARNING] Default Pipeline Type : ${PipelineType} not supported for packaging." echo $ERRMSG ;; *) rc=4 - ERRMSG=$PGM": [WARN] Inavlid Pipeline Type : ${PipelineType} specified." + ERRMSG=$PGM": [WARNING] Invalid Pipeline Type : ${PipelineType} specified." echo $ERRMSG ;; esac @@ -406,28 +510,30 @@ validatePublishingOptions() { fi } -# Call validate input options -if [ $rc -eq 0 ]; then - validateOptions -fi +# compute packaging parameters and validate publishing options +if [ $rc -eq 0 ] && [ "$publish" == "true" ]; then + # invoke function in packagingUtilities -# compute parameters -if [ $rc -eq 0 ]; then - # Compute artifactRepositoryName based on function in packageBuildOutputs.config - artifactRepositoryName=$(computeArtifactRepositoryName) - artifactRepositoryDirectory=$(computeArtifactRepositoryDirectory) + if [ ! -z "${tarFileName}" ]; then + echo $PGM": [INFO] ** Identified that tarFileName is passed into packageBuildOutputs.sh (${tarFileName}). This will be reset and recomputed based on buildIdentifier and releaseIdentifier to align with the conventions for packaging." + fi - # set default PipelineType for pipelines on main - if [ "${Branch}" == "main" ] && [ -z "${PipelineType}" ]; then - PipelineType="build" + if [ ! -z "${artifactVersionName}" ]; then + echo $PGM": [INFO] ** Identified that artifactVersionName is passed into packageBuildOutputs.sh (${artifactVersionName}). This will be reset and recomputed based on buildIdentifier and releaseIdentifier to align with the conventions for packaging." fi + + computeArchiveInformation fi -# Call validate publishing options if [ $rc -eq 0 ] && [ "$publish" == "true" ]; then validatePublishingOptions fi +# Call validate input options +if [ $rc -eq 0 ]; then + validateOptions +fi + # # Ready to go if [ $rc -eq 0 ]; then @@ -435,54 +541,65 @@ if [ $rc -eq 0 ]; then echo $PGM": [INFO] ** Started - Package Build Outputs on HOST/USER: ${SYS}/${USER}" echo $PGM": [INFO] ** WorkDir:" $(getWorkDirectory) if [ ! -z "${App}" ]; then - echo $PGM": [INFO] ** Application:" ${App} + echo $PGM": [INFO] ** Application:" ${App} fi if [ ! -z "${Branch}" ]; then - echo $PGM": [INFO] ** Branch:" ${Branch} + echo $PGM": [INFO] ** Branch:" ${Branch} + fi + + if [ ! -z "${AppDir}" ]; then + echo $PGM": [INFO] ** Application directory:" ${AppDir} fi + if [ ! -z "${PipelineType}" ]; then - echo $PGM": [INFO] ** Type of pipeline:" ${PipelineType} + echo $PGM": [INFO] ** Type of pipeline:" ${PipelineType} fi if [ ! -z "${tarFileName}" ]; then - echo $PGM": [INFO] ** Tar file Name:" ${tarFileName} + echo $PGM": [INFO] ** Tar file Name:" ${tarFileName} fi - echo $PGM": [INFO] ** BuildReport Location:" ${logDir} - echo $PGM": [INFO] ** PackagingScript Path:" ${PackagingScript} + echo $PGM": [INFO] ** BuildReport Location:" ${logDir} + echo $PGM": [INFO] ** PackagingScript Path:" ${packagingScript} if [ ! -z "${PkgPropFile}" ]; then - echo $PGM": [INFO] ** Packaging properties:" ${PkgPropFile} + echo $PGM": [INFO] ** Packaging properties:" ${PkgPropFile} fi - if [ ! -z "${artifactVersionName}" ]; then - echo $PGM": [INFO] ** Artifact name:" ${artifactVersionName} + if [ ! -z "${packageBuildIdentifier}" ]; then + echo $PGM": [INFO] ** Package Build Identifier:" ${packageBuildIdentifier} + fi + echo $PGM": [INFO] ** Generate SBOM:" ${generateSBOM} + if [ ! -z "${sbomAuthor}" ]; then + echo $PGM": [INFO] ** SBOM Author:" ${sbomAuthor} + fi + if [ ! -z "${externalDependenciesLogFile}" ]; then + echo $PGM": [INFO] ** External Dependencies log:" ${externalDependenciesLogFile} + fi + if [ ! -z "$baselineTarFile" ]; then + echo $PGM": [INFO] ** Baseline package:" ${baselineTarFile} fi - echo $PGM": [INFO] ** Publish to Artifact Repo:" ${publish} if [ "$publish" == "true" ]; then if [ ! -z "${artifactRepositoryPropertyFile}" ]; then - echo $PGM": [INFO] ** ArtifactRepo properties:" ${artifactRepositoryPropertyFile} + echo $PGM": [INFO] ** ArtifactRepo properties:" ${artifactRepositoryPropertyFile} fi if [ ! -z "${artifactRepositoryUrl}" ]; then - echo $PGM": [INFO] ** ArtifactRepo Url:" ${artifactRepositoryUrl} + echo $PGM": [INFO] ** ArtifactRepo Url:" ${artifactRepositoryUrl} fi if [ ! -z "${artifactRepositoryUser}" ]; then - echo $PGM": [INFO] ** ArtifactRepo User:" ${artifactRepositoryUser} + echo $PGM": [INFO] ** ArtifactRepo User:" ${artifactRepositoryUser} fi if [ ! -z "${artifactRepositoryPassword}" ]; then - echo $PGM": [INFO] ** ArtifactRepo Password: xxxxx" + echo $PGM": [INFO] ** ArtifactRepo Password: xxxxx" fi if [ ! -z "${artifactRepositoryName}" ]; then - echo $PGM": [INFO] ** ArtifactRepo Repo name:" ${artifactRepositoryName} + echo $PGM": [INFO] ** ArtifactRepo Repo name:" ${artifactRepositoryName} fi if [ ! -z "${artifactRepositoryDirectory}" ]; then - echo $PGM": [INFO] ** ArtifactRepo Repo Dir:" ${artifactRepositoryDirectory} + echo $PGM": [INFO] ** ArtifactRepo Repo Dir:" ${artifactRepositoryDirectory} fi fi - echo $PGM": [INFO] ** Generate SBOM:" ${generateSBOM} - if [ ! -z "${sbomAuthor}" ]; then - echo $PGM": [INFO] ** SBOM Author:" ${sbomAuthor} - fi - echo $PGM": [INFO] ** DBB_HOME:" ${DBB_HOME} + + echo $PGM": [INFO] ** DBB_HOME:" ${DBB_HOME} echo $PGM": [INFO] **************************************************************" echo "" fi @@ -491,12 +608,12 @@ fi # Invoke the Package Build Outputs script if [ $rc -eq 0 ]; then echo $PGM": [INFO] Invoking the Package Build Outputs script." - + if [ ! -z "${cycloneDXlibraries}" ]; then - cycloneDXlibraries="-cp ${cycloneDXlibraries}" + cycloneDXlibraries="-cp ${cycloneDXlibraries}" fi - CMD="$DBB_HOME/bin/groovyz ${log4j2} ${cycloneDXlibraries} ${PackagingScript} --workDir ${logDir}" + CMD="$DBB_HOME/bin/groovyz ${log4j2} ${cycloneDXlibraries} ${packagingScript} --workDir ${logDir}" # add tarfile name if [ ! -z "${tarFileName}" ]; then @@ -513,6 +630,11 @@ if [ $rc -eq 0 ]; then CMD="${CMD} --branch ${Branch}" fi + # application directory + if [ ! -z "${AppDir}" ]; then + CMD="${CMD} --applicationFolderPath $(getApplicationDir)" + fi + # packaging properties file if [ ! -z "${PkgPropFile}" ]; then CMD="${CMD} --packagingPropertiesFile ${PkgPropFile}" @@ -528,6 +650,21 @@ if [ $rc -eq 0 ]; then CMD="${CMD} --versionName ${artifactVersionName}" fi + # Wazi Deploy build identifier + if [ ! -z "${packageBuildIdentifier}" ]; then + CMD="${CMD} --buildIdentifier ${packageBuildIdentifier}" + fi + + # Pass information about externally fetched modules to packaging to document them + if [ ! -z "${externalDependenciesLogFile}" ]; then + CMD="${CMD} --externalDependenciesEvidences ${externalDependenciesLogFile}" + fi + + # Pass baseline package + if [ ! -z "$baselineTarFile" ]; then + CMD="${CMD} --baselinePackage ${baselineTarFile}" + fi + # publishing options if [ "$publish" == "true" ]; then CMD="${CMD} --publish" @@ -551,19 +688,19 @@ if [ $rc -eq 0 ]; then if [ ! -z "${artifactRepositoryDirectory}" ]; then CMD="${CMD} --artifactRepositoryDirectory ${artifactRepositoryDirectory}" fi + fi # SBOM options if [ "$generateSBOM" == "true" ]; then CMD="${CMD} --sbom" - if [ ! -z "${sbomAuthor}" ]; then - CMD="${CMD} --sbomAuthor \"${sbomAuthor}\"" - fi + if [ ! -z "${sbomAuthor}" ]; then + CMD="${CMD} --sbomAuthor \"${sbomAuthor}\"" + fi fi - echo $PGM": [INFO] ${CMD}" - /bin/env bash -c "${CMD}" + ${CMD} rc=$? if [ $rc -eq 0 ]; then diff --git a/Templates/Common-Backend-Scripts/pipelineBackend.config b/Templates/Common-Backend-Scripts/pipelineBackend.config index a2e8d526..76089d78 100644 --- a/Templates/Common-Backend-Scripts/pipelineBackend.config +++ b/Templates/Common-Backend-Scripts/pipelineBackend.config @@ -28,12 +28,13 @@ buildRootDir="$PIPELINE_WORKSPACE" # Relative directory name for outputs # default: logsDir="logs" -logsDir="logs" +logsDir="logs" ##################################################################################################### ## End of Shared Configuration parameters ######################################################## ##################################################################################################### + ##################################################################################################### ## DBB-BUILD.sh parameters ########################################################################## ##################################################################################################### @@ -107,17 +108,44 @@ featureBranchBuildBehaviour=merge-base ##################################################################################################### ##################################################################################################### -## packageBuildOutputs.sh parameters ############################################################# +## Fetch Build Dependencies based on Application Descriptor ######################################## ##################################################################################################### -# Path to PackageBuildOutputs.groovy script from the DBB community repository -# e.g. PackagingScript="/var/dbb/extensions/dbb20/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy" -PackagingScript="/var/dbb/extensions/dbb-MD/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy" +# Setting to enable fetching build dependencies configured within the Application Descriptor +# this leverages the configuration for the publishing script PackageBuildOutputs +# to retrieve build dependencies from the Artifact repo. +# default: false +fetchBuildDependencies=false + +# Settings for caching packages on build machine + +# Setting to control if packages should be stored on the build machine +# default: false +enablePackageCache=true + +# Location for cachine packages on the build machine +archiveCache=$HOME/cbs/.archiveCache + +# Internal file name documenting the fetched external dependencies, that can be tracked in the +# Wazi Deploy application manifest file +# default: externalDependenciesLogName=externalDependenciesLog.yml +externalDependenciesLogName=externalDependenciesLog.yml + +##################################################################################################### +## End of Fetch Build Dependencies based on Application Descriptor ################################# +##################################################################################################### ##################################################################################################### -#### PackageBuildOutputs.groovy configuration #################################################### +## packageBuildOutputs.sh parameters and configuration ############################################ ##################################################################################################### +# Suffix aligning to naming conventions for name of the Artifact repository within +# the Artifact repository server. Scripts will prefix it with the application name +# default: artifactRepositoryNameSuffix=repo-local +# leading to -repo-local +# ex. MortgageApplication-repo-local +artifactRepositoryNameSuffix=repo-local + # (Optional) Absolute path of a property file containing default application packaging details. # e.g. PkgPropFile="/var/dbb/extensions/dbb20/Pipeline/PackageBuildOutputs/packageBuildOutputs.properties" # PkgPropFile="" @@ -132,14 +160,13 @@ generateSBOM=false # (Optional) Name and email of the SBOM author enclosed with double quotes # Example: "Build Engineer " -sbomAuthor= +sbomAuthor="" # (Required if SBOM generation is performed) # Paths to the cycloneDX Java libraries (and their dependencies) to be added to the JAVA CLASSPATH # Example: /var/dbb/additionalLibraries/cyclonedx-core-java-8.0.3.jar:/var/dbb/additionalLibraries/jackson-annotations-2.16.1.jar:/var/dbb/additionalLibraries/jackson-core-2.16.1.jar:/var/dbb/additionalLibraries/jackson-databind-2.16.1.jar:/var/dbb/additionalLibraries/jackson-dataformat-xml-2.16.1.jar:/var/dbb/additionalLibraries/json-schema-validator-1.2.0.jar:/var/dbb/additionalLibraries/packageurl-java-1.5.0.jar cycloneDXlibraries=/var/dbb/additionalLibraries/cyclonedx-core-java-8.0.3.jar:/var/dbb/additionalLibraries/jackson-annotations-2.16.1.jar:/var/dbb/additionalLibraries/jackson-core-2.16.1.jar:/var/dbb/additionalLibraries/jackson-databind-2.16.1.jar:/var/dbb/additionalLibraries/jackson-dataformat-xml-2.16.1.jar:/var/dbb/additionalLibraries/json-schema-validator-1.2.0.jar:/var/dbb/additionalLibraries/packageurl-java-1.5.0.jar - ##################################################################################################### #### Artifact repository upload options ########################################################## ##################################################################################################### @@ -152,6 +179,7 @@ cycloneDXlibraries=/var/dbb/additionalLibraries/cyclonedx-core-java-8.0.3.jar:/v # Flag to indicate package upload to the provided Artifactory server. # Set to true to enable package upload # can be overridden by cli parm -p +# default: publish=false publish=false # URL to the Artifact repository server @@ -160,42 +188,18 @@ artifactRepositoryUrl="" # User to connect to the Artifact repository server # e.q.: artifactRepositoryUser=admin -artifactRepositoryUser="" +artifactRepositoryUser="" # Password to connect to the Artifact repository server # e.q.: artifactRepositoryPassword=xxxxx -artifactRepositoryPassword="" - -##################################################################################################### -#### Methods enforcing naming conventions ######################################################## -##################################################################################################### - -# Method to return the pattern for the artifact repository name to store the build -# e.q.: $App-repo-local -computeArtifactRepositoryName() { - echo "${App}-repo-local" -} - -# Method to return the directory path in the repository to store the build -# the default path in the repository is the branch name -# e.q.: $Branch -# If the type of pipeline is specified, we are appending it to the directory where -# the build is stored -computeArtifactRepositoryDirectory() { - if [ -z "${PipelineType}" ]; then - echo "${Branch}" - else - echo "${Branch}/${PipelineType}" - fi - -} +artifactRepositoryPassword="" # Path of a property file containing application specific artifact repository details. # application specific artifactRepositoryPropertyFile # e.q.: application-conf/packageBuildOutputs.properties # see template: # https://github.com/IBM/dbb/blob/main/Pipeline/PackageBuildOutputs/appArtifactRepository.properties -artifactRepositoryPropertyFile= +artifactRepositoryPropertyFile="" ##################################################################################################### ## End of packageBuildOutputs.sh parameters ###################################################### @@ -205,13 +209,9 @@ artifactRepositoryPropertyFile= ## ucdPackaging.sh parameters #################################################################### ##################################################################################################### -# Path to dbb-ucd-packaging.groovy script from the DBB community repository -# e.g. ucdPackagingScript="/var/dbb/extensions/dbb20/Pipeline/CreateUCDComponentVersion/dbb-ucd-packaging.groovy" -ucdPackagingScript="" - # Path to UCD buztool.sh script # e.g. BuzTool="/var/ucd-agent/bin/buztool.sh" -BuzTool="/var/ucd-agent/bin/buztool.sh" +BuzTool="" ##################################################################################################### ## End of ucdPackaging.sh parameters ############################################################# @@ -221,11 +221,6 @@ BuzTool="/var/ucd-agent/bin/buztool.sh" ## UCD-Deploy.sh parameters ###################################################################### ##################################################################################################### -# -# Location of UCD Deploy Script -# e.g. deployScriptLoc=/var/dbb/extensions/dbb20/Pipeline/DeployUCDComponentVersion -deployScriptLoc="" - # # UCD Server URL # e.g. ucdServerURL=https://ucd.server:8443/ @@ -242,23 +237,22 @@ ucdUserName="" # Protect the file by specifying the correct file permissions ucdPassword="" - ##################################################################################################### ## End of UCD-Deploy.sh parameters ############################################################### ##################################################################################################### + ##################################################################################################### ## wazideploy-generate.sh, wazideploy-deploy.sh and wazideploy-evidences.sh parameters ########### ##################################################################################################### # # Work directory for Wazi Deploy -# default: +# default: waziDeployDir wdWorkDir=waziDeployDir # # wazideploy deployment method file -# wdDeploymentMethod= wdDeploymentMethod="" # @@ -274,8 +268,8 @@ wdDeploymentPlanReportName="deploymentPlanReport.html" # to connect to Artifactory # # (Optional) Only necessary when wazideploy-generate is used to download the package -# wdDeployArtifactoryConfig= -wdDeployArtifactoryConfig= +# wdDeployArtifactoryConfig="" +wdDeployArtifactoryConfig="" # # Path to the environment configurations for Wazi Deploy @@ -291,22 +285,22 @@ wdEvidenceFileName="deploy/evidences/evidence.yaml" # # Default path of wazideploy evidence index folder -# wdIndexFolder= +# wdIndexFolder="" wdIndexFolder="index" # # wazideploy evidence search template -# wdSearchTemplate= -wdSearchTemplate="/var/WaziDeploy/wazi-deploy-samples-0.10.0/wazi-deploy-sample/plum-samples/evidences/templates/full-report.yml" +# wdSearchTemplate="" +wdSearchTemplate="" # # wazideploy evidence report renderer -# wdReportRenderer= -wdReportRenderer="/var/WaziDeploy/wazi-deploy-samples-0.10.0/wazi-deploy-sample/plum-samples/evidences/renderers/full-report.html" +# wdReportRenderer="" +wdReportRenderer="" # # Default path of wazideploy evidence output file -# wdOutputFile= +# wdOutputFile="" wdOutputFile="deploy/deployment-report.html" # diff --git a/Templates/Common-Backend-Scripts/ucdDeploy.sh b/Templates/Common-Backend-Scripts/ucdDeploy.sh index 1ffc0493..f1c56aa2 100755 --- a/Templates/Common-Backend-Scripts/ucdDeploy.sh +++ b/Templates/Common-Backend-Scripts/ucdDeploy.sh @@ -27,6 +27,7 @@ # Date Who Vers Description # ---------- --- ---- -------------------------------------------------------------- # 2023/07/18 RBS 1.00 Initial Release +# 2025/03/13 DB 1.10 Locate groovy scripts in file system #=================================================================================== Help() { echo $PGM" - Deploy UCD Component " @@ -76,6 +77,7 @@ Help() { # Either an absolute path or a relative path to the current working directory SCRIPT_HOME="`dirname "$0"`" pipelineConfiguration="${SCRIPT_HOME}/pipelineBackend.config" +deployScript="${SCRIPT_HOME}/../../../Pipeline/DeployUCDComponentVersion/ucd-deploy.groovy" # Customization - End # @@ -84,7 +86,7 @@ pipelineConfiguration="${SCRIPT_HOME}/pipelineBackend.config" #export BASH_XTRACEFD=1 # Write set -x trace to file descriptor PGM=$(basename "$0") -PGMVERS="1.00" +PGMVERS="1.10" USER=$(whoami) SYS=$(uname -Ia) @@ -269,6 +271,15 @@ if [ $rc -eq 0 ]; then fi fi +if [ $rc -eq 0 ]; then + # Validate Packaging script + if [ ! -f "${deployScript}" ]; then + rc=8 + ERRMSG=$PGM": [ERR] Unable to locate ${deployScript}. rc="$rc + echo $ERRMSG + fi +fi + # # Set up Environment if [ $rc -eq 0 ]; then @@ -280,13 +291,13 @@ fi if [ $rc -eq 0 ]; then echo $PGM": [INFO] **************************************************************" echo $PGM": [INFO] ** Start UCD Component Deploy on HOST/USER: ${SYS}/${USER}" - echo $PGM": [INFO] ** Location of ucd-deploy.groovy:" ${deployScriptLoc} - echo $PGM": [INFO] ** UCD Application Name:" ${Application} - echo $PGM": [INFO] ** UCD Environment Name:" ${Environment} - echo $PGM": [INFO] ** UCD User Name:" ${ucdUserName} - echo $PGM": [INFO] ** UCD Server URL:" ${ucdServerURL} - echo $PGM": [INFO] ** UCD Component Version:" ${Version} - echo $PGM": [INFO] ** UCD Application Process Name:" ${Process} + echo $PGM": [INFO] ** UCD deploy script :" ${deployScript} + echo $PGM": [INFO] ** UCD Application Name:" ${Application} + echo $PGM": [INFO] ** UCD Environment Name:" ${Environment} + echo $PGM": [INFO] ** UCD User Name:" ${ucdUserName} + echo $PGM": [INFO] ** UCD Server URL:" ${ucdServerURL} + echo $PGM": [INFO] ** UCD Component Version:" ${Version} + echo $PGM": [INFO] * UCD Application Process Name:" ${Process} if [ ! -z "${Timeout}" ]; then echo $PGM": [INFO] ** UCD Timeout (seconds):" ${Timeout} @@ -315,8 +326,8 @@ fi # # Set up to execute the UCD Deploy groovy script if [ $rc -eq 0 ]; then - echo $DBB_HOME/bin/groovyz ${deployScriptLoc}/ucd-deploy.groovy -a '"'${Application}'"' -e '"'${Environment}'"' -U ${ucdUserName} -P ${ucdPassword} -u ${ucdServerURL} -d '"'${Version}'"' -p ${Process} ${OptionalCommands} 2>&1 - $DBB_HOME/bin/groovyz ${deployScriptLoc}/ucd-deploy.groovy -a '"'${Application}'"' -e '"'${Environment}'"' -U ${ucdUserName} -P ${ucdPassword} -u '"'${ucdServerURL}'"' -d '"'${Version}'"' -p ${Process} ${OptionalCommands} 2>&1 + echo $DBB_HOME/bin/groovyz ${deployScript} -a '"'${Application}'"' -e '"'${Environment}'"' -U ${ucdUserName} -P ${ucdPassword} -u ${ucdServerURL} -d '"'${Version}'"' -p ${Process} ${OptionalCommands} 2>&1 + $DBB_HOME/bin/groovyz ${deployScript} -a '"'${Application}'"' -e '"'${Environment}'"' -U ${ucdUserName} -P ${ucdPassword} -u '"'${ucdServerURL}'"' -d '"'${Version}'"' -p ${Process} ${OptionalCommands} 2>&1 rc=$? if [ $rc -ne 0 ]; then diff --git a/Templates/Common-Backend-Scripts/ucdPackaging.sh b/Templates/Common-Backend-Scripts/ucdPackaging.sh index b32dd5f7..d421728d 100755 --- a/Templates/Common-Backend-Scripts/ucdPackaging.sh +++ b/Templates/Common-Backend-Scripts/ucdPackaging.sh @@ -26,9 +26,13 @@ # # 2. Review the Customization Section in the pipelineBackend.config file : # -# ucdPackagingScript - Location of the dbb-ucd-packaging.groovy # BuzTool - Location of the UCD buztool.sh # +# Maintenance Log +# Date Who Vers Description +# ---------- --- ---- -------------------------------------------------------------- +# 2023/07/18 RBS 1.00 Initial Release +# 2025/03/13 DB 1.10 Locate groovy scripts in file system #=================================================================================== Help() { echo $PGM" - Invoke DBB Build ("$PGMVERS") " @@ -96,6 +100,7 @@ Help() { # Either an absolute path or a relative path to the current working directory SCRIPT_HOME="`dirname "$0"`" pipelineConfiguration="${SCRIPT_HOME}/pipelineBackend.config" +ucdPackagingScript="${SCRIPT_HOME}/../../Pipeline/CreateUCDComponentVersion/dbb-ucd-packaging.groovy" # Customization - End # Path and File Name to the advanced debug options. @@ -107,7 +112,7 @@ pipelineConfiguration="${SCRIPT_HOME}/pipelineBackend.config" #export BASH_XTRACEFD=1 # Write set -x trace to file descriptor PGM=$(basename "$0") -PGMVERS="1.00" +PGMVERS="1.10" USER=$(whoami) SYS=$(uname -Ia) diff --git a/Templates/Common-Backend-Scripts/utilities/README.md b/Templates/Common-Backend-Scripts/utilities/README.md new file mode 100644 index 00000000..98e42a7d --- /dev/null +++ b/Templates/Common-Backend-Scripts/utilities/README.md @@ -0,0 +1,3 @@ +# Contents + +This folder contains utility scripts, that are sourced by the primary common backend scripts. \ No newline at end of file diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy new file mode 100644 index 00000000..16a41990 --- /dev/null +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependencies.groovy @@ -0,0 +1,461 @@ +@groovy.transform.BaseScript com.ibm.dbb.groovy.ScriptLoader baseScript +import com.ibm.dbb.metadata.* +import com.ibm.dbb.dependency.* +import com.ibm.dbb.build.* +import groovy.transform.* +import com.ibm.dbb.build.report.* +import com.ibm.dbb.build.report.records.* +import groovy.yaml.YamlBuilder +import groovy.lang.GroovyShell +import groovy.cli.commons.* +import groovy.util.* +import java.nio.file.* + + +// script properties +@Field Properties props = new Properties() +@Field def artifactRepositoryHelpers // Helpers to download +@Field def applicationDescriptorUtils // Helper to parse Application Descriptor +@Field def packageBuildOutputs // Helpers to download + +// Parse arguments from command-line +parseArgs(args) + +// Print parms +println("** Script configuration:") +props.sort().each { k,v-> + if ( k == "artifactRepository.password" ) + println " $k -> xxxxxx " + else + println " $k -> $v" +} + +// Load and verify helpers +def scriptDir = new File(getClass().protectionDomain.codeSource.location.path).parent + +// Artifact RepositoryHelpers are used to download the package +File artifactRepositoryHelpersScriptFile = new File("${scriptDir}/../../../Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy") +if (artifactRepositoryHelpersScriptFile.exists()) { + artifactRepositoryHelpers = loadScript(artifactRepositoryHelpersScriptFile) +} else { + println("*! [ERROR] The Artifact Repository Helper script '${props.artifactRepositoryHelpersScript}' doesn't exist. Exiting.") + System.exit(8) +} + +File applicationDescriptorUtilsFile = new File("${scriptDir}/../../../Pipeline/PackageBuildOutputs/utilities/applicationDescriptorUtils.groovy") +if (applicationDescriptorUtilsFile.exists()) { + applicationDescriptorUtils = loadScript(applicationDescriptorUtilsFile) +} else { + println("*! [ERROR] The Application Descriptor Helper script '$applicationDescriptorUtilsFile.getName()' was not found. Exiting.") + System.exit(8) +} + +File applicationDescriptorFile = new File(props.applicationDescriptor) +if (!applicationDescriptorFile.exists()) { + println("*! [ERROR] The Application Descriptor file '${applicationDescriptorFile.getName()}' doesn't exist. Exiting.") + System.exit(8) +} + +// setup import directory +File importFolder = new File("$props.workspace/imports") +File baselineFolder = new File("$props.workspace/baseline") +if (importFolder.exists()) importFolder.deleteDir() +importFolder.mkdirs() + +// setup package cache +tmpPackageDir = (props.enablePackageCache && props.enablePackageCache.toBoolean() && props.archiveCache) ? new File(props.archiveCache) : new File("$props.workspace/imports_download/") +if (!tmpPackageDir.exists()) tmpPackageDir.mkdirs() + +// Parse the application descriptor and application configurations based on the defined schema +applicationDescriptor = applicationDescriptorUtils.readApplicationDescriptor(applicationDescriptorFile) + + +ArrayList externalDependencies = new ArrayList<>() + +if (applicationDescriptor.dependencies) { + + println("*** Fetching dependent application archives") + + // Loop through all dependencies found in AD + applicationDescriptor.dependencies.each { dependency -> + + def rc=0 + + // validate dependency record + if (!dependency.name) { + rc=1 + println("*! [ERROR] Dependency record in Application Descriptor missing the 'name' attribute. rc=$rc") + } + if (!dependency.type) { + rc=1 + println("*! [ERROR] Dependency record in Application Descriptor missing the 'type' attribute. rc=$rc") + } + if (!dependency.reference) { + rc=1 + println("*! [ERROR] Dependency record in Application Descriptor missing the dependency 'reference' attribute. rc=$rc") + } + if (!dependency.buildid) { + rc=1 + println("*! [ERROR] Dependency record in Application Descriptor missing the 'buildid' attribute. rc=$rc") + } + + if (rc != 0) { + println("*! Fetching external dependencies exits after failing the validation of the dependency record.") + System.exit(1) + } + + + // compute tar file name based on build type + if (dependency.type.equalsIgnoreCase("release")) { + props.put("tarFileName","${dependency.name}-${dependency.reference}-${dependency.buildid}.tar") + } else { + props.put("tarFileName","${dependency.name}-${dependency.buildid}.tar") + } + + // Construct the path within the Artifact repo + repositoryName="${dependency.name}-${props.artifactRepositoryNameSuffix}" + + // Prepare for url computation using method in ArtifactRepoHelpers + props.put("versionName","${dependency.reference}") // compute the version name being part of the path + props.put("artifactRepository.directory", "${dependency.type}") // compute the main directory to classify builds + props.put("artifactRepository.repo", "${repositoryName}") // Artifact repository name + + // The absolute url the package in artifact repo + artifactUrl = artifactRepositoryHelpers.computeArchiveUrl(props) + + // retrieve path without artifact url + artifactRelPath = artifactUrl.replaceAll(props.get("artifactRepository.url"),"") + + // File in cache / workspace + tarFile="${tmpPackageDir}/${artifactRelPath}" + tarFileDir=tarFile.replaceAll(props.tarFileName, "") + + println("** Fetching archive for application '${dependency.name}' from '${artifactUrl}'") + + // Generating information for documentation in yaml file of retrieved dependencies + if (dependency.name != applicationDescriptor.application) { + + ExternalDependency externalDependency = new ExternalDependency() + // Map information to External Dependency Record + externalDependency.name = dependency.name // dependency name + externalDependency.properties = new HashSet() + + // Add url + Property p_uri = new Property() + p_uri.key = "uri" + p_uri.value = artifactUrl + externalDependency.properties.add(p_uri) + + // type - either build or release + Property p_type = new Property() + p_type.key = "type" + p_type.value = dependency.type + externalDependency.properties.add(p_type) + + // reference - the release name or the branch + Property p_reference = new Property() + p_reference.key = "reference" + p_reference.value = dependency.reference + externalDependency.properties.add(p_reference) + + // buildid + Property p_buildid = new Property() + p_buildid.key = "buildid" + p_buildid.value = dependency.buildid + externalDependency.properties.add(p_buildid) + + // Store external dependency information + externalDependencies.add(externalDependency) + } + + // download from artifact repo + + // foldername in workspace directory + String importSubFolder= "${importFolder}/${dependency.name}" + + if (new File(tarFile).exists()) { + println("** Archive was already found in archive cache at '${tarFile}'") + } else { + String user = props.artifactRepositoryUser + String password = props.artifactRepositoryPassword + + if (!(new File("${tarFileDir}").exists())) (new File("${tarFileDir}")).mkdirs() + + println("** Downloading archive '$artifactUrl' from artifact repository into '${tarFileDir}'.") + rc = artifactRepositoryHelpers.download(artifactUrl, tarFile, user, password, true) + + if (rc != 0) { + println("*! [ERROR] Download of archive '$artifactUrl' failed. Exiting with return code:${rc}.") + System.exit(rc) + } + } + + + File importSubFolderFile = new File(importSubFolder) + if (!importSubFolderFile.exists()) { + importSubFolderFile.mkdirs() + } + + + println("** Expanding tar file '${tarFile}' to '$importSubFolder' ") + + def processCmd = [ + "/bin/sh", + "-c", + "tar -C $importSubFolder -xvf ${tarFile}" + ] + + rc = runProcess(processCmd) + if (rc != 0) { + println("** [ERROR] Failed to untar '$tarFile' to '$importSubFolder' with rc=$rc") + System.exit(1) + } + + // Delete temporary download location if cache is not used + if (!(props.enablePackageCache && props.enablePackageCache.toBoolean())) { + tmpPackageDir.deleteDir() + } + } +} + + + +baselineRecord = applicationDescriptor.baselines.find() { baseline -> + baseline.branch.equals(props.branch) +} + +if (baselineRecord){ + println("*** Fetching baseline archive") + + def rc=0 + + // validate baseline record + if (!applicationDescriptor.application) { + rc=1 + println("*! [ERROR] Application Descriptor missing the 'application' name attribute. rc=$rc") + } + if (!baselineRecord.type) { + rc=1 + println("*! [ERROR] Baseline record in Application Descriptor missing the 'type' attribute. rc=$rc") + } + if (!baselineRecord.reference) { + rc=1 + println("*! [ERROR] Baseline record in Application Descriptor missing the dependency 'reference' attribute. rc=$rc") + } + if (!baselineRecord.buildid) { + rc=1 + println("*! [ERROR] Baseline record in Application Descriptor missing the 'buildid' attribute. rc=$rc") + } + if (rc != 0) { + // exit after validation + println("*! Fetching external dependencies exits after failing the validation of the baseline record.") + System.exit(1) + } + + def applicationName = applicationDescriptor.application + + // compute tar file name based on build type + if (baselineRecord.type.equalsIgnoreCase("release")) { + props.put("tarFileName","${applicationName}-${baselineRecord.reference}-${baselineRecord.buildid}.tar") + } else { + props.put("tarFileName","${applicationName}-${baselineRecord.buildid}.tar") + } + + // Construct the path within the Artifact repo + repositoryName="${applicationName}-${props.artifactRepositoryNameSuffix}" + + // Prepare for url computation using method in ArtifactRepoHelpers + props.put("versionName","${baselineRecord.reference}") // compute the version name being part of the path + props.put("artifactRepository.directory", "${baselineRecord.type}") // compute the main directory to classify builds + props.put("artifactRepository.repo", "${repositoryName}") // Artifact repository name + + // The absolute url the package in artifact repo + artifactUrl = artifactRepositoryHelpers.computeArchiveUrl(props) + + // retrieve path without artifact url + artifactRelPath = artifactUrl.replaceAll(props.get("artifactRepository.url"),"") + + // File in cache / workspace + tarFile="${tmpPackageDir}/${artifactRelPath}" + tarFileDir=tarFile.replaceAll(props.tarFileName, "") + if (!tmpPackageDir.exists()) tmpPackageDir.mkdirs() // create tmpDownload + + println("** Fetching baseline archive for '${applicationName}' from '${artifactUrl}' ") + + if (new File(tarFile).exists()) { + println("** Archive was already found in archive cache at '${tarFile}'") + } else { + String user = props.artifactRepositoryUser + String password = props.artifactRepositoryPassword + + if (!(new File("${tarFileDir}").exists())) (new File("${tarFileDir}")).mkdirs() + + println("** Downloading archive with '$artifactUrl' from Artifact Repository into ${tarFileDir}.") + rc = artifactRepositoryHelpers.download(artifactUrl, tarFile, user, password, true) + + if (rc != 0) { + println("** Download of archive '$artifactUrl' failed. Process exists. Return code:$rc") + System.exit(rc) + } + } + + // setup baseline directory + if (baselineFolder.exists()) baselineFolder.deleteDir() + baselineFolder.mkdirs() + + println("** Saving tar file '${tarFile}' into '${baselineFolder}'") + + def processCmd = [ + "/bin/sh", + "-c", + "cp ${tarFile} ${baselineFolder}/" + ] + + rc = runProcess(processCmd) + if (rc != 0) { + println("** [ERROR] Failed to copy '$tarFile' to '$baselineFolder' with rc=$rc") + System.exit(1) + } + + // setup import folder + String importSubFolder = "${importFolder}/${applicationName}" + File importSubFolderFile = new File(importSubFolder) + if (!importSubFolderFile.exists()) { + importSubFolderFile.mkdirs() + } + + println("** Expanding tar file '${tarFile}' to '$importSubFolder' ") + + processCmd = [ + "/bin/sh", + "-c", + "tar -C $importSubFolder -xvf ${tarFile}" + ] + + rc = runProcess(processCmd) + if (rc != 0) { + println("** [ERROR] Failed to untar '$tarFile' to '$importSubFolder' with rc=$rc") + System.exit(1) + } + + // Delete temporary download location if cache is not used + if (!(props.enablePackageCache && props.enablePackageCache.toBoolean())) { + tmpPackageDir.deleteDir() + } +} else { + println("** [INFO] No baseline record found for application '${applicationDescriptor.application}'.") +} + +def yamlBuilder = new YamlBuilder() +// write file +if (props.externalDependenciesFilePath) { + + println("** Write ExternalDependency Record to ${props.externalDependenciesFilePath}") + yamlBuilder externalDependencies + + externalDependencyFile = new File(props.externalDependenciesFilePath) + externalDependencyFile.withWriter("UTF-8") { extDepWriter -> + extDepWriter.write(yamlBuilder.toString()) + } +} + + +/** + * Parse CLI config + */ +def parseArgs(String[] args) { + + String usage = 'generateFetchStatements.groovy [options]' + + def cli = new CliBuilder(usage:usage) + // required sandbox options + cli.a(longOpt:'applicationDescriptor', args:1, 'Absolute path to the Application Descriptor file') + cli.w(longOpt:'workspace', args:1, 'Absolute path to the build workspace') + cli.d(longOpt:'externalDependenciesFilePath', args:1, 'Absolute path to the external dependencies file') + cli.p(longOpt:'pipelineBackendConfigFilePath', args:1, 'Absolute path to the pipelineBackend.config file') + cli.b(longOpt:'branch', args:1, 'Current branch of the application') + cli.c(longOpt:'archiveCache', args:1, 'Location of the Package cache') + + + def opts = cli.parse(args) + if (!opts) { + System.exit(1) + } + + if (opts.a) { + props.applicationDescriptor = opts.a + } else { + println("*! [ERROR] Missing path to the Application Descriptor file. Exiting.") + System.exit(1) + } + + if (opts.w) { + props.workspace = opts.w + } else { + println("*! [ERROR] Missing path to the Workspace directory. Exiting.") + System.exit(1) + } + + if (opts.d) { + props.externalDependenciesFilePath = opts.d + } else { + println("*! [ERROR] Missing path to the External Dependencies file. Exiting.") + System.exit(1) + } + + if (opts.b) { + props.branch = opts.b + } else { + println("*! [ERROR] Missing current Branch name. Exiting.") + System.exit(1) + } + + if(opts.c){ + props.archiveCache = opts.c + } + + if (opts.p) { + def pipelineBackendConfigFile = new File(opts.p) + if (pipelineBackendConfigFile.exists()) { + props.pipelineBackendConfigFile = opts.p + Properties temporaryProperties = new Properties() + pipelineBackendConfigFile.withInputStream { temporaryProperties.load(it) } + // artifact repo configuration properties / Map CBS pipelineBackend.config to script properties + if(temporaryProperties.get("artifactRepositoryUrl")) props.put("artifactRepository.url", temporaryProperties.get("artifactRepositoryUrl")) + if(temporaryProperties.get("artifactRepositoryUser")) props.put("artifactRepository.user", temporaryProperties.get("artifactRepositoryUser")) + if(temporaryProperties.get("artifactRepositoryPassword")) props.put("artifactRepository.password", temporaryProperties.get("artifactRepositoryPassword")) + if(temporaryProperties.get("artifactRepositoryNameSuffix")) props.put("artifactRepositoryNameSuffix", temporaryProperties.get("artifactRepositoryNameSuffix")) + if(temporaryProperties.get("enablePackageCache")) props.put("enablePackageCache", temporaryProperties.get("enablePackageCache")) + } else { + println("*! [ERROR] Configuration file ${opts.p} not found. Exiting.") + System.exit(1) + } + } else { + println("*! [ERROR] Missing path to the pipelineBackend.config file. Exiting.") + System.exit(1) + } +} + +def runProcess(ArrayList cmd){ + StringBuffer response = new StringBuffer() + StringBuffer error = new StringBuffer() + + def p = cmd.execute() + p.waitForProcessOutput(response, error) + + def rc = p.exitValue(); + if (rc != 0) { + println("*! [ERROR] Execution of command '$cmd' failed with rc=${rc} and message '${error.toString()}'") + } + return rc +} + +class ExternalDependency { + String name + HashSet properties = new HashSet<>() +} + +class Property { + String key + String value +} \ No newline at end of file diff --git a/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh new file mode 100644 index 00000000..b52351f5 --- /dev/null +++ b/Templates/Common-Backend-Scripts/utilities/fetchBuildDependenciesUtils.sh @@ -0,0 +1,104 @@ +#!/bin/env bash +#=================================================================================== +# NAME: fetchDependencies.sh +# +# DESCRIPTION: The purpose of this script is to fetch all build dependencies +# +# SYNTAX: See Help() Section Below +# +# OPTIONS: See Help() Section Below +# +# RETURNS: +# +# rc - Return Code +# +# RETURN CODES: +# +# 0 - Successful +# 4 - Warning message(s) issued. See Console messages. +# 8 - Error encountered. See Console messages. +# +# NOTE(S): +# +# None +# +# Maintenance Log +# +# Date Who Vers Description +# ---------- --- ----- -------------------------------------------------------------- +# 2025/01/30 DB 1.0.0 Initial Release +#=================================================================================== + +# Internal script to fetch +runFetchLogic() { + + # Read to go to fetch build dependencies configured in application descriptor + if [ $rc -eq 0 ]; then + echo $PGM": [INFO] **************************************************************" + echo $PGM": [INFO] ** Start Fetch Build Dependencies on HOST/USER: ${SYS}/${USER}" + echo $PGM": [INFO] ** WorkDir:" $(getWorkDirectory) + echo $PGM": [INFO] ** Application:" ${App} + echo $PGM": [INFO] ** Branch:" ${Branch} + echo $PGM": [INFO] ** Application Descriptor :" ${applicationDescriptor} + echo $PGM": [INFO] ** Use Package Cache :" ${enablePackageCache} + echo $PGM": [INFO] ** Archive Cache Location :" ${archiveCache} + echo $PGM": [INFO] ** External Dependency Log :" ${externalDependenciesLogFile} + echo $PGM": [INFO] **************************************************************" + echo "" + fi + # + + # Create import dir + if [ $rc -eq 0 ]; then + if [ ! -d "$(getWorkDirectory)/imports" ]; then + mkdir -p $(getWorkDirectory)/imports + fi + fi + + # Fetch Application Dependencies + if [ $rc -eq 0 ]; then + echo $PGM": [INFO] ** Fetch Application Dependencies from Artifact Repository" + cmd="groovyz ${PIPELINE_SCRIPTS}/utilities/fetchBuildDependencies.groovy -w $(getWorkDirectory) -a ${applicationDescriptor} -p ${pipelineConfiguration} -b ${Branch}" + # + if [ ! -z "${externalDependenciesLogFile}" ]; then + cmd="${cmd} -d ${externalDependenciesLogFile}" + fi + + if [ ! -z "${archiveCache}" ]; then + cmd="${cmd} -c ${archiveCache}" + fi + echo $PGM": [INFO] ** CMD : ${cmd}" + ${cmd} + rc=$? + fi + + if [ $rc -eq 0 ]; then + ERRMSG=$PGM": [INFO] Fetch Build Dependencies Completed. rc="$rc + echo $ERRMSG + echo "" + fi + +} + +# +# Script Logic +# + +# extracting external dependencies is based on the application descriptor +applicationDescriptor="$(getApplicationDir)/applicationDescriptor.yml" + +# this log file documents the "fetched" dependencies and their version, that is then stored in the package itself (WD application manifest) +if [ ! -z "${externalDependenciesLogName}" ]; then + externalDependenciesLogFile="$(getLogDir)/${externalDependenciesLogName}" +fi + +# create the log dir +mkdir -p "$(getLogDir)" + +# Set up to perform the clone of the Repo +if [ ! -f "${applicationDescriptor}" ]; then + ERRMSG=$PGM": [INFO] Application Descriptor file (${applicationDescriptor}) was not found. rc=" + echo $ERRMSG +else + runFetchLogic +fi diff --git a/Templates/Common-Backend-Scripts/utilities/packagingUtilities.sh b/Templates/Common-Backend-Scripts/utilities/packagingUtilities.sh new file mode 100755 index 00000000..e982a860 --- /dev/null +++ b/Templates/Common-Backend-Scripts/utilities/packagingUtilities.sh @@ -0,0 +1,152 @@ +# internal variables +mainBranchSegment="" +secondBranchSegment="" +rc=0 + +# Method implementing the conventions in the CBS for packaging using the PackageBuildOutputs.groovy script +computeArchiveInformation() { + ############################################# + # output environment variables + ############################################# + artifactRepositoryName="" # identifier of the artifact repo + artifactRepositoryDirectory="" # root directory folder in repo + artifactVersionName="" # subfolder in repo path identifying version / origin branch + tarFileName="" # computed tarFileName how it is stored in the artifact repository + archiveIdentifier="" # Identifier for Wazi Deploy Application Manifest file + ############################################# + + # configuration variable defining the Artifactory repository name pattern + artifactRepositoryRepoPattern="${App}-${artifactRepositoryNameSuffix}" + + branchConvention=(${Branch//// }) + + if [ $rc -eq 0 ]; then + if [ ${#branchConvention[@]} -gt 3 ]; then + rc=8 + ERRMSG=$PGM": [ERROR] Script is only managing branch name with up to 3 segments (${Branch}) . See recommended naming conventions. rc="$rc + echo $ERRMSG + fi + fi + + if [ $rc -eq 0 ]; then + + # split the segments + mainBranchSegment=$(echo ${Branch} | awk -F "/" ' { print $1 }') + secondBranchSegment=$(echo ${Branch} | awk -F "/" ' { print $2 }') + thirdBranchSegment=$(echo ${Branch} | awk -F "/" ' { print $3 }') + + # remove chars (. -) from the name + mainBranchSegmentTrimmed=$(echo ${mainBranchSegment} | tr -d '.-' | tr '[:lower:]' '[:upper:]') + + artifactRepositoryName=$(echo "${artifactRepositoryRepoPattern}") + + # evaluate main segment + case $mainBranchSegmentTrimmed in + "MASTER" | "MAIN" | REL*) + if [ "${PipelineType}" == "release" ]; then + ############################################# + # Conventions for release builds: + # ///--.tar + # MortgageApplication-repo-local/release/1.2.3/MortgageApplication-1.2.3-1234567890.tar + ############################################# + + # Release builds are captured in the release directory of the artifact repo + artifactRepositoryDirectory="release" + + # artifactVersionName is second identifier in the folder structure and represents the + artifactVersionName=${releaseIdentifier} + + # building up the tarFileName + tarFileName="${App}-${releaseIdentifier}-${buildIdentifier}.tar" + archiveIdentifier="${releaseIdentifier}-${buildIdentifier}" + else + ############################################# + # Conventions for snapshot builds: + # ///-.tar + # Mortgage-repo-local/build/feature/123-enhance-something/Mortgage-123456.tar + ############################################# + + artifactRepositoryDirectory="build" + artifactVersionName=${Branch} + tarFileName="${App}-${buildIdentifier}.tar" + archiveIdentifier="${buildIdentifier}" + fi + ;; + *) + ############################################# + ### similar to snapshot builds + ############################################# + artifactRepositoryDirectory="build" + artifactVersionName=${Branch} + tarFileName="${App}-${buildIdentifier}.tar" + archiveIdentifier="${buildIdentifier}" + ;; + esac + + ############################################# + ### Construct the absolute repository URL (required when downloading the package) + ############################################# + + if [ "${computeArchiveUrl}" == "true" ]; then + + artifactRepositoryHelpersScript="${SCRIPT_HOME}/../../Pipeline/PackageBuildOutputs/ArtifactRepositoryHelpers.groovy" + + # validate options + if [ ! -f "${artifactRepositoryHelpersScript}" ]; then + rc=8 + ERRMSG=$PGM": [ERROR] Unable to locate ${artifactRepositoryHelpersScript}. rc="$rc + echo $ERRMSG + fi + + # + # Invoke the Package Build Outputs script + if [ $rc -eq 0 ]; then + echo $PGM": [INFO] Invoking the ArtifactRepositoryHelper groovy script to compute Package Url." + + CMD="$DBB_HOME/bin/groovyz ${log4j2} ${artifactRepositoryHelpersScript} --computeArchiveUrl" + + # add tarfile name + if [ ! -z "${tarFileName}" ]; then + CMD="${CMD} --tarFileName ${tarFileName}" + fi + + # artifactVersionName + if [ ! -z "${artifactVersionName}" ]; then + CMD="${CMD} --versionName ${artifactVersionName}" + fi + + # Artifact repo options + if [ ! -z "${artifactRepositoryUrl}" ]; then + CMD="${CMD} --artifactRepositoryUrl \"${artifactRepositoryUrl}\"" + fi + + if [ ! -z "${artifactRepositoryName}" ]; then + CMD="${CMD} --artifactRepositoryName ${artifactRepositoryName}" + fi + if [ ! -z "${artifactRepositoryDirectory}" ]; then + CMD="${CMD} --artifactRepositoryDirectory ${artifactRepositoryDirectory}" + fi + + echo $PGM": [INFO] ${CMD}" + artifactRepositoryAbsoluteUrl=$(${CMD} | grep "url=" | awk -F "=" ' { print $2 }') + + if [ ! -z "${artifactRepositoryAbsoluteUrl}" ]; then + ERRMSG=$PGM": [INFO] Computation of Archive Url completed. rc="$rc + echo $ERRMSG + else + rc=12 + ERRMSG=$PGM": [ERR] Computation of Archive Url failed. Check Console for details. rc="$rc + echo $ERRMSG + + fi + fi + + fi + + # unset internal variables + mainBranchSegment="" + secondBranchSegment="" + thirdBranchSegment="" + + fi +} diff --git a/Templates/Common-Backend-Scripts/wazideploy-deploy.sh b/Templates/Common-Backend-Scripts/wazideploy-deploy.sh index ba18bcd2..9c91d14b 100755 --- a/Templates/Common-Backend-Scripts/wazideploy-deploy.sh +++ b/Templates/Common-Backend-Scripts/wazideploy-deploy.sh @@ -279,7 +279,7 @@ validateOptions() { echo $ERRMSG fi fi - + # validate that deployment plan exists if [ ! -f "${DeploymentPlan}" ]; then rc=8 @@ -298,7 +298,7 @@ validateOptions() { EnvironmentFile="${wdEnvironmentConfigurations}/${EnvironmentFile}" fi fi - + # validate that environment file exists if [ ! -f "${EnvironmentFile}" ]; then rc=8 @@ -337,6 +337,19 @@ validateOptions() { fi } +# When publishing is enabled, check if the tarfile exists in the expected location +if [ $rc -eq 0 ] && [ "$publish" == "true" ]; then + if [ -f "$(wdDeployPackageDir)/applicationArchive.tar" ]; then # shared convention with wazideploy-generate.sh + echo $PGM": [INFO] ** Archive was found at location '${wdDeployPackageDir}/applicationArchive.tar'." + if [ ! -z "${PackageInputFile}" ]; then + echo $PGM": [INFO] ** Package Input File was passed in as ${PackageInputFile}. It will be replaced with $(wdDeployPackageDir)/applicationArchive.tar ." + fi + PackageInputFile="$(wdDeployPackageDir)/applicationArchive.tar" + else + echo $PGM": [INFO] ** The CBS can automatically compute the Url of the archive. Wazi Deploy will then download it. Read more about the capabilities in the CBS readme." + fi +fi + # Call validate Options if [ $rc -eq 0 ]; then validateOptions diff --git a/Templates/Common-Backend-Scripts/wazideploy-generate.sh b/Templates/Common-Backend-Scripts/wazideploy-generate.sh index a84824ee..02a78f11 100755 --- a/Templates/Common-Backend-Scripts/wazideploy-generate.sh +++ b/Templates/Common-Backend-Scripts/wazideploy-generate.sh @@ -27,6 +27,8 @@ # Date Who Vers Description # ---------- ---- ---- -------------------------------------------------------------- # 2023/10/19 MDLB 1.00 Initial Release +# 2025/03/13 DB 1.10 Allow pipelines to compute the artifact location to download +# packages via wazideploy-generate #=================================================================================== Help() { echo $PGM" - Generate Wazi Deploy Deployment Plan " @@ -60,9 +62,7 @@ Help() { echo " used as input " echo " If a relative path is provided," echo " the log directory is suffixed " - echo " where PackageBuildOutputs " - echo " stores outputs " - echo " Default=None, Required. " + echo " Default=None, Required. " echo " " echo " Ex: MortgageApplication.tar " echo " " @@ -97,6 +97,33 @@ Help() { echo " See wdDeployArtifactoryConfig " echo " in pipelineBackend.config " echo " " + echo " -P - Type of the pipeline to " + echo " control in which directory builds " + echo " are stored in the artifact repo " + echo " Accepted values: " + echo " build - " + echo " development builds " + echo " release - " + echo " builds with options for " + echo " performance optimized " + echo " executables for production env " + echo " (optional) " + echo " " + echo " -b - Name of the git branch. " + echo " (optional) " + echo " " + echo " Ex: main " + echo " " + echo " " + echo " -I - A unique build identifier " + echo " typically the buildID of the " + echo " pipeline " + echo " (optional) " + echo " " + echo " -R - The release identifier for " + echo " release pipeline builds " + echo " (optional) " + echo " " echo " -d - Debug tracing flag " echo " " exit 0 @@ -108,6 +135,7 @@ Help() { # Either an absolute path or a relative path to the current working directory SCRIPT_HOME="$(dirname "$0")" pipelineConfiguration="${SCRIPT_HOME}/pipelineBackend.config" +packagingUtilities="${SCRIPT_HOME}/utilities/packagingUtilities.sh" # Customization - End # @@ -116,22 +144,40 @@ pipelineConfiguration="${SCRIPT_HOME}/pipelineBackend.config" #export BASH_XTRACEFD=1 # Write set -x trace to file descriptor PGM=$(basename "$0") -PGMVERS="1.00" +PGMVERS="1.10" USER=$(whoami) SYS=$(uname -Ia) rc=0 ERRMSG="" +# Wazi Deploy configuration variables DeploymentMethod="" DeploymentPlan="" DeploymentPlanReport="" PackageInputFile="" PackageOutputFile="" ConfigFile="" + +# CBS configuration variables Workspace="" +App="" # Application name - takes cli option a +PipelineType="" # takes cli option P +Branch="" # takes cli option b + +# Package identifier variables +buildIdentifier="" # takes cli option I +releaseIdentifier="" # takes cli option R + +# +computeArchiveUrl="true" # enables the computation of the url +artifactRepositoryAbsoluteUrl="" # Used to store the computed Url +usePackageUrl="" # Internal flag indicating if the package url was computed + Debug="" HELP=$1 +tarFileName="" # variable to store the package tar + if [ "$HELP" = "?" ]; then Help fi @@ -160,10 +206,21 @@ if [ $rc -eq 0 ]; then fi fi +# Source packaging helper +if [ $rc -eq 0 ]; then + if [ ! -f "${packagingUtilities}" ]; then + rc=8 + ERRMSG=$PGM": [ERROR] Packaging Utilities file (${packagingUtilities}) was not found. rc="$rc + echo $ERRMSG + else + source $packagingUtilities + fi +fi + # # Get Options if [ $rc -eq 0 ]; then - while getopts "hdw:m:p:r:i:o:c:" opt; do + while getopts "hdw:m:p:r:i:o:c:a:I:R:P:b:" opt; do case $opt in h) Help @@ -256,6 +313,62 @@ if [ $rc -eq 0 ]; then Debug=" -d" ;; + a) + # Application argument + argument="$OPTARG" + nextchar="$(expr substr $argument 1 1)" + if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then + rc=4 + ERRMSG=$PGM": [WARNING] Application Folder Name is required. rc="$rc + echo $ERRMSG + break + fi + App="$argument" + ;; + b) + argument="$OPTARG" + nextchar="$(expr substr $argument 1 1)" + if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then + rc=4 + ERRMSG=$PGM": [WARNING] Name of the git branch is required. rc="$rc + echo $ERRMSG + break + fi + Branch="$argument" + ;; + P) + argument="$OPTARG" + nextchar="$(expr substr $argument 1 1)" + if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then + rc=4 + INFO=$PGM": [INFO] No Pipeline type specified. rc="$rc + echo $INFO + break + fi + PipelineType="$argument" + ;; + I) + argument="$OPTARG" + nextchar="$(expr substr $argument 1 1)" + if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then + rc=4 + ERRMSG=$PGM": [WARNING] The name of the version to create is required. rc="$rc + echo $ERRMSG + break + fi + buildIdentifier="$argument" + ;; + R) + argument="$OPTARG" + nextchar="$(expr substr $argument 1 1)" + if [ -z "$argument" ] || [ "$nextchar" = "-" ]; then + rc=4 + ERRMSG=$PGM": [WARNING] The name of the release identifier is required. rc="$rc + echo $ERRMSG + break + fi + releaseIdentifier="$argument" + ;; \?) Help rc=1 @@ -316,11 +429,10 @@ validateOptions() { fi fi # if relative path - if [[ ! ${DeploymentPlan:0:1} == "/" ]] ; then + if [[ ! ${DeploymentPlan:0:1} == "/" ]]; then DeploymentPlan="$(wdDeployPackageDir)/${DeploymentPlan}" fi - # compute deployment plan report if not specified if [ -z "${DeploymentPlanReport}" ]; then # compute default based on configuration @@ -328,7 +440,7 @@ validateOptions() { DeploymentPlanReport="$(wdDeployPackageDir)/${wdDeploymentPlanReportName}" fi # if relative path - if [[ ! ${DeploymentPlanReport:0:1} == "/" ]] ; then + if [[ ! ${DeploymentPlanReport:0:1} == "/" ]]; then DeploymentPlanReport="$(wdDeployPackageDir)/${DeploymentPlanReport}" fi @@ -339,16 +451,11 @@ validateOptions() { echo $ERRMSG else # check for relative path - if [[ ! ${PackageInputFile:0:1} == "/" ]] ; then - checkWorkspace - PackageInputFile="$(getLogDir)/${PackageInputFile}" + if [ ! ${PackageInputFile:0:1} == "/" ] && [ -z "${usePackageUrl}" ]; then + checkWorkspace + PackageInputFile="$(getLogDir)/${PackageInputFile}" fi fi - if [ ! -f "${PackageInputFile}" ]; then - rc=8 - ERRMSG=$PGM": [ERROR] Package Input File (${PackageInputFile}) was not found. rc="$rc - echo $ERRMSG - fi # validate config file if [ -z "${ConfigFile}" ]; then @@ -358,23 +465,46 @@ validateOptions() { if [ ! -z "${ConfigFile}" ]; then if [ ! -f "${ConfigFile}" ]; then rc=8 - ERRMSG=$PGM": [ERROR] Specified Wazi Deploy Artifactory Configuration file (${ConfigFile}) was not found. rc="$rc + ERRMSG=$PGM": [ERROR] Specified Wazi Deploy Artifact repository configuration file (${ConfigFile}) was not found. rc="$rc echo $ERRMSG fi fi # compute the output file if [ -z "${PackageOutputFile}" ] && [ ! -z "${ConfigFile}" ]; then - # c checkWorkspace PackageOutputFile="$(wdDeployPackageDir)" fi # if relative path - if [[ ! ${PackageOutputFile:0:1} == "/" ]] && [[ ! -z "${PackageOutputFile}" ]] ; then + if [[ ! ${PackageOutputFile:0:1} == "/" ]] && [[ ! -z "${PackageOutputFile}" ]]; then PackageOutputFile="$(wdDeployPackageDir)/${PackageOutputFile}" fi } +# When publishing is enabled, try reading the wdPackageVersionFile +# that needs to be computed before this step. +if [ $rc -eq 0 ] && [ "$publish" == "true" ] && [ ! -z "${buildIdentifier}" ]; then + checkWorkspace + + # validate options + if [ -z "${PipelineType}" ]; then + rc=8 + ERRMSG=$PGM": [ERROR] To compute the Url of the stored package to enable the download of the archive file via Wazi Deploy generate, you need to provide the pipelineType. rc="$rc + echo $ERRMSG + fi + + if [ $rc -eq 0 ]; then + + # Call utilities method + computeArchiveInformation + + # Set Input and output files for Wazi Deploy + PackageInputFile="${artifactRepositoryAbsoluteUrl}" + PackageOutputFile="$(wdDeployPackageDir)/applicationArchive.tar" # shared convention with wazideploy-deploy.sh + usePackageUrl="true" + fi +fi + # Call validate Options if [ $rc -eq 0 ]; then validateOptions diff --git a/Templates/Common-Backend-Scripts/zBuilder.sh b/Templates/Common-Backend-Scripts/zBuilder.sh old mode 100644 new mode 100755 index 861638d1..ace89d1c --- a/Templates/Common-Backend-Scripts/zBuilder.sh +++ b/Templates/Common-Backend-Scripts/zBuilder.sh @@ -128,8 +128,9 @@ Help() { # Either an absolute path or a relative path to the current working directory SCRIPT_HOME="$(dirname "$0")" pipelineConfiguration="${SCRIPT_HOME}/pipelineBackend.config" +# Utility scripts buildUtilities="${SCRIPT_HOME}/utilities/dbbzBuilderUtils.sh" - +fetchBuildDependenciesUtilities="${SCRIPT_HOME}/utilities/fetchBuildDependenciesUtils.sh" # # Internal Variables @@ -137,7 +138,7 @@ buildUtilities="${SCRIPT_HOME}/utilities/dbbzBuilderUtils.sh" #export BASH_XTRACEFD=1 # Write set -x trace to file descriptor PGM=$(basename "$0") -PGMVERS="1.00" +PGMVERS="1.10" USER=$USER SYS=$(uname -Ia) @@ -205,6 +206,13 @@ if [ $rc -eq 0 ]; then source $buildUtilities fi + # Read and import utilities + if [ ! -f "${fetchBuildDependenciesUtilities}" ]; then + rc=8 + ERRMSG=$PGM": [ERROR] DBB-Build internal utilities (${fetchBuildDependenciesUtilities}) was not found. rc="$rc + echo $ERRMSG + fi + # # Get Options if [ $rc -eq 0 ]; then @@ -438,7 +446,15 @@ if [ $rc -eq 0 ]; then fi fi -# Ready to go: Suggest in the section to echo as much as possible + +# Setup build environment and pull external dependencies if an ApplicationDescriptor is found +if [ $rc -eq 0 ] && [ "$fetchBuildDependencies" == "true" ]; then + # call utilities script + . ${fetchBuildDependenciesUtilities} +fi + +# +# Echo build configuration if [ $rc -eq 0 ]; then echo $PGM": [INFO] **************************************************************" echo $PGM": [INFO] ** Started - DBB Build on HOST/USER: ${SYS}/${USER}" @@ -494,7 +510,7 @@ if [ $rc -eq 0 ]; then fi echo $PGM": [INFO] ${CMD}" - ${CMD} # run build + ${CMD} rc=$? if [ $rc -eq 0 ]; then diff --git a/Templates/GitlabCIPipeline/.gitlab-ci.yml b/Templates/GitlabCIPipeline/.gitlab-ci.yml index 1f095b16..335fdf99 100644 --- a/Templates/GitlabCIPipeline/.gitlab-ci.yml +++ b/Templates/GitlabCIPipeline/.gitlab-ci.yml @@ -569,7 +569,7 @@ Packaging: wdEvidencesDir: ${wdEvidencesRoot}/${application}/production artifacts: - name: “productionDeploymentReport-${CI_PIPELINE_ID}" + name: "productionDeploymentReport-${CI_PIPELINE_ID}" when: always paths: - "${productionReportDir}/deployment-report.html" From fce97156e2d6882776c6260e5b41593952fdac5e Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Wed, 30 Apr 2025 18:09:12 +0200 Subject: [PATCH 5/6] include reviewer feedback Signed-off-by: Dennis Behm --- .../generateCleanupCommands.sh | 21 ++++++++++++++++ .../pipelineBackend.config | 25 +++++++------------ 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/Templates/Common-Backend-Scripts/generateCleanupCommands.sh b/Templates/Common-Backend-Scripts/generateCleanupCommands.sh index d79edea2..764c8f05 100755 --- a/Templates/Common-Backend-Scripts/generateCleanupCommands.sh +++ b/Templates/Common-Backend-Scripts/generateCleanupCommands.sh @@ -240,6 +240,27 @@ validateOptions() { echo $ERRMSG fi + if [ -z "${dbbMetadataStoreType}" ]; then + rc=8 + ERRMSG=$PGM": [ERROR] DBB MetadataStore Type is a required configuration in pipelineBackend.config . rc="$rc + echo $ERRMSG + else + + case ${dbbMetadataStoreType} in + file) + cleanupDbbMetadataStoreOptions="--type file --location ${dbbFileMetadataStoreLocation}" + ;; + db2) + cleanupDbbMetadataStoreOptions="--type db2 --user ${dbbMetadataStoreJdbcId} --password-file ${dbbMetadataStoreJdbcPwdFile} --db2-config ${dbbMetadataStoreJdbcConfigFile} --url ${dbbMetadataStoreJdbcUrl}" + ;; + *) + rc=8 + ERRMSG=$PGM": [INFO] Invalid DBB MetadataStore Type: ${dbbMetadataStoreType}. Check required configuration in pipelineBackend.config. rc="$rc + echo $ERRMSG + ;; + esac + + fi } # Prepare output files diff --git a/Templates/Common-Backend-Scripts/pipelineBackend.config b/Templates/Common-Backend-Scripts/pipelineBackend.config index 76089d78..2b1ef7e1 100644 --- a/Templates/Common-Backend-Scripts/pipelineBackend.config +++ b/Templates/Common-Backend-Scripts/pipelineBackend.config @@ -86,6 +86,15 @@ dbbMetadataStoreJdbcPwdFile="" # Default = None, currently only referenced in generateCleanupCommands.sh dbbMetadataStoreJdbcConfigFile="" +# DBB Metadatastore implementation. 'db2' or 'file' +# Used to create the housekeeping instructions in generateCleanupCommands.sh +# Default: dbbMetadataStoreType=db2 +dbbMetadataStoreType="file" + +# DBB File Metadatastore location. Only relevant when using dbbMetadataStoreType="file" +# Default dbbFileMetadataStoreLocation="${HOME}" +dbbFileMetadataStoreLocation="${HOME}" + # JDBC connection server url # sample jdbc:db2://10.3.20.201:4740/MOPDBC0 # Optional if zAppBuild is configured to use the @@ -324,22 +333,6 @@ wdDeployPackageDir() { ## End of wazideploy-generate.sh, wazideploy-deploy.sh and wazideploy-evidences.sh parameters #### ##################################################################################################### -##################################################################################################### -## generateCleanupCommands.sh parameters ############################################################ -##################################################################################################### - -# DBB Metadatastore Options String to configure Db2 or File Metadatastore in cleanup commands -# e.g. -# for file metadatastore -# cleanupDbbMetadataStoreOptions="--type file --location /u/github/" -# for db2 metadatastore, reusing build settings -# cleanupDbbMetadataStoreOptions="--type db2 --user ${dbbMetadataStoreJdbcId} --password-file ${dbbMetadataStoreJdbcPwdFile} --db2-config ${dbbMetadataStoreJdbcConfigFile} --url ${dbbMetadataStoreJdbcUrl}" -cleanupDbbMetadataStoreOptions="" - -##################################################################################################### -## End of generateCleanupCommands.sh parameters #### -##################################################################################################### - ##################################################################################################### ## Central functions shared across scripts ####################################################### ##################################################################################################### From 068cac56c5d9847f1f918410cfe69125bc41bf18 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Fri, 2 May 2025 09:27:55 +0200 Subject: [PATCH 6/6] switch to default db2 Signed-off-by: Dennis Behm --- Templates/Common-Backend-Scripts/pipelineBackend.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Templates/Common-Backend-Scripts/pipelineBackend.config b/Templates/Common-Backend-Scripts/pipelineBackend.config index 2b1ef7e1..5a9c721b 100644 --- a/Templates/Common-Backend-Scripts/pipelineBackend.config +++ b/Templates/Common-Backend-Scripts/pipelineBackend.config @@ -89,7 +89,7 @@ dbbMetadataStoreJdbcConfigFile="" # DBB Metadatastore implementation. 'db2' or 'file' # Used to create the housekeeping instructions in generateCleanupCommands.sh # Default: dbbMetadataStoreType=db2 -dbbMetadataStoreType="file" +dbbMetadataStoreType="db2" # DBB File Metadatastore location. Only relevant when using dbbMetadataStoreType="file" # Default dbbFileMetadataStoreLocation="${HOME}"