From 7af329548deca008779913e663a4c5f0b4178577 Mon Sep 17 00:00:00 2001 From: Sophia Guo Date: Tue, 6 Jun 2023 11:51:18 -0400 Subject: [PATCH 01/12] Add detail error messages Signed-off-by: Sophia Guo --- tools/reproduce_comparison/Jenkinsfile | 99 +++++++++++++++++++++++++- 1 file changed, 96 insertions(+), 3 deletions(-) diff --git a/tools/reproduce_comparison/Jenkinsfile b/tools/reproduce_comparison/Jenkinsfile index d81d79864..032b43f83 100644 --- a/tools/reproduce_comparison/Jenkinsfile +++ b/tools/reproduce_comparison/Jenkinsfile @@ -13,9 +13,102 @@ pipeline { } stages { - stage('Lets Compare') { - agent { - label "${params.platform}&&build&&x64" // build label enables comparisons to be done, set to x64 for linux, might need adjust for windows/mac + stage('Prepare') { //Copy artifacts, reset parameters,trigger build and copyArtifacts + steps { + cleanWs() + checkout scm + copyArtifacts excludes: '**/OpenJDK*-sbom*metadata.json', + filter: '**/OpenJDK*-jdk*.tar.gz,**/OpenJDK*-sbom*.json', + fingerprintArtifacts: true, + flatten: true, + projectName: "${params.COMPARED_JOB_NAME}", + target: 'original/' + selector: specific("${params.COMPARED_JOB_NUMBER}") + script { + def sbomFiles = findFiles(glob: "**/*.json") + def sbomParams = readJSON file: "${sbomFiles[0].name}" + def buildTimeStamp = sbomParams.metadata.timestamp + + def sbomProperties = sbomParams.components[0].properties + def reproducedParams = [:] + sbomProperties.each { propertyItem -> + def paramFound = false + propertyItem.each { key, value -> + if (value in ["SCM Ref", "OpenJDK Source Commit", "Temurin Build Ref"]) { + paramFound = true + } + } + if (paramFound) { + reproducedParams.put("${propertyItem.name}", "${propertyItem.value}") + } + } + + def scmRef = reproducedParams["SCM Ref"] + def buildRef = reproducedParams["Temurin Build Ref"] + buildRef = buildRef.substring(buildRef.lastIndexOf('/')+1) + def scmCommit = reproducedParams["OpenJDK Source Commit"] + scmCommit = scmCommit.substring(scmCommit.lastIndexOf('/')+1) + + // Convert COMPARED_JOB_PARAMS to json formatt string + def jobParams = COMPARED_JOB_PARAMS.replaceAll("=", ':') + jobParams = jobParams.replace("ADOPT_DEFAULTS_JSON", "ADOPT_DEFAULTS_TEMP") + jobParams = jobParams.replace("DEFAULTS_JSON", "\"DEFAULTS_JSON\"") + jobParams = jobParams.replace("USER_REMOTE_CONFIGS", "\"USER_REMOTE_CONFIGS\"") + jobParams = jobParams.replace("ADOPT_DEFAULTS_TEMP", "\"ADOPT_DEFAULTS_JSON\"") + jobParams = jobParams.replace("BUILD_CONFIGURATION", "\"BUILD_CONFIGURATION\"") + + def jsonJobParams = new groovy.json.JsonSlurper().parseText(jobParams) + jsonJobParams.BUILD_CONFIGURATION.SCM_REF = scmRef + jsonJobParams.BUILD_CONFIGURATION.BUILD_REF = buildRef + jsonJobParams.BUILD_CONFIGURATION.BUILD_ARGS += " --build-reproducible-date ${buildTimeStamp}" //'${buildTimeStamp}' + jsonJobParams.BUILD_CONFIGURATION.ENABLE_TESTS = "false" + + def buildParams = [ + text(name: 'BUILD_CONFIGURATION', value: "${jsonJobParams.BUILD_CONFIGURATION}"), + text(name: 'USER_REMOTE_CONFIGS', value: "${jsonJobParams.USER_REMOTE_CONFIGS}"), + text(name: 'DEFAULTS_JSON', value: "${jsonJobParams.DEFAULTS_JSON}"), + text(name: 'ADOPT_DEFAULTS_JSON', value: "${jsonJobParams.ADOPT_DEFAULTS_JSON}") + ] + def reproducibleJob = build job: "${COMPARED_JOB_NAME}", + propagate: false, + parameters: buildParams + + def result = reproducibleJob.getResult() + if (result == 'SUCCESS') { + try { + timeout(time: 2, unit: 'HOURS') { + copyArtifacts( + projectName:"${COMPARED_JOB_NAME}", + selector:specific("${reproducibleJob.getNumber()}"), + filter: "**/*.tar.gz", + target: 'reproduced/', + fingerprintArtifacts: true, + flatten: true + ) + } + } catch (Exception e) { + echo "Cannot run copyArtifacts from job ${COMPARED_JOB_NAME}. Exception: ${e.message}. Skipping copyArtifacts..." + } + } else { + error "Reproducible jenkins job ${COMPARED_JOB_NAME} failed, exit comparision" + return + } + + def originalJDKFile = findFiles(glob: "original/*.tar.gz") + def reproducedJDKFile = findFiles(glob: "reproduced/*.tar.gz") + def result1 = sh returnStatus: true, script: "tar -xz ${originalJDKFile[0].name} -C original" + if (result1 != 0 ) { + currentBuild.result = 'UNSTABLE' + error " Untar ${originalJDKFile[0].name} failed" + return + } + def result2 = sh returnStatus: true, script: "tar -xz ${reproducedJDKFile[0].name} -C reproduced" + if (result2 != 0 ) { + currentBuild.result = 'UNSTABLE' + error " Untar ${reproducedJDKFile[0].name} failed" + return + } + } } steps { script { From ad2c904f21217f20a6e3666512944323b7373eb4 Mon Sep 17 00:00:00 2001 From: Sophia Guo Date: Tue, 6 Jun 2023 16:11:44 -0400 Subject: [PATCH 02/12] Enable nightly reproducible comparison Resubmit the changes Signed-off-by: Sophia Guo --- .../common/openjdk_build_pipeline.groovy | 31 +++++++- tools/reproduce_comparison/Jenkinsfile | 78 +++++++------------ 2 files changed, 56 insertions(+), 53 deletions(-) diff --git a/pipelines/build/common/openjdk_build_pipeline.groovy b/pipelines/build/common/openjdk_build_pipeline.groovy index 742c13920..2d9911f76 100644 --- a/pipelines/build/common/openjdk_build_pipeline.groovy +++ b/pipelines/build/common/openjdk_build_pipeline.groovy @@ -585,6 +585,33 @@ class Build { return remoteTargets } + def compareReproducibleBuild() { + // Currently only enable for jdk17, linux_x64, temurin, nightly, which shouldn't affect current build + // Move out of normal jdk** folder as it won't be regenerated automatically right now + def jobName = "${env.JOB_NAME}" + jobName = jobName.substring(jobName.lastIndexOf('/')+1) + jobName = "${jobName}_reproduce_compare" + if (getJavaVersionNumber() == 17 && + buildConfig.ARCHITECTURE.contains('x64') && + buildConfig.TARGET_OS.contains('linux') && + buildConfig.VARIANT == 'temurin' && + !isRelease) { + // For now set the build as independent, no need to wait for result as the build takes time + context.stage('Reproduce Compare') { + def buildParams = params.toString() + // passing buildParams multiline parameter to downstream job, double check the available method + context.build job: jobName, + propagate: false, + parameters: [ + context.string(name: 'COMPARED_JOB_NUMBER', value: "${env.BUILD_NUMBER}"), + context.string(name: 'COMPARED_JOB_NAME', value: "${env.JOB_NAME}"), + context.string(name: 'COMPARED_JOB_PARAMS', value: buildParams) + ], + wait: false + } + } + } + /* We use this function at the end of a build to parse a java version string and create a VersionInfo object for deployment in the metadata objects. E.g. 11.0.9+10-202010192351 would be one example of a matched string. @@ -1874,13 +1901,15 @@ class Build { // Sign job timeout managed by Jenkins job config sign(versionInfo) } catch (FlowInterruptedException e) { - throw new Exception("[ERROR] Sign job timeout (${buildTimeouts.SIGN_JOB_TIMEOUT} HOURS) has been reached OR the downstream sign job failed. Exiting...") + throw new Exception('[ERROR] Sign job timeout (${buildTimeouts.SIGN_JOB_TIMEOUT} HOURS) has been reached OR the downstream sign job failed. Exiting...') } } // Run Smoke Tests and AQA Tests if (enableTests) { try { + // Compare reproducible build, using same build pipeline with enableTests = false to avoid recursive building + compareReproducibleBuild() //Only smoke tests succeed TCK and AQA tests will be triggerred. if (runSmokeTests() == 'SUCCESS') { // Remote trigger Eclipse Temurin JCK tests diff --git a/tools/reproduce_comparison/Jenkinsfile b/tools/reproduce_comparison/Jenkinsfile index 032b43f83..66cbff532 100644 --- a/tools/reproduce_comparison/Jenkinsfile +++ b/tools/reproduce_comparison/Jenkinsfile @@ -1,15 +1,13 @@ /* -Jenkins job does comparison for two different build artifacts -If they are identical, build pass; or fail +Jenkins job does reproducible build compare. */ pipeline { agent any parameters { - choice(choices: ['linux', 'mac', 'windows'], name: 'platform', description:'select release on different platform') - string(name: 'URL1', defaultValue: '', description: 'URL to one version of the build artifacts\ne.g.: https://ci.adoptium.net/job/build-scripts/job/openjdk18-pipeline/lastSuccessfulBuild/artifact/target/linux/x64/temurin/OpenJDK18U-jdk_x64_linux_hotspot_2022-06-11-23-30.tar.gz') - string(name: 'URL2', defaultValue: '', description: 'URL to the other version of the build artifacts\ne.g.: https://github.com/adoptium/temurin19-binaries/releases/download/jdk-2022-06-09-19-11-beta/OpenJDK-jdk_x64_linux_hotspot_2022-06-09-03-31.tar.gz') - string(name: 'excludeFiles', defaultValue: 'classes_nocoops.jsa', description: 'File names to be excluded while doing the comparison, separated by `;`') + string(name: 'COMPARED_JOB_NUMBER', defaultValue: '', description: 'Compared nightly build job name') + string(name: 'COMPARED_JOB_NAME', defaultValue: '', description: 'Compared nightly build job number') + string(name: 'COMPARED_JOB_PARAMS', defaultValue: '', description: 'Compared nightly build job parameters') } stages { @@ -110,41 +108,33 @@ pipeline { } } } + } + stage('Compare') { steps { + checkout([$class: 'GitSCM', branches: [[name: 'master']], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: "temurin-build"]], submoduleCfg: [], userRemoteConfigs: [[url: "https://github.com/adoptium/temurin-build.git"]]]) script { try { - cleanWs() - // use Jenkins crendential to download JDK if source is from openjdkX-pipline - withCredentials([usernamePassword(credentialsId: 'eclipse_temurin_bot_email_and_token', passwordVariable: 'PASSWORD', usernameVariable: 'USERNAME')]) { - echo "Fetching artifact1 from ${params.URL1}" - def ret1 = sh returnStatus: true, script: "mkdir url1Dir1 && curl -s --user ${USERNAME}:${PASSWORD} ${params.URL1} | tar -xz -C url1Dir1" - if (ret1 != 0) { - currentBuild.result = 'UNSTABLE' - error 'Stopping after download and uncompress tar file' + dir('temurin-build/tools/reproduce_comparison') { + def rc = 0 + if (COMPARED_JOB_NAME.contains('linux')) { + sh "chmod 776 ./tooling/linux_repo_compare.sh" + rc = sh returnStatus: true, script: "./tooling/linux_repo_compare.sh temurin ./original temurin ./reproduced" + } else if (COMPARED_JOB_NAME.contains('mac')) { + // mac + //sh "chmod 776 ./tooling/mac_repo_compare.sh" + //sh "./tooling/mac_repo_compare.sh temurin ./original temurin ./reproduced ${self_cert_file} ${self_cert_passwd}" + echo 'todo mac build comparison' + } else if (COMPARED_JOB_NAME.contains('windows')) { + //windows + echo 'todo windows build comparison' } - echo "Fetching artifact2 from ${params.URL2}" - def ret2 = sh returnStatus: true, script: "mkdir url1Dir2 && curl -s --user ${USERNAME}:${PASSWORD} ${params.URL2} | tar -xz -C url1Dir2" - if (ret2 != 0) { - currentBuild.result = 'UNSTABLE' - error 'Stopping after download and uncompress tar file' + if (rc != 0) { + currentBuild.result = 'FAILURE' + error 'Error: two builds are not the same!' + } else { + echo 'Success: two builds are the same!' } } - // call extra platform specific function - if (params.platform != 'linux') { - "prep${platform}" ('url1Dir1', 'url1Dir2') - } - def excludeFlags="" - def excludeFileList = excludeFiles.split(';') - for(String excludeFile in excludeFileList) { - excludeFlags="${excludeFlags} --exclude=${excludeFile}" - } - def retVal = sh returnStatus: true, script: "diff -q -r url1Dir1 url1Dir2 ${excludeFlags}" - if (retVal != 0) { - currentBuild.result = 'FAILURE' - error 'Error: two builds are not the same!' - } else { - echo 'Success: two builds are the same!' - } } catch (Exception err) { echo err.getMessage() currentBuild.result = 'FAILURE' @@ -155,20 +145,4 @@ pipeline { } } } -} - -/* TODO: -set correct defaultpath for localCert -see: https://github.com/adoptium/temurin-build/issues/3015#issuecomment-1175322612 -*/ -def prepmac(String jdk1, String jdk2, String localCert='defaultpath') { - sh "chmod 776 ${WORKSPACE}/tools/reproduce_comparison/compareMacOS.sh" - sh "${WORKSPACE}/tools/reproduce_comparison/compareMacOS.sh $jdk1 $jdk2 $localCert skip" -} - -/* -TODO: placeholder for windows comparison -def prepwindows(String jdk1, String jdk2, ...) { - sh "./tools/reproduce_comparison/compareWindows.sh ..." -} -*/ +} \ No newline at end of file From f496dc4b942c112cf7dbc9df692f51ec7e51f4fa Mon Sep 17 00:00:00 2001 From: Sophia Guo Date: Tue, 6 Jun 2023 16:40:21 -0400 Subject: [PATCH 03/12] Correct lint error Signed-off-by: Sophia Guo --- pipelines/build/common/openjdk_build_pipeline.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipelines/build/common/openjdk_build_pipeline.groovy b/pipelines/build/common/openjdk_build_pipeline.groovy index 2d9911f76..cf00f23c5 100644 --- a/pipelines/build/common/openjdk_build_pipeline.groovy +++ b/pipelines/build/common/openjdk_build_pipeline.groovy @@ -1901,7 +1901,7 @@ class Build { // Sign job timeout managed by Jenkins job config sign(versionInfo) } catch (FlowInterruptedException e) { - throw new Exception('[ERROR] Sign job timeout (${buildTimeouts.SIGN_JOB_TIMEOUT} HOURS) has been reached OR the downstream sign job failed. Exiting...') + throw new Exception("[ERROR] Sign job timeout (${buildTimeouts.SIGN_JOB_TIMEOUT} HOURS) has been reached OR the downstream sign job failed. Exiting...") } } @@ -1934,7 +1934,7 @@ class Build { context.parallel testStages } } else { - context.println("[ERROR]Smoke tests are not successful! AQA and Tck tests are blocked ") + context.println('[ERROR]Smoke tests are not successful! AQA and Tck tests are blocked ') } } catch (Exception e) { context.println(e.message) From 90635ee37a1afbd1737da81f036b6869b0f1ac4e Mon Sep 17 00:00:00 2001 From: Sophia Guo Date: Wed, 7 Jun 2023 16:38:42 -0400 Subject: [PATCH 04/12] Add enableReproducibleCompare flag Signed-off-by: Sophia Guo --- docs/UsingOurScripts.md | 2 ++ pipelines/build/common/build_base_file.groovy | 24 +++++++++++++++++-- .../build/common/config_regeneration.groovy | 22 ++++++++++++++++- .../common/create_job_from_template.groovy | 1 + .../common/openjdk_build_pipeline.groovy | 8 ++++--- pipelines/build/openjdk_pipeline.groovy | 1 + .../build/prTester/pr_test_pipeline.groovy | 1 + .../regeneration/build_job_generator.groovy | 8 +++---- .../build_pipeline_generator.groovy | 2 +- .../evaluation_pipeline_generator.groovy | 2 +- pipelines/defaults.json | 1 + .../jdk17u_pipeline_config.groovy | 3 +++ pipelines/jobs/pipeline_job_template.groovy | 2 ++ .../jobs/release_pipeline_job_template.groovy | 2 ++ .../src/test/groovy/RepoHandlerTest.groovy | 3 +++ pipelines/src/test/groovy/fakeDefaults.json | 1 + 16 files changed, 71 insertions(+), 12 deletions(-) diff --git a/docs/UsingOurScripts.md b/docs/UsingOurScripts.md index eb533a521..865b93d44 100644 --- a/docs/UsingOurScripts.md +++ b/docs/UsingOurScripts.md @@ -78,6 +78,8 @@ This file contains the default constants and paths used in the build scripts for }, // Test suite and inclusion details "testDetails" : { + // Boolean denoting wether the reproducible compare is needed + "enableReproducibleCompare" :false, // Boolean denoting whether pipeline tests will be enabled or disabled by default "enableTests" : true, /* diff --git a/pipelines/build/common/build_base_file.groovy b/pipelines/build/common/build_base_file.groovy index 78650278f..12b37ad93 100644 --- a/pipelines/build/common/build_base_file.groovy +++ b/pipelines/build/common/build_base_file.groovy @@ -39,6 +39,7 @@ class Builder implements Serializable { Map DEFAULTS_JSON String activeNodeTimeout Map> dockerExcludes + boolean enableReproducibleCompare boolean enableTests boolean enableTestDynamicParallel boolean enableInstallers @@ -122,7 +123,7 @@ class Builder implements Serializable { if (additionalBuildArgs) { buildArgs += ' ' + additionalBuildArgs } - + def enableReproducibleCompare = getReproducibleCompare(platformConfig, variant) def testList = getTestList(platformConfig, variant) def dynamicTestsParameters = getDynamicParams(platformConfig, variant) @@ -178,6 +179,7 @@ class Builder implements Serializable { RELEASE: release, PUBLISH_NAME: publishName, ADOPT_BUILD_NUMBER: adoptBuildNumber, + ENABLE_REPRODUCIBLE_COMPARE: enableReproducibleCompare, ENABLE_TESTS: enableTests, ENABLE_TESTDYNAMICPARALLEL: enableTestDynamicParallel, ENABLE_INSTALLERS: enableInstallers, @@ -213,7 +215,22 @@ class Builder implements Serializable { return '' } - + /* + * Get reproduciableCompare flag from the build configurations. + */ + Boolean getReproducibleCompare(Map configuration, String variant) { + Boolean enableReproducibleCompare = DEFAULTS_JSON['testDetails']['enableReproducibleCompare'] as Boolean + if (configuration.containsKey('reproducibleCompare')) { + def reproducibleCompare + if (isMap(configuration.reproducibleCompare)) { + reproducibleCompare = (configuration.enableReproducibleCompare as Map).get(variant) + } + if (reproducibleCompare != null) { + enableReproducibleCompare = reproducibleCompare + } + } + return enableReproducibleCompare + } /* Get the list of tests to run from the build configurations. We run different test categories depending on if this build is a release or nightly. This function parses and applies this to the individual build config. @@ -781,6 +798,7 @@ class Builder implements Serializable { context.echo "Java: ${javaToBuild}" context.echo "OS: ${targetConfigurations}" + context.echo "Enable reproducible compare: ${enableReproducibleCompare}" context.echo "Enable tests: ${enableTests}" context.echo "Enable Installers: ${enableInstallers}" context.echo "Enable Signer: ${enableSigner}" @@ -925,6 +943,7 @@ return { Map DEFAULTS_JSON, String activeNodeTimeout, String dockerExcludes, + String enableReproducibleCompare, String enableTests, String enableTestDynamicParallel, String enableInstallers, @@ -989,6 +1008,7 @@ return { DEFAULTS_JSON: DEFAULTS_JSON, activeNodeTimeout: activeNodeTimeout, dockerExcludes: buildsExcludeDocker, + enableReproducibleCompare: Boolean.parseBoolean(enableReproducibleCompare), enableTests: Boolean.parseBoolean(enableTests), enableTestDynamicParallel: Boolean.parseBoolean(enableTestDynamicParallel), enableInstallers: Boolean.parseBoolean(enableInstallers), diff --git a/pipelines/build/common/config_regeneration.groovy b/pipelines/build/common/config_regeneration.groovy index 81d027728..f5c701989 100644 --- a/pipelines/build/common/config_regeneration.groovy +++ b/pipelines/build/common/config_regeneration.groovy @@ -334,7 +334,24 @@ class Regeneration implements Serializable { return '' } - + /* + * Get reproduciableCompare flag from jdk*_pipeline_config.groovy. Used when creating the IndividualBuildConfig. + * @param configuration + * @param variant + */ + Boolean getReproducibleCompare(Map configuration, String variant) { + Boolean enableReproducibleCompare = DEFAULTS_JSON['testDetails']['enableReproducibleCompare'] as Boolean + if (configuration.containsKey('reproducibleCompare')) { + def reproducibleCompare + if (isMap(configuration.reproducibleCompare)) { + reproducibleCompare = (configuration.enableReproducibleCompare as Map).get(variant) + } + if (reproducibleCompare != null) { + enableReproducibleCompare = reproducibleCompare + } + } + return enableReproducibleCompare + } /* * Get the list of tests from jdk*_pipeline_config.groovy. Used when creating the IndividualBuildConfig. Used as a placeholder since the pipelines overwrite this. * @param configuration @@ -425,6 +442,8 @@ class Regeneration implements Serializable { def numMachines = getDynamicParams().get('numMachines') + def enableReproducibleCompare = getReproducibleCompare(platformConfig, variant) + return new IndividualBuildConfig( // final build config JAVA_TO_BUILD: javaToBuild, ARCHITECTURE: platformConfig.arch as String, @@ -460,6 +479,7 @@ class Regeneration implements Serializable { RELEASE: false, PUBLISH_NAME: '', ADOPT_BUILD_NUMBER: '', + ENABLE_REPRODUCIBLE_COMPARE: enableReproducibleCompare, ENABLE_TESTS: DEFAULTS_JSON['testDetails']['enableTests'] as Boolean, ENABLE_TESTDYNAMICPARALLEL: DEFAULTS_JSON['testDetails']['enableTestDynamicParallel'] as Boolean, ENABLE_INSTALLERS: true, diff --git a/pipelines/build/common/create_job_from_template.groovy b/pipelines/build/common/create_job_from_template.groovy index b8ad372a4..0b9cfd6c8 100644 --- a/pipelines/build/common/create_job_from_template.groovy +++ b/pipelines/build/common/create_job_from_template.groovy @@ -128,6 +128,7 @@ pipelineJob("$buildFolder/$JOB_NAME") {
RELEASE
Is this build a release
PUBLISH_NAME
Set name of publish
ADOPT_BUILD_NUMBER
Adopt build number
+
ENABLE_REPRODUCIBLE_COMPARE
Run reproducible compare build
ENABLE_TESTS
Run tests
ENABLE_TESTDYNAMICPARALLEL
Run parallel
ENABLE_INSTALLERS
Run installers
diff --git a/pipelines/build/common/openjdk_build_pipeline.groovy b/pipelines/build/common/openjdk_build_pipeline.groovy index cf00f23c5..1756a084f 100644 --- a/pipelines/build/common/openjdk_build_pipeline.groovy +++ b/pipelines/build/common/openjdk_build_pipeline.groovy @@ -1726,7 +1726,7 @@ class Build { context.println "Executing tests: ${buildConfig.TEST_LIST}" context.println "Build num: ${env.BUILD_NUMBER}" context.println "File name: ${filename}" - + def enableReproducibleCompare = Boolean.valueOf(buildConfig.ENABLE_REPRODUCIBLE_COMPARE) def enableTests = Boolean.valueOf(buildConfig.ENABLE_TESTS) def enableInstallers = Boolean.valueOf(buildConfig.ENABLE_INSTALLERS) def enableSigner = Boolean.valueOf(buildConfig.ENABLE_SIGNER) @@ -1905,11 +1905,13 @@ class Build { } } + // Compare reproducible build if needed + if (enableReproducibleCompare) { + compareReproducibleBuild() + } // Run Smoke Tests and AQA Tests if (enableTests) { try { - // Compare reproducible build, using same build pipeline with enableTests = false to avoid recursive building - compareReproducibleBuild() //Only smoke tests succeed TCK and AQA tests will be triggerred. if (runSmokeTests() == 'SUCCESS') { // Remote trigger Eclipse Temurin JCK tests diff --git a/pipelines/build/openjdk_pipeline.groovy b/pipelines/build/openjdk_pipeline.groovy index 67e74237b..1da866871 100644 --- a/pipelines/build/openjdk_pipeline.groovy +++ b/pipelines/build/openjdk_pipeline.groovy @@ -150,6 +150,7 @@ if (scmVars != null || configureBuild != null || buildConfigurations != null) { DEFAULTS_JSON, activeNodeTimeout, dockerExcludes, + enableReproducibleCompare, enableTests, enableTestDynamicParallel, enableInstallers, diff --git a/pipelines/build/prTester/pr_test_pipeline.groovy b/pipelines/build/prTester/pr_test_pipeline.groovy index 907c66d18..c0343347f 100644 --- a/pipelines/build/prTester/pr_test_pipeline.groovy +++ b/pipelines/build/prTester/pr_test_pipeline.groovy @@ -52,6 +52,7 @@ class PullRequestTestPipeline implements Serializable { CHECKOUT_CREDENTIALS: '', adoptScripts : true, enableTests : false, + enableReproducibleCompare : false, enableTestDynamicParallel : false, releaseType : "pr-tester" ] diff --git a/pipelines/build/regeneration/build_job_generator.groovy b/pipelines/build/regeneration/build_job_generator.groovy index 5a0fbc2ef..bcdf1f766 100644 --- a/pipelines/build/regeneration/build_job_generator.groovy +++ b/pipelines/build/regeneration/build_job_generator.groovy @@ -49,19 +49,19 @@ node('worker') { try { // Load git url and branch and gitBranch. These determine where we will be pulling configs from. - def repoUri = (params.REPOSITORY_URL) ?: DEFAULTS_JSON['repository']['pipeline_url'] + def repoUrl = (params.REPOSITORY_URL) ?: DEFAULTS_JSON['repository']['pipeline_url'] def repoBranch = (params.REPOSITORY_BRANCH) ?: DEFAULTS_JSON['repository']['pipeline_branch'] // Load credentials to be used in checking out. This is in case we are checking out a URL that is not Adopts and they don't have their ssh key on the machine. def checkoutCreds = (params.CHECKOUT_CREDENTIALS) ?: '' def remoteConfigs = new JsonSlurper().parseText('{ "url": "" }') as Map - remoteConfigs.url = repoUri + remoteConfigs.url = repoUrl if (checkoutCreds != '') { // This currently does not work with user credentials due to https://issues.jenkins.io/browse/JENKINS-60349 remoteConfigs.credentials = "${checkoutCreds}" } else { - println "[WARNING] CHECKOUT_CREDENTIALS not specified! Checkout to $repoUri may fail if you do not have your ssh key on this machine." + println "[WARNING] CHECKOUT_CREDENTIALS not specified! Checkout to $repoUrl may fail if you do not have your ssh key on this machine." } /* @@ -186,7 +186,7 @@ node('worker') { println '[INFO] Running regeneration script with the following configuration:' println "VERSION: $javaVersion" - println "CI REPOSITORY URL: $repoUri" + println "CI REPOSITORY URL: $repoUrl" println "CI REPOSITORY BRANCH: $repoBranch" println "BUILD CONFIGURATIONS: ${JsonOutput.prettyPrint(JsonOutput.toJson(buildConfigurations))}" println "JOBS TO GENERATE: ${JsonOutput.prettyPrint(JsonOutput.toJson(targetConfigurations))}" diff --git a/pipelines/build/regeneration/build_pipeline_generator.groovy b/pipelines/build/regeneration/build_pipeline_generator.groovy index 03c86608c..8dad0d180 100644 --- a/pipelines/build/regeneration/build_pipeline_generator.groovy +++ b/pipelines/build/regeneration/build_pipeline_generator.groovy @@ -209,7 +209,7 @@ node('worker') { if (useAdoptShellScripts.toBoolean()) { config.put('adoptScripts', true) } - + config.put('enableReproducibleCompare', DEFAULTS_JSON['testDetails']['enableReproducibleCompare'] as Boolean)) config.put('enableTests', DEFAULTS_JSON['testDetails']['enableTests'] as Boolean) config.put('enableTestDynamicParallel', DEFAULTS_JSON['testDetails']['enableTestDynamicParallel'] as Boolean) diff --git a/pipelines/build/regeneration/evaluation_pipeline_generator.groovy b/pipelines/build/regeneration/evaluation_pipeline_generator.groovy index 582a1f199..72b5e6122 100644 --- a/pipelines/build/regeneration/evaluation_pipeline_generator.groovy +++ b/pipelines/build/regeneration/evaluation_pipeline_generator.groovy @@ -215,7 +215,7 @@ node('worker') { if (useAdoptShellScripts.toBoolean()) { config.put('adoptScripts', true) } - + config.put('enableReproducibleCompare', DEFAULTS_JSON['testDetails']['enableReproducibleCompare'] as Boolean) config.put('enableTests', DEFAULTS_JSON['testDetails']['enableTests'] as Boolean) config.put('enableTestDynamicParallel', DEFAULTS_JSON['testDetails']['enableTestDynamicParallel'] as Boolean) config.put('defaultsJson', DEFAULTS_JSON) diff --git a/pipelines/defaults.json b/pipelines/defaults.json index afb803d95..15f348997 100644 --- a/pipelines/defaults.json +++ b/pipelines/defaults.json @@ -37,6 +37,7 @@ "downstream" : "pipelines/build/common/openjdk_build_pipeline.groovy" }, "testDetails" : { + "enableReproducibleCompare" : false, "enableTests" : true, "nightlyDefault" : [ "sanity.openjdk", diff --git a/pipelines/jobs/configurations/jdk17u_pipeline_config.groovy b/pipelines/jobs/configurations/jdk17u_pipeline_config.groovy index db4113b7b..25a320c99 100644 --- a/pipelines/jobs/configurations/jdk17u_pipeline_config.groovy +++ b/pipelines/jobs/configurations/jdk17u_pipeline_config.groovy @@ -26,6 +26,9 @@ class Config17 { openj9 : 'pipelines/build/dockerFiles/cuda.dockerfile' ], test : 'default', + reproducibleCompare : [ + 'temurin' : true + ] additionalTestLabels: [ openj9 : '!(centos6||rhel6)' ], diff --git a/pipelines/jobs/pipeline_job_template.groovy b/pipelines/jobs/pipeline_job_template.groovy index 68915a55d..3b88db96d 100644 --- a/pipelines/jobs/pipeline_job_template.groovy +++ b/pipelines/jobs/pipeline_job_template.groovy @@ -2,6 +2,7 @@ import groovy.json.JsonOutput gitRefSpec = '' propagateFailures = true +runReproducibleCompare = enableReproducibleCompare runTests = enableTests runParallel = enableTestDynamicParallel runInstaller = true @@ -101,6 +102,7 @@ pipelineJob("${BUILD_FOLDER}/${JOB_NAME}") { stringParam('helperReference', '', 'Tag name or Branch name of jenkins-helper repo. Defaults to master') stringParam('aqaReference', '', 'Tag name or Branch name of aqa-tests. Defaults to master') booleanParam('aqaAutoGen', false, 'If set to true, force auto generate AQA test jobs. Defaults to false') + booleanParam('enableReproducibleCompare', runReproducibleCompare, 'If set to true the reproducible compare job might be triggerred') booleanParam('enableTests', runTests, 'If set to true the test pipeline will be executed') booleanParam('enableTestDynamicParallel', runParallel, 'If set to true test will be run parallel') booleanParam('enableInstallers', runInstaller, 'If set to true the installer pipeline will be executed') diff --git a/pipelines/jobs/release_pipeline_job_template.groovy b/pipelines/jobs/release_pipeline_job_template.groovy index 890df3702..2dfeb83ea 100644 --- a/pipelines/jobs/release_pipeline_job_template.groovy +++ b/pipelines/jobs/release_pipeline_job_template.groovy @@ -2,6 +2,7 @@ import groovy.json.JsonOutput String gitRefSpec = '' Boolean propagateFailures = true +Boolean runReproducibleCompare = false Boolean runTests = true Boolean runParallel = true Boolean runInstaller = true @@ -77,6 +78,7 @@ pipelineJob("${BUILD_FOLDER}/${JOB_NAME}") { stringParam('baseFilePath', '', "Relative path to where the build_base_file.groovy file is located. This runs the downstream job setup and configuration retrieval services.
Default: ${defaultsJson['baseFileDirectories']['upstream']}") stringParam('buildConfigFilePath', '', "Relative path to where the jdkxx_pipeline_config.groovy file is located. It contains the build configurations for each platform, architecture and variant.
Default: ${defaultsJson['configDirectories']['build']}/jdkxx_pipeline_config.groovy") booleanParam('aqaAutoGen', false, 'If set to true, force auto generate AQA test jobs. Defaults to false') + booleanParam('enableReproducibleCompare', runReproducibleCompare, 'If set to true the reproducible compare job might be triggerred') booleanParam('enableTests', runTests, 'If set to true the test pipeline will be executed') booleanParam('enableTestDynamicParallel', runParallel, 'If set to true test will be run parallel') booleanParam('enableInstallers', runInstaller, 'If set to true the installer pipeline will be executed') diff --git a/pipelines/src/test/groovy/RepoHandlerTest.groovy b/pipelines/src/test/groovy/RepoHandlerTest.groovy index 5213a8a59..57498ead2 100644 --- a/pipelines/src/test/groovy/RepoHandlerTest.groovy +++ b/pipelines/src/test/groovy/RepoHandlerTest.groovy @@ -63,6 +63,7 @@ class RepoHandlerTest { Assertions.assertEquals(adoptJson.defaultsUrl, 'https://raw.githubusercontent.com/adoptium/ci-jenkins-pipelines/master/pipelines/defaults.json') // Test details + Assertions.assertTrue(adoptJson.testDetails.enableReproducibleCompare instanceof Boolean) Assertions.assertTrue(adoptJson.testDetails.enableTests instanceof Boolean) Assertions.assertTrue(adoptJson.testDetails.nightlyDefault instanceof List) Assertions.assertTrue(adoptJson.testDetails.weeklyDefault instanceof List) @@ -118,6 +119,8 @@ class RepoHandlerTest { Assertions.assertEquals(userJson.defaultsUrl, '23') // Test details + Assertions.assertTrue(userJson.testDetails.enableReproducibleCompare instanceof Boolean) + Assertions.assertTrue(userJson.testDetails.enableReproducibleCompare) Assertions.assertTrue(userJson.testDetails.enableTests instanceof Boolean) Assertions.assertTrue(userJson.testDetails.enableTests) Assertions.assertEquals(userJson.testDetails.nightlyDefault, [ 'test1', 'test2', 'test3' ]) diff --git a/pipelines/src/test/groovy/fakeDefaults.json b/pipelines/src/test/groovy/fakeDefaults.json index 9b01f1c66..334317a46 100644 --- a/pipelines/src/test/groovy/fakeDefaults.json +++ b/pipelines/src/test/groovy/fakeDefaults.json @@ -33,6 +33,7 @@ "downstream" : "17" }, "testDetails" : { + "enableReproducibleCompare" : false, "enableTests" : true, "nightlyDefault" : [ "test1", "test2", "test3" ], "weeklyDefault" : [ "test4", "test5", "test6", "test7"] From e5049c501e9342902fc132ecc3ca5938e70b93d4 Mon Sep 17 00:00:00 2001 From: Sophia Guo Date: Wed, 7 Jun 2023 17:02:22 -0400 Subject: [PATCH 05/12] Set reproducible job ENABLE_REPRODUCIBLE_COMPARE false Avoid recursive build Signed-off-by: Sophia Guo --- pipelines/build/regeneration/build_pipeline_generator.groovy | 2 +- tools/reproduce_comparison/Jenkinsfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pipelines/build/regeneration/build_pipeline_generator.groovy b/pipelines/build/regeneration/build_pipeline_generator.groovy index 8dad0d180..e5ccd9856 100644 --- a/pipelines/build/regeneration/build_pipeline_generator.groovy +++ b/pipelines/build/regeneration/build_pipeline_generator.groovy @@ -209,7 +209,7 @@ node('worker') { if (useAdoptShellScripts.toBoolean()) { config.put('adoptScripts', true) } - config.put('enableReproducibleCompare', DEFAULTS_JSON['testDetails']['enableReproducibleCompare'] as Boolean)) + config.put('enableReproducibleCompare', DEFAULTS_JSON['testDetails']['enableReproducibleCompare'] as Boolean) config.put('enableTests', DEFAULTS_JSON['testDetails']['enableTests'] as Boolean) config.put('enableTestDynamicParallel', DEFAULTS_JSON['testDetails']['enableTestDynamicParallel'] as Boolean) diff --git a/tools/reproduce_comparison/Jenkinsfile b/tools/reproduce_comparison/Jenkinsfile index 66cbff532..ca190006e 100644 --- a/tools/reproduce_comparison/Jenkinsfile +++ b/tools/reproduce_comparison/Jenkinsfile @@ -59,7 +59,7 @@ pipeline { jsonJobParams.BUILD_CONFIGURATION.SCM_REF = scmRef jsonJobParams.BUILD_CONFIGURATION.BUILD_REF = buildRef jsonJobParams.BUILD_CONFIGURATION.BUILD_ARGS += " --build-reproducible-date ${buildTimeStamp}" //'${buildTimeStamp}' - jsonJobParams.BUILD_CONFIGURATION.ENABLE_TESTS = "false" + jsonJobParams.BUILD_CONFIGURATION.ENABLE_REPRODUCIBLE_COMPARE = "false" def buildParams = [ text(name: 'BUILD_CONFIGURATION', value: "${jsonJobParams.BUILD_CONFIGURATION}"), From 87b4529db7986291d3c20e737c5d710a783002b9 Mon Sep 17 00:00:00 2001 From: Sophia Guo Date: Wed, 7 Jun 2023 17:19:16 -0400 Subject: [PATCH 06/12] Update build timestamp Signed-off-by: Sophia Guo --- tools/reproduce_comparison/Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/reproduce_comparison/Jenkinsfile b/tools/reproduce_comparison/Jenkinsfile index ca190006e..497802156 100644 --- a/tools/reproduce_comparison/Jenkinsfile +++ b/tools/reproduce_comparison/Jenkinsfile @@ -25,14 +25,13 @@ pipeline { script { def sbomFiles = findFiles(glob: "**/*.json") def sbomParams = readJSON file: "${sbomFiles[0].name}" - def buildTimeStamp = sbomParams.metadata.timestamp def sbomProperties = sbomParams.components[0].properties def reproducedParams = [:] sbomProperties.each { propertyItem -> def paramFound = false propertyItem.each { key, value -> - if (value in ["SCM Ref", "OpenJDK Source Commit", "Temurin Build Ref"]) { + if (value in ["SCM Ref", "OpenJDK Source Commit", "Temurin Build Ref", "Build Timestamp"]) { paramFound = true } } @@ -43,6 +42,7 @@ pipeline { def scmRef = reproducedParams["SCM Ref"] def buildRef = reproducedParams["Temurin Build Ref"] + def buildTimeStamp = reproducedParams["Build Timestamp"] buildRef = buildRef.substring(buildRef.lastIndexOf('/')+1) def scmCommit = reproducedParams["OpenJDK Source Commit"] scmCommit = scmCommit.substring(scmCommit.lastIndexOf('/')+1) From afd791aacc3047ed5cc2960f2ea45511d561a12b Mon Sep 17 00:00:00 2001 From: Sophia Guo Date: Thu, 8 Jun 2023 11:56:35 -0400 Subject: [PATCH 07/12] Format update Signed-off-by: Sophia Guo --- docs/UsingOurScripts.md | 2 +- pipelines/build/common/build_base_file.groovy | 3 ++- pipelines/build/common/config_regeneration.groovy | 1 + pipelines/jobs/release_pipeline_job_template.groovy | 2 +- tools/reproduce_comparison/Jenkinsfile | 8 ++++---- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/UsingOurScripts.md b/docs/UsingOurScripts.md index 865b93d44..944ea4b05 100644 --- a/docs/UsingOurScripts.md +++ b/docs/UsingOurScripts.md @@ -79,7 +79,7 @@ This file contains the default constants and paths used in the build scripts for // Test suite and inclusion details "testDetails" : { // Boolean denoting wether the reproducible compare is needed - "enableReproducibleCompare" :false, + "enableReproducibleCompare" : false, // Boolean denoting whether pipeline tests will be enabled or disabled by default "enableTests" : true, /* diff --git a/pipelines/build/common/build_base_file.groovy b/pipelines/build/common/build_base_file.groovy index 12b37ad93..43823f2e1 100644 --- a/pipelines/build/common/build_base_file.groovy +++ b/pipelines/build/common/build_base_file.groovy @@ -216,7 +216,7 @@ class Builder implements Serializable { return '' } /* - * Get reproduciableCompare flag from the build configurations. + Get reproduciableCompare flag from the build configurations. */ Boolean getReproducibleCompare(Map configuration, String variant) { Boolean enableReproducibleCompare = DEFAULTS_JSON['testDetails']['enableReproducibleCompare'] as Boolean @@ -231,6 +231,7 @@ class Builder implements Serializable { } return enableReproducibleCompare } + /* Get the list of tests to run from the build configurations. We run different test categories depending on if this build is a release or nightly. This function parses and applies this to the individual build config. diff --git a/pipelines/build/common/config_regeneration.groovy b/pipelines/build/common/config_regeneration.groovy index f5c701989..b28195203 100644 --- a/pipelines/build/common/config_regeneration.groovy +++ b/pipelines/build/common/config_regeneration.groovy @@ -352,6 +352,7 @@ class Regeneration implements Serializable { } return enableReproducibleCompare } + /* * Get the list of tests from jdk*_pipeline_config.groovy. Used when creating the IndividualBuildConfig. Used as a placeholder since the pipelines overwrite this. * @param configuration diff --git a/pipelines/jobs/release_pipeline_job_template.groovy b/pipelines/jobs/release_pipeline_job_template.groovy index 2dfeb83ea..789f8dd60 100644 --- a/pipelines/jobs/release_pipeline_job_template.groovy +++ b/pipelines/jobs/release_pipeline_job_template.groovy @@ -78,7 +78,7 @@ pipelineJob("${BUILD_FOLDER}/${JOB_NAME}") { stringParam('baseFilePath', '', "Relative path to where the build_base_file.groovy file is located. This runs the downstream job setup and configuration retrieval services.
Default: ${defaultsJson['baseFileDirectories']['upstream']}") stringParam('buildConfigFilePath', '', "Relative path to where the jdkxx_pipeline_config.groovy file is located. It contains the build configurations for each platform, architecture and variant.
Default: ${defaultsJson['configDirectories']['build']}/jdkxx_pipeline_config.groovy") booleanParam('aqaAutoGen', false, 'If set to true, force auto generate AQA test jobs. Defaults to false') - booleanParam('enableReproducibleCompare', runReproducibleCompare, 'If set to true the reproducible compare job might be triggerred') + booleanParam('enableReproducibleCompare', runReproducibleCompare, 'If set to true the reproducible compare job might be triggered') booleanParam('enableTests', runTests, 'If set to true the test pipeline will be executed') booleanParam('enableTestDynamicParallel', runParallel, 'If set to true test will be run parallel') booleanParam('enableInstallers', runInstaller, 'If set to true the installer pipeline will be executed') diff --git a/tools/reproduce_comparison/Jenkinsfile b/tools/reproduce_comparison/Jenkinsfile index 497802156..a895c23bc 100644 --- a/tools/reproduce_comparison/Jenkinsfile +++ b/tools/reproduce_comparison/Jenkinsfile @@ -94,14 +94,14 @@ pipeline { def originalJDKFile = findFiles(glob: "original/*.tar.gz") def reproducedJDKFile = findFiles(glob: "reproduced/*.tar.gz") - def result1 = sh returnStatus: true, script: "tar -xz ${originalJDKFile[0].name} -C original" - if (result1 != 0 ) { + def untarOriginalJDK = sh returnStatus: true, script: "tar -xz ${originalJDKFile[0].name} -C original" + if (untarOriginalJDK != 0 ) { currentBuild.result = 'UNSTABLE' error " Untar ${originalJDKFile[0].name} failed" return } - def result2 = sh returnStatus: true, script: "tar -xz ${reproducedJDKFile[0].name} -C reproduced" - if (result2 != 0 ) { + def untarReproducedJDK = sh returnStatus: true, script: "tar -xz ${reproducedJDKFile[0].name} -C reproduced" + if (untarReproducedJDK != 0 ) { currentBuild.result = 'UNSTABLE' error " Untar ${reproducedJDKFile[0].name} failed" return From 95a4529df8dc0b8b3164e5ccd88a9a2721d09665 Mon Sep 17 00:00:00 2001 From: Sophia Guo Date: Thu, 8 Jun 2023 12:13:49 -0400 Subject: [PATCH 08/12] Update test case Signed-off-by: Sophia Guo --- pipelines/jobs/configurations/jdk17u_pipeline_config.groovy | 2 +- pipelines/src/test/groovy/RepoHandlerTest.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pipelines/jobs/configurations/jdk17u_pipeline_config.groovy b/pipelines/jobs/configurations/jdk17u_pipeline_config.groovy index 25a320c99..272e95653 100644 --- a/pipelines/jobs/configurations/jdk17u_pipeline_config.groovy +++ b/pipelines/jobs/configurations/jdk17u_pipeline_config.groovy @@ -28,7 +28,7 @@ class Config17 { test : 'default', reproducibleCompare : [ 'temurin' : true - ] + ], additionalTestLabels: [ openj9 : '!(centos6||rhel6)' ], diff --git a/pipelines/src/test/groovy/RepoHandlerTest.groovy b/pipelines/src/test/groovy/RepoHandlerTest.groovy index 57498ead2..1a410c302 100644 --- a/pipelines/src/test/groovy/RepoHandlerTest.groovy +++ b/pipelines/src/test/groovy/RepoHandlerTest.groovy @@ -120,7 +120,7 @@ class RepoHandlerTest { // Test details Assertions.assertTrue(userJson.testDetails.enableReproducibleCompare instanceof Boolean) - Assertions.assertTrue(userJson.testDetails.enableReproducibleCompare) + Assertions.assertFalse(userJson.testDetails.enableReproducibleCompare) Assertions.assertTrue(userJson.testDetails.enableTests instanceof Boolean) Assertions.assertTrue(userJson.testDetails.enableTests) Assertions.assertEquals(userJson.testDetails.nightlyDefault, [ 'test1', 'test2', 'test3' ]) From 685f068a773f7bae06a76b50ba4eb2fb547223bd Mon Sep 17 00:00:00 2001 From: Sophia Guo Date: Thu, 8 Jun 2023 12:37:32 -0400 Subject: [PATCH 09/12] Remove unnecessary file permission step Signed-off-by: Sophia Guo --- tools/reproduce_comparison/Jenkinsfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/reproduce_comparison/Jenkinsfile b/tools/reproduce_comparison/Jenkinsfile index a895c23bc..fefe9aacd 100644 --- a/tools/reproduce_comparison/Jenkinsfile +++ b/tools/reproduce_comparison/Jenkinsfile @@ -117,11 +117,9 @@ pipeline { dir('temurin-build/tools/reproduce_comparison') { def rc = 0 if (COMPARED_JOB_NAME.contains('linux')) { - sh "chmod 776 ./tooling/linux_repo_compare.sh" rc = sh returnStatus: true, script: "./tooling/linux_repo_compare.sh temurin ./original temurin ./reproduced" } else if (COMPARED_JOB_NAME.contains('mac')) { // mac - //sh "chmod 776 ./tooling/mac_repo_compare.sh" //sh "./tooling/mac_repo_compare.sh temurin ./original temurin ./reproduced ${self_cert_file} ${self_cert_passwd}" echo 'todo mac build comparison' } else if (COMPARED_JOB_NAME.contains('windows')) { From a3c64045e4cb24b472c4e5c1bed24ecf60df8d4e Mon Sep 17 00:00:00 2001 From: Sophia Guo Date: Thu, 8 Jun 2023 15:37:35 -0400 Subject: [PATCH 10/12] Typo Signed-off-by: Sophia Guo --- docs/UsingOurScripts.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/UsingOurScripts.md b/docs/UsingOurScripts.md index 944ea4b05..b98a71b53 100644 --- a/docs/UsingOurScripts.md +++ b/docs/UsingOurScripts.md @@ -78,7 +78,7 @@ This file contains the default constants and paths used in the build scripts for }, // Test suite and inclusion details "testDetails" : { - // Boolean denoting wether the reproducible compare is needed + // Boolean denoting whether the reproducible compare is needed "enableReproducibleCompare" : false, // Boolean denoting whether pipeline tests will be enabled or disabled by default "enableTests" : true, From 3efb4decb6f8203a71340409b220adf27a0c793c Mon Sep 17 00:00:00 2001 From: Sophia Guo Date: Sat, 10 Jun 2023 09:09:14 -0400 Subject: [PATCH 11/12] Fix isRelease and params correctly Signed-off-by: Sophia Guo --- pipelines/build/common/openjdk_build_pipeline.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipelines/build/common/openjdk_build_pipeline.groovy b/pipelines/build/common/openjdk_build_pipeline.groovy index 1756a084f..d4f42738d 100644 --- a/pipelines/build/common/openjdk_build_pipeline.groovy +++ b/pipelines/build/common/openjdk_build_pipeline.groovy @@ -595,10 +595,10 @@ class Build { buildConfig.ARCHITECTURE.contains('x64') && buildConfig.TARGET_OS.contains('linux') && buildConfig.VARIANT == 'temurin' && - !isRelease) { + !Boolean.valueOf(buildConfig.RELEASE)) { // For now set the build as independent, no need to wait for result as the build takes time context.stage('Reproduce Compare') { - def buildParams = params.toString() + def buildParams = context.params.toString() // passing buildParams multiline parameter to downstream job, double check the available method context.build job: jobName, propagate: false, From 094fc434ffbf36f7246796a4cceac455632d1ddf Mon Sep 17 00:00:00 2001 From: Sophia Guo Date: Mon, 12 Jun 2023 09:13:48 -0400 Subject: [PATCH 12/12] Limit json file to SBOM specific and remove unnecessary code Signed-off-by: Sophia Guo --- tools/reproduce_comparison/Jenkinsfile | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/tools/reproduce_comparison/Jenkinsfile b/tools/reproduce_comparison/Jenkinsfile index fefe9aacd..86b8d293c 100644 --- a/tools/reproduce_comparison/Jenkinsfile +++ b/tools/reproduce_comparison/Jenkinsfile @@ -15,7 +15,7 @@ pipeline { steps { cleanWs() checkout scm - copyArtifacts excludes: '**/OpenJDK*-sbom*metadata.json', + copyArtifacts excludes: '**/OpenJDK*-sbom*metadata.json,**/OpenJDK*-jdk-sources*.tar.gz', filter: '**/OpenJDK*-jdk*.tar.gz,**/OpenJDK*-sbom*.json', fingerprintArtifacts: true, flatten: true, @@ -23,15 +23,15 @@ pipeline { target: 'original/' selector: specific("${params.COMPARED_JOB_NUMBER}") script { - def sbomFiles = findFiles(glob: "**/*.json") - def sbomParams = readJSON file: "${sbomFiles[0].name}" + def sbomFiles = findFiles(glob: "**/OpenJDK*-sbom*.json") + def sbomParams = readJSON file: "${sbomFiles[0].path}" def sbomProperties = sbomParams.components[0].properties def reproducedParams = [:] sbomProperties.each { propertyItem -> def paramFound = false propertyItem.each { key, value -> - if (value in ["SCM Ref", "OpenJDK Source Commit", "Temurin Build Ref", "Build Timestamp"]) { + if (value in ["SCM Ref", "Temurin Build Ref", "Build Timestamp"]) { paramFound = true } } @@ -41,11 +41,9 @@ pipeline { } def scmRef = reproducedParams["SCM Ref"] - def buildRef = reproducedParams["Temurin Build Ref"] def buildTimeStamp = reproducedParams["Build Timestamp"] + def buildRef = reproducedParams["Temurin Build Ref"] buildRef = buildRef.substring(buildRef.lastIndexOf('/')+1) - def scmCommit = reproducedParams["OpenJDK Source Commit"] - scmCommit = scmCommit.substring(scmCommit.lastIndexOf('/')+1) // Convert COMPARED_JOB_PARAMS to json formatt string def jobParams = COMPARED_JOB_PARAMS.replaceAll("=", ':') @@ -58,7 +56,7 @@ pipeline { def jsonJobParams = new groovy.json.JsonSlurper().parseText(jobParams) jsonJobParams.BUILD_CONFIGURATION.SCM_REF = scmRef jsonJobParams.BUILD_CONFIGURATION.BUILD_REF = buildRef - jsonJobParams.BUILD_CONFIGURATION.BUILD_ARGS += " --build-reproducible-date ${buildTimeStamp}" //'${buildTimeStamp}' + jsonJobParams.BUILD_CONFIGURATION.BUILD_ARGS += " --build-reproducible-date ${buildTimeStamp}" jsonJobParams.BUILD_CONFIGURATION.ENABLE_REPRODUCIBLE_COMPARE = "false" def buildParams = [ @@ -78,7 +76,8 @@ pipeline { copyArtifacts( projectName:"${COMPARED_JOB_NAME}", selector:specific("${reproducibleJob.getNumber()}"), - filter: "**/*.tar.gz", + excludes: '**/OpenJDK*-jdk-sources*.tar.gz', + filter: '**/OpenJDK*-jdk*.tar.gz', target: 'reproduced/', fingerprintArtifacts: true, flatten: true