From 3ade0987421c85bc74ec46c53acd1fbe5796f523 Mon Sep 17 00:00:00 2001 From: amzn-sean <75276488+amzn-sean@users.noreply.github.com> Date: Mon, 26 Jul 2021 11:16:13 +0100 Subject: [PATCH] support node switching in a pipe (#2345) Signed-off-by: amzn-sean <75276488+amzn-sean@users.noreply.github.com> --- scripts/build/Jenkins/Jenkinsfile | 232 +++++++++++++++++++++--------- 1 file changed, 161 insertions(+), 71 deletions(-) diff --git a/scripts/build/Jenkins/Jenkinsfile b/scripts/build/Jenkins/Jenkinsfile index ae6ac59c3c..9aa09e136e 100644 --- a/scripts/build/Jenkins/Jenkinsfile +++ b/scripts/build/Jenkins/Jenkinsfile @@ -266,32 +266,42 @@ def CheckoutRepo(boolean disableSubmodules = false) { palRm('commitdate') } +def HandleDriveMount(String snapshot, String repositoryName, String projectName, String pipeline, String branchName, String platform, String buildType, String workspace, boolean recreateVolume = false) { + unstash name: 'incremental_build_script' + + def pythonCmd = '' + if(env.IS_UNIX) pythonCmd = 'sudo -E python3 -u ' + else pythonCmd = 'python3 -u ' + + if(recreateVolume) { + palSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action delete --repository_name ${repositoryName} --project ${projectName} --pipeline ${pipeline} --branch ${branchName} --platform ${platform} --build_type ${buildType}", 'Deleting volume', winSlashReplacement=false) + } + timeout(5) { + palSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action mount --snapshot ${snapshot} --repository_name ${repositoryName} --project ${projectName} --pipeline ${pipeline} --branch ${branchName} --platform ${platform} --build_type ${buildType}", 'Mounting volume', winSlashReplacement=false) + } + + if(env.IS_UNIX) { + sh label: 'Setting volume\'s ownership', + script: """ + if sudo test ! -d "${workspace}"; then + sudo mkdir -p ${workspace} + cd ${workspace}/.. + sudo chown -R lybuilder:root . + fi + """ + } +} + def PreBuildCommonSteps(Map pipelineConfig, String snapshot, String repositoryName, String projectName, String pipeline, String branchName, String platform, String buildType, String workspace, boolean mount = true, boolean disableSubmodules = false) { echo 'Starting pre-build common steps...' if (mount) { - unstash name: 'incremental_build_script' - - def pythonCmd = '' - if(env.IS_UNIX) pythonCmd = 'sudo -E python3 -u ' - else pythonCmd = 'python3 -u ' - - if(env.RECREATE_VOLUME?.toBoolean()) { - palSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action delete --repository_name ${repositoryName} --project ${projectName} --pipeline ${pipeline} --branch ${branchName} --platform ${platform} --build_type ${buildType}", 'Deleting volume', winSlashReplacement=false) - } - timeout(5) { - palSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action mount --snapshot ${snapshot} --repository_name ${repositoryName} --project ${projectName} --pipeline ${pipeline} --branch ${branchName} --platform ${platform} --build_type ${buildType}", 'Mounting volume', winSlashReplacement=false) - } - - if(env.IS_UNIX) { - sh label: 'Setting volume\'s ownership', - script: """ - if sudo test ! -d "${workspace}"; then - sudo mkdir -p ${workspace} - cd ${workspace}/.. - sudo chown -R lybuilder:root . - fi - """ + if(env.RECREATE_VOLUME?.toBoolean()){ + echo 'Starting to recreating drive...' + HandleDriveMount(snapshot, repositoryName, projectName, pipeline, branchName, platform, buildType, workspace, true) + } else { + echo 'Starting to mounting drive...' + HandleDriveMount(snapshot, repositoryName, projectName, pipeline, branchName, platform, buildType, workspace, false) } } @@ -399,10 +409,14 @@ def PostBuildCommonSteps(String workspace, boolean mount = true) { } } -def CreateSetupStage(Map pipelineConfig, String snapshot, String repositoryName, String projectName, String pipelineName, String branchName, String platformName, String jobName, Map environmentVars) { +def CreateSetupStage(Map pipelineConfig, String snapshot, String repositoryName, String projectName, String pipelineName, String branchName, String platformName, String jobName, Map environmentVars, boolean onlyMountEBSVolume = false) { return { stage('Setup') { - PreBuildCommonSteps(pipelineConfig, snapshot, repositoryName, projectName, pipelineName, branchName, platformName, jobName, environmentVars['WORKSPACE'], environmentVars['MOUNT_VOLUME']) + if(onlyMountEBSVolume) { + HandleDriveMount(snapshot, repositoryName, projectName, pipelineName, branchName, platformName, jobName, environmentVars['WORKSPACE'], false) + } else { + PreBuildCommonSteps(pipelineConfig, snapshot, repositoryName, projectName, pipelineName, branchName, platformName, jobName, environmentVars['WORKSPACE'], environmentVars['MOUNT_VOLUME']) + } } } } @@ -439,6 +453,128 @@ def CreateTeardownStage(Map environmentVars) { } } +def CreateSingleNode(Map pipelineConfig, def platform, def build_job, Map envVars, String branchName, String pipelineName, String repositoryName, String projectName, boolean onlyMountEBSVolume = false) { + def nodeLabel = envVars['NODE_LABEL'] + return { + node("${nodeLabel}") { + if(isUnix()) { // Has to happen inside a node + envVars['IS_UNIX'] = 1 + } + withEnv(GetEnvStringList(envVars)) { + def build_job_name = build_job.key + try { + CreateSetupStage(pipelineConfig, snapshot, repositoryName, projectName, pipelineName, branchName, platform.key, build_job.key, envVars, onlyMountEBSVolume).call() + + if(build_job.value.steps) { //this is a pipe with many steps so create all the build stages + build_job.value.steps.each { build_step -> + build_job_name = build_step + CreateBuildStage(pipelineConfig, platform.key, build_step, envVars).call() + } + } else { + CreateBuildStage(pipelineConfig, platform.key, build_job.key, envVars).call() + } + } + catch(Exception e) { + // https://github.com/jenkinsci/jenkins/blob/master/core/src/main/java/hudson/model/Result.java + // {SUCCESS,UNSTABLE,FAILURE,NOT_BUILT,ABORTED} + def currentResult = envVars['ON_FAILURE_MARK'] ?: 'FAILURE' + if (currentResult == 'FAILURE') { + currentBuild.result = 'FAILURE' + error "FAILURE: ${e}" + } else if (currentResult == 'UNSTABLE') { + currentBuild.result = 'UNSTABLE' + unstable(message: "UNSTABLE: ${e}") + } + } + finally { + def params = platform.value.build_types[build_job_name].PARAMETERS + if (env.MARS_REPO && params && params.containsKey('TEST_METRICS') && params.TEST_METRICS == 'True') { + def output_directory = params.OUTPUT_DIRECTORY + def configuration = params.CONFIGURATION + CreateTestMetricsStage(pipelineConfig, branchName, envVars, build_job_name, output_directory, configuration).call() + } + if (params && params.containsKey('TEST_RESULTS') && params.TEST_RESULTS == 'True') { + CreateExportTestResultsStage(pipelineConfig, platform.key, build_job_name, envVars, params).call() + } + CreateTeardownStage(envVars).call() + } + } + } + } +} + +// Used in CreateBuildJobs() to preprocess the build_job steps to programically create +// Node sections with a set of steps that can run on that node. +class PipeStepJobData { + String m_nodeLabel = "" + def m_steps = [] + + PipeStepJobData(String label) { + this.m_nodeLabel = label + } + + def addStep(def step) { + this.m_steps.add(step) + } +} + +def CreateBuildJobs(Map pipelineConfig, def platform, def build_job, Map envVars, String branchName, String pipelineName, String repositoryName, String projectName) { + + // if this is a pipeline, split jobs based on the NODE_LABEL + if(build_job.value.steps) { + def defaultLabel = envVars['NODE_LABEL'] + def lastNodeLable = "" + def jobList = [] + def currentIdx = -1; + + // iterate the steps to build the order of node label + steps sets. + // Order matters, as it is executed from first to last. + // example layout. + // node A + // step 1 + // step 2 + // node B + // step 3 + // node C + // step 4 + build_job.value.steps.each { build_step -> + //if node label defined + if(platform.value.build_types[build_step] && platform.value.build_types[build_step].PIPELINE_ENV && + platform.value.build_types[build_step].PIPELINE_ENV['NODE_LABEL']) { + + //if the last node label doen't match the new one, append it. + if(platform.value.build_types[build_step].PIPELINE_ENV['NODE_LABEL'] != lastNodeLable) { + lastNodeLable = platform.value.build_types[build_step].PIPELINE_ENV['NODE_LABEL'] + jobList.add(new PipeStepJobData(lastNodeLable)) + currentIdx++ + } + } + //no label define, so it needs to run on the default node label + else if(lastNodeLable != defaultLabel) { //if the last node is not the default, append default + lastNodeLable = defaultLabel + jobList.add(new PipeStepJobData(lastNodeLable)) + currentIdx++ + } + //add the build_step to the current node + jobList[currentIdx].addStep(build_step) + } + + return { + jobList.eachWithIndex{ element, idx -> + //update the node label + steps to the discovered data + envVars['NODE_LABEL'] = element.m_nodeLabel + build_job.value.steps = element.m_steps + //no any additional nodes just mount the drive, do not handle clean parameters as that will be done by the first node. + boolean onlyMountEBSVolume = idx != 0; + //add this node + CreateSingleNode(pipelineConfig, platform, build_job, envVars, branchName, pipelineName, repositoryName, projectName, onlyMountEBSVolume).call() + } + } + } else { + return CreateSingleNode(pipelineConfig, platform, build_job, envVars, branchName, pipelineName, repositoryName, projectName) + } +} + def projectName = '' def pipelineName = '' def branchName = '' @@ -527,55 +663,9 @@ try { if (IsJobEnabled(branchName, build_job, pipelineName, platform.key)) { // User can filter jobs, jobs are tagged by pipeline def envVars = GetBuildEnvVars(platform.value.PIPELINE_ENV ?: EMPTY_JSON, build_job.value.PIPELINE_ENV ?: EMPTY_JSON, pipelineName) envVars['JOB_NAME'] = "${branchName}_${platform.key}_${build_job.key}" // backwards compatibility, some scripts rely on this - def nodeLabel = envVars['NODE_LABEL'] someBuildHappened = true - buildConfigs["${platform.key} [${build_job.key}]"] = { - node("${nodeLabel}") { - if(isUnix()) { // Has to happen inside a node - envVars['IS_UNIX'] = 1 - } - withEnv(GetEnvStringList(envVars)) { - def build_job_name = build_job.key - try { - CreateSetupStage(pipelineConfig, snapshot, repositoryName, projectName, pipelineName, branchName, platform.key, build_job.key, envVars).call() - - if(build_job.value.steps) { //this is a pipe with many steps so create all the build stages - build_job.value.steps.each { build_step -> - build_job_name = build_step - CreateBuildStage(pipelineConfig, platform.key, build_step, envVars).call() - } - } else { - CreateBuildStage(pipelineConfig, platform.key, build_job.key, envVars).call() - } - } - catch(Exception e) { - // https://github.com/jenkinsci/jenkins/blob/master/core/src/main/java/hudson/model/Result.java - // {SUCCESS,UNSTABLE,FAILURE,NOT_BUILT,ABORTED} - def currentResult = envVars['ON_FAILURE_MARK'] ?: 'FAILURE' - if (currentResult == 'FAILURE') { - currentBuild.result = 'FAILURE' - error "FAILURE: ${e}" - } else if (currentResult == 'UNSTABLE') { - currentBuild.result = 'UNSTABLE' - unstable(message: "UNSTABLE: ${e}") - } - } - finally { - def params = platform.value.build_types[build_job_name].PARAMETERS - if (env.MARS_REPO && params && params.containsKey('TEST_METRICS') && params.TEST_METRICS == 'True') { - def output_directory = params.OUTPUT_DIRECTORY - def configuration = params.CONFIGURATION - CreateTestMetricsStage(pipelineConfig, branchName, envVars, build_job_name, output_directory, configuration).call() - } - if (params && params.containsKey('TEST_RESULTS') && params.TEST_RESULTS == 'True') { - CreateExportTestResultsStage(pipelineConfig, platform.key, build_job_name, envVars, params).call() - } - CreateTeardownStage(envVars).call() - } - } - } - } + buildConfigs["${platform.key} [${build_job.key}]"] = CreateBuildJobs(pipelineConfig, platform, build_job, envVars, branchName, pipelineName, repositoryName, projectName) } } }