support node switching in a pipe (#2345)
Signed-off-by: amzn-sean <75276488+amzn-sean@users.noreply.github.com>
This commit is contained in:
amzn-sean
GitHub
Vendored
+161
-71
@@ -266,32 +266,42 @@ def CheckoutRepo(boolean disableSubmodules = false) {
|
||||
palRm('commitdate')
|
||||
}
|
||||
|
||||
def HandleDriveMount(String snapshot, String repositoryName, String projectName, String pipeline, String branchName, String platform, String buildType, String workspace, boolean recreateVolume = false) {
|
||||
unstash name: 'incremental_build_script'
|
||||
|
||||
def pythonCmd = ''
|
||||
if(env.IS_UNIX) pythonCmd = 'sudo -E python3 -u '
|
||||
else pythonCmd = 'python3 -u '
|
||||
|
||||
if(recreateVolume) {
|
||||
palSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action delete --repository_name ${repositoryName} --project ${projectName} --pipeline ${pipeline} --branch ${branchName} --platform ${platform} --build_type ${buildType}", 'Deleting volume', winSlashReplacement=false)
|
||||
}
|
||||
timeout(5) {
|
||||
palSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action mount --snapshot ${snapshot} --repository_name ${repositoryName} --project ${projectName} --pipeline ${pipeline} --branch ${branchName} --platform ${platform} --build_type ${buildType}", 'Mounting volume', winSlashReplacement=false)
|
||||
}
|
||||
|
||||
if(env.IS_UNIX) {
|
||||
sh label: 'Setting volume\'s ownership',
|
||||
script: """
|
||||
if sudo test ! -d "${workspace}"; then
|
||||
sudo mkdir -p ${workspace}
|
||||
cd ${workspace}/..
|
||||
sudo chown -R lybuilder:root .
|
||||
fi
|
||||
"""
|
||||
}
|
||||
}
|
||||
|
||||
def PreBuildCommonSteps(Map pipelineConfig, String snapshot, String repositoryName, String projectName, String pipeline, String branchName, String platform, String buildType, String workspace, boolean mount = true, boolean disableSubmodules = false) {
|
||||
echo 'Starting pre-build common steps...'
|
||||
|
||||
if (mount) {
|
||||
unstash name: 'incremental_build_script'
|
||||
|
||||
def pythonCmd = ''
|
||||
if(env.IS_UNIX) pythonCmd = 'sudo -E python3 -u '
|
||||
else pythonCmd = 'python3 -u '
|
||||
|
||||
if(env.RECREATE_VOLUME?.toBoolean()) {
|
||||
palSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action delete --repository_name ${repositoryName} --project ${projectName} --pipeline ${pipeline} --branch ${branchName} --platform ${platform} --build_type ${buildType}", 'Deleting volume', winSlashReplacement=false)
|
||||
}
|
||||
timeout(5) {
|
||||
palSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action mount --snapshot ${snapshot} --repository_name ${repositoryName} --project ${projectName} --pipeline ${pipeline} --branch ${branchName} --platform ${platform} --build_type ${buildType}", 'Mounting volume', winSlashReplacement=false)
|
||||
}
|
||||
|
||||
if(env.IS_UNIX) {
|
||||
sh label: 'Setting volume\'s ownership',
|
||||
script: """
|
||||
if sudo test ! -d "${workspace}"; then
|
||||
sudo mkdir -p ${workspace}
|
||||
cd ${workspace}/..
|
||||
sudo chown -R lybuilder:root .
|
||||
fi
|
||||
"""
|
||||
if(env.RECREATE_VOLUME?.toBoolean()){
|
||||
echo 'Starting to recreating drive...'
|
||||
HandleDriveMount(snapshot, repositoryName, projectName, pipeline, branchName, platform, buildType, workspace, true)
|
||||
} else {
|
||||
echo 'Starting to mounting drive...'
|
||||
HandleDriveMount(snapshot, repositoryName, projectName, pipeline, branchName, platform, buildType, workspace, false)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -399,10 +409,14 @@ def PostBuildCommonSteps(String workspace, boolean mount = true) {
|
||||
}
|
||||
}
|
||||
|
||||
def CreateSetupStage(Map pipelineConfig, String snapshot, String repositoryName, String projectName, String pipelineName, String branchName, String platformName, String jobName, Map environmentVars) {
|
||||
def CreateSetupStage(Map pipelineConfig, String snapshot, String repositoryName, String projectName, String pipelineName, String branchName, String platformName, String jobName, Map environmentVars, boolean onlyMountEBSVolume = false) {
|
||||
return {
|
||||
stage('Setup') {
|
||||
PreBuildCommonSteps(pipelineConfig, snapshot, repositoryName, projectName, pipelineName, branchName, platformName, jobName, environmentVars['WORKSPACE'], environmentVars['MOUNT_VOLUME'])
|
||||
if(onlyMountEBSVolume) {
|
||||
HandleDriveMount(snapshot, repositoryName, projectName, pipelineName, branchName, platformName, jobName, environmentVars['WORKSPACE'], false)
|
||||
} else {
|
||||
PreBuildCommonSteps(pipelineConfig, snapshot, repositoryName, projectName, pipelineName, branchName, platformName, jobName, environmentVars['WORKSPACE'], environmentVars['MOUNT_VOLUME'])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -439,6 +453,128 @@ def CreateTeardownStage(Map environmentVars) {
|
||||
}
|
||||
}
|
||||
|
||||
def CreateSingleNode(Map pipelineConfig, def platform, def build_job, Map envVars, String branchName, String pipelineName, String repositoryName, String projectName, boolean onlyMountEBSVolume = false) {
|
||||
def nodeLabel = envVars['NODE_LABEL']
|
||||
return {
|
||||
node("${nodeLabel}") {
|
||||
if(isUnix()) { // Has to happen inside a node
|
||||
envVars['IS_UNIX'] = 1
|
||||
}
|
||||
withEnv(GetEnvStringList(envVars)) {
|
||||
def build_job_name = build_job.key
|
||||
try {
|
||||
CreateSetupStage(pipelineConfig, snapshot, repositoryName, projectName, pipelineName, branchName, platform.key, build_job.key, envVars, onlyMountEBSVolume).call()
|
||||
|
||||
if(build_job.value.steps) { //this is a pipe with many steps so create all the build stages
|
||||
build_job.value.steps.each { build_step ->
|
||||
build_job_name = build_step
|
||||
CreateBuildStage(pipelineConfig, platform.key, build_step, envVars).call()
|
||||
}
|
||||
} else {
|
||||
CreateBuildStage(pipelineConfig, platform.key, build_job.key, envVars).call()
|
||||
}
|
||||
}
|
||||
catch(Exception e) {
|
||||
// https://github.com/jenkinsci/jenkins/blob/master/core/src/main/java/hudson/model/Result.java
|
||||
// {SUCCESS,UNSTABLE,FAILURE,NOT_BUILT,ABORTED}
|
||||
def currentResult = envVars['ON_FAILURE_MARK'] ?: 'FAILURE'
|
||||
if (currentResult == 'FAILURE') {
|
||||
currentBuild.result = 'FAILURE'
|
||||
error "FAILURE: ${e}"
|
||||
} else if (currentResult == 'UNSTABLE') {
|
||||
currentBuild.result = 'UNSTABLE'
|
||||
unstable(message: "UNSTABLE: ${e}")
|
||||
}
|
||||
}
|
||||
finally {
|
||||
def params = platform.value.build_types[build_job_name].PARAMETERS
|
||||
if (env.MARS_REPO && params && params.containsKey('TEST_METRICS') && params.TEST_METRICS == 'True') {
|
||||
def output_directory = params.OUTPUT_DIRECTORY
|
||||
def configuration = params.CONFIGURATION
|
||||
CreateTestMetricsStage(pipelineConfig, branchName, envVars, build_job_name, output_directory, configuration).call()
|
||||
}
|
||||
if (params && params.containsKey('TEST_RESULTS') && params.TEST_RESULTS == 'True') {
|
||||
CreateExportTestResultsStage(pipelineConfig, platform.key, build_job_name, envVars, params).call()
|
||||
}
|
||||
CreateTeardownStage(envVars).call()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Used in CreateBuildJobs() to preprocess the build_job steps to programically create
|
||||
// Node sections with a set of steps that can run on that node.
|
||||
class PipeStepJobData {
|
||||
String m_nodeLabel = ""
|
||||
def m_steps = []
|
||||
|
||||
PipeStepJobData(String label) {
|
||||
this.m_nodeLabel = label
|
||||
}
|
||||
|
||||
def addStep(def step) {
|
||||
this.m_steps.add(step)
|
||||
}
|
||||
}
|
||||
|
||||
def CreateBuildJobs(Map pipelineConfig, def platform, def build_job, Map envVars, String branchName, String pipelineName, String repositoryName, String projectName) {
|
||||
|
||||
// if this is a pipeline, split jobs based on the NODE_LABEL
|
||||
if(build_job.value.steps) {
|
||||
def defaultLabel = envVars['NODE_LABEL']
|
||||
def lastNodeLable = ""
|
||||
def jobList = []
|
||||
def currentIdx = -1;
|
||||
|
||||
// iterate the steps to build the order of node label + steps sets.
|
||||
// Order matters, as it is executed from first to last.
|
||||
// example layout.
|
||||
// node A
|
||||
// step 1
|
||||
// step 2
|
||||
// node B
|
||||
// step 3
|
||||
// node C
|
||||
// step 4
|
||||
build_job.value.steps.each { build_step ->
|
||||
//if node label defined
|
||||
if(platform.value.build_types[build_step] && platform.value.build_types[build_step].PIPELINE_ENV &&
|
||||
platform.value.build_types[build_step].PIPELINE_ENV['NODE_LABEL']) {
|
||||
|
||||
//if the last node label doen't match the new one, append it.
|
||||
if(platform.value.build_types[build_step].PIPELINE_ENV['NODE_LABEL'] != lastNodeLable) {
|
||||
lastNodeLable = platform.value.build_types[build_step].PIPELINE_ENV['NODE_LABEL']
|
||||
jobList.add(new PipeStepJobData(lastNodeLable))
|
||||
currentIdx++
|
||||
}
|
||||
}
|
||||
//no label define, so it needs to run on the default node label
|
||||
else if(lastNodeLable != defaultLabel) { //if the last node is not the default, append default
|
||||
lastNodeLable = defaultLabel
|
||||
jobList.add(new PipeStepJobData(lastNodeLable))
|
||||
currentIdx++
|
||||
}
|
||||
//add the build_step to the current node
|
||||
jobList[currentIdx].addStep(build_step)
|
||||
}
|
||||
|
||||
return {
|
||||
jobList.eachWithIndex{ element, idx ->
|
||||
//update the node label + steps to the discovered data
|
||||
envVars['NODE_LABEL'] = element.m_nodeLabel
|
||||
build_job.value.steps = element.m_steps
|
||||
//no any additional nodes just mount the drive, do not handle clean parameters as that will be done by the first node.
|
||||
boolean onlyMountEBSVolume = idx != 0;
|
||||
//add this node
|
||||
CreateSingleNode(pipelineConfig, platform, build_job, envVars, branchName, pipelineName, repositoryName, projectName, onlyMountEBSVolume).call()
|
||||
}
|
||||
}
|
||||
} else {
|
||||
return CreateSingleNode(pipelineConfig, platform, build_job, envVars, branchName, pipelineName, repositoryName, projectName)
|
||||
}
|
||||
}
|
||||
|
||||
def projectName = ''
|
||||
def pipelineName = ''
|
||||
def branchName = ''
|
||||
@@ -527,55 +663,9 @@ try {
|
||||
if (IsJobEnabled(branchName, build_job, pipelineName, platform.key)) { // User can filter jobs, jobs are tagged by pipeline
|
||||
def envVars = GetBuildEnvVars(platform.value.PIPELINE_ENV ?: EMPTY_JSON, build_job.value.PIPELINE_ENV ?: EMPTY_JSON, pipelineName)
|
||||
envVars['JOB_NAME'] = "${branchName}_${platform.key}_${build_job.key}" // backwards compatibility, some scripts rely on this
|
||||
def nodeLabel = envVars['NODE_LABEL']
|
||||
someBuildHappened = true
|
||||
|
||||
buildConfigs["${platform.key} [${build_job.key}]"] = {
|
||||
node("${nodeLabel}") {
|
||||
if(isUnix()) { // Has to happen inside a node
|
||||
envVars['IS_UNIX'] = 1
|
||||
}
|
||||
withEnv(GetEnvStringList(envVars)) {
|
||||
def build_job_name = build_job.key
|
||||
try {
|
||||
CreateSetupStage(pipelineConfig, snapshot, repositoryName, projectName, pipelineName, branchName, platform.key, build_job.key, envVars).call()
|
||||
|
||||
if(build_job.value.steps) { //this is a pipe with many steps so create all the build stages
|
||||
build_job.value.steps.each { build_step ->
|
||||
build_job_name = build_step
|
||||
CreateBuildStage(pipelineConfig, platform.key, build_step, envVars).call()
|
||||
}
|
||||
} else {
|
||||
CreateBuildStage(pipelineConfig, platform.key, build_job.key, envVars).call()
|
||||
}
|
||||
}
|
||||
catch(Exception e) {
|
||||
// https://github.com/jenkinsci/jenkins/blob/master/core/src/main/java/hudson/model/Result.java
|
||||
// {SUCCESS,UNSTABLE,FAILURE,NOT_BUILT,ABORTED}
|
||||
def currentResult = envVars['ON_FAILURE_MARK'] ?: 'FAILURE'
|
||||
if (currentResult == 'FAILURE') {
|
||||
currentBuild.result = 'FAILURE'
|
||||
error "FAILURE: ${e}"
|
||||
} else if (currentResult == 'UNSTABLE') {
|
||||
currentBuild.result = 'UNSTABLE'
|
||||
unstable(message: "UNSTABLE: ${e}")
|
||||
}
|
||||
}
|
||||
finally {
|
||||
def params = platform.value.build_types[build_job_name].PARAMETERS
|
||||
if (env.MARS_REPO && params && params.containsKey('TEST_METRICS') && params.TEST_METRICS == 'True') {
|
||||
def output_directory = params.OUTPUT_DIRECTORY
|
||||
def configuration = params.CONFIGURATION
|
||||
CreateTestMetricsStage(pipelineConfig, branchName, envVars, build_job_name, output_directory, configuration).call()
|
||||
}
|
||||
if (params && params.containsKey('TEST_RESULTS') && params.TEST_RESULTS == 'True') {
|
||||
CreateExportTestResultsStage(pipelineConfig, platform.key, build_job_name, envVars, params).call()
|
||||
}
|
||||
CreateTeardownStage(envVars).call()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
buildConfigs["${platform.key} [${build_job.key}]"] = CreateBuildJobs(pipelineConfig, platform, build_job, envVars, branchName, pipelineName, repositoryName, projectName)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user