birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #4250 from aws-lumberyard-dev/pipelines/retry-pipeline

Browse Source 
Retry pipeline stages when a node goes offline
This commit is contained in:
Brian Herrera
2021-09-22 09:11:16 -07:00
committed by GitHub
parent 4e6c49ac33 eaddd4eee7
commit aaf292f8cd
1 changed files with 61 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/build/Jenkins/Jenkinsfile
Vendored
+61 -49
View File
@@ -9,6 +9,7 @@
import groovy.json.JsonOutput import groovy.json.JsonOutput
PIPELINE_CONFIG_FILE = 'scripts/build/Jenkins/lumberyard.json' PIPELINE_CONFIG_FILE = 'scripts/build/Jenkins/lumberyard.json'
INCREMENTAL_BUILD_SCRIPT_PATH = 'scripts/build/bootstrap/incremental_build_util.py' INCREMENTAL_BUILD_SCRIPT_PATH = 'scripts/build/bootstrap/incremental_build_util.py'
PIPELINE_RETRY_ATTEMPTS = 3
EMPTY_JSON = readJSON text: '{}' EMPTY_JSON = readJSON text: '{}'
@@ -502,64 +503,75 @@ def CreateTeardownStage(Map environmentVars) {
def CreateSingleNode(Map pipelineConfig, def platform, def build_job, Map envVars, String branchName, String pipelineName, String repositoryName, String projectName, boolean onlyMountEBSVolume = false) { def CreateSingleNode(Map pipelineConfig, def platform, def build_job, Map envVars, String branchName, String pipelineName, String repositoryName, String projectName, boolean onlyMountEBSVolume = false) {
def nodeLabel = envVars['NODE_LABEL'] def nodeLabel = envVars['NODE_LABEL']
return { return {
node("${nodeLabel}") { def currentResult = ''
if(isUnix()) { // Has to happen inside a node def currentException = ''
envVars['IS_UNIX'] = 1 retry(PIPELINE_RETRY_ATTEMPTS) {
} node("${nodeLabel}") {
withEnv(GetEnvStringList(envVars)) { if(isUnix()) { // Has to happen inside a node
def build_job_name = build_job.key envVars['IS_UNIX'] = 1
try { }
CreateSetupStage(pipelineConfig, snapshot, repositoryName, projectName, pipelineName, branchName, platform.key, build_job.key, envVars, onlyMountEBSVolume).call() withEnv(GetEnvStringList(envVars)) {
def build_job_name = build_job.key
try {
CreateSetupStage(pipelineConfig, snapshot, repositoryName, projectName, pipelineName, branchName, platform.key, build_job.key, envVars, onlyMountEBSVolume).call()
if(build_job.value.steps) { //this is a pipe with many steps so create all the build stages if(build_job.value.steps) { //this is a pipe with many steps so create all the build stages
build_job.value.steps.each { build_step -> build_job.value.steps.each { build_step ->
build_job_name = build_step build_job_name = build_step
envVars = GetBuildEnvVars(platform.value.PIPELINE_ENV ?: EMPTY_JSON, platform.value.build_types[build_step].PIPELINE_ENV ?: EMPTY_JSON, pipelineName) envVars = GetBuildEnvVars(platform.value.PIPELINE_ENV ?: EMPTY_JSON, platform.value.build_types[build_step].PIPELINE_ENV ?: EMPTY_JSON, pipelineName)
try { try {
CreateBuildStage(pipelineConfig, platform.key, build_step, envVars).call() CreateBuildStage(pipelineConfig, platform.key, build_step, envVars).call()
}
catch (Exception e) {
if (envVars['NONBLOCKING_STEP']?.toBoolean()) {
unstable(message: "Build step ${build_step} failed but it's a non-blocking step in build job ${build_job.key}")
} else {
throw e
}
}
} }
catch (Exception e) { } else {
if (envVars['NONBLOCKING_STEP']?.toBoolean()) { CreateBuildStage(pipelineConfig, platform.key, build_job.key, envVars).call()
unstable(message: "Build step ${build_step} failed but it's a non-blocking step in build job ${build_job.key}") }
} }
else { catch(Exception e) {
error "FAILURE: ${e}" if (e instanceof org.jenkinsci.plugins.workflow.steps.FlowInterruptedException) {
} def causes = e.getCauses().toString()
if (causes.contains('RemovedNodeCause')) {
error "Node disconnected during build: ${e}" // Error raised to retry stage on a new node
} }
} }
} else { // All other errors will be raised outside the retry block
CreateBuildStage(pipelineConfig, platform.key, build_job.key, envVars).call() currentResult = envVars['ON_FAILURE_MARK'] ?: 'FAILURE'
currentException = e.toString()
} }
} finally {
catch(Exception e) { def params = platform.value.build_types[build_job_name].PARAMETERS
// https://github.com/jenkinsci/jenkins/blob/master/core/src/main/java/hudson/model/Result.java if (env.MARS_REPO && params && params.containsKey('TEST_METRICS') && params.TEST_METRICS == 'True') {
// {SUCCESS,UNSTABLE,FAILURE,NOT_BUILT,ABORTED} def output_directory = params.OUTPUT_DIRECTORY
def currentResult = envVars['ON_FAILURE_MARK'] ?: 'FAILURE' def configuration = params.CONFIGURATION
if (currentResult == 'FAILURE') { CreateTestMetricsStage(pipelineConfig, branchName, envVars, build_job_name, output_directory, configuration).call()
currentBuild.result = 'FAILURE' }
error "FAILURE: ${e}" if (params && params.containsKey('TEST_RESULTS') && params.TEST_RESULTS == 'True') {
} else if (currentResult == 'UNSTABLE') { CreateExportTestResultsStage(pipelineConfig, platform.key, build_job_name, envVars, params).call()
currentBuild.result = 'UNSTABLE' }
unstable(message: "UNSTABLE: ${e}") if (params && params.containsKey('TEST_SCREENSHOTS') && params.TEST_SCREENSHOTS == 'True' && currentResult == 'FAILURE') {
CreateExportTestScreenshotsStage(pipelineConfig, platform.key, build_job_name, envVars, params).call()
}
CreateTeardownStage(envVars).call()
} }
} }
finally {
def params = platform.value.build_types[build_job_name].PARAMETERS
if (env.MARS_REPO && params && params.containsKey('TEST_METRICS') && params.TEST_METRICS == 'True') {
def output_directory = params.OUTPUT_DIRECTORY
def configuration = params.CONFIGURATION
CreateTestMetricsStage(pipelineConfig, branchName, envVars, build_job_name, output_directory, configuration).call()
}
if (params && params.containsKey('TEST_RESULTS') && params.TEST_RESULTS == 'True') {
CreateExportTestResultsStage(pipelineConfig, platform.key, build_job_name, envVars, params).call()
}
if (params && params.containsKey('TEST_SCREENSHOTS') && params.TEST_SCREENSHOTS == 'True' && currentResult == 'FAILURE') {
CreateExportTestScreenshotsStage(pipelineConfig, platform.key, build_job_name, envVars, params).call()
}
CreateTeardownStage(envVars).call()
}
} }
} }
// https://github.com/jenkinsci/jenkins/blob/master/core/src/main/java/hudson/model/Result.java
// {SUCCESS,UNSTABLE,FAILURE,NOT_BUILT,ABORTED}
if (currentResult == 'FAILURE') {
currentBuild.result = 'FAILURE'
error "FAILURE: ${currentException}"
} else if (currentResult == 'UNSTABLE') {
currentBuild.result = 'UNSTABLE'
unstable(message: "UNSTABLE: ${currentException}")
}
} }
} }