Merge pull request #4250 from aws-lumberyard-dev/pipelines/retry-pipeline

Retry pipeline stages when a node goes offline

scripts/build/Jenkins/Jenkinsfile

@@ -9,6 +9,7 @@
 import groovy.json.JsonOutput
 PIPELINE_CONFIG_FILE = 'scripts/build/Jenkins/lumberyard.json'
 INCREMENTAL_BUILD_SCRIPT_PATH = 'scripts/build/bootstrap/incremental_build_util.py'
+PIPELINE_RETRY_ATTEMPTS = 3
 
 EMPTY_JSON = readJSON text: '{}'
 
@@ -502,6 +503,9 @@ def CreateTeardownStage(Map environmentVars) {
 def CreateSingleNode(Map pipelineConfig, def platform, def build_job, Map envVars, String branchName, String pipelineName, String repositoryName, String projectName, boolean onlyMountEBSVolume = false) {
     def nodeLabel = envVars['NODE_LABEL']
     return {
+        def currentResult = ''
+        def currentException = ''
+        retry(PIPELINE_RETRY_ATTEMPTS) {
             node("${nodeLabel}") {
                 if(isUnix()) { // Has to happen inside a node
                     envVars['IS_UNIX'] = 1
@@ -521,9 +525,8 @@ def CreateSingleNode(Map pipelineConfig, def platform, def build_job, Map envVar
                                 catch (Exception e) {
                                     if (envVars['NONBLOCKING_STEP']?.toBoolean()) {
                                         unstable(message: "Build step ${build_step} failed but it's a non-blocking step in build job ${build_job.key}")
-                                }
+                                    } else {
-                                else {
+                                        throw e
-                                    error "FAILURE: ${e}"
                                     }
                                 }
                             }
@@ -532,17 +535,16 @@ def CreateSingleNode(Map pipelineConfig, def platform, def build_job, Map envVar
                         }
                     }
                     catch(Exception e) {
-                    //  https://github.com/jenkinsci/jenkins/blob/master/core/src/main/java/hudson/model/Result.java
+                        if (e instanceof org.jenkinsci.plugins.workflow.steps.FlowInterruptedException) {
-                    //  {SUCCESS,UNSTABLE,FAILURE,NOT_BUILT,ABORTED}
+                            def causes = e.getCauses().toString()
-                    def currentResult = envVars['ON_FAILURE_MARK'] ?: 'FAILURE'
+                            if (causes.contains('RemovedNodeCause')) {
-                    if (currentResult == 'FAILURE') {
+                                error "Node disconnected during build: ${e}"  // Error raised to retry stage on a new node
-                        currentBuild.result = 'FAILURE'
-                        error "FAILURE: ${e}"
-                    } else if (currentResult == 'UNSTABLE') {
-                        currentBuild.result = 'UNSTABLE'
-                        unstable(message: "UNSTABLE: ${e}")
                             }
                         }
+                        // All other errors will be raised outside the retry block
+                        currentResult = envVars['ON_FAILURE_MARK'] ?: 'FAILURE'
+                        currentException = e.toString()
+                    }
                     finally {
                         def params = platform.value.build_types[build_job_name].PARAMETERS
                         if (env.MARS_REPO && params && params.containsKey('TEST_METRICS') && params.TEST_METRICS == 'True') {
@@ -561,6 +563,16 @@ def CreateSingleNode(Map pipelineConfig, def platform, def build_job, Map envVar
                 }
             }
         }
+        // https://github.com/jenkinsci/jenkins/blob/master/core/src/main/java/hudson/model/Result.java
+        // {SUCCESS,UNSTABLE,FAILURE,NOT_BUILT,ABORTED}
+        if (currentResult == 'FAILURE') {
+            currentBuild.result = 'FAILURE'
+            error "FAILURE: ${currentException}"
+        } else if (currentResult == 'UNSTABLE') {
+            currentBuild.result = 'UNSTABLE'
+            unstable(message: "UNSTABLE: ${currentException}")
+        }
+    }
 }
 
 // Used in CreateBuildJobs() to preprocess the build_job steps to programically create