Add overall timeout for the AR pipeline #7261
This commit is contained in:
Brian Herrera
GitHub
Vendored
+118
-113
@@ -12,6 +12,9 @@ INCREMENTAL_BUILD_SCRIPT_PATH = 'scripts/build/bootstrap/incremental_build_util.
|
||||
EBS_SNAPSHOT_SCRIPT_PATH = 'scripts/build/tools/ebs_snapshot.py'
|
||||
PIPELINE_RETRY_ATTEMPTS = 3
|
||||
|
||||
// Number of minutes of inactivity in all stages of the pipeline to reach the timeout
|
||||
PIPELINE_TIMEOUT = 60
|
||||
|
||||
EMPTY_JSON = readJSON text: '{}'
|
||||
|
||||
ENGINE_REPOSITORY_NAME = 'o3de'
|
||||
@@ -753,143 +756,145 @@ def pipelineConfig = {}
|
||||
|
||||
// Start Pipeline
|
||||
try {
|
||||
stage('Setup Pipeline') {
|
||||
node('controller') {
|
||||
def envVarList = []
|
||||
if(isUnix()) {
|
||||
envVarList.add('IS_UNIX=1')
|
||||
}
|
||||
withEnv(envVarList) {
|
||||
timestamps {
|
||||
repositoryUrl = scm.getUserRemoteConfigs()[0].getUrl()
|
||||
// repositoryName is the full repository name
|
||||
repositoryName = (repositoryUrl =~ /https:\/\/github.com\/(.*)\.git/)[0][1]
|
||||
env.REPOSITORY_NAME = repositoryName
|
||||
(projectName, pipelineName) = GetRunningPipelineName(env.JOB_NAME) // env.JOB_NAME is the name of the job given by Jenkins
|
||||
env.PIPELINE_NAME = pipelineName
|
||||
if(env.BRANCH_NAME) {
|
||||
branchName = env.BRANCH_NAME
|
||||
} else {
|
||||
branchName = scm.branches[0].name // for non-multibranch pipelines
|
||||
env.BRANCH_NAME = branchName // so scripts that read this environment have it (e.g. incremental_build_util.py)
|
||||
}
|
||||
if(env.CHANGE_TARGET) {
|
||||
// PR builds
|
||||
if(BUILD_SNAPSHOTS.contains(env.CHANGE_TARGET)) {
|
||||
snapshot = env.CHANGE_TARGET
|
||||
echo "Snapshot for destination branch \"${env.CHANGE_TARGET}\" found."
|
||||
} else {
|
||||
snapshot = DEFAULT_BUILD_SNAPSHOT
|
||||
echo "Snapshot for destination branch \"${env.CHANGE_TARGET}\" does not exist, defaulting to snapshot \"${snapshot}\""
|
||||
timeout(time: PIPELINE_TIMEOUT, unit: 'MINUTES', activity: true) {
|
||||
stage('Setup Pipeline') {
|
||||
node('controller') {
|
||||
def envVarList = []
|
||||
if(isUnix()) {
|
||||
envVarList.add('IS_UNIX=1')
|
||||
}
|
||||
withEnv(envVarList) {
|
||||
timestamps {
|
||||
repositoryUrl = scm.getUserRemoteConfigs()[0].getUrl()
|
||||
// repositoryName is the full repository name
|
||||
repositoryName = (repositoryUrl =~ /https:\/\/github.com\/(.*)\.git/)[0][1]
|
||||
env.REPOSITORY_NAME = repositoryName
|
||||
(projectName, pipelineName) = GetRunningPipelineName(env.JOB_NAME) // env.JOB_NAME is the name of the job given by Jenkins
|
||||
env.PIPELINE_NAME = pipelineName
|
||||
if(env.BRANCH_NAME) {
|
||||
branchName = env.BRANCH_NAME
|
||||
} else {
|
||||
branchName = scm.branches[0].name // for non-multibranch pipelines
|
||||
env.BRANCH_NAME = branchName // so scripts that read this environment have it (e.g. incremental_build_util.py)
|
||||
}
|
||||
} else {
|
||||
// Non-PR builds
|
||||
pipelineParameters.add(choice(defaultValue: DEFAULT_BUILD_SNAPSHOT, name: 'SNAPSHOT', choices: BUILD_SNAPSHOTS_WITH_EMPTY, description: 'Selects the build snapshot to use. A more diverted snapshot will cause longer build times, but will not cause build failures.'))
|
||||
snapshot = env.SNAPSHOT
|
||||
echo "Snapshot \"${snapshot}\" selected."
|
||||
}
|
||||
pipelineProperties.add(disableConcurrentBuilds())
|
||||
if(env.CHANGE_TARGET) {
|
||||
// PR builds
|
||||
if(BUILD_SNAPSHOTS.contains(env.CHANGE_TARGET)) {
|
||||
snapshot = env.CHANGE_TARGET
|
||||
echo "Snapshot for destination branch \"${env.CHANGE_TARGET}\" found."
|
||||
} else {
|
||||
snapshot = DEFAULT_BUILD_SNAPSHOT
|
||||
echo "Snapshot for destination branch \"${env.CHANGE_TARGET}\" does not exist, defaulting to snapshot \"${snapshot}\""
|
||||
}
|
||||
} else {
|
||||
// Non-PR builds
|
||||
pipelineParameters.add(choice(defaultValue: DEFAULT_BUILD_SNAPSHOT, name: 'SNAPSHOT', choices: BUILD_SNAPSHOTS_WITH_EMPTY, description: 'Selects the build snapshot to use. A more diverted snapshot will cause longer build times, but will not cause build failures.'))
|
||||
snapshot = env.SNAPSHOT
|
||||
echo "Snapshot \"${snapshot}\" selected."
|
||||
}
|
||||
pipelineProperties.add(disableConcurrentBuilds())
|
||||
|
||||
echo "Running repository: \"${repositoryName}\", pipeline: \"${pipelineName}\", branch: \"${branchName}\", CHANGE_ID: \"${env.CHANGE_ID}\", GIT_COMMMIT: \"${scm.GIT_COMMIT}\"..."
|
||||
echo "Running repository: \"${repositoryName}\", pipeline: \"${pipelineName}\", branch: \"${branchName}\", CHANGE_ID: \"${env.CHANGE_ID}\", GIT_COMMMIT: \"${scm.GIT_COMMIT}\"..."
|
||||
|
||||
CheckoutBootstrapScripts(branchName)
|
||||
CheckoutBootstrapScripts(branchName)
|
||||
|
||||
// Load configs
|
||||
pipelineConfig = LoadPipelineConfig(pipelineName, branchName)
|
||||
// Load configs
|
||||
pipelineConfig = LoadPipelineConfig(pipelineName, branchName)
|
||||
|
||||
// Add each platform as a parameter that the user can disable if needed
|
||||
if (!IsPullRequest(branchName)) {
|
||||
pipelineParameters.add(stringParam(defaultValue: '', description: 'Filters and overrides the list of jobs to run for each of the below platforms (comma-separated). Can\'t be used during a pull request.', name: 'JOB_LIST_OVERRIDE'))
|
||||
// Add each platform as a parameter that the user can disable if needed
|
||||
if (!IsPullRequest(branchName)) {
|
||||
pipelineParameters.add(stringParam(defaultValue: '', description: 'Filters and overrides the list of jobs to run for each of the below platforms (comma-separated). Can\'t be used during a pull request.', name: 'JOB_LIST_OVERRIDE'))
|
||||
|
||||
pipelineConfig.platforms.each { platform ->
|
||||
pipelineParameters.add(booleanParam(defaultValue: true, description: '', name: platform.key))
|
||||
}
|
||||
}
|
||||
// Add additional Jenkins parameters
|
||||
pipelineConfig.platforms.each { platform ->
|
||||
pipelineParameters.add(booleanParam(defaultValue: true, description: '', name: platform.key))
|
||||
}
|
||||
}
|
||||
// Add additional Jenkins parameters
|
||||
pipelineConfig.platforms.each { platform ->
|
||||
platformEnv = platform.value.PIPELINE_ENV
|
||||
pipelineJenkinsParameters = platformEnv['PIPELINE_JENKINS_PARAMETERS'] ?: [:]
|
||||
jenkinsParametersToAdd = pipelineJenkinsParameters[pipelineName] ?: [:]
|
||||
jenkinsParametersToAdd.each{ jenkinsParameter ->
|
||||
defaultValue = jenkinsParameter['default_value']
|
||||
// Use last run's value as default value so we can save values in different Jenkins environment
|
||||
if (jenkinsParameter['use_last_run_value']?.toBoolean()) {
|
||||
defaultValue = params."${jenkinsParameter['parameter_name']}" ?: jenkinsParameter['default_value']
|
||||
}
|
||||
switch (jenkinsParameter['parameter_type']) {
|
||||
case 'string':
|
||||
pipelineParameters.add(stringParam(defaultValue: defaultValue,
|
||||
description: jenkinsParameter['description'],
|
||||
name: jenkinsParameter['parameter_name']
|
||||
))
|
||||
break
|
||||
case 'boolean':
|
||||
pipelineParameters.add(booleanParam(defaultValue: defaultValue,
|
||||
description: jenkinsParameter['description'],
|
||||
name: jenkinsParameter['parameter_name']
|
||||
))
|
||||
break
|
||||
case 'password':
|
||||
pipelineParameters.add(password(defaultValue: defaultValue,
|
||||
description: jenkinsParameter['description'],
|
||||
name: jenkinsParameter['parameter_name']
|
||||
))
|
||||
break
|
||||
platformEnv = platform.value.PIPELINE_ENV
|
||||
pipelineJenkinsParameters = platformEnv['PIPELINE_JENKINS_PARAMETERS'] ?: [:]
|
||||
jenkinsParametersToAdd = pipelineJenkinsParameters[pipelineName] ?: [:]
|
||||
jenkinsParametersToAdd.each{ jenkinsParameter ->
|
||||
defaultValue = jenkinsParameter['default_value']
|
||||
// Use last run's value as default value so we can save values in different Jenkins environment
|
||||
if (jenkinsParameter['use_last_run_value']?.toBoolean()) {
|
||||
defaultValue = params."${jenkinsParameter['parameter_name']}" ?: jenkinsParameter['default_value']
|
||||
}
|
||||
switch (jenkinsParameter['parameter_type']) {
|
||||
case 'string':
|
||||
pipelineParameters.add(stringParam(defaultValue: defaultValue,
|
||||
description: jenkinsParameter['description'],
|
||||
name: jenkinsParameter['parameter_name']
|
||||
))
|
||||
break
|
||||
case 'boolean':
|
||||
pipelineParameters.add(booleanParam(defaultValue: defaultValue,
|
||||
description: jenkinsParameter['description'],
|
||||
name: jenkinsParameter['parameter_name']
|
||||
))
|
||||
break
|
||||
case 'password':
|
||||
pipelineParameters.add(password(defaultValue: defaultValue,
|
||||
description: jenkinsParameter['description'],
|
||||
name: jenkinsParameter['parameter_name']
|
||||
))
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pipelineProperties.add(parameters(pipelineParameters.unique()))
|
||||
properties(pipelineProperties)
|
||||
pipelineProperties.add(parameters(pipelineParameters.unique()))
|
||||
properties(pipelineProperties)
|
||||
|
||||
// Stash the INCREMENTAL_BUILD_SCRIPT_PATH and EBS_SNAPSHOT_SCRIPT_PATH since all nodes will use it
|
||||
stash name: 'incremental_build_script',
|
||||
includes: INCREMENTAL_BUILD_SCRIPT_PATH
|
||||
if (fileExists(EBS_SNAPSHOT_SCRIPT_PATH)) {
|
||||
stash name: 'ebs_snapshot_script',
|
||||
includes: EBS_SNAPSHOT_SCRIPT_PATH
|
||||
// Stash the INCREMENTAL_BUILD_SCRIPT_PATH and EBS_SNAPSHOT_SCRIPT_PATH since all nodes will use it
|
||||
stash name: 'incremental_build_script',
|
||||
includes: INCREMENTAL_BUILD_SCRIPT_PATH
|
||||
if (fileExists(EBS_SNAPSHOT_SCRIPT_PATH)) {
|
||||
stash name: 'ebs_snapshot_script',
|
||||
includes: EBS_SNAPSHOT_SCRIPT_PATH
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(env.BUILD_NUMBER == '1' && !IsPullRequest(branchName)) {
|
||||
// Exit pipeline early on the intial build. This allows Jenkins to load the pipeline for the branch and enables users
|
||||
// to select build parameters on their first actual build. See https://issues.jenkins.io/browse/JENKINS-41929
|
||||
currentBuild.result = 'SUCCESS'
|
||||
return
|
||||
}
|
||||
if(env.BUILD_NUMBER == '1' && !IsPullRequest(branchName)) {
|
||||
// Exit pipeline early on the intial build. This allows Jenkins to load the pipeline for the branch and enables users
|
||||
// to select build parameters on their first actual build. See https://issues.jenkins.io/browse/JENKINS-41929
|
||||
currentBuild.result = 'SUCCESS'
|
||||
return
|
||||
}
|
||||
|
||||
def someBuildHappened = false
|
||||
def someBuildHappened = false
|
||||
|
||||
// Build and Post-Build Testing Stage
|
||||
def buildConfigs = [:]
|
||||
// Build and Post-Build Testing Stage
|
||||
def buildConfigs = [:]
|
||||
|
||||
// Platform Builds run on EC2
|
||||
pipelineConfig.platforms.each { platform ->
|
||||
platform.value.build_types.each { build_job ->
|
||||
if (IsJobEnabled(branchName, build_job, pipelineName, platform.key)) { // User can filter jobs, jobs are tagged by pipeline
|
||||
def envVars = GetBuildEnvVars(platform.value.PIPELINE_ENV ?: EMPTY_JSON, build_job.value.PIPELINE_ENV ?: EMPTY_JSON, pipelineName)
|
||||
envVars['JENKINS_JOB_NAME'] = env.JOB_NAME // Save original Jenkins job name to JENKINS_JOB_NAME
|
||||
envVars['JOB_NAME'] = "${branchName}_${platform.key}_${build_job.key}" // backwards compatibility, some scripts rely on this
|
||||
someBuildHappened = true
|
||||
// Platform Builds run on EC2
|
||||
pipelineConfig.platforms.each { platform ->
|
||||
platform.value.build_types.each { build_job ->
|
||||
if (IsJobEnabled(branchName, build_job, pipelineName, platform.key)) { // User can filter jobs, jobs are tagged by pipeline
|
||||
def envVars = GetBuildEnvVars(platform.value.PIPELINE_ENV ?: EMPTY_JSON, build_job.value.PIPELINE_ENV ?: EMPTY_JSON, pipelineName)
|
||||
envVars['JENKINS_JOB_NAME'] = env.JOB_NAME // Save original Jenkins job name to JENKINS_JOB_NAME
|
||||
envVars['JOB_NAME'] = "${branchName}_${platform.key}_${build_job.key}" // backwards compatibility, some scripts rely on this
|
||||
someBuildHappened = true
|
||||
|
||||
buildConfigs["${platform.key} [${build_job.key}]"] = CreateBuildJobs(pipelineConfig, platform, build_job, envVars, branchName, pipelineName, repositoryName, projectName)
|
||||
buildConfigs["${platform.key} [${build_job.key}]"] = CreateBuildJobs(pipelineConfig, platform, build_job, envVars, branchName, pipelineName, repositoryName, projectName)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
timestamps {
|
||||
timestamps {
|
||||
|
||||
stage('Build') {
|
||||
parallel buildConfigs // Run parallel builds
|
||||
stage('Build') {
|
||||
parallel buildConfigs // Run parallel builds
|
||||
}
|
||||
|
||||
echo 'All builds successful'
|
||||
}
|
||||
if (!someBuildHappened) {
|
||||
currentBuild.result = 'NOT_BUILT'
|
||||
}
|
||||
|
||||
echo 'All builds successful'
|
||||
}
|
||||
if (!someBuildHappened) {
|
||||
currentBuild.result = 'NOT_BUILT'
|
||||
}
|
||||
}
|
||||
catch(Exception e) {
|
||||
|
||||
Reference in New Issue
Block a user