From 576248a870fb7aab37949822a54a9b3969a6a6b1 Mon Sep 17 00:00:00 2001 From: Shirang Jia Date: Wed, 10 Nov 2021 10:26:44 -0800 Subject: [PATCH] Merge Jenkinsfile from development to stabilization (#5391) Signed-off-by: shiranj --- scripts/build/Jenkins/Jenkinsfile | 45 ++++++++++++++++++++++++++++--- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/scripts/build/Jenkins/Jenkinsfile b/scripts/build/Jenkins/Jenkinsfile index 34732a5fad..beb4a21620 100644 --- a/scripts/build/Jenkins/Jenkinsfile +++ b/scripts/build/Jenkins/Jenkinsfile @@ -16,7 +16,7 @@ EMPTY_JSON = readJSON text: '{}' ENGINE_REPOSITORY_NAME = 'o3de' // Branches with build snapshots -BUILD_SNAPSHOTS = ['development', 'stabilization/2106'] +BUILD_SNAPSHOTS = ['development', 'stabilization/2110'] // Build snapshots with empty snapshot (for use with 'SNAPSHOT' pipeline paramater) BUILD_SNAPSHOTS_WITH_EMPTY = BUILD_SNAPSHOTS + '' @@ -102,6 +102,10 @@ def IsJobEnabled(branchName, buildTypeMap, pipelineName, platformName) { } } +def IsAPLogUpload(branchName, jobName) { + return !IsPullRequest(branchName) && jobName.toLowerCase().contains('asset') && env.AP_LOGS_S3_BUCKET +} + def GetRunningPipelineName(JENKINS_JOB_NAME) { // If the job name has an underscore def job_parts = JENKINS_JOB_NAME.tokenize('/')[0].tokenize('_') @@ -267,7 +271,7 @@ def CheckoutRepo(boolean disableSubmodules = false) { palRm('commitdate') } -def HandleDriveMount(String snapshot, String repositoryName, String projectName, String pipeline, String branchName, String platform, String buildType, String workspace, boolean recreateVolume = false) { +def HandleDriveMount(String snapshot, String repositoryName, String projectName, String pipeline, String branchName, String platform, String buildType, String workspace, boolean recreateVolume = false) { unstash name: 'incremental_build_script' def pythonCmd = '' @@ -429,6 +433,27 @@ def ExportTestScreenshots(Map options, String branchName, String platformName, S } } +def UploadAPLogs(Map options, String branchName, String platformName, String jobName, String workspace, Map params) { + dir("${workspace}/${ENGINE_REPOSITORY_NAME}") { + projects = params.CMAKE_LY_PROJECTS.split(",") + projects.each{ project -> + def apLogsPath = "${project}/user/log" + def s3UploadScriptPath = "scripts/build/tools/upload_to_s3.py" + if(env.IS_UNIX) { + pythonPath = "${options.PYTHON_DIR}/python.sh" + } + else { + pythonPath = "${options.PYTHON_DIR}/python.cmd" + } + def command = "${pythonPath} -u ${s3UploadScriptPath} --base_dir ${apLogsPath} " + + "--file_regex \".*\" --bucket ${env.AP_LOGS_S3_BUCKET} " + + "--search_subdirectories True --key_prefix ${env.JENKINS_JOB_NAME}/${branchName}/${env.BUILD_NUMBER}/${platformName}/${jobName} " + + '--extra_args {\\"ACL\\":\\"bucket-owner-full-control\\"}' + palSh(command, "Uploading AP logs for job ${jobName} for branch ${branchName}", false) + } + } + } + def PostBuildCommonSteps(String workspace, boolean mount = true) { echo 'Starting post-build common steps...' @@ -492,6 +517,14 @@ def CreateExportTestScreenshotsStage(Map pipelineConfig, String branchName, Stri } } +def CreateUploadAPLogsStage(Map pipelineConfig, String branchName, String platformName, String jobName, String workspace, Map params) { + return { + stage("${jobName}_upload_ap_logs") { + UploadAPLogs(pipelineConfig, branchName, platformName, jobName, workspace, params) + } + } +} + def CreateTeardownStage(Map environmentVars) { return { stage('Teardown') { @@ -516,9 +549,11 @@ def CreateSingleNode(Map pipelineConfig, def platform, def build_job, Map envVar CreateSetupStage(pipelineConfig, snapshot, repositoryName, projectName, pipelineName, branchName, platform.key, build_job.key, envVars, onlyMountEBSVolume).call() if(build_job.value.steps) { //this is a pipe with many steps so create all the build stages + pipelineEnvVars = GetBuildEnvVars(platform.value.PIPELINE_ENV ?: EMPTY_JSON, build_job.value.PIPELINE_ENV ?: EMPTY_JSON, pipelineName) build_job.value.steps.each { build_step -> build_job_name = build_step - envVars = GetBuildEnvVars(platform.value.PIPELINE_ENV ?: EMPTY_JSON, platform.value.build_types[build_step].PIPELINE_ENV ?: EMPTY_JSON, pipelineName) + // This addition of maps makes it that the right operand will override entries if they overlap with the left operand + envVars = pipelineEnvVars + GetBuildEnvVars(platform.value.PIPELINE_ENV ?: EMPTY_JSON, platform.value.build_types[build_step].PIPELINE_ENV ?: EMPTY_JSON, pipelineName) try { CreateBuildStage(pipelineConfig, platform.key, build_step, envVars).call() } @@ -541,6 +576,9 @@ def CreateSingleNode(Map pipelineConfig, def platform, def build_job, Map envVar error "Node disconnected during build: ${e}" // Error raised to retry stage on a new node } } + if (IsAPLogUpload(branchName, build_job_name)) { + CreateUploadAPLogsStage(pipelineConfig, branchName, platform.key, build_job_name, envVars['WORKSPACE'], platform.value.build_types[build_job_name].PARAMETERS).call() + } // All other errors will be raised outside the retry block currentResult = envVars['ON_FAILURE_MARK'] ?: 'FAILURE' currentException = e.toString() @@ -768,6 +806,7 @@ try { platform.value.build_types.each { build_job -> if (IsJobEnabled(branchName, build_job, pipelineName, platform.key)) { // User can filter jobs, jobs are tagged by pipeline def envVars = GetBuildEnvVars(platform.value.PIPELINE_ENV ?: EMPTY_JSON, build_job.value.PIPELINE_ENV ?: EMPTY_JSON, pipelineName) + envVars['JENKINS_JOB_NAME'] = env.JOB_NAME // Save original Jenkins job name to JENKINS_JOB_NAME envVars['JOB_NAME'] = "${branchName}_${platform.key}_${build_job.key}" // backwards compatibility, some scripts rely on this someBuildHappened = true