From a9fc376d6cabd0fd9006b0e4b7cf014ba1c01d14 Mon Sep 17 00:00:00 2001 From: Shirang Jia Date: Mon, 25 Oct 2021 19:16:29 -0700 Subject: [PATCH] Upload AP logs when there is an AP eeror (#4896) * Updating LFS config to new endpoint (#1624) Signed-off-by: AMZN-alexpete * Updates licenses to APACHE-2.0 OR MIT (#1685) Not to be committed before 7/6/2021 Signed-off-by: lawsonamzn <70027408+lawsonamzn@users.noreply.github.com> * Updating CONTRIBUTING.md Signed-off-by: Terry Michaels * Fixed typo Signed-off-by: Terry Michaels * Updated text to be more descriptive Signed-off-by: Terry Michaels * Upload AP logs when there is an AP eeror Signed-off-by: shiranj * Add AP log upload step for single step Asset jobs Signed-off-by: shiranj * Address more comments Signed-off-by: shiranj * Only call CreateUploadAPLogsStage() once in try/catch block Signed-off-by: shiranj * Revert acciental update to README.md Signed-off-by: shiranj Co-authored-by: Alex Peterson <26804013+AMZN-alexpete@users.noreply.github.com> Co-authored-by: Chris Galvan Co-authored-by: Nicholas Lawson <70027408+lawsonamzn@users.noreply.github.com> Co-authored-by: Terry Michaels Signed-off-by: Gene Walters --- scripts/build/Jenkins/Jenkinsfile | 36 +++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/scripts/build/Jenkins/Jenkinsfile b/scripts/build/Jenkins/Jenkinsfile index 1388366661..f5e5edcf66 100644 --- a/scripts/build/Jenkins/Jenkinsfile +++ b/scripts/build/Jenkins/Jenkinsfile @@ -102,6 +102,10 @@ def IsJobEnabled(branchName, buildTypeMap, pipelineName, platformName) { } } +def IsAPLogUpload(branchName, jobName) { + return !IsPullRequest(branchName) && jobName.toLowerCase().contains('asset') && env.AP_LOGS_S3_BUCKET +} + def GetRunningPipelineName(JENKINS_JOB_NAME) { // If the job name has an underscore def job_parts = JENKINS_JOB_NAME.tokenize('/')[0].tokenize('_') @@ -431,6 +435,27 @@ def ExportTestScreenshots(Map options, String branchName, String platformName, S } } +def UploadAPLogs(Map options, String branchName, String jobName, String workspace, Map params) { + dir("${workspace}/${ENGINE_REPOSITORY_NAME}") { + projects = params.CMAKE_LY_PROJECTS.split(",") + projects.each{ project -> + def apLogsPath = "${project}/user/log" + def s3UploadScriptPath = "scripts/build/tools/upload_to_s3.py" + if(env.IS_UNIX) { + pythonPath = "${options.PYTHON_DIR}/python.sh" + } + else { + pythonPath = "${options.PYTHON_DIR}/python.cmd" + } + def command = "${pythonPath} -u ${s3UploadScriptPath} --base_dir ${apLogsPath} " + + "--file_regex \".*\" --bucket ${env.AP_LOGS_S3_BUCKET} " + + "--search_subdirectories True --key_prefix ${env.JOB_NAME}/${branchName}/${env.BUILD_NUMBER}/${jobName}" + + "--extra-args {\"ACL\": \"bucket-owner-full-control\"}" + palSh(command, "Uploading AP logs for job ${jobName} for branch ${branchName}", false) + } + } + } + def PostBuildCommonSteps(String workspace, boolean mount = true) { echo 'Starting post-build common steps...' @@ -494,6 +519,14 @@ def CreateExportTestScreenshotsStage(Map pipelineConfig, String branchName, Stri } } +def CreateUploadAPLogsStage(Map pipelineConfig, String branchName, String jobName, String workspace, Map params) { + return { + stage("${jobName}_upload_ap_logs") { + UploadAPLogs(pipelineConfig, branchName, jobName, workspace, params) + } + } +} + def CreateTeardownStage(Map environmentVars) { return { stage('Teardown') { @@ -543,6 +576,9 @@ def CreateSingleNode(Map pipelineConfig, def platform, def build_job, Map envVar error "Node disconnected during build: ${e}" // Error raised to retry stage on a new node } } + if (IsAPLogUpload(branchName, build_job_name)) { + CreateUploadAPLogsStage(pipelineConfig, branchName, build_job_name, envVars['WORKSPACE'], platform.value.build_types[build_job_name].PARAMETERS).call() + } // All other errors will be raised outside the retry block currentResult = envVars['ON_FAILURE_MARK'] ?: 'FAILURE' currentException = e.toString()