birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'stabilization/2110' of https://github.com/aws-lumberyard/o3de into Prism/AdditionalProgressFixes

Browse Source
This commit is contained in:
AMZN-Phil
2021-11-11 08:19:21 -08:00
parent 0b6c3382f1 6e70097ad7
commit c07b666bec
99 changed files with 1029 additions and 945 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/build/Jenkins/Jenkinsfile
Vendored
+42 -3
View File
@@ -16,7 +16,7 @@ EMPTY_JSON = readJSON text: '{}'
ENGINE_REPOSITORY_NAME = 'o3de'
// Branches with build snapshots
BUILD_SNAPSHOTS = ['development', 'stabilization/2106']
BUILD_SNAPSHOTS = ['development', 'stabilization/2110']
// Build snapshots with empty snapshot (for use with 'SNAPSHOT' pipeline paramater)
BUILD_SNAPSHOTS_WITH_EMPTY = BUILD_SNAPSHOTS + ''
@@ -102,6 +102,10 @@ def IsJobEnabled(branchName, buildTypeMap, pipelineName, platformName) {
}
}
def IsAPLogUpload(branchName, jobName) {
return !IsPullRequest(branchName) && jobName.toLowerCase().contains('asset') && env.AP_LOGS_S3_BUCKET
}
def GetRunningPipelineName(JENKINS_JOB_NAME) {
// If the job name has an underscore
def job_parts = JENKINS_JOB_NAME.tokenize('/')[0].tokenize('_')
@@ -267,7 +271,7 @@ def CheckoutRepo(boolean disableSubmodules = false) {
palRm('commitdate')
}
def HandleDriveMount(String snapshot, String repositoryName, String projectName, String pipeline, String branchName, String platform, String buildType, String workspace, boolean recreateVolume = false) {
def HandleDriveMount(String snapshot, String repositoryName, String projectName, String pipeline, String branchName, String platform, String buildType, String workspace, boolean recreateVolume = false) {
unstash name: 'incremental_build_script'
def pythonCmd = ''
@@ -429,6 +433,27 @@ def ExportTestScreenshots(Map options, String branchName, String platformName, S
}
}
def UploadAPLogs(Map options, String branchName, String platformName, String jobName, String workspace, Map params) {
dir("${workspace}/${ENGINE_REPOSITORY_NAME}") {
projects = params.CMAKE_LY_PROJECTS.split(",")
projects.each{ project ->
def apLogsPath = "${project}/user/log"
def s3UploadScriptPath = "scripts/build/tools/upload_to_s3.py"
if(env.IS_UNIX) {
pythonPath = "${options.PYTHON_DIR}/python.sh"
}
else {
pythonPath = "${options.PYTHON_DIR}/python.cmd"
}
def command = "${pythonPath} -u ${s3UploadScriptPath} --base_dir ${apLogsPath} " +
"--file_regex \".*\" --bucket ${env.AP_LOGS_S3_BUCKET} " +
"--search_subdirectories True --key_prefix ${env.JENKINS_JOB_NAME}/${branchName}/${env.BUILD_NUMBER}/${platformName}/${jobName} " +
'--extra_args {\\"ACL\\":\\"bucket-owner-full-control\\"}'
palSh(command, "Uploading AP logs for job ${jobName} for branch ${branchName}", false)
}
}
}
def PostBuildCommonSteps(String workspace, boolean mount = true) {
echo 'Starting post-build common steps...'
@@ -492,6 +517,14 @@ def CreateExportTestScreenshotsStage(Map pipelineConfig, String branchName, Stri
}
}
def CreateUploadAPLogsStage(Map pipelineConfig, String branchName, String platformName, String jobName, String workspace, Map params) {
return {
stage("${jobName}_upload_ap_logs") {
UploadAPLogs(pipelineConfig, branchName, platformName, jobName, workspace, params)
}
}
}
def CreateTeardownStage(Map environmentVars) {
return {
stage('Teardown') {
@@ -516,9 +549,11 @@ def CreateSingleNode(Map pipelineConfig, def platform, def build_job, Map envVar
CreateSetupStage(pipelineConfig, snapshot, repositoryName, projectName, pipelineName, branchName, platform.key, build_job.key, envVars, onlyMountEBSVolume).call()
if(build_job.value.steps) { //this is a pipe with many steps so create all the build stages
pipelineEnvVars = GetBuildEnvVars(platform.value.PIPELINE_ENV ?: EMPTY_JSON, build_job.value.PIPELINE_ENV ?: EMPTY_JSON, pipelineName)
build_job.value.steps.each { build_step ->
build_job_name = build_step
envVars = GetBuildEnvVars(platform.value.PIPELINE_ENV ?: EMPTY_JSON, platform.value.build_types[build_step].PIPELINE_ENV ?: EMPTY_JSON, pipelineName)
// This addition of maps makes it that the right operand will override entries if they overlap with the left operand
envVars = pipelineEnvVars + GetBuildEnvVars(platform.value.PIPELINE_ENV ?: EMPTY_JSON, platform.value.build_types[build_step].PIPELINE_ENV ?: EMPTY_JSON, pipelineName)
try {
CreateBuildStage(pipelineConfig, platform.key, build_step, envVars).call()
}
@@ -541,6 +576,9 @@ def CreateSingleNode(Map pipelineConfig, def platform, def build_job, Map envVar
error "Node disconnected during build: ${e}" // Error raised to retry stage on a new node
}
}
if (IsAPLogUpload(branchName, build_job_name)) {
CreateUploadAPLogsStage(pipelineConfig, branchName, platform.key, build_job_name, envVars['WORKSPACE'], platform.value.build_types[build_job_name].PARAMETERS).call()
}
// All other errors will be raised outside the retry block
currentResult = envVars['ON_FAILURE_MARK'] ?: 'FAILURE'
currentException = e.toString()
@@ -768,6 +806,7 @@ try {
platform.value.build_types.each { build_job ->
if (IsJobEnabled(branchName, build_job, pipelineName, platform.key)) { // User can filter jobs, jobs are tagged by pipeline
def envVars = GetBuildEnvVars(platform.value.PIPELINE_ENV ?: EMPTY_JSON, build_job.value.PIPELINE_ENV ?: EMPTY_JSON, pipelineName)
envVars['JENKINS_JOB_NAME'] = env.JOB_NAME // Save original Jenkins job name to JENKINS_JOB_NAME
envVars['JOB_NAME'] = "${branchName}_${platform.key}_${build_job.key}" // backwards compatibility, some scripts rely on this
someBuildHappened = true
scripts/o3de/o3de/utils.py
+1 -1
View File
@@ -128,7 +128,7 @@ def download_file(parsed_uri, download_path: pathlib.Path, force_overwrite: bool
logger.warn(f'File already downloaded to {download_path}.')
else:
try:
shutil.rmtree(download_path)
os.unlink(download_path)
except OSError:
logger.error(f'Could not remove existing download path {download_path}.')
return 1