diff --git a/scripts/build/Jenkins/Jenkinsfile b/scripts/build/Jenkins/Jenkinsfile index beb4a21620..3b5ca8fe60 100644 --- a/scripts/build/Jenkins/Jenkinsfile +++ b/scripts/build/Jenkins/Jenkinsfile @@ -102,10 +102,6 @@ def IsJobEnabled(branchName, buildTypeMap, pipelineName, platformName) { } } -def IsAPLogUpload(branchName, jobName) { - return !IsPullRequest(branchName) && jobName.toLowerCase().contains('asset') && env.AP_LOGS_S3_BUCKET -} - def GetRunningPipelineName(JENKINS_JOB_NAME) { // If the job name has an underscore def job_parts = JENKINS_JOB_NAME.tokenize('/')[0].tokenize('_') @@ -433,26 +429,35 @@ def ExportTestScreenshots(Map options, String branchName, String platformName, S } } -def UploadAPLogs(Map options, String branchName, String platformName, String jobName, String workspace, Map params) { +// All files are included by default. +// --include will only re-include files that have been excluded from an --exclude filter. +//See more details at https://docs.aws.amazon.com/cli/latest/reference/s3/#use-of-exclude-and-include-filters +def ArchiveArtifactsOnS3(String artifactsSource, String s3Prefix="", boolean recursive=false, List includes=[], List excludes=[]) { + if (!fileExists(s3Prefix)) { + palMkdir(s3Prefix) + } + palSh("echo ${env.BUILD_URL} > ${s3Prefix}/build_url.txt") + // archiveArtifacts is very slow, so we only archive one file and upload the rest artifacts to the same bucket using S3 CLI. + archiveArtifacts artifacts: "${s3Prefix}/build_url.txt" + def command = "aws s3 cp ${artifactsSource} s3://${env.JENKINS_ARTIFACTS_S3_BUCKET}/${env.JENKINS_JOB_NAME}/${env.BUILD_NUMBER}/artifacts/${s3Prefix} " + excludes.each{ exclude -> + command += "--exclude \"${exclude}\" " + } + includes.each{ include -> + command += "--include \"${include}\" " + } + if (recursive) command += "--recursive " + palSh(command, "Archiving artifacts to ${env.JENKINS_JOB_NAME}/${env.BUILD_NUMBER}/artifacts/${s3Prefix}", false) +} + +def UploadAPLogs(String platformName, String jobName, String workspace, Map params) { dir("${workspace}/${ENGINE_REPOSITORY_NAME}") { projects = params.CMAKE_LY_PROJECTS.split(",") projects.each{ project -> - def apLogsPath = "${project}/user/log" - def s3UploadScriptPath = "scripts/build/tools/upload_to_s3.py" - if(env.IS_UNIX) { - pythonPath = "${options.PYTHON_DIR}/python.sh" - } - else { - pythonPath = "${options.PYTHON_DIR}/python.cmd" - } - def command = "${pythonPath} -u ${s3UploadScriptPath} --base_dir ${apLogsPath} " + - "--file_regex \".*\" --bucket ${env.AP_LOGS_S3_BUCKET} " + - "--search_subdirectories True --key_prefix ${env.JENKINS_JOB_NAME}/${branchName}/${env.BUILD_NUMBER}/${platformName}/${jobName} " + - '--extra_args {\\"ACL\\":\\"bucket-owner-full-control\\"}' - palSh(command, "Uploading AP logs for job ${jobName} for branch ${branchName}", false) - } + ArchiveArtifactsOnS3("${project}/user/log", "ap_logs/${platformName}/${jobName}/${project}", true) } } +} def PostBuildCommonSteps(String workspace, boolean mount = true) { echo 'Starting post-build common steps...' @@ -517,10 +522,10 @@ def CreateExportTestScreenshotsStage(Map pipelineConfig, String branchName, Stri } } -def CreateUploadAPLogsStage(Map pipelineConfig, String branchName, String platformName, String jobName, String workspace, Map params) { +def CreateUploadAPLogsStage(String platformName, String jobName, String workspace, Map params) { return { stage("${jobName}_upload_ap_logs") { - UploadAPLogs(pipelineConfig, branchName, platformName, jobName, workspace, params) + UploadAPLogs(platformName, jobName, workspace, params) } } } @@ -576,8 +581,8 @@ def CreateSingleNode(Map pipelineConfig, def platform, def build_job, Map envVar error "Node disconnected during build: ${e}" // Error raised to retry stage on a new node } } - if (IsAPLogUpload(branchName, build_job_name)) { - CreateUploadAPLogsStage(pipelineConfig, branchName, platform.key, build_job_name, envVars['WORKSPACE'], platform.value.build_types[build_job_name].PARAMETERS).call() + if (build_job_name.toLowerCase().contains('asset') && env.IS_UPLOAD_AP_LOGS?.toBoolean()) { + CreateUploadAPLogsStage(platform.key, build_job_name, envVars['WORKSPACE'], platform.value.build_types[build_job_name].PARAMETERS).call() } // All other errors will be raised outside the retry block currentResult = envVars['ON_FAILURE_MARK'] ?: 'FAILURE' diff --git a/scripts/build/tools/copy_file.py b/scripts/build/tools/copy_file.py new file mode 100644 index 0000000000..45b5e39d7a --- /dev/null +++ b/scripts/build/tools/copy_file.py @@ -0,0 +1,57 @@ +# +# Copyright (c) Contributors to the Open 3D Engine Project. +# For complete copyright and license terms please see the LICENSE at the root of this distribution. +# +# SPDX-License-Identifier: Apache-2.0 OR MIT +# +# + +import argparse +import os +import sys +import glob +import shutil + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument('-s', '--src-dir', dest='src_dir', required=True, help='Source directory to copy files from, if not specified, current directory is used.') + parser.add_argument('-r', '--file-regex', dest='file_regex', required=True, help='Globbing pattern used to match file names to copy.') + parser.add_argument('-t', '--target-dir', dest="target_dir", required=True, help='Target directory to copy files to.') + args = parser.parse_args() + if not os.path.isdir(args.src_dir): + print('ERROR: src_dir is not a valid directory.') + exit(1) + return args + + +def extended_path(path): + """ + Maximum Path Length Limitation on Windows is 260 characters, use extended-length path to bypass this limitation + """ + if sys.platform in ('win32', 'cli') and len(path) >= 260: + if path.startswith('\\'): + return r'\\?\UNC\{}'.format(path.lstrip('\\')) + else: + return r'\\?\{}'.format(path) + else: + return path + + +def copy_file(src_dir, file_regex, target_dir): + if not os.path.isdir(args.target_dir): + os.makedirs(target_dir) + for f in glob.glob(os.path.join(src_dir, file_regex), recursive=True): + if os.path.isfile(f): + relative_path = os.path.relpath(f, src_dir) + target_file_path = os.path.join(target_dir, relative_path) + target_file_dir = os.path.dirname(target_file_path) + if not os.path.isdir(target_file_dir): + os.makedirs(target_file_dir) + shutil.copy2(f, extended_path(target_file_path)) + print(f'{f} -> {target_file_path}') + + +if __name__ == "__main__": + args = parse_args() + copy_file(args.src_dir, args.file_regex, args.target_dir)