Archive ap logs on s3 (#5620)

* Archive ap logs on s3 Signed-off-by: Shirang Jia <shiranj@amazon.com> * Update copy_file.py Signed-off-by: Shirang Jia <shiranj@amazon.com> * Fix syntax from bad copy Signed-off-by: Shirang Jia <shiranj@amazon.com> * Change underscore in argument name to dash Signed-off-by: Shirang Jia <shiranj@amazon.com> * Add a general ArchiveArtifactsOnS3 function so we can use it to archive any files on S3 Signed-off-by: Shirang Jia <shiranj@amazon.com> * Swap order of excludes and includes Signed-off-by: Shirang Jia <shiranj@amazon.com> * Change artifactsSourceDir to artifactsSource since the value can be either a file or a directory Signed-off-by: Shirang Jia <shiranj@amazon.com> * Fix parameter Signed-off-by: Shirang Jia <shiranj@amazon.com>
2021-11-19 10:45:09 -08:00
parent 8705b57f71
commit c0080d6b1f
2 changed files with 85 additions and 23 deletions
@@ -102,10 +102,6 @@ def IsJobEnabled(branchName, buildTypeMap, pipelineName, platformName) {
    }
 }

-def IsAPLogUpload(branchName, jobName) {
-    return !IsPullRequest(branchName) && jobName.toLowerCase().contains('asset') && env.AP_LOGS_S3_BUCKET
-}
-
 def GetRunningPipelineName(JENKINS_JOB_NAME) {
    // If the job name has an underscore
    def job_parts = JENKINS_JOB_NAME.tokenize('/')[0].tokenize('_')
@@ -433,26 +429,35 @@ def ExportTestScreenshots(Map options, String branchName, String platformName, S
    }
 }

-def UploadAPLogs(Map options, String branchName, String platformName, String jobName, String workspace, Map params) {
+// All files are included by default.
+// --include will only re-include files that have been excluded from an --exclude filter.
+//See more details at https://docs.aws.amazon.com/cli/latest/reference/s3/#use-of-exclude-and-include-filters
+def ArchiveArtifactsOnS3(String artifactsSource, String s3Prefix="", boolean recursive=false, List<String> includes=[], List<String> excludes=[]) {
+    if (!fileExists(s3Prefix)) {
+        palMkdir(s3Prefix)
+    }
+    palSh("echo ${env.BUILD_URL} > ${s3Prefix}/build_url.txt")
+    // archiveArtifacts is very slow, so we only archive one file and upload the rest artifacts to the same bucket using S3 CLI.
+    archiveArtifacts artifacts: "${s3Prefix}/build_url.txt"
+    def command = "aws s3 cp ${artifactsSource} s3://${env.JENKINS_ARTIFACTS_S3_BUCKET}/${env.JENKINS_JOB_NAME}/${env.BUILD_NUMBER}/artifacts/${s3Prefix} "
+    excludes.each{ exclude ->
+        command += "--exclude \"${exclude}\" "
+    }
+    includes.each{ include ->
+        command += "--include \"${include}\" "
+    }
+    if (recursive) command += "--recursive "
+    palSh(command, "Archiving artifacts to ${env.JENKINS_JOB_NAME}/${env.BUILD_NUMBER}/artifacts/${s3Prefix}", false)
+}
+
+def UploadAPLogs(String platformName, String jobName, String workspace, Map params) {
    dir("${workspace}/${ENGINE_REPOSITORY_NAME}") {
        projects = params.CMAKE_LY_PROJECTS.split(",")
        projects.each{ project ->
-            def apLogsPath = "${project}/user/log"
-            def s3UploadScriptPath = "scripts/build/tools/upload_to_s3.py"
-            if(env.IS_UNIX) {
-                pythonPath = "${options.PYTHON_DIR}/python.sh"
-            }
-            else {
-                pythonPath = "${options.PYTHON_DIR}/python.cmd"
-            }
-            def command = "${pythonPath} -u ${s3UploadScriptPath} --base_dir ${apLogsPath} " +
-                          "--file_regex \".*\" --bucket ${env.AP_LOGS_S3_BUCKET} " +
-                          "--search_subdirectories True --key_prefix ${env.JENKINS_JOB_NAME}/${branchName}/${env.BUILD_NUMBER}/${platformName}/${jobName} " +
-                          '--extra_args {\\"ACL\\":\\"bucket-owner-full-control\\"}'
-            palSh(command, "Uploading AP logs for job ${jobName} for branch ${branchName}", false)
-            }
+            ArchiveArtifactsOnS3("${project}/user/log", "ap_logs/${platformName}/${jobName}/${project}", true)
        }
    }
+}

 def PostBuildCommonSteps(String workspace, boolean mount = true) {
    echo 'Starting post-build common steps...'
@@ -517,10 +522,10 @@ def CreateExportTestScreenshotsStage(Map pipelineConfig, String branchName, Stri
    }
 }

-def CreateUploadAPLogsStage(Map pipelineConfig, String branchName, String platformName, String jobName, String workspace, Map params) {
+def CreateUploadAPLogsStage(String platformName, String jobName, String workspace, Map params) {
    return {
        stage("${jobName}_upload_ap_logs") {
-            UploadAPLogs(pipelineConfig, branchName, platformName, jobName, workspace, params)
+            UploadAPLogs(platformName, jobName, workspace, params)
        }
    }
 }
@@ -576,8 +581,8 @@ def CreateSingleNode(Map pipelineConfig, def platform, def build_job, Map envVar
                                error "Node disconnected during build: ${e}"  // Error raised to retry stage on a new node
                            }
                        }
-                        if (IsAPLogUpload(branchName, build_job_name)) {
-                            CreateUploadAPLogsStage(pipelineConfig, branchName, platform.key, build_job_name, envVars['WORKSPACE'], platform.value.build_types[build_job_name].PARAMETERS).call()
+                        if (build_job_name.toLowerCase().contains('asset') && env.IS_UPLOAD_AP_LOGS?.toBoolean()) {
+                            CreateUploadAPLogsStage(platform.key, build_job_name, envVars['WORKSPACE'], platform.value.build_types[build_job_name].PARAMETERS).call()
                        }
                        // All other errors will be raised outside the retry block
                        currentResult = envVars['ON_FAILURE_MARK'] ?: 'FAILURE'
@@ -0,0 +1,57 @@
+#
+# Copyright (c) Contributors to the Open 3D Engine Project.
+# For complete copyright and license terms please see the LICENSE at the root of this distribution.
+#
+# SPDX-License-Identifier: Apache-2.0 OR MIT
+#
+#
+
+import argparse
+import os
+import sys
+import glob
+import shutil
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-s', '--src-dir', dest='src_dir', required=True, help='Source directory to copy files from, if not specified, current directory is used.')
+    parser.add_argument('-r', '--file-regex', dest='file_regex', required=True, help='Globbing pattern used to match file names to copy.')
+    parser.add_argument('-t', '--target-dir', dest="target_dir", required=True, help='Target directory to copy files to.')
+    args = parser.parse_args()
+    if not os.path.isdir(args.src_dir):
+        print('ERROR: src_dir is not a valid directory.')
+        exit(1)
+    return args
+
+
+def extended_path(path):
+    """
+    Maximum Path Length Limitation on Windows is 260 characters, use extended-length path to bypass this limitation
+    """
+    if sys.platform in ('win32', 'cli') and len(path) >= 260:
+        if path.startswith('\\'):
+            return r'\\?\UNC\{}'.format(path.lstrip('\\'))
+        else:
+            return r'\\?\{}'.format(path)
+    else:
+        return path
+
+
+def copy_file(src_dir, file_regex, target_dir):
+    if not os.path.isdir(args.target_dir):
+        os.makedirs(target_dir)
+    for f in glob.glob(os.path.join(src_dir, file_regex), recursive=True):
+        if os.path.isfile(f):
+            relative_path = os.path.relpath(f, src_dir)
+            target_file_path = os.path.join(target_dir, relative_path)
+            target_file_dir = os.path.dirname(target_file_path)
+            if not os.path.isdir(target_file_dir):
+                os.makedirs(target_file_dir)
+            shutil.copy2(f, extended_path(target_file_path))
+            print(f'{f} -> {target_file_path}')
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    copy_file(args.src_dir, args.file_regex, args.target_dir)