Upload test screenshots to s3 on test failure. (#3815)

* add s3 upload on screenshot test failure, should only apply to the test_gpu_profile_vs2019 job on nightly runs

* adds support for zipping screenshot files up prior to uploading to the s3 bucket and also adds the ACL extra arg to the upload_to_s3.py execution

* remove unused json import

* remove regex to use .endswith() instead and rename variables to be more clear (PR feedback)

* rename create_zip_archive to create_screenshots_archive

Signed-off-by: jromnoa <jromnoa@amazon.com>

AutomatedTesting/Gem/PythonTests/atom_renderer/test_Atom_GPUTests.py

@@ -7,8 +7,10 @@ SPDX-License-Identifier: Apache-2.0 OR MIT
 Tests that require a GPU in order to run.
 """
 
+import datetime
 import logging
 import os
+import zipfile
 
 import pytest
 
@@ -40,6 +42,33 @@ def golden_images_directory():
     return golden_images_dir
 
 
+def create_screenshots_archive(screenshot_path):
+    """
+    Creates a new zip file archive at archive_path containing all files listed within archive_path.
+    :param screenshot_path: location containing the files to archive, the zip archive file will also be saved here.
+    :return: None, but creates a new zip file archive inside path containing all of the files inside archive_path.
+    """
+    files_to_archive = []
+
+    # Search for .png and .ppm files to add to the zip archive file.
+    for (folder_name, sub_folders, file_names) in os.walk(screenshot_path):
+        for file_name in file_names:
+            if file_name.endswith(".png") or file_name.endswith(".ppm"):
+                file_path = os.path.join(folder_name, file_name)
+                files_to_archive.append(file_path)
+
+    # Setup variables for naming the zip archive file.
+    timestamp = datetime.datetime.now().timestamp()
+    formatted_timestamp = datetime.datetime.utcfromtimestamp(timestamp).strftime("%Y-%m-%d_%H-%M-%S")
+    screenshots_file = os.path.join(screenshot_path, f'zip_archive_{formatted_timestamp}.zip')
+
+    # Write all of the valid .png and .ppm files to the archive file.
+    with zipfile.ZipFile(screenshots_file, 'w', compression=zipfile.ZIP_DEFLATED, allowZip64=True) as zip_archive:
+        for file_path in files_to_archive:
+            file_name = os.path.basename(file_path)
+            zip_archive.write(file_path, file_name)
+
+
 @pytest.mark.parametrize("project", ["AutomatedTesting"])
 @pytest.mark.parametrize("launcher_platform", ["windows_editor"])
 @pytest.mark.parametrize("level", ["auto_test"])
@@ -53,8 +82,8 @@ class TestAllComponentsIndepthTests(object):
         Tests that a basic rendering level setup can be created (lighting, meshes, materials, etc.).
         """
         # Clear existing test screenshots before starting test.
-        test_screenshots = [os.path.join(
-            workspace.paths.project(), DEFAULT_SUBFOLDER_PATH, screenshot_name)]
+        screenshot_directory = os.path.join(workspace.paths.project(), DEFAULT_SUBFOLDER_PATH)
+        test_screenshots = [os.path.join(screenshot_directory, screenshot_name)]
         file_system.delete(test_screenshots, True, True)
 
         golden_images = [os.path.join(golden_images_directory(), screenshot_name)]
@@ -86,6 +115,8 @@ class TestAllComponentsIndepthTests(object):
         for test_screenshot, golden_screenshot in zip(test_screenshots, golden_images):
             compare_screenshots(test_screenshot, golden_screenshot)
 
+        create_screenshots_archive(screenshot_directory)
+
     def test_LightComponent_ScreenshotMatchesGoldenImage(
             self, request, editor, workspace, project, launcher_platform, level):
         """
@@ -105,9 +136,10 @@ class TestAllComponentsIndepthTests(object):
             "SpotLight_5.ppm",
             "SpotLight_6.ppm",
         ]
+        screenshot_directory = os.path.join(workspace.paths.project(), DEFAULT_SUBFOLDER_PATH)
         test_screenshots = []
         for screenshot in screenshot_names:
-            screenshot_path = os.path.join(workspace.paths.project(), DEFAULT_SUBFOLDER_PATH, screenshot)
+            screenshot_path = os.path.join(screenshot_directory, screenshot)
             test_screenshots.append(screenshot_path)
         file_system.delete(test_screenshots, True, True)
 
@@ -139,6 +171,8 @@ class TestAllComponentsIndepthTests(object):
         for test_screenshot, golden_screenshot in zip(test_screenshots, golden_images):
             compare_screenshots(test_screenshot, golden_screenshot)
 
+        create_screenshots_archive(screenshot_directory)
+
 
 @pytest.mark.parametrize('rhi', ['dx12', 'vulkan'])
 @pytest.mark.parametrize("project", ["AutomatedTesting"])

scripts/build/Jenkins/Jenkinsfile

@@ -415,6 +415,19 @@ def ExportTestResults(Map options, String platform, String type, String workspac
     }
 }
 
+def ExportTestScreenshots(Map options, String workspace, String platformName, String jobName, Map params) {
+    catchError(message: "Error exporting test screenshots (this won't fail the build)", buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+        def screenshotsFolder = '${workspace}/${ENGINE_REPOSITORY_NAME}/AutomatedTesting/user/PythonTests/Automated/Screenshots'
+        def s3Uploader = '${workspace}/${ENGINE_REPOSITORY_NAME}/scripts/build/tools/upload_to_s3.py'
+        def command = '${options.PYTHON_DIR}/python.cmd -u ${s3Uploader} --base_dir ${screenshotsFolder} ' +
+                      '--file_regex "(.*zip$)" --bucket ${env.TEST_SCREENSHOT_BUCKET} ' +
+                      '--search_subdirectories True --key_prefix ${branchName}_${env.BUILD_NUMBER}' +
+                      '--extra-args {"ACL": "bucket-owner-full-control"}'
+        bat label: "Uploading test screenshots for ${jobName}",
+            script: command
+    }
+}
+
 def PostBuildCommonSteps(String workspace, boolean mount = true) {
     echo 'Starting post-build common steps...'
 
@@ -470,6 +483,14 @@ def CreateExportTestResultsStage(Map pipelineConfig, String platformName, String
     }
 }
 
+def CreateExportTestScreenshotsStage(Map pipelineConfig, String platformName, String jobName, Map environmentVars, Map params) {
+    return {
+        stage("${jobName}_screenshots") {
+            ExportTestScreenshots(pipelineConfig, platformName, jobName, environmentVars['WORKSPACE'], params)
+        }
+    }
+}
+
 def CreateTeardownStage(Map environmentVars) {
     return {
         stage('Teardown') {
@@ -532,6 +553,9 @@ def CreateSingleNode(Map pipelineConfig, def platform, def build_job, Map envVar
                     if (params && params.containsKey('TEST_RESULTS') && params.TEST_RESULTS == 'True') {
                         CreateExportTestResultsStage(pipelineConfig,  platform.key, build_job_name, envVars, params).call()
                     }
+                    if (params && params.containsKey('TEST_SCREENSHOTS') && params.TEST_SCREENSHOTS == 'True' && currentResult == 'FAILURE') {
+                        CreateExportTestScreenshotsStage(pipelineConfig,  platform.key, build_job_name, envVars, params).call()
+                    }
                     CreateTeardownStage(envVars).call()
                 }
             }