From 5c4ce8cd065f33365068120f0ba98b63a865dfa7 Mon Sep 17 00:00:00 2001 From: jromnoa <80134229+jromnoa@users.noreply.github.com> Date: Wed, 8 Sep 2021 04:11:44 -0700 Subject: [PATCH] Upload test screenshots to s3 on test failure. (#3815) * add s3 upload on screenshot test failure, should only apply to the test_gpu_profile_vs2019 job on nightly runs * adds support for zipping screenshot files up prior to uploading to the s3 bucket and also adds the ACL extra arg to the upload_to_s3.py execution * remove unused json import * remove regex to use .endswith() instead and rename variables to be more clear (PR feedback) * rename create_zip_archive to create_screenshots_archive Signed-off-by: jromnoa --- .../atom_renderer/test_Atom_GPUTests.py | 40 +++++++++++++++++-- scripts/build/Jenkins/Jenkinsfile | 24 +++++++++++ 2 files changed, 61 insertions(+), 3 deletions(-) diff --git a/AutomatedTesting/Gem/PythonTests/atom_renderer/test_Atom_GPUTests.py b/AutomatedTesting/Gem/PythonTests/atom_renderer/test_Atom_GPUTests.py index 047f46a40f..e62ab5e5dc 100644 --- a/AutomatedTesting/Gem/PythonTests/atom_renderer/test_Atom_GPUTests.py +++ b/AutomatedTesting/Gem/PythonTests/atom_renderer/test_Atom_GPUTests.py @@ -7,8 +7,10 @@ SPDX-License-Identifier: Apache-2.0 OR MIT Tests that require a GPU in order to run. """ +import datetime import logging import os +import zipfile import pytest @@ -40,6 +42,33 @@ def golden_images_directory(): return golden_images_dir +def create_screenshots_archive(screenshot_path): + """ + Creates a new zip file archive at archive_path containing all files listed within archive_path. + :param screenshot_path: location containing the files to archive, the zip archive file will also be saved here. + :return: None, but creates a new zip file archive inside path containing all of the files inside archive_path. + """ + files_to_archive = [] + + # Search for .png and .ppm files to add to the zip archive file. + for (folder_name, sub_folders, file_names) in os.walk(screenshot_path): + for file_name in file_names: + if file_name.endswith(".png") or file_name.endswith(".ppm"): + file_path = os.path.join(folder_name, file_name) + files_to_archive.append(file_path) + + # Setup variables for naming the zip archive file. + timestamp = datetime.datetime.now().timestamp() + formatted_timestamp = datetime.datetime.utcfromtimestamp(timestamp).strftime("%Y-%m-%d_%H-%M-%S") + screenshots_file = os.path.join(screenshot_path, f'zip_archive_{formatted_timestamp}.zip') + + # Write all of the valid .png and .ppm files to the archive file. + with zipfile.ZipFile(screenshots_file, 'w', compression=zipfile.ZIP_DEFLATED, allowZip64=True) as zip_archive: + for file_path in files_to_archive: + file_name = os.path.basename(file_path) + zip_archive.write(file_path, file_name) + + @pytest.mark.parametrize("project", ["AutomatedTesting"]) @pytest.mark.parametrize("launcher_platform", ["windows_editor"]) @pytest.mark.parametrize("level", ["auto_test"]) @@ -53,8 +82,8 @@ class TestAllComponentsIndepthTests(object): Tests that a basic rendering level setup can be created (lighting, meshes, materials, etc.). """ # Clear existing test screenshots before starting test. - test_screenshots = [os.path.join( - workspace.paths.project(), DEFAULT_SUBFOLDER_PATH, screenshot_name)] + screenshot_directory = os.path.join(workspace.paths.project(), DEFAULT_SUBFOLDER_PATH) + test_screenshots = [os.path.join(screenshot_directory, screenshot_name)] file_system.delete(test_screenshots, True, True) golden_images = [os.path.join(golden_images_directory(), screenshot_name)] @@ -86,6 +115,8 @@ class TestAllComponentsIndepthTests(object): for test_screenshot, golden_screenshot in zip(test_screenshots, golden_images): compare_screenshots(test_screenshot, golden_screenshot) + create_screenshots_archive(screenshot_directory) + def test_LightComponent_ScreenshotMatchesGoldenImage( self, request, editor, workspace, project, launcher_platform, level): """ @@ -105,9 +136,10 @@ class TestAllComponentsIndepthTests(object): "SpotLight_5.ppm", "SpotLight_6.ppm", ] + screenshot_directory = os.path.join(workspace.paths.project(), DEFAULT_SUBFOLDER_PATH) test_screenshots = [] for screenshot in screenshot_names: - screenshot_path = os.path.join(workspace.paths.project(), DEFAULT_SUBFOLDER_PATH, screenshot) + screenshot_path = os.path.join(screenshot_directory, screenshot) test_screenshots.append(screenshot_path) file_system.delete(test_screenshots, True, True) @@ -139,6 +171,8 @@ class TestAllComponentsIndepthTests(object): for test_screenshot, golden_screenshot in zip(test_screenshots, golden_images): compare_screenshots(test_screenshot, golden_screenshot) + create_screenshots_archive(screenshot_directory) + @pytest.mark.parametrize('rhi', ['dx12', 'vulkan']) @pytest.mark.parametrize("project", ["AutomatedTesting"]) diff --git a/scripts/build/Jenkins/Jenkinsfile b/scripts/build/Jenkins/Jenkinsfile index 328e5f8df7..5bc4b919fb 100644 --- a/scripts/build/Jenkins/Jenkinsfile +++ b/scripts/build/Jenkins/Jenkinsfile @@ -415,6 +415,19 @@ def ExportTestResults(Map options, String platform, String type, String workspac } } +def ExportTestScreenshots(Map options, String workspace, String platformName, String jobName, Map params) { + catchError(message: "Error exporting test screenshots (this won't fail the build)", buildResult: 'SUCCESS', stageResult: 'FAILURE') { + def screenshotsFolder = '${workspace}/${ENGINE_REPOSITORY_NAME}/AutomatedTesting/user/PythonTests/Automated/Screenshots' + def s3Uploader = '${workspace}/${ENGINE_REPOSITORY_NAME}/scripts/build/tools/upload_to_s3.py' + def command = '${options.PYTHON_DIR}/python.cmd -u ${s3Uploader} --base_dir ${screenshotsFolder} ' + + '--file_regex "(.*zip$)" --bucket ${env.TEST_SCREENSHOT_BUCKET} ' + + '--search_subdirectories True --key_prefix ${branchName}_${env.BUILD_NUMBER}' + + '--extra-args {"ACL": "bucket-owner-full-control"}' + bat label: "Uploading test screenshots for ${jobName}", + script: command + } +} + def PostBuildCommonSteps(String workspace, boolean mount = true) { echo 'Starting post-build common steps...' @@ -470,6 +483,14 @@ def CreateExportTestResultsStage(Map pipelineConfig, String platformName, String } } +def CreateExportTestScreenshotsStage(Map pipelineConfig, String platformName, String jobName, Map environmentVars, Map params) { + return { + stage("${jobName}_screenshots") { + ExportTestScreenshots(pipelineConfig, platformName, jobName, environmentVars['WORKSPACE'], params) + } + } +} + def CreateTeardownStage(Map environmentVars) { return { stage('Teardown') { @@ -532,6 +553,9 @@ def CreateSingleNode(Map pipelineConfig, def platform, def build_job, Map envVar if (params && params.containsKey('TEST_RESULTS') && params.TEST_RESULTS == 'True') { CreateExportTestResultsStage(pipelineConfig, platform.key, build_job_name, envVars, params).call() } + if (params && params.containsKey('TEST_SCREENSHOTS') && params.TEST_SCREENSHOTS == 'True' && currentResult == 'FAILURE') { + CreateExportTestScreenshotsStage(pipelineConfig, platform.key, build_job_name, envVars, params).call() + } CreateTeardownStage(envVars).call() } }