Upload test screenshots to s3 on test failure. (#3815)

* add s3 upload on screenshot test failure, should only apply to the test_gpu_profile_vs2019 job on nightly runs

* adds support for zipping screenshot files up prior to uploading to the s3 bucket and also adds the ACL extra arg to the upload_to_s3.py execution

* remove unused json import

* remove regex to use .endswith() instead and rename variables to be more clear (PR feedback)

* rename create_zip_archive to create_screenshots_archive

Signed-off-by: jromnoa <jromnoa@amazon.com>
This commit is contained in:
jromnoa
2021-09-08 04:11:44 -07:00
committed by GitHub
parent 052ff90c9d
commit 5c4ce8cd06
2 changed files with 61 additions and 3 deletions
@@ -7,8 +7,10 @@ SPDX-License-Identifier: Apache-2.0 OR MIT
Tests that require a GPU in order to run.
"""
import datetime
import logging
import os
import zipfile
import pytest
@@ -40,6 +42,33 @@ def golden_images_directory():
return golden_images_dir
def create_screenshots_archive(screenshot_path):
"""
Creates a new zip file archive at archive_path containing all files listed within archive_path.
:param screenshot_path: location containing the files to archive, the zip archive file will also be saved here.
:return: None, but creates a new zip file archive inside path containing all of the files inside archive_path.
"""
files_to_archive = []
# Search for .png and .ppm files to add to the zip archive file.
for (folder_name, sub_folders, file_names) in os.walk(screenshot_path):
for file_name in file_names:
if file_name.endswith(".png") or file_name.endswith(".ppm"):
file_path = os.path.join(folder_name, file_name)
files_to_archive.append(file_path)
# Setup variables for naming the zip archive file.
timestamp = datetime.datetime.now().timestamp()
formatted_timestamp = datetime.datetime.utcfromtimestamp(timestamp).strftime("%Y-%m-%d_%H-%M-%S")
screenshots_file = os.path.join(screenshot_path, f'zip_archive_{formatted_timestamp}.zip')
# Write all of the valid .png and .ppm files to the archive file.
with zipfile.ZipFile(screenshots_file, 'w', compression=zipfile.ZIP_DEFLATED, allowZip64=True) as zip_archive:
for file_path in files_to_archive:
file_name = os.path.basename(file_path)
zip_archive.write(file_path, file_name)
@pytest.mark.parametrize("project", ["AutomatedTesting"])
@pytest.mark.parametrize("launcher_platform", ["windows_editor"])
@pytest.mark.parametrize("level", ["auto_test"])
@@ -53,8 +82,8 @@ class TestAllComponentsIndepthTests(object):
Tests that a basic rendering level setup can be created (lighting, meshes, materials, etc.).
"""
# Clear existing test screenshots before starting test.
test_screenshots = [os.path.join(
workspace.paths.project(), DEFAULT_SUBFOLDER_PATH, screenshot_name)]
screenshot_directory = os.path.join(workspace.paths.project(), DEFAULT_SUBFOLDER_PATH)
test_screenshots = [os.path.join(screenshot_directory, screenshot_name)]
file_system.delete(test_screenshots, True, True)
golden_images = [os.path.join(golden_images_directory(), screenshot_name)]
@@ -86,6 +115,8 @@ class TestAllComponentsIndepthTests(object):
for test_screenshot, golden_screenshot in zip(test_screenshots, golden_images):
compare_screenshots(test_screenshot, golden_screenshot)
create_screenshots_archive(screenshot_directory)
def test_LightComponent_ScreenshotMatchesGoldenImage(
self, request, editor, workspace, project, launcher_platform, level):
"""
@@ -105,9 +136,10 @@ class TestAllComponentsIndepthTests(object):
"SpotLight_5.ppm",
"SpotLight_6.ppm",
]
screenshot_directory = os.path.join(workspace.paths.project(), DEFAULT_SUBFOLDER_PATH)
test_screenshots = []
for screenshot in screenshot_names:
screenshot_path = os.path.join(workspace.paths.project(), DEFAULT_SUBFOLDER_PATH, screenshot)
screenshot_path = os.path.join(screenshot_directory, screenshot)
test_screenshots.append(screenshot_path)
file_system.delete(test_screenshots, True, True)
@@ -139,6 +171,8 @@ class TestAllComponentsIndepthTests(object):
for test_screenshot, golden_screenshot in zip(test_screenshots, golden_images):
compare_screenshots(test_screenshot, golden_screenshot)
create_screenshots_archive(screenshot_directory)
@pytest.mark.parametrize('rhi', ['dx12', 'vulkan'])
@pytest.mark.parametrize("project", ["AutomatedTesting"])