diff --git a/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Main_GPU.py b/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Main_GPU.py index 23fb249761..534b061b1c 100644 --- a/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Main_GPU.py +++ b/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Main_GPU.py @@ -13,10 +13,10 @@ import zipfile import pytest import ly_test_tools.environment.file_system as file_system -from ly_test_tools.image.screenshot_compare_qssim import qssim as compare_screenshots from ly_test_tools.benchmark.data_aggregator import BenchmarkDataAggregator import editor_python_test_tools.hydra_test_utils as hydra +from .atom_utils.atom_component_helper import compare_screenshot_similarity, ImageComparisonTestFailure logger = logging.getLogger(__name__) DEFAULT_SUBFOLDER_PATH = 'user/PythonTests/Automated/Screenshots' @@ -92,8 +92,6 @@ class TestAllComponentsIndepthTests(object): "Exited game mode" ] unexpected_lines = [ - "Trace::Assert", - "Trace::Error", "Traceback (most recent call last):", "Screenshot failed" ] @@ -111,10 +109,12 @@ class TestAllComponentsIndepthTests(object): null_renderer=False, ) - for test_screenshot, golden_screenshot in zip(test_screenshots, golden_images): - compare_screenshots(test_screenshot, golden_screenshot) - - create_screenshots_archive(screenshot_directory) + similarity_threshold = 0.99 + for test_screenshot, golden_image in zip(test_screenshots, golden_images): + screenshot_comparison_result = compare_screenshot_similarity( + test_screenshot, golden_image, similarity_threshold, True, screenshot_directory) + if screenshot_comparison_result != "Screenshots match": + raise Exception(f"Screenshot test failed: {screenshot_comparison_result}") @pytest.mark.test_case_id("C34525095") def test_LightComponent_ScreenshotMatchesGoldenImage( @@ -168,10 +168,12 @@ class TestAllComponentsIndepthTests(object): null_renderer=False, ) - for test_screenshot, golden_screenshot in zip(test_screenshots, golden_images): - compare_screenshots(test_screenshot, golden_screenshot) - - create_screenshots_archive(screenshot_directory) + similarity_threshold = 0.99 + for test_screenshot, golden_image in zip(test_screenshots, golden_images): + screenshot_comparison_result = compare_screenshot_similarity( + test_screenshot, golden_image, similarity_threshold, True, screenshot_directory) + if screenshot_comparison_result != "Screenshots match": + raise ImageComparisonTestFailure(f"Screenshot test failed: {screenshot_comparison_result}") @pytest.mark.parametrize('rhi', ['dx12', 'vulkan']) @@ -219,7 +221,6 @@ class TestPerformanceBenchmarkSuite(object): @pytest.mark.parametrize("project", ["AutomatedTesting"]) @pytest.mark.parametrize("launcher_platform", ['windows_generic']) -@pytest.mark.system class TestMaterialEditor(object): @pytest.mark.parametrize("cfg_args,expected_lines", [ diff --git a/AutomatedTesting/Gem/PythonTests/Atom/atom_utils/atom_component_helper.py b/AutomatedTesting/Gem/PythonTests/Atom/atom_utils/atom_component_helper.py index 11832f8846..5fa558ab1a 100644 --- a/AutomatedTesting/Gem/PythonTests/Atom/atom_utils/atom_component_helper.py +++ b/AutomatedTesting/Gem/PythonTests/Atom/atom_utils/atom_component_helper.py @@ -1,5 +1,6 @@ """ -Copyright (c) Contributors to the Open 3D Engine Project. For complete copyright and license terms please see the LICENSE at the root of this distribution. +Copyright (c) Contributors to the Open 3D Engine Project. +For complete copyright and license terms please see the LICENSE at the root of this distribution. SPDX-License-Identifier: Apache-2.0 OR MIT @@ -8,12 +9,19 @@ import datetime import os import zipfile +from ly_test_tools.image.screenshot_compare_qssim import qssim as compare_screenshots + + +class ImageComparisonTestFailure(Exception): + """Custom test failure for failed image comparisons.""" + pass + def create_screenshots_archive(screenshot_path): """ Creates a new zip file archive at archive_path containing all files listed within archive_path. :param screenshot_path: location containing the files to archive, the zip archive file will also be saved here. - :return: None, but creates a new zip file archive inside path containing all of the files inside archive_path. + :return: path to the created .zip file archive. """ files_to_archive = [] @@ -27,14 +35,16 @@ def create_screenshots_archive(screenshot_path): # Setup variables for naming the zip archive file. timestamp = datetime.datetime.now().timestamp() formatted_timestamp = datetime.datetime.utcfromtimestamp(timestamp).strftime("%Y-%m-%d_%H-%M-%S") - screenshots_file = os.path.join(screenshot_path, f'screenshots_{formatted_timestamp}.zip') + screenshots_zip_file = os.path.join(screenshot_path, f'screenshots_{formatted_timestamp}.zip') # Write all of the valid .png and .ppm files to the archive file. - with zipfile.ZipFile(screenshots_file, 'w', compression=zipfile.ZIP_DEFLATED, allowZip64=True) as zip_archive: + with zipfile.ZipFile(screenshots_zip_file, 'w', compression=zipfile.ZIP_DEFLATED, allowZip64=True) as zip_archive: for file_path in files_to_archive: file_name = os.path.basename(file_path) zip_archive.write(file_path, file_name) + return screenshots_zip_file + def golden_images_directory(): """ @@ -53,6 +63,36 @@ def golden_images_directory(): return golden_images_dir +def compare_screenshot_similarity( + test_screenshot, golden_image, similarity_threshold, create_zip_archive=False, screenshot_directory=""): + """ + Compares the similarity between a test screenshot and a golden image. + It returns a "Screenshots match" string if the comparison mean value is higher than the similarity threshold. + Otherwise, it returns an error string. + :param test_screenshot: path to the test screenshot to compare. + :param golden_image: path to the golden image to compare. + :param similarity_threshold: value for the comparison mean value to be asserted against. + :param create_zip_archive: toggle to create a zip archive containing the screenshots if the assert check fails. + :param screenshot_directory: directory containing screenshots to create zip archive from. + :return: Error string if compared mean value < similarity threshold or screenshot_directory is missing for .zip, + otherwise it returns a "Screenshots match" string. + """ + error = "Screenshots match" + if create_zip_archive and not screenshot_directory: + error = 'You must specify a screenshot_directory in order to create a zip archive.\n' + + mean_similarity = compare_screenshots(test_screenshot, golden_image) + if not mean_similarity > similarity_threshold: + if create_zip_archive: + create_screenshots_archive(screenshot_directory) + error = ( + f"When comparing the test_screenshot: '{test_screenshot}' " + f"to golden_image: '{golden_image}' the mean similarity of '{mean_similarity}' " + f"was lower than the similarity threshold of '{similarity_threshold}'. ") + + return error + + def create_basic_atom_level(level_name): """ Creates a new level inside the Editor matching level_name & adds the following: diff --git a/AutomatedTesting/Gem/PythonTests/Atom/tests/hydra_GPUTest_BasicLevelSetup.py b/AutomatedTesting/Gem/PythonTests/Atom/tests/hydra_GPUTest_BasicLevelSetup.py index 9515712583..4eb942b903 100644 --- a/AutomatedTesting/Gem/PythonTests/Atom/tests/hydra_GPUTest_BasicLevelSetup.py +++ b/AutomatedTesting/Gem/PythonTests/Atom/tests/hydra_GPUTest_BasicLevelSetup.py @@ -6,18 +6,6 @@ SPDX-License-Identifier: Apache-2.0 OR MIT """ import os -import sys - -import azlmbr.asset as asset -import azlmbr.bus as bus -import azlmbr.camera -import azlmbr.entity as entity -import azlmbr.legacy.general as general -import azlmbr.math as math -import azlmbr.paths -import azlmbr.editor as editor - -sys.path.append(os.path.join(azlmbr.paths.projectroot, "Gem", "PythonTests")) import editor_python_test_tools.hydra_editor_utils as hydra from editor_python_test_tools.editor_test_helper import EditorTestHelper @@ -48,6 +36,15 @@ def run(): 12. Finally enters game mode, takes a screenshot, exits game mode, & saves the level. :return: None """ + import azlmbr.asset as asset + import azlmbr.bus as bus + import azlmbr.camera as camera + import azlmbr.entity as entity + import azlmbr.legacy.general as general + import azlmbr.math as math + import azlmbr.paths + import azlmbr.editor as editor + def initial_viewport_setup(screen_width, screen_height): general.set_viewport_size(screen_width, screen_height) general.update_viewport() @@ -147,22 +144,25 @@ def run(): parent_id=default_level.id ) azlmbr.components.TransformBus(azlmbr.bus.Event, "SetLocalUniformScale", ground_plane.id, 32.0) - ground_plane_material_asset_path = os.path.join("Materials", "Presets", "PBR", "metal_chrome.azmaterial") - ground_plane_material_asset = asset.AssetCatalogRequestBus( - bus.Broadcast, "GetAssetIdByPath", ground_plane_material_asset_path, math.Uuid(), False) - ground_plane.get_set_test(0, "Default Material|Material Asset", ground_plane_material_asset) - # Work around to add the correct Atom Mesh component + + # Work around to add the correct Atom Mesh component and asset. mesh_type_id = azlmbr.globals.property.EditorMeshComponentTypeId ground_plane.components.append( editor.EditorComponentAPIBus( bus.Broadcast, "AddComponentsOfType", ground_plane.id, [mesh_type_id] ).GetValue()[0] ) - ground_plane_mesh_asset_path = os.path.join("Objects", "plane.azmodel") + ground_plane_mesh_asset_path = os.path.join("TestData", "Objects", "plane.azmodel") ground_plane_mesh_asset = asset.AssetCatalogRequestBus( bus.Broadcast, "GetAssetIdByPath", ground_plane_mesh_asset_path, math.Uuid(), False) hydra.get_set_test(ground_plane, 1, "Controller|Configuration|Mesh Asset", ground_plane_mesh_asset) + # Add Atom Material component and asset. + ground_plane_material_asset_path = os.path.join("Materials", "Presets", "PBR", "metal_chrome.azmaterial") + ground_plane_material_asset = asset.AssetCatalogRequestBus( + bus.Broadcast, "GetAssetIdByPath", ground_plane_material_asset_path, math.Uuid(), False) + ground_plane.get_set_test(0, "Default Material|Material Asset", ground_plane_material_asset) + # Create directional_light entity and set the properties directional_light = hydra.Entity("directional_light") directional_light.create_entity( @@ -180,11 +180,8 @@ def run(): components=["Material"], parent_id=default_level.id ) - sphere_material_asset_path = os.path.join("Materials", "Presets", "PBR", "metal_brass_polished.azmaterial") - sphere_material_asset = asset.AssetCatalogRequestBus( - bus.Broadcast, "GetAssetIdByPath", sphere_material_asset_path, math.Uuid(), False) - sphere.get_set_test(0, "Default Material|Material Asset", sphere_material_asset) - # Work around to add the correct Atom Mesh component + + # Work around to add the correct Atom Mesh component and asset. sphere.components.append( editor.EditorComponentAPIBus( bus.Broadcast, "AddComponentsOfType", sphere.id, [mesh_type_id] @@ -195,6 +192,12 @@ def run(): bus.Broadcast, "GetAssetIdByPath", sphere_mesh_asset_path, math.Uuid(), False) hydra.get_set_test(sphere, 1, "Controller|Configuration|Mesh Asset", sphere_mesh_asset) + # Add Atom Material component and asset. + sphere_material_asset_path = os.path.join("Materials", "Presets", "PBR", "metal_brass_polished.azmaterial") + sphere_material_asset = asset.AssetCatalogRequestBus( + bus.Broadcast, "GetAssetIdByPath", sphere_material_asset_path, math.Uuid(), False) + sphere.get_set_test(0, "Default Material|Material Asset", sphere_material_asset) + # Create camera component and set the properties camera_entity = hydra.Entity("camera") position = math.Vector3(5.5, -12.0, 9.0) @@ -204,10 +207,9 @@ def run(): ) azlmbr.components.TransformBus(azlmbr.bus.Event, "SetLocalRotation", camera_entity.id, rotation) camera_entity.get_set_test(0, "Controller|Configuration|Field of view", 60.0) - azlmbr.camera.EditorCameraViewRequestBus(azlmbr.bus.Event, "ToggleCameraAsActiveView", camera_entity.id) + camera.EditorCameraViewRequestBus(azlmbr.bus.Event, "ToggleCameraAsActiveView", camera_entity.id) - # Save level, enter game mode, take screenshot, & exit game mode. - general.save_level() + # Enter game mode, take screenshot, & exit game mode. general.idle_wait(0.5) general.enter_game_mode() general.idle_wait(1.0)