diff --git a/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Main_GPU.py b/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Main_GPU.py index 23fb249761..f0e92c7e3e 100644 --- a/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Main_GPU.py +++ b/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Main_GPU.py @@ -13,10 +13,10 @@ import zipfile import pytest import ly_test_tools.environment.file_system as file_system -from ly_test_tools.image.screenshot_compare_qssim import qssim as compare_screenshots from ly_test_tools.benchmark.data_aggregator import BenchmarkDataAggregator import editor_python_test_tools.hydra_test_utils as hydra +from .atom_utils.atom_component_helper import compare_screenshot_similarity, ImageComparisonTestFailure logger = logging.getLogger(__name__) DEFAULT_SUBFOLDER_PATH = 'user/PythonTests/Automated/Screenshots' @@ -69,7 +69,7 @@ def create_screenshots_archive(screenshot_path): @pytest.mark.parametrize("project", ["AutomatedTesting"]) @pytest.mark.parametrize("launcher_platform", ["windows_editor"]) -@pytest.mark.parametrize("level", ["auto_test"]) +@pytest.mark.parametrize("level", ["Base"]) class TestAllComponentsIndepthTests(object): @pytest.mark.parametrize("screenshot_name", ["AtomBasicLevelSetup.ppm"]) @@ -91,12 +91,7 @@ class TestAllComponentsIndepthTests(object): "Viewport is set to the expected size: True", "Exited game mode" ] - unexpected_lines = [ - "Trace::Assert", - "Trace::Error", - "Traceback (most recent call last):", - "Screenshot failed" - ] + unexpected_lines = ["Traceback (most recent call last):"] hydra.launch_and_validate_results( request, @@ -111,10 +106,12 @@ class TestAllComponentsIndepthTests(object): null_renderer=False, ) - for test_screenshot, golden_screenshot in zip(test_screenshots, golden_images): - compare_screenshots(test_screenshot, golden_screenshot) - - create_screenshots_archive(screenshot_directory) + similarity_threshold = 0.99 + for test_screenshot, golden_image in zip(test_screenshots, golden_images): + screenshot_comparison_result = compare_screenshot_similarity( + test_screenshot, golden_image, similarity_threshold, True, screenshot_directory) + if screenshot_comparison_result != "Screenshots match": + raise Exception(f"Screenshot test failed: {screenshot_comparison_result}") @pytest.mark.test_case_id("C34525095") def test_LightComponent_ScreenshotMatchesGoldenImage( @@ -149,12 +146,7 @@ class TestAllComponentsIndepthTests(object): golden_images.append(golden_image_path) expected_lines = ["spot_light Controller|Configuration|Shadows|Shadowmap size: SUCCESS"] - unexpected_lines = [ - "Trace::Assert", - "Trace::Error", - "Traceback (most recent call last):", - "Screenshot failed", - ] + unexpected_lines = ["Traceback (most recent call last):"] hydra.launch_and_validate_results( request, TEST_DIRECTORY, @@ -168,10 +160,12 @@ class TestAllComponentsIndepthTests(object): null_renderer=False, ) - for test_screenshot, golden_screenshot in zip(test_screenshots, golden_images): - compare_screenshots(test_screenshot, golden_screenshot) - - create_screenshots_archive(screenshot_directory) + similarity_threshold = 0.99 + for test_screenshot, golden_image in zip(test_screenshots, golden_images): + screenshot_comparison_result = compare_screenshot_similarity( + test_screenshot, golden_image, similarity_threshold, True, screenshot_directory) + if screenshot_comparison_result != "Screenshots match": + raise ImageComparisonTestFailure(f"Screenshot test failed: {screenshot_comparison_result}") @pytest.mark.parametrize('rhi', ['dx12', 'vulkan']) @@ -219,7 +213,6 @@ class TestPerformanceBenchmarkSuite(object): @pytest.mark.parametrize("project", ["AutomatedTesting"]) @pytest.mark.parametrize("launcher_platform", ['windows_generic']) -@pytest.mark.system class TestMaterialEditor(object): @pytest.mark.parametrize("cfg_args,expected_lines", [ diff --git a/AutomatedTesting/Gem/PythonTests/Atom/atom_utils/atom_component_helper.py b/AutomatedTesting/Gem/PythonTests/Atom/atom_utils/atom_component_helper.py index 11832f8846..50cc15f7e8 100644 --- a/AutomatedTesting/Gem/PythonTests/Atom/atom_utils/atom_component_helper.py +++ b/AutomatedTesting/Gem/PythonTests/Atom/atom_utils/atom_component_helper.py @@ -1,5 +1,6 @@ """ -Copyright (c) Contributors to the Open 3D Engine Project. For complete copyright and license terms please see the LICENSE at the root of this distribution. +Copyright (c) Contributors to the Open 3D Engine Project. +For complete copyright and license terms please see the LICENSE at the root of this distribution. SPDX-License-Identifier: Apache-2.0 OR MIT @@ -8,12 +9,19 @@ import datetime import os import zipfile +from ly_test_tools.image.screenshot_compare_qssim import qssim as compare_screenshots + + +class ImageComparisonTestFailure(Exception): + """Custom test failure for failed image comparisons.""" + pass + def create_screenshots_archive(screenshot_path): """ Creates a new zip file archive at archive_path containing all files listed within archive_path. :param screenshot_path: location containing the files to archive, the zip archive file will also be saved here. - :return: None, but creates a new zip file archive inside path containing all of the files inside archive_path. + :return: path to the created .zip file archive. """ files_to_archive = [] @@ -27,14 +35,16 @@ def create_screenshots_archive(screenshot_path): # Setup variables for naming the zip archive file. timestamp = datetime.datetime.now().timestamp() formatted_timestamp = datetime.datetime.utcfromtimestamp(timestamp).strftime("%Y-%m-%d_%H-%M-%S") - screenshots_file = os.path.join(screenshot_path, f'screenshots_{formatted_timestamp}.zip') + screenshots_zip_file = os.path.join(screenshot_path, f'screenshots_{formatted_timestamp}.zip') # Write all of the valid .png and .ppm files to the archive file. - with zipfile.ZipFile(screenshots_file, 'w', compression=zipfile.ZIP_DEFLATED, allowZip64=True) as zip_archive: + with zipfile.ZipFile(screenshots_zip_file, 'w', compression=zipfile.ZIP_DEFLATED, allowZip64=True) as zip_archive: for file_path in files_to_archive: file_name = os.path.basename(file_path) zip_archive.write(file_path, file_name) + return screenshots_zip_file + def golden_images_directory(): """ @@ -53,6 +63,36 @@ def golden_images_directory(): return golden_images_dir +def compare_screenshot_similarity( + test_screenshot, golden_image, similarity_threshold, create_zip_archive=False, screenshot_directory=""): + """ + Compares the similarity between a test screenshot and a golden image. + It returns a "Screenshots match" string if the comparison mean value is higher than the similarity threshold. + Otherwise, it returns an error string. + :param test_screenshot: path to the test screenshot to compare. + :param golden_image: path to the golden image to compare. + :param similarity_threshold: value for the comparison mean value to be asserted against. + :param create_zip_archive: toggle to create a zip archive containing the screenshots if the assert check fails. + :param screenshot_directory: directory containing screenshots to create zip archive from. + :return: Error string if compared mean value < similarity threshold or screenshot_directory is missing for .zip, + otherwise it returns a "Screenshots match" string. + """ + result = "Screenshots match" + if create_zip_archive and not screenshot_directory: + result = 'You must specify a screenshot_directory in order to create a zip archive.\n' + + mean_similarity = compare_screenshots(test_screenshot, golden_image) + if not mean_similarity > similarity_threshold: + if create_zip_archive: + create_screenshots_archive(screenshot_directory) + result = ( + f"When comparing the test_screenshot: '{test_screenshot}' " + f"to golden_image: '{golden_image}' the mean similarity of '{mean_similarity}' " + f"was lower than the similarity threshold of '{similarity_threshold}'. ") + + return result + + def create_basic_atom_level(level_name): """ Creates a new level inside the Editor matching level_name & adds the following: @@ -76,29 +116,11 @@ def create_basic_atom_level(level_name): helper = EditorTestHelper(log_prefix="Atom_EditorTestHelper") - # Create a new level. - new_level_name = level_name - heightmap_resolution = 512 - heightmap_meters_per_pixel = 1 - terrain_texture_resolution = 412 - use_terrain = False - - # Return codes are ECreateLevelResult defined in CryEdit.h - return_code = general.create_level_no_prompt( - new_level_name, heightmap_resolution, heightmap_meters_per_pixel, terrain_texture_resolution, use_terrain) - if return_code == 1: - general.log(f"{new_level_name} level already exists") - elif return_code == 2: - general.log("Failed to create directory") - elif return_code == 3: - general.log("Directory length is too long") - elif return_code != 0: - general.log("Unknown error, failed to create level") - else: - general.log(f"{new_level_name} level created successfully") - - # Enable idle and update viewport. + # Wait for Editor idle loop before executing Python hydra scripts. general.idle_enable(True) + + # Basic setup for opened level. + helper.open_level(level_name="Base") general.idle_wait(1.0) general.update_viewport() general.idle_wait(0.5) # half a second is more than enough for updating the viewport. @@ -165,24 +187,25 @@ def create_basic_atom_level(level_name): components=["Material"], parent_id=default_level.id) azlmbr.components.TransformBus(azlmbr.bus.Event, "SetLocalUniformScale", ground_plane.id, 32.0) - ground_plane_material_asset_path = os.path.join( - "Materials", "Presets", "PBR", "metal_chrome.azmaterial") - ground_plane_material_asset_value = asset.AssetCatalogRequestBus( - bus.Broadcast, "GetAssetIdByPath", ground_plane_material_asset_path, math.Uuid(), False) - ground_plane.get_set_test(0, "Default Material|Material Asset", ground_plane_material_asset_value) - # Work around to add the correct Atom Mesh component + # Work around to add the correct Atom Mesh component and asset. mesh_type_id = azlmbr.globals.property.EditorMeshComponentTypeId ground_plane.components.append( editor.EditorComponentAPIBus( bus.Broadcast, "AddComponentsOfType", ground_plane.id, [mesh_type_id] ).GetValue()[0] ) - ground_plane_mesh_asset_path = os.path.join("Models", "plane.azmodel") + ground_plane_mesh_asset_path = os.path.join("TestData", "Objects", "plane.azmodel") ground_plane_mesh_asset_value = asset.AssetCatalogRequestBus( bus.Broadcast, "GetAssetIdByPath", ground_plane_mesh_asset_path, math.Uuid(), False) ground_plane.get_set_test(1, "Controller|Configuration|Mesh Asset", ground_plane_mesh_asset_value) + # Add Atom Material component and asset. + ground_plane_material_asset_path = os.path.join("Materials", "Presets", "PBR", "metal_chrome.azmaterial") + ground_plane_material_asset_value = asset.AssetCatalogRequestBus( + bus.Broadcast, "GetAssetIdByPath", ground_plane_material_asset_path, math.Uuid(), False) + ground_plane.get_set_test(0, "Default Material|Material Asset", ground_plane_material_asset_value) + # Create directional_light entity and set the properties directional_light = hydra.Entity("directional_light") directional_light.create_entity( @@ -199,12 +222,8 @@ def create_basic_atom_level(level_name): entity_position=math.Vector3(0.0, 0.0, 1.0), components=["Material"], parent_id=default_level.id) - sphere_material_asset_path = os.path.join("Materials", "Presets", "PBR", "metal_brass_polished.azmaterial") - sphere_material_asset_value = asset.AssetCatalogRequestBus( - bus.Broadcast, "GetAssetIdByPath", sphere_material_asset_path, math.Uuid(), False) - sphere_entity.get_set_test(0, "Default Material|Material Asset", sphere_material_asset_value) - # Work around to add the correct Atom Mesh component + # Work around to add the correct Atom Mesh component and asset. sphere_entity.components.append( editor.EditorComponentAPIBus( bus.Broadcast, "AddComponentsOfType", sphere_entity.id, [mesh_type_id] @@ -215,6 +234,12 @@ def create_basic_atom_level(level_name): bus.Broadcast, "GetAssetIdByPath", sphere_mesh_asset_path, math.Uuid(), False) sphere_entity.get_set_test(1, "Controller|Configuration|Mesh Asset", sphere_mesh_asset_value) + # Add Atom Material component and asset. + sphere_material_asset_path = os.path.join("Materials", "Presets", "PBR", "metal_brass_polished.azmaterial") + sphere_material_asset_value = asset.AssetCatalogRequestBus( + bus.Broadcast, "GetAssetIdByPath", sphere_material_asset_path, math.Uuid(), False) + sphere_entity.get_set_test(0, "Default Material|Material Asset", sphere_material_asset_value) + # Create camera component and set the properties camera_entity = hydra.Entity("camera") camera_entity.create_entity( diff --git a/AutomatedTesting/Gem/PythonTests/Atom/tests/hydra_GPUTest_BasicLevelSetup.py b/AutomatedTesting/Gem/PythonTests/Atom/tests/hydra_GPUTest_BasicLevelSetup.py index 9515712583..645447e6de 100644 --- a/AutomatedTesting/Gem/PythonTests/Atom/tests/hydra_GPUTest_BasicLevelSetup.py +++ b/AutomatedTesting/Gem/PythonTests/Atom/tests/hydra_GPUTest_BasicLevelSetup.py @@ -6,18 +6,6 @@ SPDX-License-Identifier: Apache-2.0 OR MIT """ import os -import sys - -import azlmbr.asset as asset -import azlmbr.bus as bus -import azlmbr.camera -import azlmbr.entity as entity -import azlmbr.legacy.general as general -import azlmbr.math as math -import azlmbr.paths -import azlmbr.editor as editor - -sys.path.append(os.path.join(azlmbr.paths.projectroot, "Gem", "PythonTests")) import editor_python_test_tools.hydra_editor_utils as hydra from editor_python_test_tools.editor_test_helper import EditorTestHelper @@ -45,9 +33,18 @@ def run(): 11. Adds a "camera" entity to "default_level" & adds a Camera component with 80 degree FOV and Transform values: Translate - x:5.5m, y:-12.0m, z:9.0m Rotate - x:-27.0, y:-12.0, z:25.0 - 12. Finally enters game mode, takes a screenshot, exits game mode, & saves the level. + 12. Finally enters game mode, takes a screenshot, & exits game mode. :return: None """ + import azlmbr.asset as asset + import azlmbr.bus as bus + import azlmbr.camera as camera + import azlmbr.entity as entity + import azlmbr.legacy.general as general + import azlmbr.math as math + import azlmbr.paths + import azlmbr.editor as editor + def initial_viewport_setup(screen_width, screen_height): general.set_viewport_size(screen_width, screen_height) general.update_viewport() @@ -84,33 +81,11 @@ def run(): general.run_console("r_displayInfo=0") general.idle_wait(1.0) - return True - # Wait for Editor idle loop before executing Python hydra scripts. general.idle_enable(True) - # Open the auto_test level. - new_level_name = "auto_test" # Specified in class TestAllComponentsIndepthTests() - heightmap_resolution = 512 - heightmap_meters_per_pixel = 1 - terrain_texture_resolution = 412 - use_terrain = False - - # Return codes are ECreateLevelResult defined in CryEdit.h - return_code = general.create_level_no_prompt( - new_level_name, heightmap_resolution, heightmap_meters_per_pixel, terrain_texture_resolution, use_terrain) - if return_code == 1: - general.log(f"{new_level_name} level already exists") - elif return_code == 2: - general.log("Failed to create directory") - elif return_code == 3: - general.log("Directory length is too long") - elif return_code != 0: - general.log("Unknown error, failed to create level") - else: - general.log(f"{new_level_name} level created successfully") - - # Basic setup for newly created level. + # Basic setup for opened level. + helper.open_level(level_name="Base") after_level_load() initial_viewport_setup(SCREEN_WIDTH, SCREEN_HEIGHT) @@ -147,22 +122,25 @@ def run(): parent_id=default_level.id ) azlmbr.components.TransformBus(azlmbr.bus.Event, "SetLocalUniformScale", ground_plane.id, 32.0) - ground_plane_material_asset_path = os.path.join("Materials", "Presets", "PBR", "metal_chrome.azmaterial") - ground_plane_material_asset = asset.AssetCatalogRequestBus( - bus.Broadcast, "GetAssetIdByPath", ground_plane_material_asset_path, math.Uuid(), False) - ground_plane.get_set_test(0, "Default Material|Material Asset", ground_plane_material_asset) - # Work around to add the correct Atom Mesh component + + # Work around to add the correct Atom Mesh component and asset. mesh_type_id = azlmbr.globals.property.EditorMeshComponentTypeId ground_plane.components.append( editor.EditorComponentAPIBus( bus.Broadcast, "AddComponentsOfType", ground_plane.id, [mesh_type_id] ).GetValue()[0] ) - ground_plane_mesh_asset_path = os.path.join("Objects", "plane.azmodel") + ground_plane_mesh_asset_path = os.path.join("TestData", "Objects", "plane.azmodel") ground_plane_mesh_asset = asset.AssetCatalogRequestBus( bus.Broadcast, "GetAssetIdByPath", ground_plane_mesh_asset_path, math.Uuid(), False) hydra.get_set_test(ground_plane, 1, "Controller|Configuration|Mesh Asset", ground_plane_mesh_asset) + # Add Atom Material component and asset. + ground_plane_material_asset_path = os.path.join("Materials", "Presets", "PBR", "metal_chrome.azmaterial") + ground_plane_material_asset = asset.AssetCatalogRequestBus( + bus.Broadcast, "GetAssetIdByPath", ground_plane_material_asset_path, math.Uuid(), False) + ground_plane.get_set_test(0, "Default Material|Material Asset", ground_plane_material_asset) + # Create directional_light entity and set the properties directional_light = hydra.Entity("directional_light") directional_light.create_entity( @@ -180,11 +158,8 @@ def run(): components=["Material"], parent_id=default_level.id ) - sphere_material_asset_path = os.path.join("Materials", "Presets", "PBR", "metal_brass_polished.azmaterial") - sphere_material_asset = asset.AssetCatalogRequestBus( - bus.Broadcast, "GetAssetIdByPath", sphere_material_asset_path, math.Uuid(), False) - sphere.get_set_test(0, "Default Material|Material Asset", sphere_material_asset) - # Work around to add the correct Atom Mesh component + + # Work around to add the correct Atom Mesh component and asset. sphere.components.append( editor.EditorComponentAPIBus( bus.Broadcast, "AddComponentsOfType", sphere.id, [mesh_type_id] @@ -195,6 +170,12 @@ def run(): bus.Broadcast, "GetAssetIdByPath", sphere_mesh_asset_path, math.Uuid(), False) hydra.get_set_test(sphere, 1, "Controller|Configuration|Mesh Asset", sphere_mesh_asset) + # Add Atom Material component and asset. + sphere_material_asset_path = os.path.join("Materials", "Presets", "PBR", "metal_brass_polished.azmaterial") + sphere_material_asset = asset.AssetCatalogRequestBus( + bus.Broadcast, "GetAssetIdByPath", sphere_material_asset_path, math.Uuid(), False) + sphere.get_set_test(0, "Default Material|Material Asset", sphere_material_asset) + # Create camera component and set the properties camera_entity = hydra.Entity("camera") position = math.Vector3(5.5, -12.0, 9.0) @@ -204,10 +185,9 @@ def run(): ) azlmbr.components.TransformBus(azlmbr.bus.Event, "SetLocalRotation", camera_entity.id, rotation) camera_entity.get_set_test(0, "Controller|Configuration|Field of view", 60.0) - azlmbr.camera.EditorCameraViewRequestBus(azlmbr.bus.Event, "ToggleCameraAsActiveView", camera_entity.id) + camera.EditorCameraViewRequestBus(azlmbr.bus.Event, "ToggleCameraAsActiveView", camera_entity.id) - # Save level, enter game mode, take screenshot, & exit game mode. - general.save_level() + # Enter game mode, take screenshot, & exit game mode. general.idle_wait(0.5) general.enter_game_mode() general.idle_wait(1.0) diff --git a/AutomatedTesting/Gem/PythonTests/Atom/tests/hydra_GPUTest_LightComponent.py b/AutomatedTesting/Gem/PythonTests/Atom/tests/hydra_GPUTest_LightComponent.py index 1c3e6226c1..f69ceb2120 100644 --- a/AutomatedTesting/Gem/PythonTests/Atom/tests/hydra_GPUTest_LightComponent.py +++ b/AutomatedTesting/Gem/PythonTests/Atom/tests/hydra_GPUTest_LightComponent.py @@ -22,7 +22,7 @@ from editor_python_test_tools.editor_test_helper import EditorTestHelper helper = EditorTestHelper(log_prefix="Atom_EditorTestHelper") -LEVEL_NAME = "auto_test" +LEVEL_NAME = "Base" LIGHT_COMPONENT = "Light" LIGHT_TYPE_PROPERTY = 'Controller|Configuration|Light type' DEGREE_RADIAN_FACTOR = 0.0174533 diff --git a/Gems/Atom/TestData/TestData/Objects/plane.fbx b/Gems/Atom/TestData/TestData/Objects/plane.fbx new file mode 100644 index 0000000000..b274bfa282 --- /dev/null +++ b/Gems/Atom/TestData/TestData/Objects/plane.fbx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a1f8d75dcd85e8b4aa57f6c0c81af0300ff96915ba3c2b591095c215d5e1d8c +size 12072