diff --git a/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Main_GPU.py b/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Main_GPU.py index 14d850245c..ad2662113b 100644 --- a/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Main_GPU.py +++ b/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Main_GPU.py @@ -12,7 +12,7 @@ import pytest import editor_python_test_tools.hydra_test_utils as hydra import ly_test_tools.environment.file_system as file_system from ly_test_tools.benchmark.data_aggregator import BenchmarkDataAggregator -from ly_test_tools.o3de.editor_test import EditorSharedTest, EditorTestSuite +from ly_test_tools.o3de.editor_test import EditorSingleTest, EditorTestSuite from Atom.atom_utils.atom_component_helper import compare_screenshot_to_golden_image, golden_images_directory DEFAULT_SUBFOLDER_PATH = 'user/PythonTests/Automated/Screenshots' @@ -23,89 +23,137 @@ logger = logging.getLogger(__name__) @pytest.mark.parametrize("launcher_platform", ['windows_editor']) class TestAutomation(EditorTestSuite): # Remove -autotest_mode from global_extra_cmdline_args since we need rendering for these tests. - global_extra_cmdline_args = ["-BatchMode"] # Default is ["-BatchMode", "-autotest_mode"] - + global_extra_cmdline_args = ["-autotest_mode"] # Default is ["-BatchMode", "-autotest_mode"] + use_null_renderer = False # Default is True enable_prefab_system = False - @pytest.mark.test_case_id("C34603773") - class AtomGPU_BasicLevelSetup_SetsUpLevel(EditorSharedTest): - use_null_renderer = False # Default is True - screenshot_name = "AtomBasicLevelSetup.ppm" - test_screenshots = [] # Gets set by setup() - screenshot_directory = "" # Gets set by setup() - - # Clear existing test screenshots before starting test. - def setup(self, workspace): - screenshot_directory = os.path.join(workspace.paths.project(), DEFAULT_SUBFOLDER_PATH) - test_screenshots = [os.path.join(screenshot_directory, self.screenshot_name)] - file_system.delete(test_screenshots, True, True) - - golden_images = [os.path.join(golden_images_directory(), screenshot_name)] - - from Atom.tests import hydra_AtomGPU_BasicLevelSetup as test_module - - assert compare_screenshot_to_golden_image(screenshot_directory, test_screenshots, golden_images, 0.99) is True - - @pytest.mark.test_case_id("C34525095") - class AtomGPU_LightComponent_AreaLightScreenshotsMatchGoldenImages(EditorSharedTest): - use_null_renderer = False # Default is True - screenshot_names = [ - "AreaLight_1.ppm", - "AreaLight_2.ppm", - "AreaLight_3.ppm", - "AreaLight_4.ppm", - "AreaLight_5.ppm", - ] - test_screenshots = [] # Gets set by setup() - screenshot_directory = "" # Gets set by setup() - - # Clear existing test screenshots before starting test. - def setup(self, workspace): - screenshot_directory = os.path.join(workspace.paths.project(), DEFAULT_SUBFOLDER_PATH) - for screenshot in self.screenshot_names: - screenshot_path = os.path.join(screenshot_directory, screenshot) - self.test_screenshots.append(screenshot_path) - file_system.delete(self.test_screenshots, True, True) - + @staticmethod + def screenshot_setup(screenshot_directory, screenshot_names): + """ + :param screenshot_names: list of screenshot file names with extensions + :return: tuple test_screenshots, golden_images each a list of full file paths + """ + test_screenshots = [] golden_images = [] + for screenshot in screenshot_names: + screenshot_path = os.path.join(screenshot_directory, screenshot) + test_screenshots.append(screenshot_path) + file_system.delete(test_screenshots, True, True) for golden_image in screenshot_names: golden_image_path = os.path.join(golden_images_directory(), golden_image) golden_images.append(golden_image_path) + return test_screenshots, golden_images + + @pytest.mark.test_case_id("C34525095") + class AtomGPU_LightComponent_AreaLightScreenshotsMatchGoldenImages_DX12(EditorSingleTest): from Atom.tests import hydra_AtomGPU_AreaLightScreenshotTest as test_module - assert compare_screenshot_to_golden_image(screenshot_directory, test_screenshots, golden_images, 0.99) is True + extra_cmdline_args = ["-rhi=dx12"] + + # Custom setup/teardown to remove old screenshots and establish paths to golden images + def setup(self, request, workspace, editor, editor_test_results, launcher_platform): + self.screenshot_directory = os.path.join(workspace.paths.project(), DEFAULT_SUBFOLDER_PATH) + self.screenshot_names = [ + "AreaLight_1.ppm", + "AreaLight_2.ppm", + "AreaLight_3.ppm", + "AreaLight_4.ppm", + "AreaLight_5.ppm", + ] + self.test_screenshots, self.golden_images = TestAutomation.screenshot_setup( + screenshot_directory=self.screenshot_directory, + screenshot_names=self.screenshot_names) + + def wrap_run(self, request, workspace, editor, editor_test_results, launcher_platform): + yield + assert compare_screenshot_to_golden_image(self.screenshot_directory, + self.test_screenshots, + self.golden_images, + similarity_threshold=0.96) is True - @pytest.mark.test_case_id("C34525110") - class AtomGPU_LightComponent_SpotLightScreenshotsMatchGoldenImages(EditorSharedTest): - use_null_renderer = False # Default is True - screenshot_names = [ - "SpotLight_1.ppm", - "SpotLight_2.ppm", - "SpotLight_3.ppm", - "SpotLight_4.ppm", - "SpotLight_5.ppm", - "SpotLight_6.ppm", - ] - test_screenshots = [] # Gets set by setup() - screenshot_directory = "" # Gets set by setup() + @pytest.mark.test_case_id("C34525095") + class AtomGPU_LightComponent_AreaLightScreenshotsMatchGoldenImages_Vulkan(EditorSingleTest): + from Atom.tests import hydra_AtomGPU_AreaLightScreenshotTest as test_module - # Clear existing test screenshots before starting test. - def setup(self, workspace): - screenshot_directory = os.path.join(workspace.paths.project(), DEFAULT_SUBFOLDER_PATH) - for screenshot in self.screenshot_names: - screenshot_path = os.path.join(screenshot_directory, screenshot) - self.test_screenshots.append(screenshot_path) - file_system.delete(self.test_screenshots, True, True) + extra_cmdline_args = ["-rhi=vulkan"] + + # Custom setup/teardown to remove old screenshots and establish paths to golden images + def setup(self, request, workspace, editor, editor_test_results, launcher_platform): + self.screenshot_directory = os.path.join(workspace.paths.project(), DEFAULT_SUBFOLDER_PATH) + self.screenshot_names = [ + "AreaLight_1.ppm", + "AreaLight_2.ppm", + "AreaLight_3.ppm", + "AreaLight_4.ppm", + "AreaLight_5.ppm", + ] + self.test_screenshots, self.golden_images = TestAutomation.screenshot_setup( + screenshot_directory=self.screenshot_directory, + screenshot_names=self.screenshot_names) + + def wrap_run(self, request, workspace, editor, editor_test_results, launcher_platform): + yield + assert compare_screenshot_to_golden_image(self.screenshot_directory, + self.test_screenshots, + self.golden_images, + similarity_threshold=0.96) is True - golden_images = [] - for golden_image in screenshot_names: - golden_image_path = os.path.join(golden_images_directory(), golden_image) - golden_images.append(golden_image_path) + @pytest.mark.test_case_id("C34525110") + class AtomGPU_LightComponent_SpotLightScreenshotsMatchGoldenImages_DX12(EditorSingleTest): + from Atom.tests import hydra_AtomGPU_SpotLightScreenshotTest as test_module + + extra_cmdline_args = ["-rhi=dx12"] + + # Custom setup/teardown to remove old screenshots and establish paths to golden images + def setup(self, request, workspace, editor, editor_test_results, launcher_platform): + self.screenshot_directory = os.path.join(workspace.paths.project(), DEFAULT_SUBFOLDER_PATH) + self.screenshot_names = [ + "SpotLight_1.ppm", + "SpotLight_2.ppm", + "SpotLight_3.ppm", + "SpotLight_4.ppm", + "SpotLight_5.ppm", + "SpotLight_6.ppm", + ] + self.test_screenshots, self.golden_images = TestAutomation.screenshot_setup( + screenshot_directory=self.screenshot_directory, + screenshot_names=self.screenshot_names) + + def wrap_run(self, request, workspace, editor, editor_test_results, launcher_platform): + yield + assert compare_screenshot_to_golden_image(self.screenshot_directory, + self.test_screenshots, + self.golden_images, + similarity_threshold=0.96) is True + @pytest.mark.test_case_id("C34525110") + class AtomGPU_LightComponent_SpotLightScreenshotsMatchGoldenImages_Vulkan(EditorSingleTest): from Atom.tests import hydra_AtomGPU_SpotLightScreenshotTest as test_module - assert compare_screenshot_to_golden_image(screenshot_directory, test_screenshots, golden_images, 0.99) is True + extra_cmdline_args = ["-rhi=vulkan"] + + # Custom setup/teardown to remove old screenshots and establish paths to golden images + def setup(self, request, workspace, editor, editor_test_results, launcher_platform): + self.screenshot_directory = os.path.join(workspace.paths.project(), DEFAULT_SUBFOLDER_PATH) + self.screenshot_names = [ + "SpotLight_1.ppm", + "SpotLight_2.ppm", + "SpotLight_3.ppm", + "SpotLight_4.ppm", + "SpotLight_5.ppm", + "SpotLight_6.ppm", + ] + self.test_screenshots, self.golden_images = TestAutomation.screenshot_setup( + screenshot_directory=self.screenshot_directory, + screenshot_names=self.screenshot_names) + + def wrap_run(self, request, workspace, editor, editor_test_results, launcher_platform): + yield + assert compare_screenshot_to_golden_image(self.screenshot_directory, + self.test_screenshots, + self.golden_images, + similarity_threshold=0.96) is True @pytest.mark.parametrize('rhi', ['dx12', 'vulkan']) diff --git a/AutomatedTesting/Gem/PythonTests/Atom/atom_utils/atom_component_helper.py b/AutomatedTesting/Gem/PythonTests/Atom/atom_utils/atom_component_helper.py index 5fdd4c810d..42310ee983 100644 --- a/AutomatedTesting/Gem/PythonTests/Atom/atom_utils/atom_component_helper.py +++ b/AutomatedTesting/Gem/PythonTests/Atom/atom_utils/atom_component_helper.py @@ -86,9 +86,8 @@ def compare_screenshot_similarity( if create_zip_archive: create_screenshots_archive(screenshot_directory) result = ( - f"When comparing the test_screenshot: '{test_screenshot}' " - f"to golden_image: '{golden_image}' the mean similarity of '{mean_similarity}' " - f"was lower than the similarity threshold of '{similarity_threshold}'. ") + f"When comparing the test_screenshot: '{test_screenshot}' to golden_image: '{golden_image}'.\n" + f"The mean similarity ({mean_similarity}) was lower than the similarity threshold ({similarity_threshold})") return result @@ -123,7 +122,9 @@ def initial_viewport_setup(screen_width=1280, screen_height=720): import azlmbr.legacy.general as general general.set_viewport_size(screen_width, screen_height) + general.idle_wait_frames(1) general.update_viewport() + general.idle_wait_frames(1) def enter_exit_game_mode_take_screenshot(screenshot_name, enter_game_tuple, exit_game_tuple, timeout_in_seconds=4): @@ -137,13 +138,18 @@ def enter_exit_game_mode_take_screenshot(screenshot_name, enter_game_tuple, exit """ import azlmbr.legacy.general as general - from editor_python_test_tools.utils import TestHelper + from editor_python_test_tools.utils import TestHelper, Report from Atom.atom_utils.screenshot_utils import ScreenshotHelper + screenshot_helper = ScreenshotHelper(general.idle_wait_frames) TestHelper.enter_game_mode(enter_game_tuple) TestHelper.wait_for_condition(function=lambda: general.is_in_game_mode(), timeout_in_seconds=timeout_in_seconds) - ScreenshotHelper(general.idle_wait_frames).capture_screenshot_blocking(screenshot_name) + screenshot_helper.prepare_viewport_for_screenshot(1920, 1080) + success_screenshot = TestHelper.wait_for_condition( + function=lambda: screenshot_helper.capture_screenshot_blocking(screenshot_name), + timeout_in_seconds=timeout_in_seconds) + Report.result(("Screenshot taken", "Screenshot failed to be taken"), success_screenshot) TestHelper.exit_game_mode(exit_game_tuple) TestHelper.wait_for_condition(function=lambda: not general.is_in_game_mode(), timeout_in_seconds=timeout_in_seconds) diff --git a/AutomatedTesting/Gem/PythonTests/Atom/tests/hydra_AtomGPU_BasicLevelSetup.py b/AutomatedTesting/Gem/PythonTests/Atom/tests/hydra_AtomGPU_BasicLevelSetup.py index b15e5b2131..c910662222 100644 --- a/AutomatedTesting/Gem/PythonTests/Atom/tests/hydra_AtomGPU_BasicLevelSetup.py +++ b/AutomatedTesting/Gem/PythonTests/Atom/tests/hydra_AtomGPU_BasicLevelSetup.py @@ -107,10 +107,8 @@ def AtomGPU_BasicLevelSetup_SetsUpLevel(): 18. Add Mesh component to Sphere Entity and set the Mesh Asset property for the Mesh component. 19. Create a Camera Entity as a child entity of the Default Level Entity then add a Camera component. 20. Set the Camera Entity rotation value and set the Camera component Field of View value. - 21. Enter game mode. - 22. Take screenshot. - 23. Exit game mode. - 24. Look for errors. + 21. Enter/Exit game mode taking screenshot. + 22. Look for errors. :return: None """ @@ -127,7 +125,7 @@ def AtomGPU_BasicLevelSetup_SetsUpLevel(): from Atom.atom_utils.atom_constants import AtomComponentProperties from Atom.atom_utils.atom_component_helper import initial_viewport_setup - from Atom.atom_utils.screenshot_utils import ScreenshotHelper + from Atom.atom_utils.atom_component_helper import enter_exit_game_mode_take_screenshot DEGREE_RADIAN_FACTOR = 0.0174533 SCREENSHOT_NAME = "AtomBasicLevelSetup" @@ -300,18 +298,10 @@ def AtomGPU_BasicLevelSetup_SetsUpLevel(): Report.result(Tests.camera_fov_set, camera_component.get_component_property_value( AtomComponentProperties.camera('Field of view')) == camera_fov_value) - # 21. Enter game mode. - TestHelper.enter_game_mode(Tests.enter_game_mode) - TestHelper.wait_for_condition(function=lambda: general.is_in_game_mode(), timeout_in_seconds=4.0) + # 21. Enter/Exit game mode taking screenshot. + enter_exit_game_mode_take_screenshot(f"{SCREENSHOT_NAME}.ppm", Tests.enter_game_mode, Tests.exit_game_mode) - # 22. Take screenshot. - ScreenshotHelper(general.idle_wait_frames).capture_screenshot_blocking(f"{SCREENSHOT_NAME}.ppm") - - # 23. Exit game mode. - TestHelper.exit_game_mode(Tests.exit_game_mode) - TestHelper.wait_for_condition(function=lambda: not general.is_in_game_mode(), timeout_in_seconds=4.0) - - # 24. Look for errors. + # 22. Look for errors. TestHelper.wait_for_condition(lambda: error_tracer.has_errors or error_tracer.has_asserts, 1.0) for error_info in error_tracer.errors: Report.info(f"Error: {error_info.filename} {error_info.function} | {error_info.message}")