From d904f6ab409055b8499e3bc32cb15ec9fd4ae7d1 Mon Sep 17 00:00:00 2001 From: jromnoa Date: Fri, 18 Jun 2021 12:27:16 -0700 Subject: [PATCH] adds GPU test to the nightly test runs for o3de --- .../PythonTests/atom_renderer/CMakeLists.txt | 12 + .../hydra_GPUTest_BasicLevelSetup.py | 232 ++++++++++++++++++ .../atom_renderer/atom_utils/__init__.py | 10 + .../atom_utils/screenshot_utils.py | 97 ++++++++ .../golden_images/AtomBasicLevelSetup.ppm | 3 + .../atom_renderer/test_Atom_GPUTests.py | 87 +++++++ 6 files changed, 441 insertions(+) create mode 100644 AutomatedTesting/Gem/PythonTests/atom_renderer/atom_hydra_scripts/hydra_GPUTest_BasicLevelSetup.py create mode 100644 AutomatedTesting/Gem/PythonTests/atom_renderer/atom_utils/__init__.py create mode 100644 AutomatedTesting/Gem/PythonTests/atom_renderer/atom_utils/screenshot_utils.py create mode 100644 AutomatedTesting/Gem/PythonTests/atom_renderer/golden_images/AtomBasicLevelSetup.ppm create mode 100644 AutomatedTesting/Gem/PythonTests/atom_renderer/test_Atom_GPUTests.py diff --git a/AutomatedTesting/Gem/PythonTests/atom_renderer/CMakeLists.txt b/AutomatedTesting/Gem/PythonTests/atom_renderer/CMakeLists.txt index 91228bc71c..46d62cb54e 100644 --- a/AutomatedTesting/Gem/PythonTests/atom_renderer/CMakeLists.txt +++ b/AutomatedTesting/Gem/PythonTests/atom_renderer/CMakeLists.txt @@ -37,4 +37,16 @@ if(PAL_TRAIT_BUILD_HOST_TOOLS AND PAL_TRAIT_BUILD_TESTS_SUPPORTED AND AutomatedT AutomatedTesting.Assets Editor ) + ly_add_pytest( + NAME AutomatedTesting::AtomRenderer_HydraTests_GPUTests + TEST_SUITE sandbox + TEST_REQUIRES gpu + TEST_SERIAL + TIMEOUT 400 + PATH ${CMAKE_CURRENT_LIST_DIR}/test_Atom_GPUTests.py + RUNTIME_DEPENDENCIES + AssetProcessor + AutomatedTesting.Assets + Editor + ) endif() diff --git a/AutomatedTesting/Gem/PythonTests/atom_renderer/atom_hydra_scripts/hydra_GPUTest_BasicLevelSetup.py b/AutomatedTesting/Gem/PythonTests/atom_renderer/atom_hydra_scripts/hydra_GPUTest_BasicLevelSetup.py new file mode 100644 index 0000000000..7f63d1f2bc --- /dev/null +++ b/AutomatedTesting/Gem/PythonTests/atom_renderer/atom_hydra_scripts/hydra_GPUTest_BasicLevelSetup.py @@ -0,0 +1,232 @@ +""" +All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +its licensors. + +For complete copyright and license terms please see the LICENSE at the root of this +distribution (the "License"). All use of this software is governed by the License, +or, if provided, by the license below or the license accompanying this file. Do not +remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + +Hydra script that is used to create a new level with a default rendering setup. +After the level is setup, screenshots are diffed against golden images are used to verify pass/fail results of the test. + +See the run() function for more in-depth test info. +""" + +import os +import sys + +import azlmbr.asset as asset +import azlmbr.bus as bus +import azlmbr.camera +import azlmbr.entity as entity +import azlmbr.legacy.general as general +import azlmbr.math as math +import azlmbr.paths +import azlmbr.editor as editor + +sys.path.append(os.path.join(azlmbr.paths.devroot, "AutomatedTesting", "Gem", "PythonTests")) + +import editor_python_test_tools.hydra_editor_utils as hydra +from editor_python_test_tools.editor_test_helper import EditorTestHelper +from atom_renderer.atom_utils.screenshot_utils import ScreenshotHelper + +SCREEN_WIDTH = 1280 +SCREEN_HEIGHT = 720 +DEGREE_RADIAN_FACTOR = 0.0174533 + +helper = EditorTestHelper(log_prefix="Test_Atom_BasicLevelSetup") + + +def run(): + """ + 1. View -> Layouts -> Restore Default Layout, sets the viewport to ratio 16:9 @ 1280 x 720 + 2. Runs console command r_DisplayInfo = 0 + 3. Deletes all entities currently present in the level. + 4. Creates a "default_level" entity to hold all other entities, setting the translate values to x:0, y:0, z:0 + 5. Adds a Grid component to the "default_level" & updates its Grid Spacing to 1.0m + 6. Adds a "global_skylight" entity to "default_level", attaching an HDRi Skybox w/ a Cubemap Texture. + 7. Adds a Global Skylight (IBL) component w/ diffuse image and specular image to "global_skylight" entity. + 8. Adds a "ground_plane" entity to "default_level", attaching a Mesh component & Material component. + 9. Adds a "directional_light" entity to "default_level" & adds a Directional Light component. + 10. Adds a "sphere" entity to "default_level" & adds a Mesh component with a Material component to it. + 11. Adds a "camera" entity to "default_level" & adds a Camera component with 80 degree FOV and Transform values: + Translate - x:5.5m, y:-12.0m, z:9.0m + Rotate - x:-27.0, y:-12.0, z:25.0 + 12. Finally enters game mode, takes a screenshot, exits game mode, & saves the level. + :return: None + """ + def initial_viewport_setup(screen_width, screen_height): + general.set_viewport_size(screen_width, screen_height) + general.update_viewport() + helper.wait_for_condition( + function=lambda: helper.isclose(a=general.get_viewport_size().x, b=SCREEN_WIDTH, rel_tol=0.1) + and helper.isclose(a=general.get_viewport_size().y, b=SCREEN_HEIGHT, rel_tol=0.1), + timeout_in_seconds=4.0 + ) + result = helper.isclose(a=general.get_viewport_size().x, b=SCREEN_WIDTH, rel_tol=0.1) and helper.isclose( + a=general.get_viewport_size().y, b=SCREEN_HEIGHT, rel_tol=0.1) + general.log(general.get_viewport_size().x) + general.log(general.get_viewport_size().y) + general.log(general.get_viewport_size().z) + general.log(f"Viewport is set to the expected size: {result}") + general.run_console("r_DisplayInfo = 0") + + def after_level_load(): + """Function to call after creating/opening a level to ensure it loads.""" + # Give everything a second to initialize. + general.idle_enable(True) + general.idle_wait(1.0) + general.update_viewport() + general.idle_wait(0.5) # half a second is more than enough for updating the viewport. + + # Close out problematic windows, FPS meters, and anti-aliasing. + if general.is_helpers_shown(): # Turn off the helper gizmos if visible + general.toggle_helpers() + general.idle_wait(1.0) + if general.is_pane_visible("Error Report"): # Close Error Report windows that block focus. + general.close_pane("Error Report") + if general.is_pane_visible("Error Log"): # Close Error Log windows that block focus. + general.close_pane("Error Log") + general.idle_wait(1.0) + general.run_console("r_displayInfo=0") + general.run_console("r_antialiasingmode=0") + general.idle_wait(1.0) + + return True + + # Wait for Editor idle loop before executing Python hydra scripts. + general.idle_enable(True) + + # Create a new level. + new_level_name = "all_components_indepth_level" # Specified in class TestAllComponentsIndepthTests() + heightmap_resolution = 512 + heightmap_meters_per_pixel = 1 + terrain_texture_resolution = 412 + use_terrain = False + + # Return codes are ECreateLevelResult defined in CryEdit.h + return_code = general.create_level_no_prompt( + new_level_name, heightmap_resolution, heightmap_meters_per_pixel, terrain_texture_resolution, use_terrain) + if return_code == 1: + general.log(f"{new_level_name} level already exists") + elif return_code == 2: + general.log("Failed to create directory") + elif return_code == 3: + general.log("Directory length is too long") + elif return_code != 0: + general.log("Unknown error, failed to create level") + else: + general.log(f"{new_level_name} level created successfully") + + # Basic setup for newly created level. + after_level_load() + initial_viewport_setup(SCREEN_WIDTH, SCREEN_HEIGHT) + + # Create default_level entity + search_filter = azlmbr.entity.SearchFilter() + all_entities = entity.SearchBus(azlmbr.bus.Broadcast, "SearchEntities", search_filter) + editor.ToolsApplicationRequestBus(bus.Broadcast, "DeleteEntities", all_entities) + + default_level = hydra.Entity("default_level") + position = math.Vector3(0.0, 0.0, 0.0) + default_level.create_entity(position, ["Grid"]) + default_level.get_set_test(0, "Controller|Configuration|Secondary Grid Spacing", 1.0) + + # Create global_skylight entity and set the properties + global_skylight = hydra.Entity("global_skylight") + global_skylight.create_entity( + entity_position=math.Vector3(0.0, 0.0, 0.0), + components=["HDRi Skybox", "Global Skylight (IBL)"], + parent_id=default_level.id + ) + global_skylight_image_asset_path = os.path.join( + "LightingPresets", "greenwich_park_02_4k_iblskyboxcm_iblspecular.exr.streamingimage") + global_skylight_image_asset = asset.AssetCatalogRequestBus( + bus.Broadcast, "GetAssetIdByPath", global_skylight_image_asset_path, math.Uuid(), False) + global_skylight.get_set_test(0, "Controller|Configuration|Cubemap Texture", global_skylight_image_asset) + hydra.get_set_test(global_skylight, 1, "Controller|Configuration|Diffuse Image", global_skylight_image_asset) + hydra.get_set_test(global_skylight, 1, "Controller|Configuration|Specular Image", global_skylight_image_asset) + + # Create ground_plane entity and set the properties + ground_plane = hydra.Entity("ground_plane") + ground_plane.create_entity( + entity_position=math.Vector3(0.0, 0.0, 0.0), + components=["Material"], + parent_id=default_level.id + ) + azlmbr.components.TransformBus(azlmbr.bus.Event, "SetLocalUniformScale", ground_plane.id, 32.0) + ground_plane_material_asset_path = os.path.join("Materials", "Presets", "PBR", "metal_chrome.azmaterial") + ground_plane_material_asset = asset.AssetCatalogRequestBus( + bus.Broadcast, "GetAssetIdByPath", ground_plane_material_asset_path, math.Uuid(), False) + ground_plane.get_set_test(0, "Default Material|Material Asset", ground_plane_material_asset) + # Work around to add the correct Atom Mesh component + mesh_type_id = azlmbr.globals.property.EditorMeshComponentTypeId + ground_plane.components.append( + editor.EditorComponentAPIBus( + bus.Broadcast, "AddComponentsOfType", ground_plane.id, [mesh_type_id] + ).GetValue()[0] + ) + ground_plane_mesh_asset_path = os.path.join("Objects", "plane.azmodel") + ground_plane_mesh_asset = asset.AssetCatalogRequestBus( + bus.Broadcast, "GetAssetIdByPath", ground_plane_mesh_asset_path, math.Uuid(), False) + hydra.get_set_test(ground_plane, 1, "Controller|Configuration|Mesh Asset", ground_plane_mesh_asset) + + # Create directional_light entity and set the properties + directional_light = hydra.Entity("directional_light") + directional_light.create_entity( + entity_position=math.Vector3(0.0, 0.0, 10.0), + components=["Directional Light"], + parent_id=default_level.id + ) + rotation = math.Vector3(DEGREE_RADIAN_FACTOR * -90.0, 0.0, 0.0) + azlmbr.components.TransformBus(azlmbr.bus.Event, "SetLocalRotation", directional_light.id, rotation) + + # Create sphere entity and set the properties + sphere = hydra.Entity("sphere") + sphere.create_entity( + entity_position=math.Vector3(0.0, 0.0, 1.0), + components=["Material"], + parent_id=default_level.id + ) + sphere_material_asset_path = os.path.join("Materials", "Presets", "PBR", "metal_brass_polished.azmaterial") + sphere_material_asset = asset.AssetCatalogRequestBus( + bus.Broadcast, "GetAssetIdByPath", sphere_material_asset_path, math.Uuid(), False) + sphere.get_set_test(0, "Default Material|Material Asset", sphere_material_asset) + # Work around to add the correct Atom Mesh component + sphere.components.append( + editor.EditorComponentAPIBus( + bus.Broadcast, "AddComponentsOfType", sphere.id, [mesh_type_id] + ).GetValue()[0] + ) + sphere_mesh_asset_path = os.path.join("Models", "sphere.azmodel") + sphere_mesh_asset = asset.AssetCatalogRequestBus( + bus.Broadcast, "GetAssetIdByPath", sphere_mesh_asset_path, math.Uuid(), False) + hydra.get_set_test(sphere, 1, "Controller|Configuration|Mesh Asset", sphere_mesh_asset) + + # Create camera component and set the properties + camera_entity = hydra.Entity("camera") + position = math.Vector3(5.5, -12.0, 9.0) + camera_entity.create_entity(components=["Camera"], entity_position=position, parent_id=default_level.id) + rotation = math.Vector3( + DEGREE_RADIAN_FACTOR * -27.0, DEGREE_RADIAN_FACTOR * -12.0, DEGREE_RADIAN_FACTOR * 25.0 + ) + azlmbr.components.TransformBus(azlmbr.bus.Event, "SetLocalRotation", camera_entity.id, rotation) + camera_entity.get_set_test(0, "Controller|Configuration|Field of view", 60.0) + azlmbr.camera.EditorCameraViewRequestBus(azlmbr.bus.Event, "ToggleCameraAsActiveView", camera_entity.id) + + # Save level, enter game mode, take screenshot, & exit game mode. + general.save_level() + general.idle_wait(0.5) + general.enter_game_mode() + general.idle_wait(1.0) + helper.wait_for_condition(function=lambda: general.is_in_game_mode(), timeout_in_seconds=2.0) + ScreenshotHelper(general.idle_wait_frames).capture_screenshot_blocking(f"{'AtomBasicLevelSetup'}.ppm") + general.exit_game_mode() + helper.wait_for_condition(function=lambda: not general.is_in_game_mode(), timeout_in_seconds=2.0) + general.log("Basic level created") + + +if __name__ == "__main__": + run() diff --git a/AutomatedTesting/Gem/PythonTests/atom_renderer/atom_utils/__init__.py b/AutomatedTesting/Gem/PythonTests/atom_renderer/atom_utils/__init__.py new file mode 100644 index 0000000000..6ed3dc4bda --- /dev/null +++ b/AutomatedTesting/Gem/PythonTests/atom_renderer/atom_utils/__init__.py @@ -0,0 +1,10 @@ +""" +All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +its licensors. + +For complete copyright and license terms please see the LICENSE at the root of this +distribution (the "License"). All use of this software is governed by the License, +or, if provided, by the license below or the license accompanying this file. Do not +remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +""" \ No newline at end of file diff --git a/AutomatedTesting/Gem/PythonTests/atom_renderer/atom_utils/screenshot_utils.py b/AutomatedTesting/Gem/PythonTests/atom_renderer/atom_utils/screenshot_utils.py new file mode 100644 index 0000000000..6e4afc9239 --- /dev/null +++ b/AutomatedTesting/Gem/PythonTests/atom_renderer/atom_utils/screenshot_utils.py @@ -0,0 +1,97 @@ +""" +All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +its licensors. + +For complete copyright and license terms please see the LICENSE at the root of this +distribution (the "License"). All use of this software is governed by the License, +or, if provided, by the license below or the license accompanying this file. Do not +remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +""" +import azlmbr.atom +import azlmbr.legacy.general as general + +from editor_python_test_tools.editor_test_helper import EditorTestHelper + +DEFAULT_FRAME_WIDTH = 1920 +DEFAULT_FRAME_HEIGHT = 1080 +FOLDER_PATH = '@user@/PythonTests/Automated/Screenshots' + +helper = EditorTestHelper(log_prefix="Atom_ScreenshotHelper") + + +class ScreenshotHelper(object): + """ + A helper to capture screenshots and wait for them. + """ + + def __init__(self, idle_wait_frames_callback, frame_width=DEFAULT_FRAME_WIDTH, frame_height=DEFAULT_FRAME_HEIGHT): + super().__init__() + self.done = False + self.capturedScreenshot = False + self.max_frames_to_wait = 60 + self.prepare_viewport_for_screenshot(frame_width, frame_height) + self.idle_wait_frames_callback = idle_wait_frames_callback + + def capture_screenshot_blocking_in_game_mode(self, filename): + helper.enter_game_mode(["", ""]) + general.idle_wait_frames(120) + self.capture_screenshot_blocking(filename) + helper.exit_game_mode(["", ""]) + + def prepare_viewport_for_screenshot(self, frame_width, frame_height): + cur_viewport_size = general.get_viewport_size() + if int(cur_viewport_size.x) != frame_width or int(cur_viewport_size.y) != frame_height: + general.set_viewport_expansion_policy('FixedSize') + general.idle_wait_frames(1) + general.set_viewport_size(frame_width, frame_height) + general.set_cvar_integer('r_DisplayInfo', 0) + general.update_viewport() + general.idle_wait_frames(120) + + new_viewport_size = general.get_viewport_size() + if int(new_viewport_size.x) != frame_width or int(new_viewport_size.y) != frame_height: + general.log("Resolution is incorrect!") + general.log(f"width: {int(new_viewport_size.x)}") + general.log(f"height: {int(new_viewport_size.y)}") + + def capture_screenshot_blocking(self, filename, folder_path=FOLDER_PATH): + """ + Capture a screenshot and block the execution until the screenshot has been written to the disk. + """ + self.handler = azlmbr.atom.FrameCaptureNotificationBusHandler() + self.handler.connect() + self.handler.add_callback('OnCaptureFinished', self.on_screenshot_captured) + + self.done = False + self.capturedScreenshot = False + success = azlmbr.atom.FrameCaptureRequestBus( + azlmbr.bus.Broadcast, "CaptureScreenshot", f"{folder_path}/{filename}") + if success: + self.wait_until_screenshot() + general.log("Screenshot taken.") + else: + general.log("Screenshot failed") + return self.capturedScreenshot + + def on_screenshot_captured(self, parameters): + # the parameters come in as a tuple + if parameters[0] == azlmbr.atom.FrameCaptureResult_Success: + general.log(f"screenshot saved: {parameters[1]}") + self.capturedScreenshot = True + else: + general.log(f"screenshot failed: {parameters[1]}") + self.done = True + self.handler.disconnect() + + def wait_until_screenshot(self): + frames_waited = 0 + while self.done == False: + self.idle_wait_frames_callback(1) + if frames_waited > self.max_frames_to_wait: + general.log("timeout while waiting for the screenshot to be written") + self.handler.disconnect() + break + else: + frames_waited = frames_waited + 1 + general.log(f"(waited {frames_waited} frames)") diff --git a/AutomatedTesting/Gem/PythonTests/atom_renderer/golden_images/AtomBasicLevelSetup.ppm b/AutomatedTesting/Gem/PythonTests/atom_renderer/golden_images/AtomBasicLevelSetup.ppm new file mode 100644 index 0000000000..ef41b6cf77 --- /dev/null +++ b/AutomatedTesting/Gem/PythonTests/atom_renderer/golden_images/AtomBasicLevelSetup.ppm @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07e09d3eb5bf0cee3d9b3752aaad40f3ead1dcc5ddd837a6226fadde55d57274 +size 6220817 diff --git a/AutomatedTesting/Gem/PythonTests/atom_renderer/test_Atom_GPUTests.py b/AutomatedTesting/Gem/PythonTests/atom_renderer/test_Atom_GPUTests.py new file mode 100644 index 0000000000..1371fed4ad --- /dev/null +++ b/AutomatedTesting/Gem/PythonTests/atom_renderer/test_Atom_GPUTests.py @@ -0,0 +1,87 @@ +""" +All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +its licensors. + +For complete copyright and license terms please see the LICENSE at the root of this +distribution (the "License"). All use of this software is governed by the License, +or, if provided, by the license below or the license accompanying this file. Do not +remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + +Tests that require a GPU in order to run. +""" + +import logging +import os + +import pytest + +import ly_test_tools.environment.file_system as file_system +import editor_python_test_tools.hydra_test_utils as hydra + +logger = logging.getLogger(__name__) +DEFAULT_SUBFOLDER_PATH = 'user/PythonTests/Automated/Screenshots' +EDITOR_TIMEOUT = 300 +TEST_DIRECTORY = os.path.join(os.path.dirname(__file__), "atom_hydra_scripts") + + +def golden_images_directory(): + """ + Uses this file location to return the valid location for golden image files. + :return: The path to the golden_images directory, but raises an IOError if the golden_images directory is missing. + """ + current_file_directory = os.path.join(os.path.dirname(__file__)) + golden_images_dir = os.path.join(current_file_directory, 'golden_images') + + if not os.path.exists(golden_images_dir): + raise IOError( + f'golden_images" directory was not found at path "{golden_images_dir}"' + f'Please add a "golden_images" directory inside: "{current_file_directory}"' + ) + + return golden_images_dir + + +@pytest.mark.parametrize("project", ["AutomatedTesting"]) +@pytest.mark.parametrize("launcher_platform", ["windows_editor"]) +@pytest.mark.parametrize("level", ["auto_test"]) +class TestAllComponentsIndepthTests(object): + + @pytest.mark.parametrize("screenshot_name", ["AtomBasicLevelSetup.ppm"]) + def test_BasicLevelSetup_SetsUpLevel( + self, request, editor, workspace, project, launcher_platform, level, screenshot_name): + """ + Please review the hydra script run by this test for more specific test info. + Tests that a basic rendering level setup can be created (lighting, meshes, materials, etc.). + """ + # Clear existing test screenshots before starting test. + test_screenshots = [os.path.join( + workspace.paths.project(), DEFAULT_SUBFOLDER_PATH, screenshot_name)] + file_system.delete(test_screenshots, True, True) + + golden_images = [os.path.join(golden_images_directory(), screenshot_name)] + + level_creation_expected_lines = [ + "Viewport is set to the expected size: True", + "Basic level created" + ] + unexpected_lines = [ + "Trace::Assert", + "Trace::Error", + "Traceback (most recent call last):", + ] + + hydra.launch_and_validate_results( + request, + TEST_DIRECTORY, + editor, + "hydra_GPUTest_BasicLevelSetup.py", + timeout=EDITOR_TIMEOUT, + expected_lines=level_creation_expected_lines, + unexpected_lines=unexpected_lines, + halt_on_unexpected=True, + cfg_args=[level], + ) + + for test_screenshot, golden_screenshot in zip(test_screenshots, golden_images): + self.compare_screenshots(test_screenshot, golden_screenshot)