birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #7596 from aws-lumberyard-dev/Atom/scottmur/editor_test_screenshot

Browse Source 
Atom editor screenshot test refactor for nightly GPU
This commit is contained in:
jromnoa
2022-02-15 16:07:19 -08:00
committed by GitHub
parent 16d6cbca2a 5124a6322d
commit 932f0a4d84
3 changed files with 138 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Main_GPU.py
+121 -73
View File
@@ -12,7 +12,7 @@ import pytest
import editor_python_test_tools.hydra_test_utils as hydra
import ly_test_tools.environment.file_system as file_system
from ly_test_tools.benchmark.data_aggregator import BenchmarkDataAggregator
from ly_test_tools.o3de.editor_test import EditorSharedTest, EditorTestSuite
from ly_test_tools.o3de.editor_test import EditorSingleTest, EditorTestSuite
from Atom.atom_utils.atom_component_helper import compare_screenshot_to_golden_image, golden_images_directory
DEFAULT_SUBFOLDER_PATH = 'user/PythonTests/Automated/Screenshots'
@@ -23,89 +23,137 @@ logger = logging.getLogger(__name__)
@pytest.mark.parametrize("launcher_platform", ['windows_editor'])
class TestAutomation(EditorTestSuite):
# Remove -autotest_mode from global_extra_cmdline_args since we need rendering for these tests.
global_extra_cmdline_args = ["-BatchMode"] # Default is ["-BatchMode", "-autotest_mode"]
global_extra_cmdline_args = ["-autotest_mode"] # Default is ["-BatchMode", "-autotest_mode"]
use_null_renderer = False # Default is True
enable_prefab_system = False
@pytest.mark.test_case_id("C34603773")
class AtomGPU_BasicLevelSetup_SetsUpLevel(EditorSharedTest):
use_null_renderer = False # Default is True
screenshot_name = "AtomBasicLevelSetup.ppm"
test_screenshots = [] # Gets set by setup()
screenshot_directory = "" # Gets set by setup()
@staticmethod
def screenshot_setup(screenshot_directory, screenshot_names):
"""
:param screenshot_names: list of screenshot file names with extensions
:return: tuple test_screenshots, golden_images each a list of full file paths
"""
test_screenshots = []
golden_images = []
for screenshot in screenshot_names:
screenshot_path = os.path.join(screenshot_directory, screenshot)
test_screenshots.append(screenshot_path)
file_system.delete(test_screenshots, True, True)
for golden_image in screenshot_names:
golden_image_path = os.path.join(golden_images_directory(), golden_image)
golden_images.append(golden_image_path)
return test_screenshots, golden_images
# Clear existing test screenshots before starting test.
def setup(self, workspace):
screenshot_directory = os.path.join(workspace.paths.project(), DEFAULT_SUBFOLDER_PATH)
test_screenshots = [os.path.join(screenshot_directory, self.screenshot_name)]
file_system.delete(test_screenshots, True, True)
golden_images = [os.path.join(golden_images_directory(), screenshot_name)]
from Atom.tests import hydra_AtomGPU_BasicLevelSetup as test_module
assert compare_screenshot_to_golden_image(screenshot_directory, test_screenshots, golden_images, 0.99) is True
@pytest.mark.test_case_id("C34525095")
class AtomGPU_LightComponent_AreaLightScreenshotsMatchGoldenImages(EditorSharedTest):
use_null_renderer = False # Default is True
screenshot_names = [
"AreaLight_1.ppm",
"AreaLight_2.ppm",
"AreaLight_3.ppm",
"AreaLight_4.ppm",
"AreaLight_5.ppm",
]
test_screenshots = [] # Gets set by setup()
screenshot_directory = "" # Gets set by setup()
# Clear existing test screenshots before starting test.
def setup(self, workspace):
screenshot_directory = os.path.join(workspace.paths.project(), DEFAULT_SUBFOLDER_PATH)
for screenshot in self.screenshot_names:
screenshot_path = os.path.join(screenshot_directory, screenshot)
self.test_screenshots.append(screenshot_path)
file_system.delete(self.test_screenshots, True, True)
golden_images = []
for golden_image in screenshot_names:
golden_image_path = os.path.join(golden_images_directory(), golden_image)
golden_images.append(golden_image_path)
class AtomGPU_LightComponent_AreaLightScreenshotsMatchGoldenImages_DX12(EditorSingleTest):
from Atom.tests import hydra_AtomGPU_AreaLightScreenshotTest as test_module
assert compare_screenshot_to_golden_image(screenshot_directory, test_screenshots, golden_images, 0.99) is True
extra_cmdline_args = ["-rhi=dx12"]
# Custom setup/teardown to remove old screenshots and establish paths to golden images
def setup(self, request, workspace, editor, editor_test_results, launcher_platform):
self.screenshot_directory = os.path.join(workspace.paths.project(), DEFAULT_SUBFOLDER_PATH)
self.screenshot_names = [
"AreaLight_1.ppm",
"AreaLight_2.ppm",
"AreaLight_3.ppm",
"AreaLight_4.ppm",
"AreaLight_5.ppm",
]
self.test_screenshots, self.golden_images = TestAutomation.screenshot_setup(
screenshot_directory=self.screenshot_directory,
screenshot_names=self.screenshot_names)
def wrap_run(self, request, workspace, editor, editor_test_results, launcher_platform):
yield
assert compare_screenshot_to_golden_image(self.screenshot_directory,
self.test_screenshots,
self.golden_images,
similarity_threshold=0.96) is True
@pytest.mark.test_case_id("C34525095")
class AtomGPU_LightComponent_AreaLightScreenshotsMatchGoldenImages_Vulkan(EditorSingleTest):
from Atom.tests import hydra_AtomGPU_AreaLightScreenshotTest as test_module
extra_cmdline_args = ["-rhi=vulkan"]
# Custom setup/teardown to remove old screenshots and establish paths to golden images
def setup(self, request, workspace, editor, editor_test_results, launcher_platform):
self.screenshot_directory = os.path.join(workspace.paths.project(), DEFAULT_SUBFOLDER_PATH)
self.screenshot_names = [
"AreaLight_1.ppm",
"AreaLight_2.ppm",
"AreaLight_3.ppm",
"AreaLight_4.ppm",
"AreaLight_5.ppm",
]
self.test_screenshots, self.golden_images = TestAutomation.screenshot_setup(
screenshot_directory=self.screenshot_directory,
screenshot_names=self.screenshot_names)
def wrap_run(self, request, workspace, editor, editor_test_results, launcher_platform):
yield
assert compare_screenshot_to_golden_image(self.screenshot_directory,
self.test_screenshots,
self.golden_images,
similarity_threshold=0.96) is True
@pytest.mark.test_case_id("C34525110")
class AtomGPU_LightComponent_SpotLightScreenshotsMatchGoldenImages(EditorSharedTest):
use_null_renderer = False # Default is True
screenshot_names = [
"SpotLight_1.ppm",
"SpotLight_2.ppm",
"SpotLight_3.ppm",
"SpotLight_4.ppm",
"SpotLight_5.ppm",
"SpotLight_6.ppm",
]
test_screenshots = [] # Gets set by setup()
screenshot_directory = "" # Gets set by setup()
# Clear existing test screenshots before starting test.
def setup(self, workspace):
screenshot_directory = os.path.join(workspace.paths.project(), DEFAULT_SUBFOLDER_PATH)
for screenshot in self.screenshot_names:
screenshot_path = os.path.join(screenshot_directory, screenshot)
self.test_screenshots.append(screenshot_path)
file_system.delete(self.test_screenshots, True, True)
golden_images = []
for golden_image in screenshot_names:
golden_image_path = os.path.join(golden_images_directory(), golden_image)
golden_images.append(golden_image_path)
class AtomGPU_LightComponent_SpotLightScreenshotsMatchGoldenImages_DX12(EditorSingleTest):
from Atom.tests import hydra_AtomGPU_SpotLightScreenshotTest as test_module
assert compare_screenshot_to_golden_image(screenshot_directory, test_screenshots, golden_images, 0.99) is True
extra_cmdline_args = ["-rhi=dx12"]
# Custom setup/teardown to remove old screenshots and establish paths to golden images
def setup(self, request, workspace, editor, editor_test_results, launcher_platform):
self.screenshot_directory = os.path.join(workspace.paths.project(), DEFAULT_SUBFOLDER_PATH)
self.screenshot_names = [
"SpotLight_1.ppm",
"SpotLight_2.ppm",
"SpotLight_3.ppm",
"SpotLight_4.ppm",
"SpotLight_5.ppm",
"SpotLight_6.ppm",
]
self.test_screenshots, self.golden_images = TestAutomation.screenshot_setup(
screenshot_directory=self.screenshot_directory,
screenshot_names=self.screenshot_names)
def wrap_run(self, request, workspace, editor, editor_test_results, launcher_platform):
yield
assert compare_screenshot_to_golden_image(self.screenshot_directory,
self.test_screenshots,
self.golden_images,
similarity_threshold=0.96) is True
@pytest.mark.test_case_id("C34525110")
class AtomGPU_LightComponent_SpotLightScreenshotsMatchGoldenImages_Vulkan(EditorSingleTest):
from Atom.tests import hydra_AtomGPU_SpotLightScreenshotTest as test_module
extra_cmdline_args = ["-rhi=vulkan"]
# Custom setup/teardown to remove old screenshots and establish paths to golden images
def setup(self, request, workspace, editor, editor_test_results, launcher_platform):
self.screenshot_directory = os.path.join(workspace.paths.project(), DEFAULT_SUBFOLDER_PATH)
self.screenshot_names = [
"SpotLight_1.ppm",
"SpotLight_2.ppm",
"SpotLight_3.ppm",
"SpotLight_4.ppm",
"SpotLight_5.ppm",
"SpotLight_6.ppm",
]
self.test_screenshots, self.golden_images = TestAutomation.screenshot_setup(
screenshot_directory=self.screenshot_directory,
screenshot_names=self.screenshot_names)
def wrap_run(self, request, workspace, editor, editor_test_results, launcher_platform):
yield
assert compare_screenshot_to_golden_image(self.screenshot_directory,
self.test_screenshots,
self.golden_images,
similarity_threshold=0.96) is True
@pytest.mark.parametrize('rhi', ['dx12', 'vulkan'])
AutomatedTesting/Gem/PythonTests/Atom/atom_utils/atom_component_helper.py
+11 -5
View File
@@ -86,9 +86,8 @@ def compare_screenshot_similarity(
if create_zip_archive:
create_screenshots_archive(screenshot_directory)
result = (
f"When comparing the test_screenshot: '{test_screenshot}' "
f"to golden_image: '{golden_image}' the mean similarity of '{mean_similarity}' "
f"was lower than the similarity threshold of '{similarity_threshold}'. ")
f"When comparing the test_screenshot: '{test_screenshot}' to golden_image: '{golden_image}'.\n"
f"The mean similarity ({mean_similarity}) was lower than the similarity threshold ({similarity_threshold})")
return result
@@ -123,7 +122,9 @@ def initial_viewport_setup(screen_width=1280, screen_height=720):
import azlmbr.legacy.general as general
general.set_viewport_size(screen_width, screen_height)
general.idle_wait_frames(1)
general.update_viewport()
general.idle_wait_frames(1)
def enter_exit_game_mode_take_screenshot(screenshot_name, enter_game_tuple, exit_game_tuple, timeout_in_seconds=4):
@@ -137,13 +138,18 @@ def enter_exit_game_mode_take_screenshot(screenshot_name, enter_game_tuple, exit
"""
import azlmbr.legacy.general as general
from editor_python_test_tools.utils import TestHelper
from editor_python_test_tools.utils import TestHelper, Report
from Atom.atom_utils.screenshot_utils import ScreenshotHelper
screenshot_helper = ScreenshotHelper(general.idle_wait_frames)
TestHelper.enter_game_mode(enter_game_tuple)
TestHelper.wait_for_condition(function=lambda: general.is_in_game_mode(), timeout_in_seconds=timeout_in_seconds)
ScreenshotHelper(general.idle_wait_frames).capture_screenshot_blocking(screenshot_name)
screenshot_helper.prepare_viewport_for_screenshot(1920, 1080)
success_screenshot = TestHelper.wait_for_condition(
function=lambda: screenshot_helper.capture_screenshot_blocking(screenshot_name),
timeout_in_seconds=timeout_in_seconds)
Report.result(("Screenshot taken", "Screenshot failed to be taken"), success_screenshot)
TestHelper.exit_game_mode(exit_game_tuple)
TestHelper.wait_for_condition(function=lambda: not general.is_in_game_mode(), timeout_in_seconds=timeout_in_seconds)
AutomatedTesting/Gem/PythonTests/Atom/tests/hydra_AtomGPU_BasicLevelSetup.py
+6 -16
View File
@@ -107,10 +107,8 @@ def AtomGPU_BasicLevelSetup_SetsUpLevel():
18. Add Mesh component to Sphere Entity and set the Mesh Asset property for the Mesh component.
19. Create a Camera Entity as a child entity of the Default Level Entity then add a Camera component.
20. Set the Camera Entity rotation value and set the Camera component Field of View value.
21. Enter game mode.
22. Take screenshot.
23. Exit game mode.
24. Look for errors.
21. Enter/Exit game mode taking screenshot.
22. Look for errors.
:return: None
"""
@@ -127,7 +125,7 @@ def AtomGPU_BasicLevelSetup_SetsUpLevel():
from Atom.atom_utils.atom_constants import AtomComponentProperties
from Atom.atom_utils.atom_component_helper import initial_viewport_setup
from Atom.atom_utils.screenshot_utils import ScreenshotHelper
from Atom.atom_utils.atom_component_helper import enter_exit_game_mode_take_screenshot
DEGREE_RADIAN_FACTOR = 0.0174533
SCREENSHOT_NAME = "AtomBasicLevelSetup"
@@ -300,18 +298,10 @@ def AtomGPU_BasicLevelSetup_SetsUpLevel():
Report.result(Tests.camera_fov_set, camera_component.get_component_property_value(
AtomComponentProperties.camera('Field of view')) == camera_fov_value)
# 21. Enter game mode.
TestHelper.enter_game_mode(Tests.enter_game_mode)
TestHelper.wait_for_condition(function=lambda: general.is_in_game_mode(), timeout_in_seconds=4.0)
# 21. Enter/Exit game mode taking screenshot.
enter_exit_game_mode_take_screenshot(f"{SCREENSHOT_NAME}.ppm", Tests.enter_game_mode, Tests.exit_game_mode)
# 22. Take screenshot.
ScreenshotHelper(general.idle_wait_frames).capture_screenshot_blocking(f"{SCREENSHOT_NAME}.ppm")
# 23. Exit game mode.
TestHelper.exit_game_mode(Tests.exit_game_mode)
TestHelper.wait_for_condition(function=lambda: not general.is_in_game_mode(), timeout_in_seconds=4.0)
# 24. Look for errors.
# 22. Look for errors.
TestHelper.wait_for_condition(lambda: error_tracer.has_errors or error_tracer.has_asserts, 1.0)
for error_info in error_tracer.errors:
Report.info(f"Error: {error_info.filename} {error_info.function} | {error_info.message}")