birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #5033 from aws-lumberyard-dev/Atom/jromnoa/assert-for-screenshot-comparisons

Browse Source 
Fixes the test_BasicLevelSetup_SetsUpLevel() test and adds proper asserts for screenshot comparisons.
This commit is contained in:
jromnoa
2021-11-03 12:26:56 -07:00
committed by GitHub
parent 0e8edfa702 1085aed137
commit f0ba0ff7eb
5 changed files with 114 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Main_GPU.py
+16 -23
View File
@@ -13,10 +13,10 @@ import zipfile
import pytest
import ly_test_tools.environment.file_system as file_system
from ly_test_tools.image.screenshot_compare_qssim import qssim as compare_screenshots
from ly_test_tools.benchmark.data_aggregator import BenchmarkDataAggregator
import editor_python_test_tools.hydra_test_utils as hydra
from .atom_utils.atom_component_helper import compare_screenshot_similarity, ImageComparisonTestFailure
logger = logging.getLogger(__name__)
DEFAULT_SUBFOLDER_PATH = 'user/PythonTests/Automated/Screenshots'
@@ -69,7 +69,7 @@ def create_screenshots_archive(screenshot_path):
@pytest.mark.parametrize("project", ["AutomatedTesting"])
@pytest.mark.parametrize("launcher_platform", ["windows_editor"])
@pytest.mark.parametrize("level", ["auto_test"])
@pytest.mark.parametrize("level", ["Base"])
class TestAllComponentsIndepthTests(object):
@pytest.mark.parametrize("screenshot_name", ["AtomBasicLevelSetup.ppm"])
@@ -91,12 +91,7 @@ class TestAllComponentsIndepthTests(object):
"Viewport is set to the expected size: True",
"Exited game mode"
]
unexpected_lines = [
"Trace::Assert",
"Trace::Error",
"Traceback (most recent call last):",
"Screenshot failed"
]
unexpected_lines = ["Traceback (most recent call last):"]
hydra.launch_and_validate_results(
request,
@@ -111,10 +106,12 @@ class TestAllComponentsIndepthTests(object):
null_renderer=False,
)
for test_screenshot, golden_screenshot in zip(test_screenshots, golden_images):
compare_screenshots(test_screenshot, golden_screenshot)
create_screenshots_archive(screenshot_directory)
similarity_threshold = 0.99
for test_screenshot, golden_image in zip(test_screenshots, golden_images):
screenshot_comparison_result = compare_screenshot_similarity(
test_screenshot, golden_image, similarity_threshold, True, screenshot_directory)
if screenshot_comparison_result != "Screenshots match":
raise Exception(f"Screenshot test failed: {screenshot_comparison_result}")
@pytest.mark.test_case_id("C34525095")
def test_LightComponent_ScreenshotMatchesGoldenImage(
@@ -149,12 +146,7 @@ class TestAllComponentsIndepthTests(object):
golden_images.append(golden_image_path)
expected_lines = ["spot_light Controller|Configuration|Shadows|Shadowmap size: SUCCESS"]
unexpected_lines = [
"Trace::Assert",
"Trace::Error",
"Traceback (most recent call last):",
"Screenshot failed",
]
unexpected_lines = ["Traceback (most recent call last):"]
hydra.launch_and_validate_results(
request,
TEST_DIRECTORY,
@@ -168,10 +160,12 @@ class TestAllComponentsIndepthTests(object):
null_renderer=False,
)
for test_screenshot, golden_screenshot in zip(test_screenshots, golden_images):
compare_screenshots(test_screenshot, golden_screenshot)
create_screenshots_archive(screenshot_directory)
similarity_threshold = 0.99
for test_screenshot, golden_image in zip(test_screenshots, golden_images):
screenshot_comparison_result = compare_screenshot_similarity(
test_screenshot, golden_image, similarity_threshold, True, screenshot_directory)
if screenshot_comparison_result != "Screenshots match":
raise ImageComparisonTestFailure(f"Screenshot test failed: {screenshot_comparison_result}")
@pytest.mark.parametrize('rhi', ['dx12', 'vulkan'])
@@ -219,7 +213,6 @@ class TestPerformanceBenchmarkSuite(object):
@pytest.mark.parametrize("project", ["AutomatedTesting"])
@pytest.mark.parametrize("launcher_platform", ['windows_generic'])
@pytest.mark.system
class TestMaterialEditor(object):
@pytest.mark.parametrize("cfg_args,expected_lines", [
AutomatedTesting/Gem/PythonTests/Atom/atom_utils/atom_component_helper.py
+63 -38
View File
@@ -1,5 +1,6 @@
"""
Copyright (c) Contributors to the Open 3D Engine Project. For complete copyright and license terms please see the LICENSE at the root of this distribution.
Copyright (c) Contributors to the Open 3D Engine Project.
For complete copyright and license terms please see the LICENSE at the root of this distribution.
SPDX-License-Identifier: Apache-2.0 OR MIT
@@ -8,12 +9,19 @@ import datetime
import os
import zipfile
from ly_test_tools.image.screenshot_compare_qssim import qssim as compare_screenshots
class ImageComparisonTestFailure(Exception):
"""Custom test failure for failed image comparisons."""
pass
def create_screenshots_archive(screenshot_path):
"""
Creates a new zip file archive at archive_path containing all files listed within archive_path.
:param screenshot_path: location containing the files to archive, the zip archive file will also be saved here.
:return: None, but creates a new zip file archive inside path containing all of the files inside archive_path.
:return: path to the created .zip file archive.
"""
files_to_archive = []
@@ -27,14 +35,16 @@ def create_screenshots_archive(screenshot_path):
# Setup variables for naming the zip archive file.
timestamp = datetime.datetime.now().timestamp()
formatted_timestamp = datetime.datetime.utcfromtimestamp(timestamp).strftime("%Y-%m-%d_%H-%M-%S")
screenshots_file = os.path.join(screenshot_path, f'screenshots_{formatted_timestamp}.zip')
screenshots_zip_file = os.path.join(screenshot_path, f'screenshots_{formatted_timestamp}.zip')
# Write all of the valid .png and .ppm files to the archive file.
with zipfile.ZipFile(screenshots_file, 'w', compression=zipfile.ZIP_DEFLATED, allowZip64=True) as zip_archive:
with zipfile.ZipFile(screenshots_zip_file, 'w', compression=zipfile.ZIP_DEFLATED, allowZip64=True) as zip_archive:
for file_path in files_to_archive:
file_name = os.path.basename(file_path)
zip_archive.write(file_path, file_name)
return screenshots_zip_file
def golden_images_directory():
"""
@@ -53,6 +63,36 @@ def golden_images_directory():
return golden_images_dir
def compare_screenshot_similarity(
test_screenshot, golden_image, similarity_threshold, create_zip_archive=False, screenshot_directory=""):
"""
Compares the similarity between a test screenshot and a golden image.
It returns a "Screenshots match" string if the comparison mean value is higher than the similarity threshold.
Otherwise, it returns an error string.
:param test_screenshot: path to the test screenshot to compare.
:param golden_image: path to the golden image to compare.
:param similarity_threshold: value for the comparison mean value to be asserted against.
:param create_zip_archive: toggle to create a zip archive containing the screenshots if the assert check fails.
:param screenshot_directory: directory containing screenshots to create zip archive from.
:return: Error string if compared mean value < similarity threshold or screenshot_directory is missing for .zip,
otherwise it returns a "Screenshots match" string.
"""
result = "Screenshots match"
if create_zip_archive and not screenshot_directory:
result = 'You must specify a screenshot_directory in order to create a zip archive.\n'
mean_similarity = compare_screenshots(test_screenshot, golden_image)
if not mean_similarity > similarity_threshold:
if create_zip_archive:
create_screenshots_archive(screenshot_directory)
result = (
f"When comparing the test_screenshot: '{test_screenshot}' "
f"to golden_image: '{golden_image}' the mean similarity of '{mean_similarity}' "
f"was lower than the similarity threshold of '{similarity_threshold}'. ")
return result
def create_basic_atom_level(level_name):
"""
Creates a new level inside the Editor matching level_name & adds the following:
@@ -76,29 +116,11 @@ def create_basic_atom_level(level_name):
helper = EditorTestHelper(log_prefix="Atom_EditorTestHelper")
# Create a new level.
new_level_name = level_name
heightmap_resolution = 512
heightmap_meters_per_pixel = 1
terrain_texture_resolution = 412
use_terrain = False
# Return codes are ECreateLevelResult defined in CryEdit.h
return_code = general.create_level_no_prompt(
new_level_name, heightmap_resolution, heightmap_meters_per_pixel, terrain_texture_resolution, use_terrain)
if return_code == 1:
general.log(f"{new_level_name} level already exists")
elif return_code == 2:
general.log("Failed to create directory")
elif return_code == 3:
general.log("Directory length is too long")
elif return_code != 0:
general.log("Unknown error, failed to create level")
else:
general.log(f"{new_level_name} level created successfully")
# Enable idle and update viewport.
# Wait for Editor idle loop before executing Python hydra scripts.
general.idle_enable(True)
# Basic setup for opened level.
helper.open_level(level_name="Base")
general.idle_wait(1.0)
general.update_viewport()
general.idle_wait(0.5) # half a second is more than enough for updating the viewport.
@@ -165,24 +187,25 @@ def create_basic_atom_level(level_name):
components=["Material"],
parent_id=default_level.id)
azlmbr.components.TransformBus(azlmbr.bus.Event, "SetLocalUniformScale", ground_plane.id, 32.0)
ground_plane_material_asset_path = os.path.join(
"Materials", "Presets", "PBR", "metal_chrome.azmaterial")
ground_plane_material_asset_value = asset.AssetCatalogRequestBus(
bus.Broadcast, "GetAssetIdByPath", ground_plane_material_asset_path, math.Uuid(), False)
ground_plane.get_set_test(0, "Default Material|Material Asset", ground_plane_material_asset_value)
# Work around to add the correct Atom Mesh component
# Work around to add the correct Atom Mesh component and asset.
mesh_type_id = azlmbr.globals.property.EditorMeshComponentTypeId
ground_plane.components.append(
editor.EditorComponentAPIBus(
bus.Broadcast, "AddComponentsOfType", ground_plane.id, [mesh_type_id]
).GetValue()[0]
)
ground_plane_mesh_asset_path = os.path.join("Models", "plane.azmodel")
ground_plane_mesh_asset_path = os.path.join("TestData", "Objects", "plane.azmodel")
ground_plane_mesh_asset_value = asset.AssetCatalogRequestBus(
bus.Broadcast, "GetAssetIdByPath", ground_plane_mesh_asset_path, math.Uuid(), False)
ground_plane.get_set_test(1, "Controller|Configuration|Mesh Asset", ground_plane_mesh_asset_value)
# Add Atom Material component and asset.
ground_plane_material_asset_path = os.path.join("Materials", "Presets", "PBR", "metal_chrome.azmaterial")
ground_plane_material_asset_value = asset.AssetCatalogRequestBus(
bus.Broadcast, "GetAssetIdByPath", ground_plane_material_asset_path, math.Uuid(), False)
ground_plane.get_set_test(0, "Default Material|Material Asset", ground_plane_material_asset_value)
# Create directional_light entity and set the properties
directional_light = hydra.Entity("directional_light")
directional_light.create_entity(
@@ -199,12 +222,8 @@ def create_basic_atom_level(level_name):
entity_position=math.Vector3(0.0, 0.0, 1.0),
components=["Material"],
parent_id=default_level.id)
sphere_material_asset_path = os.path.join("Materials", "Presets", "PBR", "metal_brass_polished.azmaterial")
sphere_material_asset_value = asset.AssetCatalogRequestBus(
bus.Broadcast, "GetAssetIdByPath", sphere_material_asset_path, math.Uuid(), False)
sphere_entity.get_set_test(0, "Default Material|Material Asset", sphere_material_asset_value)
# Work around to add the correct Atom Mesh component
# Work around to add the correct Atom Mesh component and asset.
sphere_entity.components.append(
editor.EditorComponentAPIBus(
bus.Broadcast, "AddComponentsOfType", sphere_entity.id, [mesh_type_id]
@@ -215,6 +234,12 @@ def create_basic_atom_level(level_name):
bus.Broadcast, "GetAssetIdByPath", sphere_mesh_asset_path, math.Uuid(), False)
sphere_entity.get_set_test(1, "Controller|Configuration|Mesh Asset", sphere_mesh_asset_value)
# Add Atom Material component and asset.
sphere_material_asset_path = os.path.join("Materials", "Presets", "PBR", "metal_brass_polished.azmaterial")
sphere_material_asset_value = asset.AssetCatalogRequestBus(
bus.Broadcast, "GetAssetIdByPath", sphere_material_asset_path, math.Uuid(), False)
sphere_entity.get_set_test(0, "Default Material|Material Asset", sphere_material_asset_value)
# Create camera component and set the properties
camera_entity = hydra.Entity("camera")
camera_entity.create_entity(
AutomatedTesting/Gem/PythonTests/Atom/tests/hydra_GPUTest_BasicLevelSetup.py
+31 -51
View File
@@ -6,18 +6,6 @@ SPDX-License-Identifier: Apache-2.0 OR MIT
"""
import os
import sys
import azlmbr.asset as asset
import azlmbr.bus as bus
import azlmbr.camera
import azlmbr.entity as entity
import azlmbr.legacy.general as general
import azlmbr.math as math
import azlmbr.paths
import azlmbr.editor as editor
sys.path.append(os.path.join(azlmbr.paths.projectroot, "Gem", "PythonTests"))
import editor_python_test_tools.hydra_editor_utils as hydra
from editor_python_test_tools.editor_test_helper import EditorTestHelper
@@ -45,9 +33,18 @@ def run():
11. Adds a "camera" entity to "default_level" & adds a Camera component with 80 degree FOV and Transform values:
Translate - x:5.5m, y:-12.0m, z:9.0m
Rotate - x:-27.0, y:-12.0, z:25.0
12. Finally enters game mode, takes a screenshot, exits game mode, & saves the level.
12. Finally enters game mode, takes a screenshot, & exits game mode.
:return: None
"""
import azlmbr.asset as asset
import azlmbr.bus as bus
import azlmbr.camera as camera
import azlmbr.entity as entity
import azlmbr.legacy.general as general
import azlmbr.math as math
import azlmbr.paths
import azlmbr.editor as editor
def initial_viewport_setup(screen_width, screen_height):
general.set_viewport_size(screen_width, screen_height)
general.update_viewport()
@@ -84,33 +81,11 @@ def run():
general.run_console("r_displayInfo=0")
general.idle_wait(1.0)
return True
# Wait for Editor idle loop before executing Python hydra scripts.
general.idle_enable(True)
# Open the auto_test level.
new_level_name = "auto_test" # Specified in class TestAllComponentsIndepthTests()
heightmap_resolution = 512
heightmap_meters_per_pixel = 1
terrain_texture_resolution = 412
use_terrain = False
# Return codes are ECreateLevelResult defined in CryEdit.h
return_code = general.create_level_no_prompt(
new_level_name, heightmap_resolution, heightmap_meters_per_pixel, terrain_texture_resolution, use_terrain)
if return_code == 1:
general.log(f"{new_level_name} level already exists")
elif return_code == 2:
general.log("Failed to create directory")
elif return_code == 3:
general.log("Directory length is too long")
elif return_code != 0:
general.log("Unknown error, failed to create level")
else:
general.log(f"{new_level_name} level created successfully")
# Basic setup for newly created level.
# Basic setup for opened level.
helper.open_level(level_name="Base")
after_level_load()
initial_viewport_setup(SCREEN_WIDTH, SCREEN_HEIGHT)
@@ -147,22 +122,25 @@ def run():
parent_id=default_level.id
)
azlmbr.components.TransformBus(azlmbr.bus.Event, "SetLocalUniformScale", ground_plane.id, 32.0)
ground_plane_material_asset_path = os.path.join("Materials", "Presets", "PBR", "metal_chrome.azmaterial")
ground_plane_material_asset = asset.AssetCatalogRequestBus(
bus.Broadcast, "GetAssetIdByPath", ground_plane_material_asset_path, math.Uuid(), False)
ground_plane.get_set_test(0, "Default Material|Material Asset", ground_plane_material_asset)
# Work around to add the correct Atom Mesh component
# Work around to add the correct Atom Mesh component and asset.
mesh_type_id = azlmbr.globals.property.EditorMeshComponentTypeId
ground_plane.components.append(
editor.EditorComponentAPIBus(
bus.Broadcast, "AddComponentsOfType", ground_plane.id, [mesh_type_id]
).GetValue()[0]
)
ground_plane_mesh_asset_path = os.path.join("Objects", "plane.azmodel")
ground_plane_mesh_asset_path = os.path.join("TestData", "Objects", "plane.azmodel")
ground_plane_mesh_asset = asset.AssetCatalogRequestBus(
bus.Broadcast, "GetAssetIdByPath", ground_plane_mesh_asset_path, math.Uuid(), False)
hydra.get_set_test(ground_plane, 1, "Controller|Configuration|Mesh Asset", ground_plane_mesh_asset)
# Add Atom Material component and asset.
ground_plane_material_asset_path = os.path.join("Materials", "Presets", "PBR", "metal_chrome.azmaterial")
ground_plane_material_asset = asset.AssetCatalogRequestBus(
bus.Broadcast, "GetAssetIdByPath", ground_plane_material_asset_path, math.Uuid(), False)
ground_plane.get_set_test(0, "Default Material|Material Asset", ground_plane_material_asset)
# Create directional_light entity and set the properties
directional_light = hydra.Entity("directional_light")
directional_light.create_entity(
@@ -180,11 +158,8 @@ def run():
components=["Material"],
parent_id=default_level.id
)
sphere_material_asset_path = os.path.join("Materials", "Presets", "PBR", "metal_brass_polished.azmaterial")
sphere_material_asset = asset.AssetCatalogRequestBus(
bus.Broadcast, "GetAssetIdByPath", sphere_material_asset_path, math.Uuid(), False)
sphere.get_set_test(0, "Default Material|Material Asset", sphere_material_asset)
# Work around to add the correct Atom Mesh component
# Work around to add the correct Atom Mesh component and asset.
sphere.components.append(
editor.EditorComponentAPIBus(
bus.Broadcast, "AddComponentsOfType", sphere.id, [mesh_type_id]
@@ -195,6 +170,12 @@ def run():
bus.Broadcast, "GetAssetIdByPath", sphere_mesh_asset_path, math.Uuid(), False)
hydra.get_set_test(sphere, 1, "Controller|Configuration|Mesh Asset", sphere_mesh_asset)
# Add Atom Material component and asset.
sphere_material_asset_path = os.path.join("Materials", "Presets", "PBR", "metal_brass_polished.azmaterial")
sphere_material_asset = asset.AssetCatalogRequestBus(
bus.Broadcast, "GetAssetIdByPath", sphere_material_asset_path, math.Uuid(), False)
sphere.get_set_test(0, "Default Material|Material Asset", sphere_material_asset)
# Create camera component and set the properties
camera_entity = hydra.Entity("camera")
position = math.Vector3(5.5, -12.0, 9.0)
@@ -204,10 +185,9 @@ def run():
)
azlmbr.components.TransformBus(azlmbr.bus.Event, "SetLocalRotation", camera_entity.id, rotation)
camera_entity.get_set_test(0, "Controller|Configuration|Field of view", 60.0)
azlmbr.camera.EditorCameraViewRequestBus(azlmbr.bus.Event, "ToggleCameraAsActiveView", camera_entity.id)
camera.EditorCameraViewRequestBus(azlmbr.bus.Event, "ToggleCameraAsActiveView", camera_entity.id)
# Save level, enter game mode, take screenshot, & exit game mode.
general.save_level()
# Enter game mode, take screenshot, & exit game mode.
general.idle_wait(0.5)
general.enter_game_mode()
general.idle_wait(1.0)
AutomatedTesting/Gem/PythonTests/Atom/tests/hydra_GPUTest_LightComponent.py
+1 -1
View File
@@ -22,7 +22,7 @@ from editor_python_test_tools.editor_test_helper import EditorTestHelper
helper = EditorTestHelper(log_prefix="Atom_EditorTestHelper")
LEVEL_NAME = "auto_test"
LEVEL_NAME = "Base"
LIGHT_COMPONENT = "Light"
LIGHT_TYPE_PROPERTY = 'Controller|Configuration|Light type'
DEGREE_RADIAN_FACTOR = 0.0174533
Gems/Atom/TestData/TestData/Objects/plane.fbx
+3
View File
@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:0a1f8d75dcd85e8b4aa57f6c0c81af0300ff96915ba3c2b591095c215d5e1d8c
size 12072