birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fixes the basic level GPU test screenshot comparison failure

Browse Source 
Signed-off-by: jromnoa <jromnoa@amazon.com>
This commit is contained in:
jromnoa
2021-10-26 18:51:09 -07:00
parent 7a0246530a
commit 6d62dbba76
3 changed files with 85 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Main_GPU.py
+13 -12
View File
@@ -13,10 +13,10 @@ import zipfile
import pytest
import ly_test_tools.environment.file_system as file_system
from ly_test_tools.image.screenshot_compare_qssim import qssim as compare_screenshots
from ly_test_tools.benchmark.data_aggregator import BenchmarkDataAggregator
import editor_python_test_tools.hydra_test_utils as hydra
from .atom_utils.atom_component_helper import compare_screenshot_similarity, ImageComparisonTestFailure
logger = logging.getLogger(__name__)
DEFAULT_SUBFOLDER_PATH = 'user/PythonTests/Automated/Screenshots'
@@ -92,8 +92,6 @@ class TestAllComponentsIndepthTests(object):
"Exited game mode"
]
unexpected_lines = [
"Trace::Assert",
"Trace::Error",
"Traceback (most recent call last):",
"Screenshot failed"
]
@@ -111,10 +109,12 @@ class TestAllComponentsIndepthTests(object):
null_renderer=False,
)
for test_screenshot, golden_screenshot in zip(test_screenshots, golden_images):
compare_screenshots(test_screenshot, golden_screenshot)
create_screenshots_archive(screenshot_directory)
similarity_threshold = 0.99
for test_screenshot, golden_image in zip(test_screenshots, golden_images):
screenshot_comparison_result = compare_screenshot_similarity(
test_screenshot, golden_image, similarity_threshold, True, screenshot_directory)
if screenshot_comparison_result != "Screenshots match":
raise Exception(f"Screenshot test failed: {screenshot_comparison_result}")
@pytest.mark.test_case_id("C34525095")
def test_LightComponent_ScreenshotMatchesGoldenImage(
@@ -168,10 +168,12 @@ class TestAllComponentsIndepthTests(object):
null_renderer=False,
)
for test_screenshot, golden_screenshot in zip(test_screenshots, golden_images):
compare_screenshots(test_screenshot, golden_screenshot)
create_screenshots_archive(screenshot_directory)
similarity_threshold = 0.99
for test_screenshot, golden_image in zip(test_screenshots, golden_images):
screenshot_comparison_result = compare_screenshot_similarity(
test_screenshot, golden_image, similarity_threshold, True, screenshot_directory)
if screenshot_comparison_result != "Screenshots match":
raise ImageComparisonTestFailure(f"Screenshot test failed: {screenshot_comparison_result}")
@pytest.mark.parametrize('rhi', ['dx12', 'vulkan'])
@@ -219,7 +221,6 @@ class TestPerformanceBenchmarkSuite(object):
@pytest.mark.parametrize("project", ["AutomatedTesting"])
@pytest.mark.parametrize("launcher_platform", ['windows_generic'])
@pytest.mark.system
class TestMaterialEditor(object):
@pytest.mark.parametrize("cfg_args,expected_lines", [
AutomatedTesting/Gem/PythonTests/Atom/atom_utils/atom_component_helper.py
+44 -4
View File
@@ -1,5 +1,6 @@
"""
Copyright (c) Contributors to the Open 3D Engine Project. For complete copyright and license terms please see the LICENSE at the root of this distribution.
Copyright (c) Contributors to the Open 3D Engine Project.
For complete copyright and license terms please see the LICENSE at the root of this distribution.
SPDX-License-Identifier: Apache-2.0 OR MIT
@@ -8,12 +9,19 @@ import datetime
import os
import zipfile
from ly_test_tools.image.screenshot_compare_qssim import qssim as compare_screenshots
class ImageComparisonTestFailure(Exception):
"""Custom test failure for failed image comparisons."""
pass
def create_screenshots_archive(screenshot_path):
"""
Creates a new zip file archive at archive_path containing all files listed within archive_path.
:param screenshot_path: location containing the files to archive, the zip archive file will also be saved here.
:return: None, but creates a new zip file archive inside path containing all of the files inside archive_path.
:return: path to the created .zip file archive.
"""
files_to_archive = []
@@ -27,14 +35,16 @@ def create_screenshots_archive(screenshot_path):
# Setup variables for naming the zip archive file.
timestamp = datetime.datetime.now().timestamp()
formatted_timestamp = datetime.datetime.utcfromtimestamp(timestamp).strftime("%Y-%m-%d_%H-%M-%S")
screenshots_file = os.path.join(screenshot_path, f'screenshots_{formatted_timestamp}.zip')
screenshots_zip_file = os.path.join(screenshot_path, f'screenshots_{formatted_timestamp}.zip')
# Write all of the valid .png and .ppm files to the archive file.
with zipfile.ZipFile(screenshots_file, 'w', compression=zipfile.ZIP_DEFLATED, allowZip64=True) as zip_archive:
with zipfile.ZipFile(screenshots_zip_file, 'w', compression=zipfile.ZIP_DEFLATED, allowZip64=True) as zip_archive:
for file_path in files_to_archive:
file_name = os.path.basename(file_path)
zip_archive.write(file_path, file_name)
return screenshots_zip_file
def golden_images_directory():
"""
@@ -53,6 +63,36 @@ def golden_images_directory():
return golden_images_dir
def compare_screenshot_similarity(
test_screenshot, golden_image, similarity_threshold, create_zip_archive=False, screenshot_directory=""):
"""
Compares the similarity between a test screenshot and a golden image.
It returns a "Screenshots match" string if the comparison mean value is higher than the similarity threshold.
Otherwise, it returns an error string.
:param test_screenshot: path to the test screenshot to compare.
:param golden_image: path to the golden image to compare.
:param similarity_threshold: value for the comparison mean value to be asserted against.
:param create_zip_archive: toggle to create a zip archive containing the screenshots if the assert check fails.
:param screenshot_directory: directory containing screenshots to create zip archive from.
:return: Error string if compared mean value < similarity threshold or screenshot_directory is missing for .zip,
otherwise it returns a "Screenshots match" string.
"""
error = "Screenshots match"
if create_zip_archive and not screenshot_directory:
error = 'You must specify a screenshot_directory in order to create a zip archive.\n'
mean_similarity = compare_screenshots(test_screenshot, golden_image)
if not mean_similarity > similarity_threshold:
if create_zip_archive:
create_screenshots_archive(screenshot_directory)
error = (
f"When comparing the test_screenshot: '{test_screenshot}' "
f"to golden_image: '{golden_image}' the mean similarity of '{mean_similarity}' "
f"was lower than the similarity threshold of '{similarity_threshold}'. ")
return error
def create_basic_atom_level(level_name):
"""
Creates a new level inside the Editor matching level_name & adds the following:
AutomatedTesting/Gem/PythonTests/Atom/tests/hydra_GPUTest_BasicLevelSetup.py
+28 -26
View File
@@ -6,18 +6,6 @@ SPDX-License-Identifier: Apache-2.0 OR MIT
"""
import os
import sys
import azlmbr.asset as asset
import azlmbr.bus as bus
import azlmbr.camera
import azlmbr.entity as entity
import azlmbr.legacy.general as general
import azlmbr.math as math
import azlmbr.paths
import azlmbr.editor as editor
sys.path.append(os.path.join(azlmbr.paths.projectroot, "Gem", "PythonTests"))
import editor_python_test_tools.hydra_editor_utils as hydra
from editor_python_test_tools.editor_test_helper import EditorTestHelper
@@ -48,6 +36,15 @@ def run():
12. Finally enters game mode, takes a screenshot, exits game mode, & saves the level.
:return: None
"""
import azlmbr.asset as asset
import azlmbr.bus as bus
import azlmbr.camera as camera
import azlmbr.entity as entity
import azlmbr.legacy.general as general
import azlmbr.math as math
import azlmbr.paths
import azlmbr.editor as editor
def initial_viewport_setup(screen_width, screen_height):
general.set_viewport_size(screen_width, screen_height)
general.update_viewport()
@@ -147,22 +144,25 @@ def run():
parent_id=default_level.id
)
azlmbr.components.TransformBus(azlmbr.bus.Event, "SetLocalUniformScale", ground_plane.id, 32.0)
ground_plane_material_asset_path = os.path.join("Materials", "Presets", "PBR", "metal_chrome.azmaterial")
ground_plane_material_asset = asset.AssetCatalogRequestBus(
bus.Broadcast, "GetAssetIdByPath", ground_plane_material_asset_path, math.Uuid(), False)
ground_plane.get_set_test(0, "Default Material|Material Asset", ground_plane_material_asset)
# Work around to add the correct Atom Mesh component
# Work around to add the correct Atom Mesh component and asset.
mesh_type_id = azlmbr.globals.property.EditorMeshComponentTypeId
ground_plane.components.append(
editor.EditorComponentAPIBus(
bus.Broadcast, "AddComponentsOfType", ground_plane.id, [mesh_type_id]
).GetValue()[0]
)
ground_plane_mesh_asset_path = os.path.join("Objects", "plane.azmodel")
ground_plane_mesh_asset_path = os.path.join("TestData", "Objects", "plane.azmodel")
ground_plane_mesh_asset = asset.AssetCatalogRequestBus(
bus.Broadcast, "GetAssetIdByPath", ground_plane_mesh_asset_path, math.Uuid(), False)
hydra.get_set_test(ground_plane, 1, "Controller|Configuration|Mesh Asset", ground_plane_mesh_asset)
# Add Atom Material component and asset.
ground_plane_material_asset_path = os.path.join("Materials", "Presets", "PBR", "metal_chrome.azmaterial")
ground_plane_material_asset = asset.AssetCatalogRequestBus(
bus.Broadcast, "GetAssetIdByPath", ground_plane_material_asset_path, math.Uuid(), False)
ground_plane.get_set_test(0, "Default Material|Material Asset", ground_plane_material_asset)
# Create directional_light entity and set the properties
directional_light = hydra.Entity("directional_light")
directional_light.create_entity(
@@ -180,11 +180,8 @@ def run():
components=["Material"],
parent_id=default_level.id
)
sphere_material_asset_path = os.path.join("Materials", "Presets", "PBR", "metal_brass_polished.azmaterial")
sphere_material_asset = asset.AssetCatalogRequestBus(
bus.Broadcast, "GetAssetIdByPath", sphere_material_asset_path, math.Uuid(), False)
sphere.get_set_test(0, "Default Material|Material Asset", sphere_material_asset)
# Work around to add the correct Atom Mesh component
# Work around to add the correct Atom Mesh component and asset.
sphere.components.append(
editor.EditorComponentAPIBus(
bus.Broadcast, "AddComponentsOfType", sphere.id, [mesh_type_id]
@@ -195,6 +192,12 @@ def run():
bus.Broadcast, "GetAssetIdByPath", sphere_mesh_asset_path, math.Uuid(), False)
hydra.get_set_test(sphere, 1, "Controller|Configuration|Mesh Asset", sphere_mesh_asset)
# Add Atom Material component and asset.
sphere_material_asset_path = os.path.join("Materials", "Presets", "PBR", "metal_brass_polished.azmaterial")
sphere_material_asset = asset.AssetCatalogRequestBus(
bus.Broadcast, "GetAssetIdByPath", sphere_material_asset_path, math.Uuid(), False)
sphere.get_set_test(0, "Default Material|Material Asset", sphere_material_asset)
# Create camera component and set the properties
camera_entity = hydra.Entity("camera")
position = math.Vector3(5.5, -12.0, 9.0)
@@ -204,10 +207,9 @@ def run():
)
azlmbr.components.TransformBus(azlmbr.bus.Event, "SetLocalRotation", camera_entity.id, rotation)
camera_entity.get_set_test(0, "Controller|Configuration|Field of view", 60.0)
azlmbr.camera.EditorCameraViewRequestBus(azlmbr.bus.Event, "ToggleCameraAsActiveView", camera_entity.id)
camera.EditorCameraViewRequestBus(azlmbr.bus.Event, "ToggleCameraAsActiveView", camera_entity.id)
# Save level, enter game mode, take screenshot, & exit game mode.
general.save_level()
# Enter game mode, take screenshot, & exit game mode.
general.idle_wait(0.5)
general.enter_game_mode()
general.idle_wait(1.0)