birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'stabilization/2106' into gitflow_210609

Browse Source
This commit is contained in:
daimini
2021-06-09 14:26:29 -07:00
parent 548f65e5d8 3c23f5fead
commit d6d2cbae7e
46 changed files with 138929 additions and 329 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/ctest/CMakeLists.txt
-29
View File
@@ -16,35 +16,6 @@ if(NOT PAL_TRAIT_BUILD_TESTS_SUPPORTED)
return()
endif()
################################################################################
# Asset Processing Target
# i.e. Tests depend on AutomatedTesting.Assets
################################################################################
if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS)
get_property(LY_PROJECTS_TARGET_NAME GLOBAL PROPERTY LY_PROJECTS_TARGET_NAME)
foreach(project_target_name project_path IN ZIP_LISTS LY_PROJECTS_TARGET_NAME LY_PROJECTS)
file(REAL_PATH ${project_path} project_real_path BASE_DIRECTORY ${LY_ROOT_FOLDER})
# With the lock file, asset processing jobs are serialized to avoid race conditions
# on files that are created temporarily in source folders during shader processing.
add_custom_target(${project_target_name}.Assets
COMMENT "Processing ${project_target_name} assets..."
COMMAND "${CMAKE_COMMAND}"
-DLY_LOCK_FILE=$<TARGET_FILE_DIR:AZ::AssetProcessorBatch>/project_assets.lock
-P ${LY_ROOT_FOLDER}/cmake/CommandExecution.cmake
EXEC_COMMAND $<TARGET_FILE:AZ::AssetProcessorBatch>
--zeroAnalysisMode
--project-path=${project_real_path}
--platforms=${LY_ASSET_DEPLOY_ASSET_TYPE}
)
set_target_properties(${project_target_name}.Assets
PROPERTIES
EXCLUDE_FROM_ALL TRUE
FOLDER ${project_target_name}
)
endforeach()
endif()
################################################################################
# Tests
################################################################################
scripts/o3de/o3de/cmake.py
+79 -21
View File
@@ -40,26 +40,52 @@ def add_gem_dependency(cmake_file: pathlib.Path,
# on a line by basis, see if there already is {gem_name}
# find the first occurrence of a gem, copy its formatting and replace
# the gem name with the new one and append it
# if the gem is already present fail
t_data = []
added = False
line_index_to_append = None
with open(cmake_file, 'r') as s:
start_marker_line_index = None
end_marker_line_index = None
with cmake_file.open('r') as s:
in_gem_list = False
line_index = 0
for line in s:
if line.strip().startswith(enable_gem_start_marker):
line_index_to_append = line_index
if f'{gem_name}' == line.strip():
logger.warning(f'{gem_name} is already enabled in file {str(cmake_file)}.')
return 0
parsed_line = line.strip()
if parsed_line.startswith(enable_gem_start_marker):
# Skip pass the 'set(ENABLED_GEMS' marker just in case their are gems declared on the same line
parsed_line = parsed_line[len(enable_gem_start_marker):]
# Set the flag to indicate that we are in the ENABLED_GEMS variable
in_gem_list = True
start_marker_line_index = line_index
if in_gem_list:
# Since we are inside the ENABLED_GEMS variable determine if the line has the end_marker of ')'
if parsed_line.endswith(enable_gem_end_marker):
# Strip away the line end marker
parsed_line = parsed_line[:-len(enable_gem_end_marker)]
# Set the flag to indicate that we are no longer in the ENABLED_GEMS variable after this line
in_gem_list = False
end_marker_line_index = line_index
# Split the rest of the line on whitespace just in case there are multiple gems in a line
gem_name_list = map(lambda gem_name: gem_name.strip('"'), parsed_line.split())
if gem_name in gem_name_list:
logger.warning(f'{gem_name} is already enabled in file {str(cmake_file)}.')
return 0
t_data.append(line)
line_index += 1
indent = 4
if line_index_to_append:
# Insert the gem after the 'set(ENABLED_GEMS)...` line
t_data.insert(line_index_to_append + 1, f'{" " * indent}{gem_name}\n')
if start_marker_line_index:
# Make sure if there is a enable gem start marker, there is an end marker as well
if not end_marker_line_index:
logger.error(f'The Enable Gem start marker of "{enable_gem_start_marker}" has been found, but not the'
f' Enable Gem end marker of "{enable_gem_end_marker}"')
return 1
# Insert the gem before the ')' end marker
end_marker_partition = list(t_data[end_marker_line_index].rpartition(enable_gem_end_marker))
end_marker_partition[1] = f'{" " * indent}{gem_name}\n' + end_marker_partition[1]
t_data[end_marker_line_index] = ''.join(end_marker_partition)
added = True
# if we didn't add, then create a new set(ENABLED_GEMS) variable
@@ -71,7 +97,7 @@ def add_gem_dependency(cmake_file: pathlib.Path,
t_data.append(f'{enable_gem_end_marker}\n')
# write the cmake
with open(cmake_file, 'w') as s:
with cmake_file.open('w') as s:
s.writelines(t_data)
return 0
@@ -90,12 +116,44 @@ def remove_gem_dependency(cmake_file: pathlib.Path,
# on a line by basis, remove any line with {gem_name}
t_data = []
# Remove the gem from the enabled_gem file by skipping the gem name entry
removed = False
with open(cmake_file, 'r') as s:
with cmake_file.open('r') as s:
in_gem_list = False
for line in s:
if gem_name == line.strip():
removed = True
# Strip whitespace from both ends of the line, but keep track of the leading whitespace
# for indenting the result line
parsed_line = line.lstrip()
indent = line[:-len(parsed_line)]
parsed_line = parsed_line.rstrip()
result_line = indent
if parsed_line.startswith(enable_gem_start_marker):
# Skip pass the 'set(ENABLED_GEMS' marker just in case their are gems declared on the same line
parsed_line = parsed_line[len(enable_gem_start_marker):]
result_line += enable_gem_start_marker
# Set the flag to indicate that we are in the ENABLED_GEMS variable
in_gem_list = True
if in_gem_list:
# Since we are inside the ENABLED_GEMS variable determine if the line has the end_marker of ')'
if parsed_line.endswith(enable_gem_end_marker):
# Strip away the line end marker
parsed_line = parsed_line[:-len(enable_gem_end_marker)]
# Set the flag to indicate that we are no longer in the ENABLED_GEMS variable after this line
in_gem_list = False
# Split the rest of the line on whitespace just in case there are multiple gems in a line
# Strip double quotes surround any gem name
gem_name_list = list(map(lambda gem_name: gem_name.strip('"'), parsed_line.split()))
while gem_name in gem_name_list:
gem_name_list.remove(gem_name)
removed = True
# Append the renaming gems to the line
result_line += ' '.join(gem_name_list)
# If the in_gem_list was flipped to false, that means the currently parsed line contained the
# line end marker, so append that to the result_line
result_line += enable_gem_end_marker if not in_gem_list else ''
t_data.append(result_line + '\n')
else:
t_data.append(line)
@@ -104,14 +162,14 @@ def remove_gem_dependency(cmake_file: pathlib.Path,
return 1
# write the cmake
with open(cmake_file, 'w') as s:
with cmake_file.open('w') as s:
s.writelines(t_data)
return 0
def get_project_gems(project_path: pathlib.Path,
platform: str = 'Common') -> set:
platform: str = 'Common') -> set:
return get_gems_from_cmake_file(get_enabled_gem_cmake_file(project_path=project_path, platform=platform))
@@ -145,7 +203,7 @@ def get_enabled_gems(cmake_file: pathlib.Path) -> set:
# Set the flag to indicate that we are no longer in the ENABLED_GEMS variable after this line
in_gem_list = False
# Split the rest of the line on whitespace just in case there are multiple gems in a line
gem_name_list = line.split()
gem_name_list = list(map(lambda gem_name: gem_name.strip('"'), line.split()))
gem_target_set.update(gem_name_list)
return gem_target_set
@@ -156,7 +214,7 @@ def get_project_gem_paths(project_path: pathlib.Path,
gem_names = get_project_gems(project_path, platform)
gem_paths = set()
for gem_name in gem_names:
gem_paths.add(manifest.get_registered(gem_name=gem_name))
gem_paths.add(manifest.get_registered(gem_name=gem_name, project_path=project_path))
return gem_paths
scripts/o3de/o3de/disable_gem.py
+2 -2
View File
@@ -64,7 +64,7 @@ def disable_gem_in_project(gem_name: str = None,
# if gem name resolve it into a path
if gem_name and not gem_path:
gem_path = manifest.get_registered(gem_name=gem_name)
gem_path = manifest.get_registered(gem_name=gem_name, project_path=project_path)
if not gem_path:
logger.error(f'Unable to locate gem path from the registered manifest.json files:'
f' {str(pathlib.Path.home() / ".o3de/manifest.json")},'
@@ -78,7 +78,7 @@ def disable_gem_in_project(gem_name: str = None,
# Read gem.json from the gem path
gem_json_data = manifest.get_gem_json_data(gem_path=gem_path)
gem_json_data = manifest.get_gem_json_data(gem_path=gem_path, project_path=project_path)
if not gem_json_data:
logger.error(f'Could not read gem.json content under {gem_path}.')
return 1
scripts/o3de/o3de/enable_gem.py
+2 -2
View File
@@ -64,7 +64,7 @@ def enable_gem_in_project(gem_name: str = None,
# if gem name resolve it into a path
if gem_name and not gem_path:
gem_path = manifest.get_registered(gem_name=gem_name)
gem_path = manifest.get_registered(gem_name=gem_name, project_path=project_path)
if not gem_path:
logger.error(f'Unable to locate gem path from the registered manifest.json files:'
f' {str(pathlib.Path( "~/.o3de/o3de_manifest.json").expanduser())},'
@@ -78,7 +78,7 @@ def enable_gem_in_project(gem_name: str = None,
return 1
# Read gem.json from the gem path
gem_json_data = manifest.get_gem_json_data(gem_path=gem_path)
gem_json_data = manifest.get_gem_json_data(gem_path=gem_path, project_path=project_path)
if not gem_json_data:
logger.error(f'Could not read gem.json content under {gem_path}.')
return 1
scripts/o3de/o3de/manifest.py
+39 -14
View File
@@ -354,7 +354,7 @@ def get_all_templates(project_path: pathlib.Path = None) -> list:
return list(dict.fromkeys(templates_data))
def get_all_restricted() -> list:
def get_all_restricted(project_path: pathlib.Path = None) -> list:
restricted_data = get_restricted()
restricted_data.extend(get_engine_restricted())
if project_path:
@@ -488,14 +488,14 @@ def get_project_json_data(project_name: str = None,
return None
def get_gem_json_data(gem_name: str = None,
gem_path: str or pathlib.Path = None) -> dict or None:
def get_gem_json_data(gem_name: str = None, gem_path: str or pathlib.Path = None,
project_path: pathlib.Path = None) -> dict or None:
if not gem_name and not gem_path:
logger.error('Must specify either a Gem name or Gem Path.')
return None
if gem_name and not gem_path:
gem_path = get_registered(gem_name=gem_name)
gem_path = get_registered(gem_name=gem_name, project_path=project_path)
if not gem_path:
logger.error(f'Gem Path {gem_path} has not been registered.')
@@ -521,14 +521,14 @@ def get_gem_json_data(gem_name: str = None,
return None
def get_template_json_data(template_name: str = None,
template_path: str or pathlib.Path = None) -> dict or None:
def get_template_json_data(template_name: str = None, template_path: str or pathlib.Path = None,
project_path: pathlib.Path = None) -> dict or None:
if not template_name and not template_path:
logger.error('Must specify either a Template name or Template Path.')
return None
if template_name and not template_path:
template_path = get_registered(template_name=template_name)
template_path = get_registered(template_name=template_name, project_path=project_path)
if not template_path:
logger.error(f'Template Path {template_path} has not been registered.')
@@ -554,14 +554,14 @@ def get_template_json_data(template_name: str = None,
return None
def get_restricted_json_data(restricted_name: str = None,
restricted_path: str or pathlib.Path = None) -> dict or None:
def get_restricted_json_data(restricted_name: str = None, restricted_path: str or pathlib.Path = None,
project_path: pathlib.Path = None) -> dict or None:
if not restricted_name and not restricted_path:
logger.error('Must specify either a Restricted name or Restricted Path.')
return None
if restricted_name and not restricted_path:
restricted_path = get_registered(restricted_name=restricted_name)
restricted_path = get_registered(restricted_name=restricted_name, project_path=project_path)
if not restricted_path:
logger.error(f'Restricted Path {restricted_path} has not been registered.')
@@ -593,7 +593,32 @@ def get_registered(engine_name: str = None,
template_name: str = None,
default_folder: str = None,
repo_name: str = None,
restricted_name: str = None) -> pathlib.Path or None:
restricted_name: str = None,
project_path: pathlib.Path = None) -> pathlib.Path or None:
"""
Looks up a registered entry in either the ~/.o3de/o3de_manifest.json, <this-engine-root>/engine.json
or the <project-path>/project.json (if the project_path parameter is supplied)
:param engine_name: Name of a registered engine to lookup in the ~/.o3de/o3de_manifest.json file
:param project_name: Name of a project to lookup in either the ~/.o3de/o3de_manifest.json or
<this-engine-root>/engine.json file
:param gem_name: Name of a gem to lookup in either the ~/.o3de/o3de_manifest.json, <this-engine-root>/engine.json
or <project-path>/project.json. NOTE: The project_path parameter must be supplied to lookup the registration
with the project.json
:param template_name: Name of a template to lookup in either the ~/.o3de/o3de_manifest.json, <this-engine-root>/engine.json
or <project-path>/project.json. NOTE: The project_path parameter must be supplied to lookup the registration
with the project.json
:param repo_name: Name of a repo to lookup in the ~/.o3de/o3de_manifest.json
:param default_folder: Type of "default" folder to lookup in the ~/.o3de/o3de_manifest.json
Valid values are "engines", "projects", "gems", "templates,", "restricted"
:param restricted_name: Name of a restricted directory object to lookup in either the ~/.o3de/o3de_manifest.json,
<this-engine-root>/engine.json or <project-path>/project.json.
NOTE: The project_path parameter must be supplied to lookup the registration with the project.json
:param project_path: Path to project root, which is used to examined the project.json file in order to
query either gems, templates or restricted directories registered with the project
:return path value associated with the registered object name if found. Otherwise None is returned
"""
json_data = load_o3de_manifest()
# check global first then this engine
@@ -627,7 +652,7 @@ def get_registered(engine_name: str = None,
return project_path
elif isinstance(gem_name, str):
gems = get_all_gems()
gems = get_all_gems(project_path)
for gem_path in gems:
gem_path = pathlib.Path(gem_path).resolve()
gem_json = gem_path / 'gem.json'
@@ -642,7 +667,7 @@ def get_registered(engine_name: str = None,
return gem_path
elif isinstance(template_name, str):
templates = get_all_templates()
templates = get_all_templates(project_path)
for template_path in templates:
template_path = pathlib.Path(template_path).resolve()
template_json = template_path / 'template.json'
@@ -657,7 +682,7 @@ def get_registered(engine_name: str = None,
return template_path
elif isinstance(restricted_name, str):
restricted = get_all_restricted()
restricted = get_all_restricted(project_path)
for restricted_path in restricted:
restricted_path = pathlib.Path(restricted_path).resolve()
restricted_json = restricted_path / 'restricted.json'
scripts/o3de/tests/unit_test_cmake.py
+163
View File
@@ -12,6 +12,8 @@
import io
import json
import logging
import unittest.mock
import pytest
import pathlib
from unittest.mock import patch
@@ -67,3 +69,164 @@ class TestGetEnabledGems:
enabled_gems_set = cmake.get_enabled_gems(pathlib.Path('enabled_gems.cmake'))
assert enabled_gems_set == expected_set
class TestAddGemDependency:
@pytest.mark.parametrize(
"enable_gems_cmake_data, expected_set, expected_return", [
pytest.param("""
# Comment
set(ENABLED_GEMS foo bar baz)
""", set(['foo', 'bar', 'baz', 'TestGem']), 0),
pytest.param("""
# Comment
set(ENABLED_GEMS
foo
bar
baz
)
""", set(['foo', 'bar', 'baz', 'TestGem']), 0),
pytest.param("""
# Comment
set(ENABLED_GEMS
foo
bar
baz)
""", set(['foo', 'bar', 'baz', 'TestGem']), 0),
pytest.param("""
# Comment
set(ENABLED_GEMS
foo bar
baz)
""", set(['foo', 'bar', 'baz', 'TestGem']), 0),
pytest.param("""
""", set(['TestGem']), 0),
pytest.param("""
# Comment
set(RANDOM_VARIABLE TestGame, TestProject Test Engine)
set(ENABLED_GEMS HelloWorld IceCream
foo
baz bar
baz baz baz baz baz morebaz lessbaz
)
Random Text
""", set(['HelloWorld', 'IceCream', 'foo', 'bar', 'baz', 'morebaz', 'lessbaz', 'TestGem']),
0),
pytest.param("""
set(ENABLED_GEMS foo bar baz
""", set(['foo', 'bar', 'baz']), 1),
]
)
def test_add_gem_dependency(self, enable_gems_cmake_data, expected_set, expected_return):
enabled_gems_set = set()
add_gem_return = None
class StringBufferIOWrapper(io.StringIO):
def __init__(self):
nonlocal enable_gems_cmake_data
super().__init__(enable_gems_cmake_data)
def __enter__(self):
return super().__enter__()
def __exit__(self, exc_type, exc_val, exc_tb):
nonlocal enable_gems_cmake_data
enable_gems_cmake_data = super().getvalue()
super().__exit__(exc_tb, exc_val, exc_tb)
with patch('pathlib.Path.resolve', return_value=pathlib.Path('enabled_gems.cmake')) as pathlib_is_resolve_mock,\
patch('pathlib.Path.is_file', return_value=True) as pathlib_is_file_mock,\
patch('pathlib.Path.open', side_effect=lambda mode: StringBufferIOWrapper()) as pathlib_open_mock:
add_gem_return = cmake.add_gem_dependency(pathlib.Path('enabled_gems.cmake'), 'TestGem')
enabled_gems_set = cmake.get_enabled_gems(pathlib.Path('enabled_gems.cmake'))
assert add_gem_return == expected_return
assert enabled_gems_set == expected_set
class TestRemoveGemDependency:
@pytest.mark.parametrize(
"enable_gems_cmake_data, expected_set, expected_return", [
pytest.param("""
# Comment
set(ENABLED_GEMS foo bar baz TestGem)
""", set(['foo', 'bar', 'baz']), 0),
pytest.param("""
# Comment
set(ENABLED_GEMS
foo
bar
baz
TestGem
)
""", set(['foo', 'bar', 'baz']), 0),
pytest.param("""
# Comment
set(ENABLED_GEMS
foo
bar
baz
TestGem)
""", set(['foo', 'bar', 'baz']), 0),
pytest.param("""
# Comment
set(ENABLED_GEMS
foo bar
baz TestGem)
""", set(['foo', 'bar', 'baz']), 0),
pytest.param("""
# Comment
set(ENABLED_GEMS
foo
TestGem
bar
TestGem
baz
)
Random Text
""", set(['foo', 'bar', 'baz']),
0),
pytest.param("""
set(ENABLED_GEMS
foo
bar
baz
"TestGem"
)
""", set(['foo', 'bar', 'baz']), 0),
pytest.param("""
""", set(), 1),
pytest.param("""
set(ENABLED_GEMS
foo
bar
baz
)
""", set(['foo', 'bar', 'baz']), 1),
]
)
def test_remove_gem_dependency(self, enable_gems_cmake_data, expected_set, expected_return):
enabled_gems_set = set()
add_gem_return = None
class StringBufferIOWrapper(io.StringIO):
def __init__(self):
nonlocal enable_gems_cmake_data
super().__init__(enable_gems_cmake_data)
def __enter__(self):
return super().__enter__()
def __exit__(self, exc_type, exc_val, exc_tb):
nonlocal enable_gems_cmake_data
enable_gems_cmake_data = super().getvalue()
super().__exit__(exc_tb, exc_val, exc_tb)
with patch('pathlib.Path.resolve', return_value=pathlib.Path('enabled_gems.cmake')) as pathlib_is_resolve_mock,\
patch('pathlib.Path.is_file', return_value=True) as pathlib_is_file_mock,\
patch('pathlib.Path.open', side_effect=lambda mode: StringBufferIOWrapper()) as pathlib_open_mock:
add_gem_return = cmake.remove_gem_dependency(pathlib.Path('enabled_gems.cmake'), 'TestGem')
enabled_gems_set = cmake.get_enabled_gems(pathlib.Path('enabled_gems.cmake'))
assert add_gem_return == expected_return
assert enabled_gems_set == expected_set