Merge branch 'development' into TIF/Runtime

This commit is contained in:
John
2021-06-16 17:23:18 +01:00
925 changed files with 201921 additions and 111266 deletions
@@ -20,9 +20,6 @@ IF NOT EXIST "%LY_3RDPARTY_PATH%" (
GOTO :error
)
IF NOT EXIST "%LY_ANDROID_SDK%" (
SET LY_ANDROID_SDK=!LY_3RDPARTY_PATH!/android-sdk/platform-29
)
IF NOT EXIST "%LY_ANDROID_SDK%" (
ECHO [ci_build] FAIL: LY_ANDROID_SDK=!LY_ANDROID_SDK!
GOTO :error
@@ -24,7 +24,7 @@
"COMMAND": "python_linux.sh",
"PARAMETERS": {
"SCRIPT_PATH": "scripts/build/ci_build_metrics.py",
"SCRIPT_PARAMETERS": "--platform Linux --jobname '${JOB_NAME}' --jobnumber '${BUILD_NUMBER}' --jobnode '${NODE_NAME}' --changelist '${CHANGE_ID}'"
"SCRIPT_PARAMETERS": "--platform Linux --jobname=\\'${JOB_NAME}\\' --jobnumber \\'${BUILD_NUMBER}\\' --jobnode \\'${NODE_LABEL}\\' --changelist \\'${CHANGE_ID}\\'"
}
},
"debug": {
@@ -83,7 +83,7 @@
"CMAKE_OPTIONS": "-G 'Ninja Multi-Config' -DCMAKE_C_COMPILER=clang-6.0 -DCMAKE_CXX_COMPILER=clang++-6.0 -DLY_UNITY_BUILD=TRUE -DLY_PARALLEL_LINK_JOBS=4",
"CMAKE_LY_PROJECTS": "AutomatedTesting",
"CMAKE_TARGET": "all",
"CTEST_OPTIONS": "-E (Gem::EMotionFX.Editor.Tests|Gem::AWSClientAuth.Tests|Gem::AWSCore.Editor.Tests) -LE SUITE_sandbox -L FRAMEWORK_googletest"
"CTEST_OPTIONS": "-E (Gem::EMotionFX.Editor.Tests|Gem::AWSCore.Editor.Tests) -LE SUITE_sandbox -L FRAMEWORK_googletest"
}
},
"test_profile_nounity": {
@@ -95,7 +95,7 @@
"CMAKE_OPTIONS": "-G 'Ninja Multi-Config' -DCMAKE_C_COMPILER=clang-6.0 -DCMAKE_CXX_COMPILER=clang++-6.0 -DLY_UNITY_BUILD=FALSE -DLY_PARALLEL_LINK_JOBS=4",
"CMAKE_LY_PROJECTS": "AutomatedTesting",
"CMAKE_TARGET": "all",
"CTEST_OPTIONS": "-E (Gem::EMotionFX.Editor.Tests|Gem::AWSClientAuth.Tests|Gem::AWSCore.Editor.Tests) -LE SUITE_sandbox -L FRAMEWORK_googletest"
"CTEST_OPTIONS": "-E (Gem::EMotionFX.Editor.Tests|Gem::AWSCore.Editor.Tests) -LE SUITE_sandbox -L FRAMEWORK_googletest"
}
},
"asset_profile": {
+4 -20
View File
@@ -13,27 +13,11 @@
set -o errexit # exit on the first failure encountered
if ! command -v cmake &> /dev/null; then
if [[ -z $LY_CMAKE_PATH ]]; then LY_CMAKE_PATH=${LY_3RDPARTY_PATH}/CMake/3.19.1/Linux/bin; fi
if [[ ! -d $LY_CMAKE_PATH ]]; then
echo "[ci_build] CMake path not found"
exit 1
fi
PATH=${LY_CMAKE_PATH}:${PATH}
if ! command -v cmake &> /dev/null; then
echo "[ci_build] CMake not found"
exit 1
fi
echo "[ci_build] CMake not found"
exit 1
fi
if ! command -v ninja &> /dev/null; then
if [[ -z $LY_NINJA_PATH ]]; then LY_NINJA_PATH=${LY_3RDPARTY_PATH}/ninja/1.10.1/Linux; fi
if [[ ! -d $LY_NINJA_PATH ]]; then
echo "[ci_build] Ninja path not found"
exit 1
fi
PATH=${LY_NINJA_PATH}:${PATH}
if ! command -v ninja &> /dev/null; then
echo "[ci_build] Ninja not found"
exit 1
fi
echo "[ci_build] Ninja not found"
exit 1
fi
+2 -2
View File
@@ -12,5 +12,5 @@
set -o errexit # exit on the first failure encountered
echo [ci_build] python/python.sh -u ${SCRIPT_PATH} ${SCRIPT_PARAMETERS}
python/python.sh -u ${SCRIPT_PATH} ${SCRIPT_PARAMETERS}
echo [ci_build] python/python.sh -u ${SCRIPT_PATH} $(eval echo ${SCRIPT_PARAMETERS})
python/python.sh -u ${SCRIPT_PATH} $(eval echo ${SCRIPT_PARAMETERS})
+1 -1
View File
@@ -24,7 +24,7 @@
"COMMAND": "python_mac.sh",
"PARAMETERS": {
"SCRIPT_PATH": "scripts/build/ci_build_metrics.py",
"SCRIPT_PARAMETERS": "--platform Mac --jobname '${JOB_NAME}' --jobnumber '${BUILD_NUMBER}' --jobnode '${NODE_NAME}' --changelist '${CHANGE_ID}'"
"SCRIPT_PARAMETERS": "--platform Mac --jobname \\'${JOB_NAME}\\' --jobnumber \\'${BUILD_NUMBER}\\' --jobnode \\'${NODE_LABEL}\\' --changelist \\'${CHANGE_ID}\\'"
}
},
"debug": {
+2 -10
View File
@@ -13,14 +13,6 @@
set -o errexit # exit on the first failure encountered
if ! command -v cmake &> /dev/null; then
if [[ -z $LY_CMAKE_PATH ]]; then LY_CMAKE_PATH=${LY_3RDPARTY_PATH}/CMake/3.19.1/Mac/CMake.app/Contents/bin; fi
if [[ ! -d $LY_CMAKE_PATH ]]; then
echo "[ci_build] CMake path not found"
exit 1
fi
PATH=${LY_CMAKE_PATH}:${PATH}
if ! command -v cmake &> /dev/null; then
echo "[ci_build] CMake not found"
exit 1
fi
echo "[ci_build] CMake not found"
exit 1
fi
+1 -1
View File
@@ -1,6 +1,6 @@
{
"ENV": {
"NODE_LABEL": "mac-catalina-7ad2e45b",
"NODE_LABEL": "mac-bigsur-2fc3a22",
"LY_3RDPARTY_PATH": "/Users/lybuilder/3rdParty",
"TIMEOUT": 30,
"WORKSPACE": "/Users/lybuilder/workspace",
+2 -2
View File
@@ -12,5 +12,5 @@
set -o errexit # exit on the first failure encountered
echo [ci_build] python/python.sh -u ${SCRIPT_PATH} ${SCRIPT_PARAMETERS}
python/python.sh -u ${SCRIPT_PATH} ${SCRIPT_PARAMETERS}
echo [ci_build] python/python.sh -u ${SCRIPT_PATH} $(eval echo ${SCRIPT_PARAMETERS})
python/python.sh -u ${SCRIPT_PATH} $(eval echo ${SCRIPT_PARAMETERS})
@@ -59,7 +59,7 @@
"COMMAND": "python_windows.cmd",
"PARAMETERS": {
"SCRIPT_PATH": "scripts/build/ci_build_metrics.py",
"SCRIPT_PARAMETERS": "--platform Windows --jobname \"!JOB_NAME!\" --jobnumber \"!BUILD_NUMBER!\" --jobnode \"!NODE_NAME!\" --changelist \"!CHANGE_ID!\""
"SCRIPT_PARAMETERS": "--platform Windows --jobname \"!JOB_NAME!\" --jobnumber \"!BUILD_NUMBER!\" --jobnode \"!NODE_LABEL!\" --changelist \"!CHANGE_ID!\""
}
},
"windows_packaging_all": {
@@ -12,19 +12,10 @@ REM
SETLOCAL EnableDelayedExpansion
IF NOT EXIST "%LY_NINJA_PATH%" (
SET LY_NINJA_PATH=%LY_3RDPARTY_PATH%/ninja/1.10.1/Windows
)
IF NOT EXIST "%LY_NINJA_PATH%" (
ECHO [ci_build] FAIL: LY_NINJA_PATH=%LY_NINJA_PATH%
GOTO :error
)
PATH %LY_NINJA_PATH%;%PATH%
CALL "%~dp0build_windows.cmd"
IF NOT %ERRORLEVEL%==0 GOTO :error
EXIT /b 0
:error
EXIT /b 1
EXIT /b 1
+2 -11
View File
@@ -12,17 +12,8 @@ REM
where /Q cmake
IF NOT %ERRORLEVEL%==0 (
IF "%LY_CMAKE_PATH%"=="" (SET LY_CMAKE_PATH=%LY_3RDPARTY_PATH%/CMake/3.19.1/Windows/bin)
IF NOT EXIST !LY_CMAKE_PATH! (
ECHO [ci_build] CMake path not found
GOTO :error
)
PATH !LY_CMAKE_PATH!;!PATH!
where /Q cmake
IF NOT !ERRORLEVEL!==0 (
ECHO [ci_build] CMake not found
GOTO :error
)
ECHO [ci_build] CMake not found
GOTO :error
)
EXIT /b 0
+20 -1
View File
@@ -14,7 +14,7 @@
"COMMAND": "../Mac/python_mac.sh",
"PARAMETERS": {
"SCRIPT_PATH": "scripts/build/ci_build_metrics.py",
"SCRIPT_PARAMETERS": "--platform iOS --jobname '${JOB_NAME}' --jobnumber '${BUILD_NUMBER}' --jobnode '${NODE_NAME}' --changelist '${CHANGE_ID}'"
"SCRIPT_PARAMETERS": "--platform iOS --jobname '${JOB_NAME}' --jobname \\'${JOB_NAME}\\' --jobnumber \\'${BUILD_NUMBER}\\' --jobnode \\'${NODE_LABEL}\\' --changelist \\'${CHANGE_ID}\\'"
}
},
"debug": {
@@ -99,5 +99,24 @@
"CMAKE_TARGET": "ALL_BUILD",
"CMAKE_NATIVE_BUILD_ARGS": "-destination generic/platform=iOS"
}
},
"device_test_profile": {
"TAGS": [
],
"PIPELINE_ENV":{
"NODE_LABEL":"Catalina_DeviceFarm_1",
"LY_3RDPARTY_PATH": "/Volumes/Data/jenkins/3rdParty",
"WORKSPACE": "/Volumes/Data/jenkins/workspace"
},
"COMMAND": "./build_ios_test.sh",
"PARAMETERS": {
"CONFIGURATION": "profile",
"OUTPUT_DIRECTORY": "build/ios_test",
"CMAKE_OPTIONS": "-G Xcode -DCMAKE_TOOLCHAIN_FILE=cmake/Platform/iOS/Toolchain_ios.cmake -DLY_MONOLITHIC_GAME=FALSE -DCMAKE_XCODE_ATTRIBUTE_CODE_SIGNING_REQUIRED=TRUE -DLY_IOS_CODE_SIGNING_IDENTITY=\"\" -DCMAKE_XCODE_ATTRIBUTE_CODE_SIGN_ENTITLEMENTS=\"\" -DCMAKE_XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED=TRUE -DLY_UNITY_BUILD=TRUE -DO3DE_HOME_PATH=\"${WORKSPACE}/home\" -DO3DE_REGISTER_ENGINE_PATH=\"${WORKSPACE}/o3de\" -DO3DE_REGISTER_THIS_ENGINE=TRUE",
"CMAKE_TARGET": "ALL_BUILD",
"CMAKE_NATIVE_BUILD_ARGS": "",
"TARGET_DEVICE_NAME": "Lumberyard",
"TEST_MODULE_TIMEOUT": "600"
}
}
}
+76
View File
@@ -0,0 +1,76 @@
#!/usr/bin/env bash
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
set -o errexit # exit on the first failure encountered
BASEDIR=$(dirname "$0")
source $BASEDIR/../Mac/env_mac.sh
mkdir -p ${OUTPUT_DIRECTORY}
SOURCE_DIRECTORY=${PWD}
ECHO Configuring for iOS Testing
LAST_CONFIGURE_CMD_FILE=ci_last_configure_cmd.txt
CONFIGURE_CMD="cmake -B ${OUTPUT_DIRECTORY} ${SOURCE_DIRECTORY} ${CMAKE_OPTIONS} ${EXTRA_CMAKE_OPTIONS} -DLY_3RDPARTY_PATH=${LY_3RDPARTY_PATH}"
if [[ ! -e "CMakeCache.txt" ]]; then
echo [ci_build] First run, generating
RUN_CONFIGURE=1
elif [[ ! -e ${LAST_CONFIGURE_CMD_FILE} ]]; then
echo [ci_build] Last run command not found, generating
RUN_CONFIGURE=1
else
# Detect if the input has changed
LAST_CMD=$(<${LAST_CONFIGURE_CMD_FILE})
if [[ "${LAST_CMD}" != "${CONFIGURE_CMD}" ]]; then
echo [ci_build] Last run command different, generating
RUN_CONFIGURE=1
fi
fi
if [[ ! -z "$RUN_CONFIGURE" ]]; then
# have to use eval since $CMAKE_OPTIONS (${EXTRA_CMAKE_OPTIONS}) contains quotes that need to be processed
echo [ci_build] ${CONFIGURE_CMD}
eval ${CONFIGURE_CMD}
# Save the run only if success
echo "${CONFIGURE_CMD}" > ${LAST_CONFIGURE_CMD_FILE}
fi
if [ $? -ne 0 ]
then
echo "CMake configuration failed"
exit 1
fi
echo Building for iOS Testing
echo
echo [ci+build] ${SOURCE_DIRECTORY}/python/python.sh ${SOURCE_DIRECTORY}/cmake/Tools/Platform/iOS/build_ios_test.py -b ${OUTPUT_DIRECTORY} -c ${CONFIGURATION}
${SOURCE_DIRECTORY}/python/python.sh ${SOURCE_DIRECTORY}/cmake/Tools/Platform/iOS/build_ios_test.py -b ${OUTPUT_DIRECTORY} -c ${CONFIGURATION}
if [ $? -ne 0 ]
then
echo "iOS Test Build failed"
exit 1
fi
ECHO Launching iOS Test
echo [ci+build] ${SOURCE_DIRECTORY}/python/python.sh ${SOURCE_DIRECTORY}/cmake/Tools/Platform/iOS/launch_ios_test.py -b ${OUTPUT_DIRECTORY} --device-name ${TARGET_DEVICE_NAME} --timeout ${TEST_MODULE_TIMEOUT}
${SOURCE_DIRECTORY}/python/python.sh ${SOURCE_DIRECTORY}/cmake/Tools/Platform/iOS/launch_ios_test.py -b ${OUTPUT_DIRECTORY} --device-name ${TARGET_DEVICE_NAME} --timeout ${TEST_MODULE_TIMEOUT}
if [ $? -ne 0 ]
then
echo "iOS Test failed"
exit 1
fi
echo "iOS Tests passed"
exit 0
+1 -1
View File
@@ -1,6 +1,6 @@
{
"ENV": {
"NODE_LABEL": "mac-catalina-7ad2e45b",
"NODE_LABEL": "mac-bigsur-2fc3a22",
"LY_3RDPARTY_PATH": "/Users/lybuilder/3rdParty",
"TIMEOUT": 30,
"WORKSPACE": "/Users/lybuilder/workspace",
+15 -42
View File
@@ -16,53 +16,26 @@ if(NOT PAL_TRAIT_BUILD_TESTS_SUPPORTED)
return()
endif()
################################################################################
# Asset Processing Target
# i.e. Tests depend on AutomatedTesting.Assets
################################################################################
if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS)
get_property(LY_PROJECTS_TARGET_NAME GLOBAL PROPERTY LY_PROJECTS_TARGET_NAME)
foreach(project_target_name project_path IN ZIP_LISTS LY_PROJECTS_TARGET_NAME LY_PROJECTS)
file(REAL_PATH ${project_path} project_real_path BASE_DIRECTORY ${LY_ROOT_FOLDER})
# With the lock file, asset processing jobs are serialized to avoid race conditions
# on files that are created temporarily in source folders during shader processing.
add_custom_target(${project_target_name}.Assets
COMMENT "Processing ${project_target_name} assets..."
COMMAND "${CMAKE_COMMAND}"
-DLY_LOCK_FILE=$<TARGET_FILE_DIR:AZ::AssetProcessorBatch>/project_assets.lock
-P ${LY_ROOT_FOLDER}/cmake/CommandExecution.cmake
EXEC_COMMAND $<TARGET_FILE:AZ::AssetProcessorBatch>
--zeroAnalysisMode
--project-path=${project_real_path}
--platforms=${LY_ASSET_DEPLOY_ASSET_TYPE}
)
set_target_properties(${project_target_name}.Assets
PROPERTIES
EXCLUDE_FROM_ALL TRUE
FOLDER ${project_target_name}
)
endforeach()
endif()
################################################################################
# Tests
################################################################################
foreach(suite_name ${LY_TEST_GLOBAL_KNOWN_SUITE_NAMES})
ly_add_pytest(
NAME pytest_sanity_${suite_name}_no_gpu
PATH ${CMAKE_CURRENT_LIST_DIR}/sanity_test.py
TEST_SUITE ${suite_name}
)
if(PAL_TRAIT_TEST_LYTESTTOOLS_SUPPORTED)
foreach(suite_name ${LY_TEST_GLOBAL_KNOWN_SUITE_NAMES})
ly_add_pytest(
NAME pytest_sanity_${suite_name}_no_gpu
PATH ${CMAKE_CURRENT_LIST_DIR}/sanity_test.py
TEST_SUITE ${suite_name}
)
ly_add_pytest(
NAME pytest_sanity_${suite_name}_requires_gpu
PATH ${CMAKE_CURRENT_LIST_DIR}/sanity_test.py
TEST_SUITE ${suite_name}
TEST_REQUIRES gpu
)
endforeach()
ly_add_pytest(
NAME pytest_sanity_${suite_name}_requires_gpu
PATH ${CMAKE_CURRENT_LIST_DIR}/sanity_test.py
TEST_SUITE ${suite_name}
TEST_REQUIRES gpu
)
endforeach()
endif()
# EPB Sanity test is being registered here to validate that the ly_add_editor_python_test function works.
#if(PAL_TRAIT_BUILD_HOST_TOOLS AND PAL_TRAIT_BUILD_TESTS_SUPPORTED AND AutomatedTesting IN_LIST LY_PROJECTS_TARGET_NAME)
+79 -21
View File
@@ -40,26 +40,52 @@ def add_gem_dependency(cmake_file: pathlib.Path,
# on a line by basis, see if there already is {gem_name}
# find the first occurrence of a gem, copy its formatting and replace
# the gem name with the new one and append it
# if the gem is already present fail
t_data = []
added = False
line_index_to_append = None
with open(cmake_file, 'r') as s:
start_marker_line_index = None
end_marker_line_index = None
with cmake_file.open('r') as s:
in_gem_list = False
line_index = 0
for line in s:
if line.strip().startswith(enable_gem_start_marker):
line_index_to_append = line_index
if f'{gem_name}' == line.strip():
logger.warning(f'{gem_name} is already enabled in file {str(cmake_file)}.')
return 0
parsed_line = line.strip()
if parsed_line.startswith(enable_gem_start_marker):
# Skip pass the 'set(ENABLED_GEMS' marker just in case their are gems declared on the same line
parsed_line = parsed_line[len(enable_gem_start_marker):]
# Set the flag to indicate that we are in the ENABLED_GEMS variable
in_gem_list = True
start_marker_line_index = line_index
if in_gem_list:
# Since we are inside the ENABLED_GEMS variable determine if the line has the end_marker of ')'
if parsed_line.endswith(enable_gem_end_marker):
# Strip away the line end marker
parsed_line = parsed_line[:-len(enable_gem_end_marker)]
# Set the flag to indicate that we are no longer in the ENABLED_GEMS variable after this line
in_gem_list = False
end_marker_line_index = line_index
# Split the rest of the line on whitespace just in case there are multiple gems in a line
gem_name_list = map(lambda gem_name: gem_name.strip('"'), parsed_line.split())
if gem_name in gem_name_list:
logger.warning(f'{gem_name} is already enabled in file {str(cmake_file)}.')
return 0
t_data.append(line)
line_index += 1
indent = 4
if line_index_to_append:
# Insert the gem after the 'set(ENABLED_GEMS)...` line
t_data.insert(line_index_to_append + 1, f'{" " * indent}{gem_name}\n')
if start_marker_line_index:
# Make sure if there is a enable gem start marker, there is an end marker as well
if not end_marker_line_index:
logger.error(f'The Enable Gem start marker of "{enable_gem_start_marker}" has been found, but not the'
f' Enable Gem end marker of "{enable_gem_end_marker}"')
return 1
# Insert the gem before the ')' end marker
end_marker_partition = list(t_data[end_marker_line_index].rpartition(enable_gem_end_marker))
end_marker_partition[1] = f'{" " * indent}{gem_name}\n' + end_marker_partition[1]
t_data[end_marker_line_index] = ''.join(end_marker_partition)
added = True
# if we didn't add, then create a new set(ENABLED_GEMS) variable
@@ -71,7 +97,7 @@ def add_gem_dependency(cmake_file: pathlib.Path,
t_data.append(f'{enable_gem_end_marker}\n')
# write the cmake
with open(cmake_file, 'w') as s:
with cmake_file.open('w') as s:
s.writelines(t_data)
return 0
@@ -90,12 +116,44 @@ def remove_gem_dependency(cmake_file: pathlib.Path,
# on a line by basis, remove any line with {gem_name}
t_data = []
# Remove the gem from the enabled_gem file by skipping the gem name entry
removed = False
with open(cmake_file, 'r') as s:
with cmake_file.open('r') as s:
in_gem_list = False
for line in s:
if gem_name == line.strip():
removed = True
# Strip whitespace from both ends of the line, but keep track of the leading whitespace
# for indenting the result line
parsed_line = line.lstrip()
indent = line[:-len(parsed_line)]
parsed_line = parsed_line.rstrip()
result_line = indent
if parsed_line.startswith(enable_gem_start_marker):
# Skip pass the 'set(ENABLED_GEMS' marker just in case their are gems declared on the same line
parsed_line = parsed_line[len(enable_gem_start_marker):]
result_line += enable_gem_start_marker
# Set the flag to indicate that we are in the ENABLED_GEMS variable
in_gem_list = True
if in_gem_list:
# Since we are inside the ENABLED_GEMS variable determine if the line has the end_marker of ')'
if parsed_line.endswith(enable_gem_end_marker):
# Strip away the line end marker
parsed_line = parsed_line[:-len(enable_gem_end_marker)]
# Set the flag to indicate that we are no longer in the ENABLED_GEMS variable after this line
in_gem_list = False
# Split the rest of the line on whitespace just in case there are multiple gems in a line
# Strip double quotes surround any gem name
gem_name_list = list(map(lambda gem_name: gem_name.strip('"'), parsed_line.split()))
while gem_name in gem_name_list:
gem_name_list.remove(gem_name)
removed = True
# Append the renaming gems to the line
result_line += ' '.join(gem_name_list)
# If the in_gem_list was flipped to false, that means the currently parsed line contained the
# line end marker, so append that to the result_line
result_line += enable_gem_end_marker if not in_gem_list else ''
t_data.append(result_line + '\n')
else:
t_data.append(line)
@@ -104,14 +162,14 @@ def remove_gem_dependency(cmake_file: pathlib.Path,
return 1
# write the cmake
with open(cmake_file, 'w') as s:
with cmake_file.open('w') as s:
s.writelines(t_data)
return 0
def get_project_gems(project_path: pathlib.Path,
platform: str = 'Common') -> set:
platform: str = 'Common') -> set:
return get_gems_from_cmake_file(get_enabled_gem_cmake_file(project_path=project_path, platform=platform))
@@ -145,7 +203,7 @@ def get_enabled_gems(cmake_file: pathlib.Path) -> set:
# Set the flag to indicate that we are no longer in the ENABLED_GEMS variable after this line
in_gem_list = False
# Split the rest of the line on whitespace just in case there are multiple gems in a line
gem_name_list = line.split()
gem_name_list = list(map(lambda gem_name: gem_name.strip('"'), line.split()))
gem_target_set.update(gem_name_list)
return gem_target_set
@@ -156,7 +214,7 @@ def get_project_gem_paths(project_path: pathlib.Path,
gem_names = get_project_gems(project_path, platform)
gem_paths = set()
for gem_name in gem_names:
gem_paths.add(manifest.get_registered(gem_name=gem_name))
gem_paths.add(manifest.get_registered(gem_name=gem_name, project_path=project_path))
return gem_paths
+2 -2
View File
@@ -64,7 +64,7 @@ def disable_gem_in_project(gem_name: str = None,
# if gem name resolve it into a path
if gem_name and not gem_path:
gem_path = manifest.get_registered(gem_name=gem_name)
gem_path = manifest.get_registered(gem_name=gem_name, project_path=project_path)
if not gem_path:
logger.error(f'Unable to locate gem path from the registered manifest.json files:'
f' {str(pathlib.Path.home() / ".o3de/manifest.json")},'
@@ -78,7 +78,7 @@ def disable_gem_in_project(gem_name: str = None,
# Read gem.json from the gem path
gem_json_data = manifest.get_gem_json_data(gem_path=gem_path)
gem_json_data = manifest.get_gem_json_data(gem_path=gem_path, project_path=project_path)
if not gem_json_data:
logger.error(f'Could not read gem.json content under {gem_path}.')
return 1
+2 -2
View File
@@ -64,7 +64,7 @@ def enable_gem_in_project(gem_name: str = None,
# if gem name resolve it into a path
if gem_name and not gem_path:
gem_path = manifest.get_registered(gem_name=gem_name)
gem_path = manifest.get_registered(gem_name=gem_name, project_path=project_path)
if not gem_path:
logger.error(f'Unable to locate gem path from the registered manifest.json files:'
f' {str(pathlib.Path( "~/.o3de/o3de_manifest.json").expanduser())},'
@@ -78,7 +78,7 @@ def enable_gem_in_project(gem_name: str = None,
return 1
# Read gem.json from the gem path
gem_json_data = manifest.get_gem_json_data(gem_path=gem_path)
gem_json_data = manifest.get_gem_json_data(gem_path=gem_path, project_path=project_path)
if not gem_json_data:
logger.error(f'Could not read gem.json content under {gem_path}.')
return 1
+11
View File
@@ -1232,10 +1232,12 @@ def create_from_template(destination_path: str,
with_this = replace.pop(0)
replacements.append((replace_this, with_this))
sanitized_cpp_name = utils.sanitize_identifier_for_cpp(destination_name)
# dst name is Name
replacements.append(("${Name}", destination_name))
replacements.append(("${NameUpper}", destination_name.upper()))
replacements.append(("${NameLower}", destination_name.lower()))
replacements.append(("${SanitizedCppName}", sanitized_cpp_name))
if _instantiate_template(template_json_data,
destination_name,
@@ -1536,10 +1538,12 @@ def create_project(project_path: str,
with_this = replace.pop(0)
replacements.append((replace_this, with_this))
sanitized_cpp_name = utils.sanitize_identifier_for_cpp(project_name)
# project name
replacements.append(("${Name}", project_name))
replacements.append(("${NameUpper}", project_name.upper()))
replacements.append(("${NameLower}", project_name.lower()))
replacements.append(("${SanitizedCppName}", sanitized_cpp_name))
# module id is a uuid with { and -
if module_id:
@@ -1890,6 +1894,10 @@ def create_gem(gem_path: str,
# gem name is now the last component of the gem_path
gem_name = os.path.basename(gem_path)
if not utils.validate_identifier(gem_name):
logger.error(f'Gem name must be fewer than 64 characters, contain only alphanumeric, "_" or "-" characters, and start with a letter. {gem_name}')
return 1
# gem name cannot be the same as a restricted platform name
if gem_name in restricted_platforms:
logger.error(f'Gem path cannot be a restricted name. {gem_name}')
@@ -1927,10 +1935,13 @@ def create_gem(gem_path: str,
with_this = replace.pop(0)
replacements.append((replace_this, with_this))
sanitized_cpp_name = utils.sanitize_identifier_for_cpp(gem_name)
# gem name
replacements.append(("${Name}", gem_name))
replacements.append(("${NameUpper}", gem_name.upper()))
replacements.append(("${NameLower}", gem_name.lower()))
replacements.append(("${SanitizedCppName}", sanitized_cpp_name))
# module id is a uuid with { and -
if module_id:
+39 -21
View File
@@ -354,7 +354,7 @@ def get_all_templates(project_path: pathlib.Path = None) -> list:
return list(dict.fromkeys(templates_data))
def get_all_restricted() -> list:
def get_all_restricted(project_path: pathlib.Path = None) -> list:
restricted_data = get_restricted()
restricted_data.extend(get_engine_restricted())
if project_path:
@@ -401,13 +401,6 @@ def get_templates_for_generic_creation(): # temporary until we have a better wa
return list(filter(filter_project_and_gem_templates_out, get_all_templates()))
def get_all_restricted() -> list:
engine_restricted = get_engine_restricted()
restricted_data = get_restricted()
restricted_data.extend(engine_restricted)
return restricted_data
def find_engine_data(json_data: dict,
engine_path: str or pathlib.Path = None) -> dict or None:
if not engine_path:
@@ -488,14 +481,14 @@ def get_project_json_data(project_name: str = None,
return None
def get_gem_json_data(gem_name: str = None,
gem_path: str or pathlib.Path = None) -> dict or None:
def get_gem_json_data(gem_name: str = None, gem_path: str or pathlib.Path = None,
project_path: pathlib.Path = None) -> dict or None:
if not gem_name and not gem_path:
logger.error('Must specify either a Gem name or Gem Path.')
return None
if gem_name and not gem_path:
gem_path = get_registered(gem_name=gem_name)
gem_path = get_registered(gem_name=gem_name, project_path=project_path)
if not gem_path:
logger.error(f'Gem Path {gem_path} has not been registered.')
@@ -521,14 +514,14 @@ def get_gem_json_data(gem_name: str = None,
return None
def get_template_json_data(template_name: str = None,
template_path: str or pathlib.Path = None) -> dict or None:
def get_template_json_data(template_name: str = None, template_path: str or pathlib.Path = None,
project_path: pathlib.Path = None) -> dict or None:
if not template_name and not template_path:
logger.error('Must specify either a Template name or Template Path.')
return None
if template_name and not template_path:
template_path = get_registered(template_name=template_name)
template_path = get_registered(template_name=template_name, project_path=project_path)
if not template_path:
logger.error(f'Template Path {template_path} has not been registered.')
@@ -554,14 +547,14 @@ def get_template_json_data(template_name: str = None,
return None
def get_restricted_json_data(restricted_name: str = None,
restricted_path: str or pathlib.Path = None) -> dict or None:
def get_restricted_json_data(restricted_name: str = None, restricted_path: str or pathlib.Path = None,
project_path: pathlib.Path = None) -> dict or None:
if not restricted_name and not restricted_path:
logger.error('Must specify either a Restricted name or Restricted Path.')
return None
if restricted_name and not restricted_path:
restricted_path = get_registered(restricted_name=restricted_name)
restricted_path = get_registered(restricted_name=restricted_name, project_path=project_path)
if not restricted_path:
logger.error(f'Restricted Path {restricted_path} has not been registered.')
@@ -593,7 +586,32 @@ def get_registered(engine_name: str = None,
template_name: str = None,
default_folder: str = None,
repo_name: str = None,
restricted_name: str = None) -> pathlib.Path or None:
restricted_name: str = None,
project_path: pathlib.Path = None) -> pathlib.Path or None:
"""
Looks up a registered entry in either the ~/.o3de/o3de_manifest.json, <this-engine-root>/engine.json
or the <project-path>/project.json (if the project_path parameter is supplied)
:param engine_name: Name of a registered engine to lookup in the ~/.o3de/o3de_manifest.json file
:param project_name: Name of a project to lookup in either the ~/.o3de/o3de_manifest.json or
<this-engine-root>/engine.json file
:param gem_name: Name of a gem to lookup in either the ~/.o3de/o3de_manifest.json, <this-engine-root>/engine.json
or <project-path>/project.json. NOTE: The project_path parameter must be supplied to lookup the registration
with the project.json
:param template_name: Name of a template to lookup in either the ~/.o3de/o3de_manifest.json, <this-engine-root>/engine.json
or <project-path>/project.json. NOTE: The project_path parameter must be supplied to lookup the registration
with the project.json
:param repo_name: Name of a repo to lookup in the ~/.o3de/o3de_manifest.json
:param default_folder: Type of "default" folder to lookup in the ~/.o3de/o3de_manifest.json
Valid values are "engines", "projects", "gems", "templates,", "restricted"
:param restricted_name: Name of a restricted directory object to lookup in either the ~/.o3de/o3de_manifest.json,
<this-engine-root>/engine.json or <project-path>/project.json.
NOTE: The project_path parameter must be supplied to lookup the registration with the project.json
:param project_path: Path to project root, which is used to examined the project.json file in order to
query either gems, templates or restricted directories registered with the project
:return path value associated with the registered object name if found. Otherwise None is returned
"""
json_data = load_o3de_manifest()
# check global first then this engine
@@ -627,7 +645,7 @@ def get_registered(engine_name: str = None,
return project_path
elif isinstance(gem_name, str):
gems = get_all_gems()
gems = get_all_gems(project_path)
for gem_path in gems:
gem_path = pathlib.Path(gem_path).resolve()
gem_json = gem_path / 'gem.json'
@@ -642,7 +660,7 @@ def get_registered(engine_name: str = None,
return gem_path
elif isinstance(template_name, str):
templates = get_all_templates()
templates = get_all_templates(project_path)
for template_path in templates:
template_path = pathlib.Path(template_path).resolve()
template_json = template_path / 'template.json'
@@ -657,7 +675,7 @@ def get_registered(engine_name: str = None,
return template_path
elif isinstance(restricted_name, str):
restricted = get_all_restricted()
restricted = get_all_restricted(project_path)
for restricted_path in restricted:
restricted_path = pathlib.Path(restricted_path).resolve()
restricted_json = restricted_path / 'restricted.json'
+5 -5
View File
@@ -45,20 +45,20 @@ def edit_project_props(proj_path, proj_name, new_origin, new_display,
if new_icon:
proj_json['icon_path'] = new_icon
if new_tags:
tag_list = [new_tags] if isinstance(new_tags, str) else new_tags
tag_list = new_tags.split() if isinstance(new_tags, str) else new_tags
proj_json.setdefault('user_tags', []).extend(tag_list)
if delete_tags:
removal_list = [delete_tags] if isinstance(delete_tags, str) else delete_tags
removal_list = delete_tags.split() if isinstance(delete_tags, str) else delete_tags
if 'user_tags' in proj_json:
for tag in removal_list:
if tag in proj_json['user_tags']:
proj_json['user_tags'].remove(tag)
else:
logger.warn(f'{tag} not found in user_tags for removal.')
logger.warning(f'{tag} not found in user_tags for removal.')
else:
logger.warn(f'user_tags property not found for removal of {remove_tags}.')
logger.warning(f'user_tags property not found for removal of {delete_tags}.')
if replace_tags:
tag_list = [replace_tags] if isinstance(replace_tags, str) else replace_tags
tag_list = replace_tags.split() if isinstance(replace_tags, str) else replace_tags
proj_json['user_tags'] = tag_list
manifest.save_o3de_manifest(proj_json, pathlib.Path(proj_path) / 'project.json')
+17
View File
@@ -36,6 +36,23 @@ def validate_identifier(identifier: str) -> bool:
return True
def sanitize_identifier_for_cpp(identifier: str) -> str:
"""
Convert the provided identifier to a valid C++ identifier
:param identifier: the name which needs to to sanitized
:return: str: sanitized identifier
"""
if not identifier:
return ''
sanitized_identifier = list(identifier)
for index, character in enumerate(sanitized_identifier):
if not (character.isalnum() or character == '_'):
sanitized_identifier[index] = '_'
return "".join(sanitized_identifier)
def validate_uuid4(uuid_string: str) -> bool:
"""
Determine if the uuid supplied is valid.
+7
View File
@@ -41,3 +41,10 @@ ly_add_pytest(
TEST_SUITE smoke
EXCLUDE_TEST_RUN_TARGET_FROM_IDE
)
ly_add_pytest(
NAME o3de_project_properties
PATH ${CMAKE_CURRENT_LIST_DIR}/unit_test_project_properties.py
TEST_SUITE smoke
EXCLUDE_TEST_RUN_TARGET_FROM_IDE
)
+163
View File
@@ -12,6 +12,8 @@
import io
import json
import logging
import unittest.mock
import pytest
import pathlib
from unittest.mock import patch
@@ -67,3 +69,164 @@ class TestGetEnabledGems:
enabled_gems_set = cmake.get_enabled_gems(pathlib.Path('enabled_gems.cmake'))
assert enabled_gems_set == expected_set
class TestAddGemDependency:
@pytest.mark.parametrize(
"enable_gems_cmake_data, expected_set, expected_return", [
pytest.param("""
# Comment
set(ENABLED_GEMS foo bar baz)
""", set(['foo', 'bar', 'baz', 'TestGem']), 0),
pytest.param("""
# Comment
set(ENABLED_GEMS
foo
bar
baz
)
""", set(['foo', 'bar', 'baz', 'TestGem']), 0),
pytest.param("""
# Comment
set(ENABLED_GEMS
foo
bar
baz)
""", set(['foo', 'bar', 'baz', 'TestGem']), 0),
pytest.param("""
# Comment
set(ENABLED_GEMS
foo bar
baz)
""", set(['foo', 'bar', 'baz', 'TestGem']), 0),
pytest.param("""
""", set(['TestGem']), 0),
pytest.param("""
# Comment
set(RANDOM_VARIABLE TestGame, TestProject Test Engine)
set(ENABLED_GEMS HelloWorld IceCream
foo
baz bar
baz baz baz baz baz morebaz lessbaz
)
Random Text
""", set(['HelloWorld', 'IceCream', 'foo', 'bar', 'baz', 'morebaz', 'lessbaz', 'TestGem']),
0),
pytest.param("""
set(ENABLED_GEMS foo bar baz
""", set(['foo', 'bar', 'baz']), 1),
]
)
def test_add_gem_dependency(self, enable_gems_cmake_data, expected_set, expected_return):
enabled_gems_set = set()
add_gem_return = None
class StringBufferIOWrapper(io.StringIO):
def __init__(self):
nonlocal enable_gems_cmake_data
super().__init__(enable_gems_cmake_data)
def __enter__(self):
return super().__enter__()
def __exit__(self, exc_type, exc_val, exc_tb):
nonlocal enable_gems_cmake_data
enable_gems_cmake_data = super().getvalue()
super().__exit__(exc_tb, exc_val, exc_tb)
with patch('pathlib.Path.resolve', return_value=pathlib.Path('enabled_gems.cmake')) as pathlib_is_resolve_mock,\
patch('pathlib.Path.is_file', return_value=True) as pathlib_is_file_mock,\
patch('pathlib.Path.open', side_effect=lambda mode: StringBufferIOWrapper()) as pathlib_open_mock:
add_gem_return = cmake.add_gem_dependency(pathlib.Path('enabled_gems.cmake'), 'TestGem')
enabled_gems_set = cmake.get_enabled_gems(pathlib.Path('enabled_gems.cmake'))
assert add_gem_return == expected_return
assert enabled_gems_set == expected_set
class TestRemoveGemDependency:
@pytest.mark.parametrize(
"enable_gems_cmake_data, expected_set, expected_return", [
pytest.param("""
# Comment
set(ENABLED_GEMS foo bar baz TestGem)
""", set(['foo', 'bar', 'baz']), 0),
pytest.param("""
# Comment
set(ENABLED_GEMS
foo
bar
baz
TestGem
)
""", set(['foo', 'bar', 'baz']), 0),
pytest.param("""
# Comment
set(ENABLED_GEMS
foo
bar
baz
TestGem)
""", set(['foo', 'bar', 'baz']), 0),
pytest.param("""
# Comment
set(ENABLED_GEMS
foo bar
baz TestGem)
""", set(['foo', 'bar', 'baz']), 0),
pytest.param("""
# Comment
set(ENABLED_GEMS
foo
TestGem
bar
TestGem
baz
)
Random Text
""", set(['foo', 'bar', 'baz']),
0),
pytest.param("""
set(ENABLED_GEMS
foo
bar
baz
"TestGem"
)
""", set(['foo', 'bar', 'baz']), 0),
pytest.param("""
""", set(), 1),
pytest.param("""
set(ENABLED_GEMS
foo
bar
baz
)
""", set(['foo', 'bar', 'baz']), 1),
]
)
def test_remove_gem_dependency(self, enable_gems_cmake_data, expected_set, expected_return):
enabled_gems_set = set()
add_gem_return = None
class StringBufferIOWrapper(io.StringIO):
def __init__(self):
nonlocal enable_gems_cmake_data
super().__init__(enable_gems_cmake_data)
def __enter__(self):
return super().__enter__()
def __exit__(self, exc_type, exc_val, exc_tb):
nonlocal enable_gems_cmake_data
enable_gems_cmake_data = super().getvalue()
super().__exit__(exc_tb, exc_val, exc_tb)
with patch('pathlib.Path.resolve', return_value=pathlib.Path('enabled_gems.cmake')) as pathlib_is_resolve_mock,\
patch('pathlib.Path.is_file', return_value=True) as pathlib_is_file_mock,\
patch('pathlib.Path.open', side_effect=lambda mode: StringBufferIOWrapper()) as pathlib_open_mock:
add_gem_return = cmake.remove_gem_dependency(pathlib.Path('enabled_gems.cmake'), 'TestGem')
enabled_gems_set = cmake.get_enabled_gems(pathlib.Path('enabled_gems.cmake'))
assert add_gem_return == expected_return
assert enabled_gems_set == expected_set
@@ -0,0 +1,83 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import pytest
import pathlib
from unittest.mock import patch
from o3de import project_properties
TEST_DEFAULT_PROJECT_DATA = {
"template_name": "DefaultProject",
"restricted_name": "o3de",
"restricted_platform_relative_path": "Templates",
"origin": "The primary repo for DefaultProject goes here: i.e. http://www.mydomain.com",
"license": "What license DefaultProject uses goes here: i.e. https://opensource.org/licenses/MIT",
"display_name": "Default",
"summary": "A short description of DefaultProject.",
"included_gems": ["Atom","Camera","EMotionFX","UI","Maestro","Input","ImGui"],
"canonical_tags": [],
"user_tags": [
"DefaultProject"
],
"icon_path": "preview.png"
}
@pytest.fixture(scope='class')
def init_project_json_data(request):
class ProjectJsonData:
def __init__(self):
self.data = TEST_DEFAULT_PROJECT_DATA
request.cls.project_json = ProjectJsonData()
@pytest.mark.usefixtures('init_project_json_data')
class TestEditProjectProperties:
@pytest.mark.parametrize("project_path, project_name, project_origin, project_display,\
project_summary, project_icon, add_tags, delete_tags,\
replace_tags, expected_result", [
pytest.param(pathlib.PurePath('E:/TestProject'),
'test', 'editing by pytest', 'Unit Test', 'pyTest project', 'pytest.bmp', 'A B C',
'B', 'D E F', 0),
pytest.param('',
'test', 'editing by pytest', 'Unit Test', 'pyTest project', 'pytest.bmp', 'A B C',
'B', 'D E F', 1)
]
)
def test_edit_project_properties(self, project_path, project_name, project_origin, project_display,
project_summary, project_icon, add_tags, delete_tags,
replace_tags, expected_result):
def get_project_json_data(project_name: str, project_path) -> dict:
if not project_path:
self.project_json.data = None
return None
return self.project_json.data
def save_o3de_manifest(new_proj_data: dict, project_path) -> None:
self.project_json.data = new_proj_data
with patch('o3de.manifest.get_project_json_data', side_effect=get_project_json_data) as get_project_json_data_patch, \
patch('o3de.manifest.save_o3de_manifest', side_effect=save_o3de_manifest) as save_o3de_manifest_patch:
result = project_properties.edit_project_props(project_path, project_name, project_origin,
project_display, project_summary, project_icon,
add_tags, delete_tags, replace_tags)
assert result == expected_result
if project_path:
assert self.project_json.data
assert self.project_json.data.get('origin', '') == project_origin
assert self.project_json.data.get('display_name', '') == project_display
assert self.project_json.data.get('summary', '') == project_summary
assert self.project_json.data.get('icon_path', '') == project_icon
expected_tag_set = set(replace_tags.split())
project_json_tag_set = set(self.project_json.data.get('user_tags', []))
assert project_json_tag_set == expected_tag_set
else:
assert not self.project_json.data