From 62775add6d2a3a225b8d7dc975095e48990c929d Mon Sep 17 00:00:00 2001 From: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com> Date: Wed, 26 Jan 2022 10:44:35 -0600 Subject: [PATCH] Implemented Support to allow project's to reference gems via the gem name (#7109) * Implemented Support to allow project's to reference gems via the gem name Updated the enable-gem command to add the name of the enabled gem to the "gem_names" array in the project.json Updated the enable-gem test to validate this functionality Centralized the CMake logic for locating external subdirectories to the Subdirectories.cmake script Added an option to the edit-project-properties and edit-engine-properties o3de.py commands to add/remove/replace the "gem_names" field in the project.json and engine.json respectively Added a CMake function to determine the root CMake "subdirectory" of any input path which is a parent of it. This logic has been used to improve the installation of external gems to the /External directory. Tested out the install layout before submitting PR fixes #7108 Signed-off-by: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com> * Fixed the enable-gem test on Linux to resolve the mock path. Renamed all of the o3de python test from "unit_test*.py" to "test*.py" to faciliate the python unittest module picking up the test automatically. Signed-off-by: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com> * Adding test for the disable_gem command. Fixed some typos in engine_properties.py scrip. Signed-off-by: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com> --- CMakeLists.txt | 56 +---- Gems/PhysXDebug/Code/CMakeLists.txt | 2 +- cmake/FileUtil.cmake | 44 +++- cmake/PAL.cmake | 46 +--- cmake/Platform/Common/Install_common.cmake | 55 +++-- cmake/Projects.cmake | 30 +-- cmake/Subdirectories.cmake | 197 +++++++++++++++++ cmake/cmake_files.cmake | 1 + engine.json | 2 +- scripts/o3de/o3de/cmake.py | 19 +- scripts/o3de/o3de/disable_gem.py | 17 +- scripts/o3de/o3de/enable_gem.py | 17 +- scripts/o3de/o3de/engine_properties.py | 46 +++- scripts/o3de/o3de/project_properties.py | 41 +++- scripts/o3de/tests/CMakeLists.txt | 27 ++- .../{unit_test_cmake.py => test_cmake.py} | 0 scripts/o3de/tests/test_disable_gem.py | 200 ++++++++++++++++++ ..._test_enable_gem.py => test_enable_gem.py} | 31 +-- ...roperties.py => test_engine_properties.py} | 0 ...ne_template.py => test_engine_template.py} | 0 ...m_properties.py => test_gem_properties.py} | 0 ...obal_project.py => test_global_project.py} | 0 ...unit_test_manifest.py => test_manifest.py} | 0 ...stration.py => test_print_registration.py} | 0 ...operties.py => test_project_properties.py} | 0 ...unit_test_register.py => test_register.py} | 0 .../{unit_test_utils.py => test_utils.py} | 0 27 files changed, 635 insertions(+), 196 deletions(-) create mode 100644 cmake/Subdirectories.cmake rename scripts/o3de/tests/{unit_test_cmake.py => test_cmake.py} (100%) create mode 100644 scripts/o3de/tests/test_disable_gem.py rename scripts/o3de/tests/{unit_test_enable_gem.py => test_enable_gem.py} (82%) rename scripts/o3de/tests/{unit_test_engine_properties.py => test_engine_properties.py} (100%) rename scripts/o3de/tests/{unit_test_engine_template.py => test_engine_template.py} (100%) rename scripts/o3de/tests/{unit_test_gem_properties.py => test_gem_properties.py} (100%) rename scripts/o3de/tests/{unit_test_global_project.py => test_global_project.py} (100%) rename scripts/o3de/tests/{unit_test_manifest.py => test_manifest.py} (100%) rename scripts/o3de/tests/{unit_test_print_registration.py => test_print_registration.py} (100%) rename scripts/o3de/tests/{unit_test_project_properties.py => test_project_properties.py} (100%) rename scripts/o3de/tests/{unit_test_register.py => test_register.py} (100%) rename scripts/o3de/tests/{unit_test_utils.py => test_utils.py} (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index efcc866195..3f9f56a606 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -44,51 +44,11 @@ include(cmake/SettingsRegistry.cmake) include(cmake/TestImpactFramework/LYTestImpactFramework.cmake) include(cmake/CMakeFiles.cmake) include(cmake/O3DEJson.cmake) +include(cmake/Subdirectories.cmake) -################################################################################ -# Subdirectory processing -################################################################################ - -# this function is building up the LY_EXTERNAL_SUBDIRS global property -function(add_engine_gem_json_external_subdirectories gem_path) - set(gem_json_path ${gem_path}/gem.json) - if(EXISTS ${gem_json_path}) - read_json_external_subdirs(gem_external_subdirs ${gem_path}/gem.json) - foreach(gem_external_subdir ${gem_external_subdirs}) - file(REAL_PATH ${gem_external_subdir} real_external_subdir BASE_DIRECTORY ${gem_path}) - set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS ${real_external_subdir}) - add_engine_gem_json_external_subdirectories(${real_external_subdir}) - endforeach() - endif() -endfunction() - -function(add_engine_json_external_subdirectories) - read_json_external_subdirs(engine_external_subdirs ${LY_ROOT_FOLDER}/engine.json) - foreach(engine_external_subdir ${engine_external_subdirs}) - file(REAL_PATH ${engine_external_subdir} real_external_subdir BASE_DIRECTORY ${LY_ROOT_FOLDER}) - set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS ${real_external_subdir}) - add_engine_gem_json_external_subdirectories(${real_external_subdir}) - endforeach() -endfunction() - -function(add_subdirectory_on_externalsubdirs) - get_property(external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS) - list(APPEND LY_EXTERNAL_SUBDIRS ${external_subdirs}) - # Loop over the additional external subdirectories and invoke add_subdirectory on them - foreach(external_directory ${LY_EXTERNAL_SUBDIRS}) - # Hash the external_directory name and append it to the Binary Directory section of add_subdirectory - # This is to deal with potential situations where multiple external directories has the same last directory name - # For example if D:/Company1/RayTracingGem and F:/Company2/Path/RayTracingGem were both added as a subdirectory - file(REAL_PATH ${external_directory} full_directory_path) - string(SHA256 full_directory_hash ${full_directory_path}) - # Truncate the full_directory_hash down to 8 characters to avoid hitting the Windows 260 character path limit - # when the external subdirectory contains relative paths of significant length - string(SUBSTRING ${full_directory_hash} 0 8 full_directory_hash) - # Use the last directory as the suffix path to use for the Binary Directory - get_filename_component(directory_name ${external_directory} NAME) - add_subdirectory(${external_directory} ${CMAKE_BINARY_DIR}/External/${directory_name}-${full_directory_hash}) - endforeach() -endfunction() +# Gather the list of o3de_manifest external Subdirectories +# into the LY_EXTERNAL_SUBDIRS_O3DE_MANIFEST_PROPERTY +add_o3de_manifest_json_external_subdirectories() # Add the projects first so the Launcher can find them include(cmake/Projects.cmake) @@ -99,9 +59,11 @@ endif() if(NOT INSTALLED_ENGINE) # Add external subdirectories listed in the engine.json. LY_EXTERNAL_SUBDIRS is a cache variable so the user can add extra - # external subdirectories. This should go before adding the rest of the targets so the targets are availbe to the launcher. + # external subdirectories. This should go before adding the rest of the targets so the targets are available to the launcher. add_engine_json_external_subdirectories() - add_subdirectory_on_externalsubdirs() + + # Invoke add_subdirectory on external subdirectories that should be used a this point + add_subdirectory_on_external_subdirs() # Add the rest of the targets add_subdirectory(Assets) @@ -114,7 +76,7 @@ if(NOT INSTALLED_ENGINE) else() ly_find_o3de_packages() - add_subdirectory_on_externalsubdirs() + add_subdirectory_on_external_subdirs() endif() ################################################################################ diff --git a/Gems/PhysXDebug/Code/CMakeLists.txt b/Gems/PhysXDebug/Code/CMakeLists.txt index 0886f2bf34..1f5d266fbf 100644 --- a/Gems/PhysXDebug/Code/CMakeLists.txt +++ b/Gems/PhysXDebug/Code/CMakeLists.txt @@ -10,7 +10,7 @@ o3de_find_gem("PhysX" physx_gem_path) set(physx_gem_json ${physx_gem_path}/gem.json) o3de_restricted_path(${physx_gem_json} physx_gem_restricted_path physx_gem_parent_relative_path) -o3de_pal_dir(physx_pal_source_dir ${physx_gem_path}/Code/Source/Platform/${PAL_PLATFORM_NAME} ${physx_gem_restricted_path} ${physx_gem_path} ${physx_gem_parent_relative_path}) +o3de_pal_dir(physx_pal_source_dir ${physx_gem_path}/Code/Source/Platform/${PAL_PLATFORM_NAME} "${physx_gem_restricted_path}" "${physx_gem_path}" "${physx_gem_parent_relative_path}") include(${physx_pal_source_dir}/PAL_${PAL_PLATFORM_NAME_LOWERCASE}.cmake) # for PAL_TRAIT_PHYSX_SUPPORTED diff --git a/cmake/FileUtil.cmake b/cmake/FileUtil.cmake index 5721cf452c..aa63b97c9d 100644 --- a/cmake/FileUtil.cmake +++ b/cmake/FileUtil.cmake @@ -153,6 +153,36 @@ function(ly_get_last_path_segment_concat_sha256 absolute_path output_path) set(${output_path} ${last_path_segment_sha256_path} PARENT_SCOPE) endfunction() +#! ly_get_root_subdirectory_which_is_parent: Locates the root source directory added the input directory +# as a subdirectory of the build, which an actual prefix of the input directory +# This is done by recursing through the PARENT_DIRECTORY "DIRECTORY" property +# The use for this is to locate the top most directory which called add_subdirectory from any input path +# i.e Given an +# LY_ROOT_FOLDER = D:\o3de +# EXTERNAL_SUBDIRS = [D:\TestGem, D:\o3de\Gems\MyGem] +# The LY_ROOT_FOLDER is responsible for invoking add_subdirectory on the external subdirectories +# so it in the PARENT_DIRECTORY property, of the subdirectory, though it might not be an actual "parent" +# If the input path to this function is D:\TestGem\Code, then the return value is D:\TestGem +# If the input path to this function is D:\o3de\Gems\MyGem, then the return value is D:\o3de + +# \arg:absolute_path - directory to locate top most parent "subdirectory", which is an "parent" of the input +# \return:output_path- top most parent subdirectory, which is actual parent(i.e a prefix) +function(ly_get_root_subdirectory_which_is_parent absolute_path output_path) + # Walk up the parent add_subdirectory calls until a parent directory which is not a prefix of the target directory + # is found + cmake_path(SET candidate_path ${absolute_path}) + get_property(parent_subdir DIRECTORY ${candidate_path} PROPERTY PARENT_DIRECTORY) + cmake_path(IS_PREFIX parent_subdir ${candidate_path} is_parent_subdir) + while(parent_subdir AND is_parent_subdir) + cmake_path(SET candidate_path "${parent_subdir}") + get_property(parent_subdir DIRECTORY ${candidate_path} PROPERTY PARENT_DIRECTORY) + cmake_path(IS_PREFIX parent_subdir ${candidate_path} is_parent_subdir) + endwhile() + + message(DEBUG "Root subdirectory of path \"${absolute_path}\" is \"${candidate_path}\"") + set(${output_path} ${candidate_path} PARENT_SCOPE) +endfunction() + #! ly_get_engine_relative_source_dir: Attempts to form a path relative to the BASE_DIRECTORY. # If that fails the last path segment of the absolute_target_source_dir concatenated with a SHA256 hash to form a target directory # \arg:BASE_DIRECTORY - Directory to base relative path against. Defaults to LY_ROOT_FOLDER @@ -167,14 +197,16 @@ function(ly_get_engine_relative_source_dir absolute_target_source_dir output_sou endif() # Get a relative target source directory to the LY root folder if possible - # Otherwise use the final component name + # Otherwise use the top most source directory which led to calling add_subdirectory on the input directory + ly_get_root_subdirectory_which_is_parent(${absolute_target_source_dir} root_subdir_of_target) + cmake_path(RELATIVE_PATH absolute_target_source_dir BASE_DIRECTORY ${root_subdir_of_target} OUTPUT_VARIABLE relative_target_source_dir) + cmake_path(IS_PREFIX LY_ROOT_FOLDER ${absolute_target_source_dir} is_target_source_dir_subdirectory_of_engine) - if(is_target_source_dir_subdirectory_of_engine) - cmake_path(RELATIVE_PATH absolute_target_source_dir BASE_DIRECTORY ${LY_ROOT_FOLDER} OUTPUT_VARIABLE relative_target_source_dir) - else() - ly_get_last_path_segment_concat_sha256(${absolute_target_source_dir} target_source_dir_last_path_segment) + if(NOT is_target_source_dir_subdirectory_of_engine) + cmake_path(GET root_subdir_of_target FILENAME root_subdir_dirname) + set(relative_subdir ${relative_target_source_dir}) unset(relative_target_source_dir) - cmake_path(APPEND relative_target_source_dir "External" ${target_source_dir_last_path_segment}) + cmake_path(APPEND relative_target_source_dir "External" ${root_subdir_dirname} ${relative_subdir}) endif() set(${output_source_dir} ${relative_target_source_dir} PARENT_SCOPE) diff --git a/cmake/PAL.cmake b/cmake/PAL.cmake index a43e601c74..b76fb8ac81 100644 --- a/cmake/PAL.cmake +++ b/cmake/PAL.cmake @@ -51,49 +51,21 @@ function(o3de_read_manifest o3de_manifest_json_data) endif() endfunction() -#! o3de_recurse_gems: returns the gem paths -# -# \arg:object json path -# \arg:gems returns the gems from the external subdirectory elements from the manifest -function(o3de_recurse_gems object_json_path gems) - get_filename_component(object_json_parent_path ${object_json_path} DIRECTORY) - ly_file_read(${object_json_path} json_data) - string(JSON external_subdirectories_count ERROR_VARIABLE json_error LENGTH ${json_data} "external_subdirectories") - if(NOT json_error) - if(external_subdirectories_count GREATER 0) - math(EXPR external_subdirectories_range "${external_subdirectories_count}-1") - foreach(external_subdirectories_index RANGE ${external_subdirectories_range}) - string(JSON external_subdirectories_entry ERROR_VARIABLE json_error GET ${json_data} "external_subdirectories" "${external_subdirectories_index}") - cmake_path(IS_RELATIVE external_subdirectories_entry is_relative) - if(${is_relative}) - cmake_path(ABSOLUTE_PATH external_subdirectories_entry BASE_DIRECTORY ${object_json_parent_path} NORMALIZE OUTPUT_VARIABLE external_subdirectories_entry) - endif() - if(EXISTS ${external_subdirectories_entry}/gem.json) - list(APPEND gem_entries ${external_subdirectories_entry}) - o3de_recurse_gems(${external_subdirectories_entry}/gem.json gem_entries) - endif() - endforeach() - endif() - endif() - set(${gems} ${gem_entries} PARENT_SCOPE) -endfunction() #! o3de_find_gem: returns the gem path # # \arg:gem_name the gem name to find # \arg:the path of the gem function(o3de_find_gem gem_name gem_path) - o3de_get_manifest_path(manifest_path) - if(EXISTS ${manifest_path}) - o3de_recurse_gems(${manifest_path} gems) - endif() - o3de_recurse_gems(${LY_ROOT_FOLDER}/engine.json gems) - foreach(gem ${gems}) - ly_file_read(${gem}/gem.json json_data) - string(JSON gem_json_name ERROR_VARIABLE json_error GET ${json_data} "gem_name") - if(gem_json_name STREQUAL gem_name) - set(${gem_path} ${gem} PARENT_SCOPE) - return() + get_all_external_subdirectories(all_external_subdirs) + foreach(external_subdir IN LISTS all_external_subdirs) + set(candidate_gem_path ${external_subdir}/gem.json) + if(EXISTS ${candidate_gem_path}) + o3de_read_json_key(gem_json_name ${candidate_gem_path} "gem_name") + if(gem_json_name STREQUAL gem_name) + set(${gem_path} ${external_subdir} PARENT_SCOPE) + return() + endif() endif() endforeach() endfunction() diff --git a/cmake/Platform/Common/Install_common.cmake b/cmake/Platform/Common/Install_common.cmake index e42664c107..5d92b9944e 100644 --- a/cmake/Platform/Common/Install_common.cmake +++ b/cmake/Platform/Common/Install_common.cmake @@ -448,9 +448,27 @@ function(ly_setup_cmake_install) # Transform the LY_EXTERNAL_SUBDIRS global property list into a json array set(indent " ") get_property(external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS) + list(REMOVE_DUPLICATES external_subdirs) foreach(external_subdir ${external_subdirs}) - cmake_path(RELATIVE_PATH external_subdir BASE_DIRECTORY ${LY_ROOT_FOLDER} OUTPUT_VARIABLE engine_rel_external_subdir) - list(APPEND relative_external_subdirs "\"${engine_rel_external_subdir}\"") + # If an external subdirectory is not a subdirectory of the engine root, then + # prepend "External" to its subdirectory root + ly_get_root_subdirectory_which_is_parent(${external_subdir} root_subdir_of_external_subdir) + cmake_path(RELATIVE_PATH external_subdir BASE_DIRECTORY ${root_subdir_of_external_subdir} OUTPUT_VARIABLE engine_rel_external_subdir) + + cmake_path(IS_PREFIX LY_ROOT_FOLDER ${external_subdir} is_subdirectory_of_engine) + if(NOT is_subdirectory_of_engine) + cmake_path(GET root_subdir_of_external_subdir FILENAME root_subdir_dirname) + set(relative_subdir ${engine_rel_external_subdir}) + unset(engine_rel_external_subdir) + cmake_path(APPEND engine_rel_external_subdir "External" ${root_subdir_dirname} ${relative_subdir}) + endif() + + set(quoted_engine_rel_external_subdir "\"${engine_rel_external_subdir}\"") + if (quoted_engine_rel_external_subdir IN_LIST relative_external_subdirs) + message(WARNING "An external subdirectory \"${external_subdir}\" has been found twice when generating the engine.json for the install layout") + else() + list(APPEND relative_external_subdirs "\"${engine_rel_external_subdir}\"") + endif() endforeach() list(JOIN relative_external_subdirs ",\n${indent}" LY_INSTALL_EXTERNAL_SUBDIRS) @@ -507,7 +525,17 @@ function(ly_setup_cmake_install) # Add to find_subdirectories all directories in which ly_add_target were called in get_property(all_subdirectories GLOBAL PROPERTY LY_ALL_TARGET_DIRECTORIES) foreach(target_subdirectory IN LISTS all_subdirectories) - cmake_path(RELATIVE_PATH target_subdirectory BASE_DIRECTORY ${LY_ROOT_FOLDER} OUTPUT_VARIABLE relative_target_subdirectory) + ly_get_root_subdirectory_which_is_parent(${target_subdirectory} root_subdir_of_target) + cmake_path(RELATIVE_PATH target_subdirectory BASE_DIRECTORY ${root_subdir_of_target} OUTPUT_VARIABLE relative_target_subdirectory) + + cmake_path(IS_PREFIX LY_ROOT_FOLDER ${target_subdirectory} is_subdirectory_of_engine) + if(NOT is_subdirectory_of_engine) + cmake_path(GET root_subdir_of_target FILENAME root_subdir_dirname) + set(relative_subdir ${relative_target_subdirectory}) + unset(relative_target_subdirectory) + cmake_path(APPEND relative_target_subdirectory "External" ${root_subdir_dirname} ${relative_subdir}) + endif() + string(APPEND find_subdirectories "add_subdirectory(${relative_target_subdirectory})\n") endforeach() set(permutation_find_subdirectories ${CMAKE_CURRENT_BINARY_DIR}/cmake/Platform/${PAL_PLATFORM_NAME}/${LY_BUILD_PERMUTATION}/o3de_subdirectories_${PAL_PLATFORM_NAME_LOWERCASE}.cmake) @@ -657,12 +685,12 @@ function(ly_setup_assets) set_property(GLOBAL APPEND PROPERTY global_gem_candidate_dirs_prop ${gem_candidate_dir}) endforeach() - # Iterate over each gem candidate directories and read populate a directory property + # Iterate over each gem candidate directories and populate a directory property # containing the files to copy over get_property(gem_candidate_dirs GLOBAL PROPERTY global_gem_candidate_dirs_prop) foreach(gem_candidate_dir IN LISTS gem_candidate_dirs) get_property(filtered_asset_paths DIRECTORY ${gem_candidate_dir} PROPERTY directory_filtered_asset_paths) - ly_get_last_path_segment_concat_sha256(${gem_candidate_dir} last_gem_root_path_segment) + # Check if the gem is a subdirectory of the engine cmake_path(IS_PREFIX LY_ROOT_FOLDER ${gem_candidate_dir} is_gem_subdirectory_of_engine) @@ -697,15 +725,16 @@ function(ly_setup_assets) # gem directories and files to install get_property(gems_assets_paths DIRECTORY ${gem_candidate_dir} PROPERTY gems_assets_paths) foreach(gem_absolute_path IN LISTS gems_assets_paths) - if(is_gem_subdirectory_of_engine) - cmake_path(RELATIVE_PATH gem_absolute_path BASE_DIRECTORY ${LY_ROOT_FOLDER} OUTPUT_VARIABLE gem_install_dest_dir) - else() - # The gem resides outside of the LY_ROOT_FOLDER, so the destination is made relative to the - # gem candidate directory and placed under the "External" directory" - # directory - cmake_path(RELATIVE_PATH gem_absolute_path BASE_DIRECTORY ${gem_candidate_dir} OUTPUT_VARIABLE gem_relative_path) + # If an external subdirectory is not a subdirectory of the engine root, then + # prepend "External" to its subdirectory root + ly_get_root_subdirectory_which_is_parent(${gem_candidate_dir} root_subdir_of_gem) + cmake_path(RELATIVE_PATH gem_absolute_path BASE_DIRECTORY ${root_subdir_of_gem} OUTPUT_VARIABLE gem_install_dest_dir) + + if(NOT is_gem_subdirectory_of_engine) + cmake_path(GET root_subdir_of_gem FILENAME root_subdir_dirname) + set(relative_subdir ${gem_install_dest_dir}) unset(gem_install_dest_dir) - cmake_path(APPEND gem_install_dest_dir "External" ${last_gem_root_path_segment} ${gem_relative_path}) + cmake_path(APPEND gem_install_dest_dir "External" ${root_subdir_dirname} ${relative_subdir}) endif() cmake_path(GET gem_install_dest_dir PARENT_PATH gem_install_dest_dir) diff --git a/cmake/Projects.cmake b/cmake/Projects.cmake index c6aa6c382a..acfa921ef1 100644 --- a/cmake/Projects.cmake +++ b/cmake/Projects.cmake @@ -118,30 +118,6 @@ function(ly_generate_project_build_path_setreg project_real_path) file(GENERATE OUTPUT ${project_user_build_path_setreg_file} CONTENT ${project_build_path_setreg_content}) endfunction() -function(add_gem_json_external_subdirectories gem_path) - set(gem_json_path ${gem_path}/gem.json) - if(EXISTS ${gem_json_path}) - read_json_external_subdirs(gem_external_subdirs ${gem_path}/gem.json) - foreach(gem_external_subdir ${gem_external_subdirs}) - file(REAL_PATH ${gem_external_subdir} real_external_subdir BASE_DIRECTORY ${gem_path}) - set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS ${real_external_subdir}) - add_gem_json_external_subdirectories(${real_external_subdir}) - endforeach() - endif() -endfunction() - -function(add_project_json_external_subdirectories project_path) - set(project_json_path ${project_path}/project.json) - if(EXISTS ${project_json_path}) - read_json_external_subdirs(project_external_subdirs ${project_path}/project.json) - foreach(project_external_subdir ${project_external_subdirs}) - file(REAL_PATH ${project_external_subdir} real_external_subdir BASE_DIRECTORY ${project_path}) - set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS ${real_external_subdir}) - add_gem_json_external_subdirectories(${real_external_subdir}) - endforeach() - endif() -endfunction() - function(install_project_asset_artifacts project_real_path) # The cmake tar command has a bit of a flaw # Any paths within the archive files it creates are relative to the current working directory. @@ -212,16 +188,16 @@ foreach(project ${LY_PROJECTS}) # when the external subdirectory contains relative paths of significant length string(SUBSTRING ${full_directory_hash} 0 8 full_directory_hash) - get_filename_component(project_folder_name ${project} NAME) + cmake_path(GET project FILENAME project_folder_name ) list(APPEND LY_PROJECTS_FOLDER_NAME ${project_folder_name}) add_subdirectory(${project} "${project_folder_name}-${full_directory_hash}") ly_generate_project_build_path_setreg(${full_directory_path}) - add_project_json_external_subdirectories(${full_directory_path}) # Get project name o3de_read_json_key(project_name ${full_directory_path}/project.json "project_name") + add_project_json_external_subdirectories(${full_directory_path} "${project_name}") - install_project_asset_artifacts(${full_directory_path}) + install_project_asset_artifacts(${full_directory_path}) endforeach() diff --git a/cmake/Subdirectories.cmake b/cmake/Subdirectories.cmake new file mode 100644 index 0000000000..3580710c82 --- /dev/null +++ b/cmake/Subdirectories.cmake @@ -0,0 +1,197 @@ +# +# Copyright (c) Contributors to the Open 3D Engine Project. +# For complete copyright and license terms please see the LICENSE at the root of this distribution. +# +# SPDX-License-Identifier: Apache-2.0 OR MIT +# +# + +include_guard() + +################################################################################ +# Subdirectory processing +################################################################################ + +# this function is building up the LY_EXTERNAL_SUBDIRS global property +function(add_engine_gem_json_external_subdirectories gem_path) + set(gem_json_path ${gem_path}/gem.json) + if(EXISTS ${gem_json_path}) + read_json_external_subdirs(gem_external_subdirs ${gem_path}/gem.json) + foreach(gem_external_subdir ${gem_external_subdirs}) + file(REAL_PATH ${gem_external_subdir} real_external_subdir BASE_DIRECTORY ${gem_path}) + + # Append external subdirectory if it is not in global property + get_property(current_external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS) + if(NOT real_external_subdir IN_LIST current_external_subdirs) + set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS ${real_external_subdir}) + # Also append the project external subdirectores to the LY_EXTERNAL_SUBDIRS_ENGINE property + set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS_ENGINE ${real_external_subdir}) + add_engine_gem_json_external_subdirectories(${real_external_subdir}) + endif() + endforeach() + endif() +endfunction() + +function(add_engine_json_external_subdirectories) + set(engine_json_path ${LY_ROOT_FOLDER}/engine.json) + if(EXISTS ${engine_json_path}) + read_json_external_subdirs(engine_external_subdirs ${engine_json_path}) + foreach(engine_external_subdir ${engine_external_subdirs}) + file(REAL_PATH ${engine_external_subdir} real_external_subdir BASE_DIRECTORY ${LY_ROOT_FOLDER}) + + # Append external subdirectory if it is not in global property + get_property(current_external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS) + if(NOT real_external_subdir IN_LIST current_external_subdirs) + set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS ${real_external_subdir}) + # Also append the project external subdirectores to the LY_EXTERNAL_SUBDIRS_ENGINE property + set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS_ENGINE ${real_external_subdir}) + add_engine_gem_json_external_subdirectories(${real_external_subdir}) + endif() + endforeach() + endif() +endfunction() + + +function(add_project_gem_json_external_subdirectories gem_path project_name) + set(gem_json_path ${gem_path}/gem.json) + if(EXISTS ${gem_json_path}) + read_json_external_subdirs(gem_external_subdirs ${gem_path}/gem.json) + foreach(gem_external_subdir ${gem_external_subdirs}) + file(REAL_PATH ${gem_external_subdir} real_external_subdir BASE_DIRECTORY ${gem_path}) + + # Append external subdirectory if it is not in global property + get_property(current_external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS) + if(NOT real_external_subdir IN_LIST current_external_subdirs) + set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS ${real_external_subdir}) + # Also append the project external subdirectores to the LY_EXTERNAL_SUBDIRS_${project_name} property + set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS_${project_name} ${real_external_subdir}) + add_project_gem_json_external_subdirectories(${real_external_subdir} "${project_name}") + endif() + endforeach() + endif() +endfunction() + +function(add_project_json_external_subdirectories project_path project_name) + set(project_json_path ${project_path}/project.json) + if(EXISTS ${project_json_path}) + read_json_external_subdirs(project_external_subdirs ${project_path}/project.json) + foreach(project_external_subdir ${project_external_subdirs}) + file(REAL_PATH ${project_external_subdir} real_external_subdir BASE_DIRECTORY ${project_path}) + + # Append external subdirectory if it is not in global property + get_property(current_external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS) + if(NOT real_external_subdir IN_LIST current_external_subdirs) + set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS ${real_external_subdir}) + # Also append the project external subdirectores to the LY_EXTERNAL_SUBDIRS_${project_name} property + set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS_${project_name} ${real_external_subdir}) + add_project_gem_json_external_subdirectories(${real_external_subdir} "${project_name}") + endif() + endforeach() + endif() +endfunction() + + +#! add_o3de_manifest_gem_json_external_subdirectories : Recurses through external subdirectories +#! originally found in the add_o3de_manifest_json_external_subdirectories command +function(add_o3de_manifest_gem_json_external_subdirectories gem_path) + set(gem_json_path ${gem_path}/gem.json) + if(EXISTS ${gem_json_path}) + read_json_external_subdirs(gem_external_subdirs ${gem_path}/gem.json) + foreach(gem_external_subdir ${gem_external_subdirs}) + file(REAL_PATH ${gem_external_subdir} real_external_subdir BASE_DIRECTORY ${gem_path}) + + # Append external subdirectory ONLY to LY_EXTERNAL_SUBDIRS_O3DE_MANIFEST PROPERTY + # It is not appended to LY_EXTERNAL_SUBDIRS unless that gem is used by the project + get_property(current_external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS_O3DE_MANIFEST) + if(NOT real_external_subdir IN_LIST current_external_subdirs) + set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS_O3DE_MANIFEST ${real_external_subdir}) + add_o3de_manifest_gem_json_external_subdirectories(${real_external_subdir}) + endif() + endforeach() + endif() +endfunction() + +#! add_o3de_manifest_json_external_subdirectories : Adds the list of external_subdirectories +#! in the user o3de_manifest.json to the LY_EXTERNAL_SUBDIRS_O3DE_MANIFEST property +function(add_o3de_manifest_json_external_subdirectories) + o3de_get_manifest_path(manifest_path) + if(EXISTS ${manifest_path}) + read_json_external_subdirs(o3de_manifest_external_subdirs ${manifest_path}) + foreach(manifest_external_subdir ${o3de_manifest_external_subdirs}) + file(REAL_PATH ${manifest_external_subdir} real_external_subdir BASE_DIRECTORY ${LY_ROOT_FOLDER}) + + # Append external subdirectory ONLY to LY_EXTERNAL_SUBDIRS_O3DE_MANIFEST PROPERTY + # It is not appended to LY_EXTERNAL_SUBDIRS unless that gem is used by the project + get_property(current_external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS_O3DE_MANIFEST) + if(NOT real_external_subdir IN_LIST current_external_subdirs) + set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS_O3DE_MANIFEST ${real_external_subdir}) + add_o3de_manifest_gem_json_external_subdirectories(${real_external_subdir}) + endif() + endforeach() + endif() +endfunction() + +#! Gather unique_list of all external subdirectories that is union +#! of the engine.json, project.json, o3de_manifest.json and any gem.json files found visiting +function(get_all_external_subdirectories output_subdirs) + get_property(all_external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS) + get_property(manifest_external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS_O3DE_MANIFEST) + list(APPEND all_external_subdirs ${manifest_external_subdirs}) + list(REMOVE_DUPLICATES all_external_subdirs) + set(${output_subdirs} ${all_external_subdirs} PARENT_SCOPE) +endfunction() + +#! add_registered_gems_to_external_subdirs: +#! Accepts a list of gem_names (which can be read from the project.json or engine.json) +#! and cross checks them against union of all external subdirectories to determine the gem path. +#! If that gem exist it is appended to LY_EXTERNAL_SUBDIRS so that that the build generator +#! adds to the generated build project. +#! Otherwise a fatal error is logged indicating that is not gem could not be found in the list of external subdirectories +function(add_registered_gems_to_external_subdirs gem_names) + if (gem_names) + get_all_external_subdirectories(all_external_subdirs) + foreach(gem_name IN LISTS gem_names) + unset(gem_path) + o3de_find_gem(${gem_name} gem_path) + if (gem_path) + set_property(GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS ${gem_path} APPEND) + else() + list(JOIN all_external_subdirs "\n" external_subdirs_formatted) + message(SEND_ERROR "The gem \"${gem_name}\" from the \"gem_names\" field in the engine.json/project.json " + " could not be found in any gem.json from the following list of registered external subdirectories:\n" + "${external_subdirs_formatted}") + break() + endif() + endforeach() + endif() +endfunction() + +function(add_subdirectory_on_external_subdirs) + # Lookup the paths of "gem_names" array all project.json files and engine.json + # and append them to the LY_EXTERNAL_SUBDIRS property + foreach(project ${LY_PROJECTS}) + file(REAL_PATH ${project} full_directory_path BASE_DIRECTORY ${CMAKE_SOURCE_DIR}) + o3de_read_json_array(gem_names ${full_directory_path}/project.json "gem_names") + add_registered_gems_to_external_subdirs("${gem_names}") + endforeach() + o3de_read_json_array(gem_names ${LY_ROOT_FOLDER}/engine.json "gem_names") + add_registered_gems_to_external_subdirs("${gem_names}") + + get_property(external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS) + list(APPEND LY_EXTERNAL_SUBDIRS ${external_subdirs}) + list(REMOVE_DUPLICATES LY_EXTERNAL_SUBDIRS) + # Loop over the additional external subdirectories and invoke add_subdirectory on them + foreach(external_directory ${LY_EXTERNAL_SUBDIRS}) + # Hash the external_directory name and append it to the Binary Directory section of add_subdirectory + # This is to deal with potential situations where multiple external directories has the same last directory name + # For example if D:/Company1/RayTracingGem and F:/Company2/Path/RayTracingGem were both added as a subdirectory + file(REAL_PATH ${external_directory} full_directory_path) + string(SHA256 full_directory_hash ${full_directory_path}) + # Truncate the full_directory_hash down to 8 characters to avoid hitting the Windows 260 character path limit + # when the external subdirectory contains relative paths of significant length + string(SUBSTRING ${full_directory_hash} 0 8 full_directory_hash) + # Use the last directory as the suffix path to use for the Binary Directory + cmake_path(GET external_directory FILENAME directory_name) + add_subdirectory(${external_directory} ${CMAKE_BINARY_DIR}/External/${directory_name}-${full_directory_hash}) + endforeach() +endfunction() diff --git a/cmake/cmake_files.cmake b/cmake/cmake_files.cmake index caeab9d12e..90f1e9a2d8 100644 --- a/cmake/cmake_files.cmake +++ b/cmake/cmake_files.cmake @@ -35,6 +35,7 @@ set(FILES Projects.cmake RuntimeDependencies.cmake SettingsRegistry.cmake + Subdirectories.cmake UnitTest.cmake Version.cmake ) diff --git a/engine.json b/engine.json index 51b6738001..5c64eed308 100644 --- a/engine.json +++ b/engine.json @@ -54,7 +54,7 @@ "Gems/NvCloth", "Gems/PhysX", "Gems/PhysXDebug", - "Gems/Prefab", + "Gems/Prefab/PrefabBuilder", "Gems/Presence", "Gems/PrimitiveAssets", "Gems/Profiler", diff --git a/scripts/o3de/o3de/cmake.py b/scripts/o3de/o3de/cmake.py index 7a820a9677..163d77f2f4 100644 --- a/scripts/o3de/o3de/cmake.py +++ b/scripts/o3de/o3de/cmake.py @@ -150,7 +150,10 @@ def remove_gem_dependency(cmake_file: pathlib.Path, # If the in_gem_list was flipped to false, that means the currently parsed line contained the # line end marker, so append that to the result_line result_line += enable_gem_end_marker if not in_gem_list else '' - t_data.append(result_line + '\n') + # Strip of trailing whitespace. This also strips result lines which are empty of the indent + result_line = result_line.rstrip() + if result_line: + t_data.append(result_line + '\n') else: t_data.append(line) @@ -165,11 +168,6 @@ def remove_gem_dependency(cmake_file: pathlib.Path, return 0 -def get_project_gems(project_path: pathlib.Path, - platform: str = 'Common') -> set: - return get_gems_from_cmake_file(get_enabled_gem_cmake_file(project_path=project_path, platform=platform)) - - def get_enabled_gems(cmake_file: pathlib.Path) -> set: """ Gets a list of enabled gems from the cmake file @@ -206,15 +204,6 @@ def get_enabled_gems(cmake_file: pathlib.Path) -> set: return gem_target_set -def get_project_gem_paths(project_path: pathlib.Path, - platform: str = 'Common') -> set: - gem_names = get_project_gems(project_path, platform) - gem_paths = set() - for gem_name in gem_names: - gem_paths.add(manifest.get_registered(gem_name=gem_name, project_path=project_path)) - return gem_paths - - def get_enabled_gem_cmake_file(project_name: str = None, project_path: str or pathlib.Path = None, platform: str = 'Common') -> pathlib.Path or None: diff --git a/scripts/o3de/o3de/disable_gem.py b/scripts/o3de/o3de/disable_gem.py index eecc765d32..617ad8ba41 100644 --- a/scripts/o3de/o3de/disable_gem.py +++ b/scripts/o3de/o3de/disable_gem.py @@ -15,7 +15,7 @@ import os import pathlib import sys -from o3de import cmake, manifest, utils +from o3de import cmake, manifest, project_properties, utils logger = logging.getLogger('o3de.disable_gem') logging.basicConfig(format=utils.LOG_FORMAT) @@ -68,8 +68,8 @@ def disable_gem_in_project(gem_name: str = None, f' {project_path / "project.json"}, engine.json') return 1 gem_path = pathlib.Path(gem_path).resolve() - # make sure this gem already exists if we're adding. We can always remove a gem. - if not gem_path.exists(): + # make sure the gem path is a directory + if not gem_path.is_dir(): logger.error(f'Gem Path {gem_path} does not exist.') return 1 @@ -79,9 +79,6 @@ def disable_gem_in_project(gem_name: str = None, logger.error(f'Could not read gem.json content under {gem_path}.') return 1 - # when removing we will try to do as much as possible even with failures so ret_val will be the last error code - ret_val = 0 - if not enabled_gem_file: enabled_gem_file = cmake.get_enabled_gem_cmake_file(project_path=project_path) @@ -89,10 +86,14 @@ def disable_gem_in_project(gem_name: str = None, if not enabled_gem_file.is_file(): logger.error(f'Enabled gem file {enabled_gem_file} is not present.') return 1 + # remove the gem error_code = cmake.remove_gem_dependency(enabled_gem_file, gem_json_data['gem_name']) - if error_code: - ret_val = error_code + + # Remove the name of the gem from the project.json "gem_names" field if the gem is neither + # registered with the project.json nor engine.json + ret_val = project_properties.edit_project_props(project_path, + delete_gem_names=gem_json_data['gem_name']) or error_code return ret_val diff --git a/scripts/o3de/o3de/enable_gem.py b/scripts/o3de/o3de/enable_gem.py index ea79f6c73f..bcfcd7157a 100644 --- a/scripts/o3de/o3de/enable_gem.py +++ b/scripts/o3de/o3de/enable_gem.py @@ -16,7 +16,7 @@ import os import pathlib import sys -from o3de import cmake, manifest, register, validation, utils +from o3de import cmake, manifest, project_properties, register, validation, utils logger = logging.getLogger('o3de.enable_gem') logging.basicConfig(format=utils.LOG_FORMAT) @@ -33,7 +33,7 @@ def enable_gem_in_project(gem_name: str = None, :param gem_path: path to the gem to add :param project_name: name of to the project to add the gem to :param project_path: path to the project to add the gem to - :param enabled_gem_file_file: if this dependency goes/is in a specific file + :param enabled_gem_file: if this dependency goes/is in a specific file :return: 0 for success or non 0 failure code """ # we need either a project name or path @@ -80,8 +80,6 @@ def enable_gem_in_project(gem_name: str = None, logger.error(f'Could not read gem.json content under {gem_path}.') return 1 - - ret_val = 0 if enabled_gem_file: # make sure this is a project has an enabled gems file if not enabled_gem_file.is_file(): @@ -96,17 +94,16 @@ def enable_gem_in_project(gem_name: str = None, if not project_enabled_gem_file.is_file(): project_enabled_gem_file.touch() - # Before adding the gem_dependency check if the project is registered in either the project or engine - # manifest + # Before adding the gem_dependency check if the project is registered in either the project or engine manifest buildable_gems = manifest.get_engine_gems() buildable_gems.extend(manifest.get_project_gems(project_path)) - # Convert each path to pathlib.Path object and filter out duplictes using dict.fromkeys + # Convert each path to pathlib.Path object and filter out duplicates using dict.fromkeys buildable_gems = list(dict.fromkeys(map(lambda gem_path_string: pathlib.Path(gem_path_string), buildable_gems))) ret_val = 0 - # If the gem is not part of buildable set, it needs to be registered - if not gem_path in buildable_gems: - ret_val = register.register(gem_path=gem_path, external_subdir_project_path=project_path) + # If the gem is not part of buildable set, it's gem_name should be registered to the "gem_names" field + if gem_path not in buildable_gems: + ret_val = project_properties.edit_project_props(project_path, new_gem_names=gem_json_data['gem_name']) # add the gem if it is registered in either the project.json or engine.json ret_val = ret_val or cmake.add_gem_dependency(project_enabled_gem_file, gem_json_data['gem_name']) diff --git a/scripts/o3de/o3de/engine_properties.py b/scripts/o3de/o3de/engine_properties.py index 636e32704e..06bff88d64 100644 --- a/scripts/o3de/o3de/engine_properties.py +++ b/scripts/o3de/o3de/engine_properties.py @@ -18,11 +18,35 @@ from o3de import manifest, utils logger = logging.getLogger('o3de.engine_properties') logging.basicConfig(format=utils.LOG_FORMAT) +def _edit_gem_names(engine_json: dict, + new_gem_names: str or list = None, + delete_gem_names: str or list = None, + replace_gem_names: str or list = None): + if new_gem_names: + tag_list = new_gem_names.split() if isinstance(new_gem_names, str) else new_gem_names + engine_json.setdefault('gem_names', []).extend(tag_list) + if delete_gem_names: + removal_list = delete_gem_names.split() if isinstance(delete_gem_names, str) else delete_gem_names + if 'gem_names' in engine_json: + for tag in removal_list: + if tag in engine_json['gem_names']: + engine_json['gem_names'].remove(tag) + if replace_gem_names: + tag_list = replace_gem_names.split() if isinstance(replace_gem_names, str) else replace_gem_names + engine_json['gem_names'] = tag_list + + # Remove duplicates from list + engine_json['gem_names'] = list(dict.fromkeys(engine_json.get('gem_names', []))) + def edit_engine_props(engine_path: pathlib.Path = None, engine_name: str = None, new_name: str = None, - new_version: str = None) -> int: + new_version: str = None, + new_gem_names: str or list = None, + delete_gem_names: str or list = None, + replace_gem_names: str or list = None + ) -> int: if not engine_path and not engine_name: logger.error(f'Either a engine path or a engine name must be supplied to lookup engine.json') return 1 @@ -51,13 +75,20 @@ def edit_engine_props(engine_path: pathlib.Path = None, if new_version: engine_json_data['O3DEVersion'] = new_version + # Update the gem_names field in the engine.json + _edit_gem_names(engine_json_data, new_gem_names, delete_gem_names, replace_gem_names) + return 0 if manifest.save_o3de_manifest(engine_json_data, pathlib.Path(engine_path) / 'engine.json') else 1 def _edit_engine_props(args: argparse) -> int: return edit_engine_props(args.engine_path, - args.engine_name, - args.engine_new_name, - args.engine_version) + args.engine_name, + args.engine_new_name, + args.engine_version, + args.add_gem_names, + args.delete_gem_names, + args.replace_gem_names + ) def add_parser_args(parser): group = parser.add_mutually_exclusive_group(required=True) @@ -70,6 +101,13 @@ def add_parser_args(parser): help='Sets the name for the engine.') group.add_argument('-ev', '--engine-version', type=str, required=False, help='Sets the version for the engine.') + group = parser.add_mutually_exclusive_group(required=False) + group.add_argument('-agn', '--add-gem-names', type=str, nargs='*', required=False, + help='Adds gem name(s) to gem_names field. Space delimited list (ex. -at A B C)') + group.add_argument('-dgn', '--delete-gem-names', type=str, nargs='*', required=False, + help='Removes gem name(s) from the gem_names field. Space delimited list (ex. -dt A B C') + group.add_argument('-rgn', '--replace-gem-names', type=str, nargs='*', required=False, + help='Replace entirety of gem_names field with space delimited list of values') parser.set_defaults(func=_edit_engine_props) def add_args(subparsers) -> None: diff --git a/scripts/o3de/o3de/project_properties.py b/scripts/o3de/o3de/project_properties.py index 9ee55773b5..f809bc8153 100644 --- a/scripts/o3de/o3de/project_properties.py +++ b/scripts/o3de/o3de/project_properties.py @@ -28,6 +28,27 @@ def get_project_props(name: str = None, path: pathlib.Path = None) -> dict: return proj_json +def _edit_gem_names(proj_json: dict, + new_gem_names: str or list = None, + delete_gem_names: str or list = None, + replace_gem_names: str or list = None): + if new_gem_names: + tag_list = new_gem_names.split() if isinstance(new_gem_names, str) else new_gem_names + proj_json.setdefault('gem_names', []).extend(tag_list) + if delete_gem_names: + removal_list = delete_gem_names.split() if isinstance(delete_gem_names, str) else delete_gem_names + if 'gem_names' in proj_json: + for tag in removal_list: + if tag in proj_json['gem_names']: + proj_json['gem_names'].remove(tag) + if replace_gem_names: + tag_list = replace_gem_names.split() if isinstance(replace_gem_names, str) else replace_gem_names + proj_json['gem_names'] = tag_list + + # Remove duplicates from list + proj_json['gem_names'] = list(dict.fromkeys(proj_json.get('gem_names', []))) + + def edit_project_props(proj_path: pathlib.Path = None, proj_name: str = None, new_name: str = None, @@ -38,7 +59,11 @@ def edit_project_props(proj_path: pathlib.Path = None, new_icon: str = None, new_tags: str or list = None, delete_tags: str or list = None, - replace_tags: str or list = None) -> int: + replace_tags: str or list = None, + new_gem_names: str or list = None, + delete_gem_names: str or list = None, + replace_gem_names: str or list = None + ) -> int: proj_json = get_project_props(proj_name, proj_path) if not proj_json: @@ -74,6 +99,8 @@ def edit_project_props(proj_path: pathlib.Path = None, if replace_tags: tag_list = replace_tags.split() if isinstance(replace_tags, str) else replace_tags proj_json['user_tags'] = tag_list + # Update the gem_names field in the project.json + _edit_gem_names(proj_json, new_gem_names, delete_gem_names, replace_gem_names) return 0 if manifest.save_o3de_manifest(proj_json, pathlib.Path(proj_path) / 'project.json') else 1 @@ -89,7 +116,10 @@ def _edit_project_props(args: argparse) -> int: args.project_icon, args.add_tags, args.delete_tags, - args.replace_tags) + args.replace_tags, + args.add_gem_names, + args.delete_gem_names, + args.replace_gem_names) def add_parser_args(parser): @@ -118,6 +148,13 @@ def add_parser_args(parser): help='Removes tag(s) from the user_tags property. Space delimited list (ex. -dt A B C') group.add_argument('-rt', '--replace-tags', type=str, nargs ='*', required=False, help='Replace entirety of user_tags property with space delimited list of values') + group = parser.add_mutually_exclusive_group(required=False) + group.add_argument('-agn', '--add-gem-names', type=str, nargs='*', required=False, + help='Adds gem name(s) to gem_names field. Space delimited list (ex. -at A B C)') + group.add_argument('-dgn', '--delete-gem-names', type=str, nargs='*', required=False, + help='Removes gem name(s) from the gem_names field. Space delimited list (ex. -dt A B C') + group.add_argument('-rgn', '--replace-gem-names', type=str, nargs='*', required=False, + help='Replace entirety of gem_names field with space delimited list of values') parser.set_defaults(func=_edit_project_props) diff --git a/scripts/o3de/tests/CMakeLists.txt b/scripts/o3de/tests/CMakeLists.txt index de8e9e4974..4c75e5e7df 100644 --- a/scripts/o3de/tests/CMakeLists.txt +++ b/scripts/o3de/tests/CMakeLists.txt @@ -13,70 +13,77 @@ endif() # Add a test to test out the o3de package `o3de.py register` command ly_add_pytest( NAME o3de_register - PATH ${CMAKE_CURRENT_LIST_DIR}/unit_test_register.py + PATH ${CMAKE_CURRENT_LIST_DIR}/test_register.py TEST_SUITE smoke EXCLUDE_TEST_RUN_TARGET_FROM_IDE ) ly_add_pytest( NAME o3de_cmake - PATH ${CMAKE_CURRENT_LIST_DIR}/unit_test_cmake.py + PATH ${CMAKE_CURRENT_LIST_DIR}/test_cmake.py + TEST_SUITE smoke + EXCLUDE_TEST_RUN_TARGET_FROM_IDE +) + +ly_add_pytest( + NAME o3de_disable_gem + PATH ${CMAKE_CURRENT_LIST_DIR}/test_disable_gem.py TEST_SUITE smoke EXCLUDE_TEST_RUN_TARGET_FROM_IDE ) ly_add_pytest( NAME o3de_enable_gem - PATH ${CMAKE_CURRENT_LIST_DIR}/unit_test_enable_gem.py + PATH ${CMAKE_CURRENT_LIST_DIR}/test_enable_gem.py TEST_SUITE smoke EXCLUDE_TEST_RUN_TARGET_FROM_IDE ) ly_add_pytest( NAME o3de_global_project - PATH ${CMAKE_CURRENT_LIST_DIR}/unit_test_global_project.py + PATH ${CMAKE_CURRENT_LIST_DIR}/test_global_project.py TEST_SUITE smoke EXCLUDE_TEST_RUN_TARGET_FROM_IDE ) ly_add_pytest( NAME o3de_manifest - PATH ${CMAKE_CURRENT_LIST_DIR}/unit_test_manifest.py + PATH ${CMAKE_CURRENT_LIST_DIR}/test_manifest.py TEST_SUITE smoke EXCLUDE_TEST_RUN_TARGET_FROM_IDE ) ly_add_pytest( NAME o3de_engine_properties - PATH ${CMAKE_CURRENT_LIST_DIR}/unit_test_engine_properties.py + PATH ${CMAKE_CURRENT_LIST_DIR}/test_engine_properties.py TEST_SUITE smoke EXCLUDE_TEST_RUN_TARGET_FROM_IDE ) ly_add_pytest( NAME o3de_project_properties - PATH ${CMAKE_CURRENT_LIST_DIR}/unit_test_project_properties.py + PATH ${CMAKE_CURRENT_LIST_DIR}/test_project_properties.py TEST_SUITE smoke EXCLUDE_TEST_RUN_TARGET_FROM_IDE ) ly_add_pytest( NAME o3de_gem_properties - PATH ${CMAKE_CURRENT_LIST_DIR}/unit_test_gem_properties.py + PATH ${CMAKE_CURRENT_LIST_DIR}/test_gem_properties.py TEST_SUITE smoke EXCLUDE_TEST_RUN_TARGET_FROM_IDE ) ly_add_pytest( NAME o3de_template - PATH ${CMAKE_CURRENT_LIST_DIR}/unit_test_engine_template.py + PATH ${CMAKE_CURRENT_LIST_DIR}/test_engine_template.py TEST_SUITE smoke EXCLUDE_TEST_RUN_TARGET_FROM_IDE ) ly_add_pytest( NAME o3de_register_show - PATH ${CMAKE_CURRENT_LIST_DIR}/unit_test_print_registration.py + PATH ${CMAKE_CURRENT_LIST_DIR}/test_print_registration.py TEST_SUITE smoke EXCLUDE_TEST_RUN_TARGET_FROM_IDE ) diff --git a/scripts/o3de/tests/unit_test_cmake.py b/scripts/o3de/tests/test_cmake.py similarity index 100% rename from scripts/o3de/tests/unit_test_cmake.py rename to scripts/o3de/tests/test_cmake.py diff --git a/scripts/o3de/tests/test_disable_gem.py b/scripts/o3de/tests/test_disable_gem.py new file mode 100644 index 0000000000..18e0402185 --- /dev/null +++ b/scripts/o3de/tests/test_disable_gem.py @@ -0,0 +1,200 @@ +# +# Copyright (c) Contributors to the Open 3D Engine Project. +# For complete copyright and license terms please see the LICENSE at the root of this distribution. +# +# SPDX-License-Identifier: Apache-2.0 OR MIT +# +# + +import json + +import pytest +import pathlib +from unittest.mock import patch + +from o3de import cmake, disable_gem, enable_gem + + +TEST_PROJECT_JSON_PAYLOAD = ''' +{ + "project_name": "TestProject", + "origin": "The primary repo for TestProject goes here: i.e. http://www.mydomain.com", + "license": "What license TestProject uses goes here: i.e. https://opensource.org/licenses/MIT", + "display_name": "TestProject", + "summary": "A short description of TestProject.", + "canonical_tags": [ + "Project" + ], + "user_tags": [ + "TestProject" + ], + "icon_path": "preview.png", + "engine": "o3de-install", + "restricted_name": "projects", + "external_subdirectories": [ + ] +} +''' + +TEST_GEM_JSON_PAYLOAD = ''' +{ + "gem_name": "TestGem", + "display_name": "TestGem", + "license": "Apache-2.0 Or MIT", + "license_url": "https://github.com/o3de/o3de/blob/development/LICENSE.txt", + "origin": "Open 3D Engine - o3de.org", + "origin_url": "https://github.com/o3de/o3de", + "type": "Code", + "summary": "A short description of TestGem.", + "canonical_tags": [ + "Gem" + ], + "user_tags": [ + "TestGem" + ], + "icon_path": "preview.png", + "requirements": "Any requirement goes here.", + "documentation_url": "The link to the documentation goes here.", + "dependencies": [ + ] +} +''' + +TEST_O3DE_MANIFEST_JSON_PAYLOAD = ''' +{ + "o3de_manifest_name": "testuser", + "origin": "C:/Users/testuser/.o3de", + "default_engines_folder": "C:/Users/testuser/.o3de/Engines", + "default_projects_folder": "C:/Users/testuser/.o3de/Projects", + "default_gems_folder": "C:/Users/testuser/.o3de/Gems", + "default_templates_folder": "C:/Users/testuser/.o3de/Templates", + "default_restricted_folder": "C:/Users/testuser/.o3de/Restricted", + "default_third_party_folder": "C:/Users/testuser/.o3de/3rdParty", + "projects": [ + "D:/MinimalProject" + ], + "external_subdirectories": [], + "templates": [], + "restricted": [], + "repos": [], + "engines": [ + "D:/o3de/o3de" + ], + "engines_path": { + "o3de": "D:/o3de/o3de" + } +} +''' + +@pytest.fixture(scope='class') +def init_disable_gem_data(request): + class DisableGemData: + def __init__(self): + self.project_data = json.loads(TEST_PROJECT_JSON_PAYLOAD) + self.gem_data = json.loads(TEST_GEM_JSON_PAYLOAD) + request.cls.disable_gem = DisableGemData() + + +@pytest.mark.usefixtures('init_disable_gem_data') +class TestDisableGemCommand: + @pytest.mark.parametrize("gem_path, project_path, gem_registered_with_project, gem_registered_with_engine," + "expected_result", [ + pytest.param(pathlib.PurePath('TestProject/TestGem'), pathlib.PurePath('TestProject'), False, True, 0), + pytest.param(pathlib.PurePath('TestProject/TestGem'), pathlib.PurePath('TestProject'), False, False, 0), + pytest.param(pathlib.PurePath('TestProject/TestGem'), pathlib.PurePath('TestProject'), True, False, 0), + pytest.param(pathlib.PurePath('TestGem'), pathlib.PurePath('TestProject'), False, False, 0), + ] + ) + def test_disable_gem_registers_gem_name_with_project_json(self, gem_path, project_path, gem_registered_with_project, + gem_registered_with_engine, expected_result): + + project_gem_dependencies = [] + + def get_registered_path(project_name: str = None, gem_name: str = None) -> pathlib.Path or None: + if project_name: + return project_path + elif gem_name: + return gem_path + return None + + def save_o3de_manifest(new_project_data: dict, manifest_path: pathlib.Path = None) -> bool: + if manifest_path == project_path / 'project.json': + self.disable_gem.project_data = new_project_data + return True + + def load_o3de_manifest(manifest_path: pathlib.Path = None) -> dict or None: + if not manifest_path: + return json.loads(TEST_O3DE_MANIFEST_JSON_PAYLOAD) + return None + + def get_project_json_data(project_name: str = None, project_path: pathlib.Path = None): + return self.disable_gem.project_data + + def get_gem_json_data(gem_path: pathlib.Path, project_path: pathlib.Path): + return self.disable_gem.gem_data + + def get_project_gems(project_path: pathlib.Path): + return [pathlib.Path(gem_path).resolve()] if gem_registered_with_project else [] + + def get_engine_gems(): + return [pathlib.Path(gem_path).resolve()] if gem_registered_with_engine else [] + + def add_gem_dependency(enable_gem_cmake_file: pathlib.Path, gem_name: str): + project_gem_dependencies.append(gem_name) + return 0 + + def remove_gem_dependency(enable_gem_cmake_file: pathlib.Path, gem_name: str): + project_gem_dependencies.remove(gem_name) + return 0 + + def get_enabled_gems(enable_gem_cmake_file: pathlib.Path) -> list: + return project_gem_dependencies + + + with patch('pathlib.Path.is_dir', return_value=True) as pathlib_is_dir_patch,\ + patch('pathlib.Path.is_file', return_value=True) as pathlib_is_file_patch, \ + patch('o3de.manifest.load_o3de_manifest', side_effect=load_o3de_manifest) as load_o3de_manifest_patch, \ + patch('o3de.manifest.save_o3de_manifest', side_effect=save_o3de_manifest) as save_o3de_manifest_patch,\ + patch('o3de.manifest.get_registered', side_effect=get_registered_path) as get_registered_patch,\ + patch('o3de.manifest.get_gem_json_data', side_effect=get_gem_json_data) as get_gem_json_data_patch,\ + patch('o3de.manifest.get_project_json_data', side_effect=get_project_json_data) as get_gem_json_data_patch,\ + patch('o3de.manifest.get_project_gems', side_effect=get_project_gems) as get_project_gems_patch,\ + patch('o3de.manifest.get_engine_gems', side_effect=get_engine_gems) as get_engine_gems_patch,\ + patch('o3de.cmake.add_gem_dependency', side_effect=add_gem_dependency) as add_gem_dependency_patch, \ + patch('o3de.cmake.remove_gem_dependency', + side_effect=remove_gem_dependency) as remove_gem_dependency_patch, \ + patch('o3de.cmake.get_enabled_gems', + side_effect=get_enabled_gems) as get_enabled_gems, \ + patch('o3de.validation.valid_o3de_gem_json', return_value=True) as valid_gem_json_patch: + + # Clear out any "gem_names" from the previous iterations + self.disable_gem.project_data.pop('gem_names', None) + + # First enable the gem + assert enable_gem.enable_gem_in_project(gem_path=gem_path, project_path=project_path) == 0 + + # Check that the gem is enabled + gem_json = get_gem_json_data(gem_path, project_path) + project_json = get_project_json_data(project_path=project_path) + enabled_gems_list = cmake.get_enabled_gems(project_path / "Gem/enabled_gems.cmake") + assert gem_json.get('gem_name', '') in enabled_gems_list + + # If the gem that is neither registered in the project.json nor engine.json, + # then it must appear in the "gem_names" field. + if not gem_registered_with_engine and not gem_registered_with_project: + assert gem_json.get('gem_name', '') in project_json.get('gem_names', []) + else: + assert gem_json.get('gem_name', '') not in project_json.get('gem_names', []) + + # Now disable the gem + result = disable_gem.disable_gem_in_project(gem_path=gem_path, project_path=project_path) + assert result == expected_result + + # Refresh the enabled_gems list and check for removal of the gem + gem_json = get_gem_json_data(gem_path, project_path) + project_json = get_project_json_data(project_path=project_path) + enabled_gems_list = cmake.get_enabled_gems(project_path / "Gem/enabled_gems.cmake") + assert gem_json.get('gem_name', '') not in enabled_gems_list + + # If gem name should no longer appear in the "gem_names" field + assert gem_json.get('gem_name', '') not in project_json.get('gem_names', []) diff --git a/scripts/o3de/tests/unit_test_enable_gem.py b/scripts/o3de/tests/test_enable_gem.py similarity index 82% rename from scripts/o3de/tests/unit_test_enable_gem.py rename to scripts/o3de/tests/test_enable_gem.py index 8067fc329d..9e9fe0a713 100644 --- a/scripts/o3de/tests/unit_test_enable_gem.py +++ b/scripts/o3de/tests/test_enable_gem.py @@ -104,10 +104,11 @@ class TestEnableGemCommand: pytest.param(pathlib.PurePath('TestProject/TestGem'), pathlib.PurePath('TestProject'), False, True, 0), pytest.param(pathlib.PurePath('TestProject/TestGem'), pathlib.PurePath('TestProject'), False, False, 0), pytest.param(pathlib.PurePath('TestProject/TestGem'), pathlib.PurePath('TestProject'), True, False, 0), + pytest.param(pathlib.PurePath('TestGem'), pathlib.PurePath('TestProject'), False, False, 0), ] ) - def test_enable_gem_registers_gem_as_well(self, gem_path, project_path, gem_registered_with_project, gem_registered_with_engine, - expected_result): + def test_enable_gem_registers_gem_name_with_project_json(self, gem_path, project_path, gem_registered_with_project, + gem_registered_with_engine, expected_result): def get_registered_path(project_name: str = None, gem_name: str = None) -> pathlib.Path: if project_name: @@ -116,11 +117,8 @@ class TestEnableGemCommand: return gem_path return None - def get_registered_gem_path(gem_name: str) -> pathlib.Path: - return gem_path - def save_o3de_manifest(new_project_data: dict, manifest_path: pathlib.Path = None) -> bool: - if manifest_path == project_path: + if manifest_path == project_path / 'project.json': self.enable_gem.project_data = new_project_data return True @@ -129,17 +127,17 @@ class TestEnableGemCommand: return json.loads(TEST_O3DE_MANIFEST_JSON_PAYLOAD) return None - def get_project_json_data(project_path: pathlib.Path): + def get_project_json_data(project_name: str = None, project_path: pathlib.Path = None): return self.enable_gem.project_data def get_gem_json_data(gem_path: pathlib.Path, project_path: pathlib.Path): return self.enable_gem.gem_data def get_project_gems(project_path: pathlib.Path): - return [gem_path] if gem_registered_with_project else [] + return [pathlib.Path(gem_path).resolve()] if gem_registered_with_project else [] def get_engine_gems(): - return [gem_path] if gem_registered_with_engine else [] + return [pathlib.Path(gem_path).resolve()] if gem_registered_with_engine else [] def add_gem_dependency(enable_gem_cmake_file: pathlib.Path, gem_name: str): return 0 @@ -155,11 +153,14 @@ class TestEnableGemCommand: patch('o3de.manifest.get_engine_gems', side_effect=get_engine_gems) as get_engine_gems_patch,\ patch('o3de.cmake.add_gem_dependency', side_effect=add_gem_dependency) as add_gem_dependency_patch,\ patch('o3de.validation.valid_o3de_gem_json', return_value=True) as valid_gem_json_patch: + + self.enable_gem.project_data.pop('gem_names', None) result = enable_gem.enable_gem_in_project(gem_path=gem_path, project_path=project_path) assert result == expected_result - # If the gem isn't registered with the engine or project already it should now be registered with the project - if not gem_registered_with_engine and gem_registered_with_project: - # Prepend the project path to each external subdirectory - project_relative_subdirs = map(lambda subdir: (pathlib.Path(project_path) / subdir).as_posix(), - self.enable_gem.project_data.get('external_subdirectories', [])) - assert gem_path.as_posix() in project_relative_subdirs + + gem_json = get_gem_json_data(gem_path, project_path) + project_json = get_project_json_data(project_path=project_path) + if not gem_registered_with_engine and not gem_registered_with_project: + assert gem_json.get('gem_name', '') in project_json.get('gem_names', []) + else: + assert gem_json.get('gem_name', '') not in project_json.get('gem_names', []) diff --git a/scripts/o3de/tests/unit_test_engine_properties.py b/scripts/o3de/tests/test_engine_properties.py similarity index 100% rename from scripts/o3de/tests/unit_test_engine_properties.py rename to scripts/o3de/tests/test_engine_properties.py diff --git a/scripts/o3de/tests/unit_test_engine_template.py b/scripts/o3de/tests/test_engine_template.py similarity index 100% rename from scripts/o3de/tests/unit_test_engine_template.py rename to scripts/o3de/tests/test_engine_template.py diff --git a/scripts/o3de/tests/unit_test_gem_properties.py b/scripts/o3de/tests/test_gem_properties.py similarity index 100% rename from scripts/o3de/tests/unit_test_gem_properties.py rename to scripts/o3de/tests/test_gem_properties.py diff --git a/scripts/o3de/tests/unit_test_global_project.py b/scripts/o3de/tests/test_global_project.py similarity index 100% rename from scripts/o3de/tests/unit_test_global_project.py rename to scripts/o3de/tests/test_global_project.py diff --git a/scripts/o3de/tests/unit_test_manifest.py b/scripts/o3de/tests/test_manifest.py similarity index 100% rename from scripts/o3de/tests/unit_test_manifest.py rename to scripts/o3de/tests/test_manifest.py diff --git a/scripts/o3de/tests/unit_test_print_registration.py b/scripts/o3de/tests/test_print_registration.py similarity index 100% rename from scripts/o3de/tests/unit_test_print_registration.py rename to scripts/o3de/tests/test_print_registration.py diff --git a/scripts/o3de/tests/unit_test_project_properties.py b/scripts/o3de/tests/test_project_properties.py similarity index 100% rename from scripts/o3de/tests/unit_test_project_properties.py rename to scripts/o3de/tests/test_project_properties.py diff --git a/scripts/o3de/tests/unit_test_register.py b/scripts/o3de/tests/test_register.py similarity index 100% rename from scripts/o3de/tests/unit_test_register.py rename to scripts/o3de/tests/test_register.py diff --git a/scripts/o3de/tests/unit_test_utils.py b/scripts/o3de/tests/test_utils.py similarity index 100% rename from scripts/o3de/tests/unit_test_utils.py rename to scripts/o3de/tests/test_utils.py