diff --git a/CMakeLists.txt b/CMakeLists.txt index efcc866195..3f9f56a606 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -44,51 +44,11 @@ include(cmake/SettingsRegistry.cmake) include(cmake/TestImpactFramework/LYTestImpactFramework.cmake) include(cmake/CMakeFiles.cmake) include(cmake/O3DEJson.cmake) +include(cmake/Subdirectories.cmake) -################################################################################ -# Subdirectory processing -################################################################################ - -# this function is building up the LY_EXTERNAL_SUBDIRS global property -function(add_engine_gem_json_external_subdirectories gem_path) - set(gem_json_path ${gem_path}/gem.json) - if(EXISTS ${gem_json_path}) - read_json_external_subdirs(gem_external_subdirs ${gem_path}/gem.json) - foreach(gem_external_subdir ${gem_external_subdirs}) - file(REAL_PATH ${gem_external_subdir} real_external_subdir BASE_DIRECTORY ${gem_path}) - set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS ${real_external_subdir}) - add_engine_gem_json_external_subdirectories(${real_external_subdir}) - endforeach() - endif() -endfunction() - -function(add_engine_json_external_subdirectories) - read_json_external_subdirs(engine_external_subdirs ${LY_ROOT_FOLDER}/engine.json) - foreach(engine_external_subdir ${engine_external_subdirs}) - file(REAL_PATH ${engine_external_subdir} real_external_subdir BASE_DIRECTORY ${LY_ROOT_FOLDER}) - set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS ${real_external_subdir}) - add_engine_gem_json_external_subdirectories(${real_external_subdir}) - endforeach() -endfunction() - -function(add_subdirectory_on_externalsubdirs) - get_property(external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS) - list(APPEND LY_EXTERNAL_SUBDIRS ${external_subdirs}) - # Loop over the additional external subdirectories and invoke add_subdirectory on them - foreach(external_directory ${LY_EXTERNAL_SUBDIRS}) - # Hash the external_directory name and append it to the Binary Directory section of add_subdirectory - # This is to deal with potential situations where multiple external directories has the same last directory name - # For example if D:/Company1/RayTracingGem and F:/Company2/Path/RayTracingGem were both added as a subdirectory - file(REAL_PATH ${external_directory} full_directory_path) - string(SHA256 full_directory_hash ${full_directory_path}) - # Truncate the full_directory_hash down to 8 characters to avoid hitting the Windows 260 character path limit - # when the external subdirectory contains relative paths of significant length - string(SUBSTRING ${full_directory_hash} 0 8 full_directory_hash) - # Use the last directory as the suffix path to use for the Binary Directory - get_filename_component(directory_name ${external_directory} NAME) - add_subdirectory(${external_directory} ${CMAKE_BINARY_DIR}/External/${directory_name}-${full_directory_hash}) - endforeach() -endfunction() +# Gather the list of o3de_manifest external Subdirectories +# into the LY_EXTERNAL_SUBDIRS_O3DE_MANIFEST_PROPERTY +add_o3de_manifest_json_external_subdirectories() # Add the projects first so the Launcher can find them include(cmake/Projects.cmake) @@ -99,9 +59,11 @@ endif() if(NOT INSTALLED_ENGINE) # Add external subdirectories listed in the engine.json. LY_EXTERNAL_SUBDIRS is a cache variable so the user can add extra - # external subdirectories. This should go before adding the rest of the targets so the targets are availbe to the launcher. + # external subdirectories. This should go before adding the rest of the targets so the targets are available to the launcher. add_engine_json_external_subdirectories() - add_subdirectory_on_externalsubdirs() + + # Invoke add_subdirectory on external subdirectories that should be used a this point + add_subdirectory_on_external_subdirs() # Add the rest of the targets add_subdirectory(Assets) @@ -114,7 +76,7 @@ if(NOT INSTALLED_ENGINE) else() ly_find_o3de_packages() - add_subdirectory_on_externalsubdirs() + add_subdirectory_on_external_subdirs() endif() ################################################################################ diff --git a/Code/Framework/AzCore/AzCore/std/string/string_view.h b/Code/Framework/AzCore/AzCore/std/string/string_view.h index c333e2cea1..daa00d98d3 100644 --- a/Code/Framework/AzCore/AzCore/std/string/string_view.h +++ b/Code/Framework/AzCore/AzCore/std/string/string_view.h @@ -309,78 +309,87 @@ namespace AZStd } static constexpr bool eq(char_type left, char_type right) noexcept { return left == right; } static constexpr bool lt(char_type left, char_type right) noexcept { return left < right; } - static constexpr int compare(const char_type* s1, const char_type* s2, size_t count) noexcept - { - // In GCC versions prior to major version 10, __builtin_memcmp fails in valid checks in constexpr evaluation -#if !defined(AZ_COMPILER_GCC) || AZ_COMPILER_GCC >= 100000 - if constexpr (AZStd::is_same_v) - { - return __builtin_memcmp(s1, s2, count); - } - else if constexpr (AZStd::is_same_v) - { - return __builtin_wmemcmp(s1, s2, count); - } else -#endif - { - if (az_builtin_is_constant_evaluated()) - { - for (; count; --count, ++s1, ++s2) + static constexpr int compare(const char_type* s1, const char_type* s2, size_t count) noexcept + { + // In GCC versions , __builtin_memcmp fails in valid checks in constexpr evaluation +#if !defined(AZ_COMPILER_GCC) + if constexpr (AZStd::is_same_v) + { + return __builtin_memcmp(s1, s2, count); + } + else if constexpr (AZStd::is_same_v) + { + return __builtin_wmemcmp(s1, s2, count); + } + else +#endif + { + if (az_builtin_is_constant_evaluated()) + { + for (; count; --count, ++s1, ++s2) { - if (lt(*s1, *s2)) - { - return -1; - } - else if (lt(*s2, *s1)) - { - return 1; - } - } - return 0; - } - else - { - return ::memcmp(s1, s2, count * sizeof(char_type)); - } - } + if (lt(*s1, *s2)) + { + return -1; + } + else if (lt(*s2, *s1)) + { + return 1; + } + } + return 0; + } + else + { + return ::memcmp(s1, s2, count * sizeof(char_type)); + } + } } static constexpr size_t length(const char_type* s) noexcept { // For GCC versions less than 10, __builtin_strlen and __builtin_wcslen is not supported as const expressions // so for that case it will need to manually count the characters (at compile time) instead -#if defined(AZ_COMPILER_GCC) && AZ_COMPILER_GCC < 100000 if constexpr (AZStd::is_same_v) { +#if defined(AZ_COMPILER_GCC) && AZ_COMPILER_GCC < 100000 if (!az_builtin_is_constant_evaluated()) { return strlen(s); } + else + { + size_t strLength{}; + for (; *s; ++s, ++strLength) + { + ; + } + return strLength; + } +#else + return __builtin_strlen(s); +#endif } else if constexpr (AZStd::is_same_v) { +#if defined(AZ_COMPILER_GCC) if (!az_builtin_is_constant_evaluated()) { return wcslen(s); } - } - - size_t strLength{}; - for (; *s; ++s, ++strLength) - { - ; - } - return strLength; + else + { + size_t strLength{}; + for (; *s; ++s, ++strLength) + { + ; + } + return strLength; + } #else - - if constexpr (AZStd::is_same_v) - { - return __builtin_strlen(s); - } - else if constexpr (AZStd::is_same_v) - { return __builtin_wcslen(s); +#endif } else { @@ -391,46 +400,59 @@ namespace AZStd } return strLength; } -#endif // defined(AZ_COMPILER_GCC) && AZ_COMPILER_GCC < 100000 + } static constexpr const char_type* find(const char_type* s, size_t count, const char_type& ch) noexcept { - // For GCC versions less than 10, __builtin_char_memchr and __builtin_wmemchr is not supported, and - // __builtin_memchr is not supported as const expressions. In those cases we will manually locate and + // For GCC versions less than 10, __builtin_char_memchr and __builtin_wmemchr is not supported, and + // __builtin_memchr is not supported as const expressions. In those cases we will manually locate and // return the pointer to 's' (at compile time) -#if defined(AZ_COMPILER_GCC) && AZ_COMPILER_GCC < 100000 if constexpr (AZStd::is_same_v) { +#if defined(AZ_COMPILER_GCC) if (!az_builtin_is_constant_evaluated()) { return static_cast(__builtin_memchr(s, ch, count)); } + else + { + for (; count; --count, ++s) + { + if (eq(*s, ch)) + { + return s; + } + } + + return nullptr; + } +#else + return __builtin_char_memchr(s, ch, count); +#endif // defined(AZ_COMPILER_GCC)AZ_COMPILER_GCC < 100000 } else if constexpr (AZStd::is_same_v) { +#if defined(AZ_COMPILER_GCC) if (!az_builtin_is_constant_evaluated()) { return wmemchr(s, ch, count); } - } - - for (; count; --count, ++s) - { - if (eq(*s, ch)) + else { - return s; + for (; count; --count, ++s) + { + if (eq(*s, ch)) + { + return s; + } + } + + return nullptr; } - } - return nullptr; #else - if constexpr (AZStd::is_same_v) - { - return __builtin_char_memchr(s, ch, count); - } - else if constexpr (AZStd::is_same_v) - { return __builtin_wmemchr(s, ch, count); +#endif } else { @@ -441,9 +463,9 @@ namespace AZStd return s; } } + return nullptr; } -#endif } static constexpr char_type* move(char_type* dest, const char_type* src, size_t count) noexcept { @@ -453,7 +475,7 @@ namespace AZStd return dest; } - #if az_has_builtin_memmove + #if !defined(AZ_COMPILER_GCC) && az_has_builtin_memmove __builtin_memmove(dest, src, count * sizeof(char_type)); #else auto NonBuiltinMove = [](char_type* dest1, const char_type* src1, size_t count1) constexpr @@ -506,7 +528,7 @@ namespace AZStd } static constexpr char_type* copy(char_type* dest, const char_type* src, size_t count) noexcept { - #if az_has_builtin_memcpy + #if !defined(AZ_COMPILER_GCC) && az_has_builtin_memcpy __builtin_memcpy(dest, src, count * sizeof(char_type)); #else auto NonBuiltinCopy = [](char_type* dest1, const char_type* src1, size_t count1) constexpr @@ -536,7 +558,7 @@ namespace AZStd static constexpr char_type* copy_backward(char_type* dest, const char_type* src, size_t count) noexcept { char_type* result = dest; - #if az_has_builtin_memmove + #if !defined(AZ_COMPILER_GCC) && az_has_builtin_memmove __builtin_memmove(dest, src, count * sizeof(char_type)); #else if (az_builtin_is_constant_evaluated()) diff --git a/Code/Tools/AssetProcessor/native/utilities/ApplicationManager.cpp b/Code/Tools/AssetProcessor/native/utilities/ApplicationManager.cpp index 57884b304f..06536921d9 100644 --- a/Code/Tools/AssetProcessor/native/utilities/ApplicationManager.cpp +++ b/Code/Tools/AssetProcessor/native/utilities/ApplicationManager.cpp @@ -412,7 +412,7 @@ void ApplicationManager::PopulateApplicationDependencies() // Note that its not necessary for any of these files to actually exist. It is considered a "change" if they // change their file modtime, or if they go from existing to not existing, or if they go from not existing, to existing. // any of those should cause AP to drop. - for (const QString& pathName : { "CrySystem", + for (QString pathName : { "CrySystem", "SceneCore", "SceneData", "SceneBuilder", "AzQtComponents" }) diff --git a/Gems/Atom/RHI/Code/Include/Atom/RHI/ThreadLocalContext.h b/Gems/Atom/RHI/Code/Include/Atom/RHI/ThreadLocalContext.h index 4d3534b845..d45c7bf93e 100644 --- a/Gems/Atom/RHI/Code/Include/Atom/RHI/ThreadLocalContext.h +++ b/Gems/Atom/RHI/Code/Include/Atom/RHI/ThreadLocalContext.h @@ -19,7 +19,7 @@ namespace AZ { /** * This class is a container of thread local storage. It allows for multiple instances - * of thread local storage to exist simultaneously (a property not possible with the + * of thread local storage to exist simultaneously (a property not possible with the * thread_local modifier, which is really a thread global). The context tracks AZ thread * lifetime through a bus in order to clean up storage for exiting threads. The context * allows thread-safe iteration of all thread contexts, which is also a property not possible @@ -36,7 +36,11 @@ namespace AZ public: using InitFunction = AZStd::function; - ThreadLocalContext(InitFunction initFunction = [] (Storage&) {}); + static void DefaultFunction(Storage&) + { + } + + ThreadLocalContext(InitFunction initFunction = &DefaultFunction); ~ThreadLocalContext(); // No copying or moving allowed. diff --git a/Gems/PhysXDebug/Code/CMakeLists.txt b/Gems/PhysXDebug/Code/CMakeLists.txt index 0886f2bf34..1f5d266fbf 100644 --- a/Gems/PhysXDebug/Code/CMakeLists.txt +++ b/Gems/PhysXDebug/Code/CMakeLists.txt @@ -10,7 +10,7 @@ o3de_find_gem("PhysX" physx_gem_path) set(physx_gem_json ${physx_gem_path}/gem.json) o3de_restricted_path(${physx_gem_json} physx_gem_restricted_path physx_gem_parent_relative_path) -o3de_pal_dir(physx_pal_source_dir ${physx_gem_path}/Code/Source/Platform/${PAL_PLATFORM_NAME} ${physx_gem_restricted_path} ${physx_gem_path} ${physx_gem_parent_relative_path}) +o3de_pal_dir(physx_pal_source_dir ${physx_gem_path}/Code/Source/Platform/${PAL_PLATFORM_NAME} "${physx_gem_restricted_path}" "${physx_gem_path}" "${physx_gem_parent_relative_path}") include(${physx_pal_source_dir}/PAL_${PAL_PLATFORM_NAME_LOWERCASE}.cmake) # for PAL_TRAIT_PHYSX_SUPPORTED diff --git a/Gems/ScriptEvents/Code/Include/ScriptEvents/ScriptEventDefinition.cpp b/Gems/ScriptEvents/Code/Include/ScriptEvents/ScriptEventDefinition.cpp index e6c40cf789..3a6a411611 100644 --- a/Gems/ScriptEvents/Code/Include/ScriptEvents/ScriptEventDefinition.cpp +++ b/Gems/ScriptEvents/Code/Include/ScriptEvents/ScriptEventDefinition.cpp @@ -131,10 +131,7 @@ namespace ScriptEvents return AZ::Failure(AZStd::string::format("%s, invalid name specified, event name must only have alpha numeric characters, may not start with a number and may not have white space", name.c_str())); } - if (m_methods.empty()) - { - return AZ::Failure(AZStd::string::format("Script Events (%s) must provide at least one event otherwise they are unusable, be sure to add an event before saving.", name.c_str())); - } + AZ_Warning("Script Events", !m_methods.empty(), AZStd::string::format("Script Events (%s) must provide at least one event, otherwise they are unusable.", name.c_str()).c_str()); // Validate each method AZStd::string methodName; diff --git a/Gems/Terrain/Code/Source/TerrainRenderer/Aabb2i.cpp b/Gems/Terrain/Code/Source/TerrainRenderer/Aabb2i.cpp index c38651f296..82d5a6bf2d 100644 --- a/Gems/Terrain/Code/Source/TerrainRenderer/Aabb2i.cpp +++ b/Gems/Terrain/Code/Source/TerrainRenderer/Aabb2i.cpp @@ -18,12 +18,40 @@ namespace Terrain Aabb2i Aabb2i::operator+(const Vector2i& rhs) const { - return { m_min + rhs, m_max + rhs }; + Aabb2i returnValue = *this; + returnValue += rhs; + return returnValue; + } + + Aabb2i& Aabb2i::operator+=(const Vector2i& rhs) + { + m_min += rhs; + m_max += rhs; + return *this; } Aabb2i Aabb2i::operator-(const Vector2i& rhs) const { - return *this + -rhs; + Aabb2i returnValue = *this; + returnValue -= rhs; + return returnValue; + } + + Aabb2i& Aabb2i::operator-=(const Vector2i& rhs) + { + m_min -= rhs; + m_max -= rhs; + return *this; + } + + bool Aabb2i::operator==(const Aabb2i& other) const + { + return m_min == other.m_min && m_max == other.m_max; + } + + bool Aabb2i::operator!=(const Aabb2i& other) const + { + return !(*this == other); } Aabb2i Aabb2i::GetClamped(Aabb2i rhs) const diff --git a/Gems/Terrain/Code/Source/TerrainRenderer/Aabb2i.h b/Gems/Terrain/Code/Source/TerrainRenderer/Aabb2i.h index 88f735564d..3d350c8c3a 100644 --- a/Gems/Terrain/Code/Source/TerrainRenderer/Aabb2i.h +++ b/Gems/Terrain/Code/Source/TerrainRenderer/Aabb2i.h @@ -21,7 +21,11 @@ namespace Terrain Aabb2i(const Vector2i& min, const Vector2i& max); Aabb2i operator+(const Vector2i& offset) const; + Aabb2i& operator+=(const Vector2i& offset); Aabb2i operator-(const Vector2i& offset) const; + Aabb2i& operator-=(const Vector2i& offset); + bool operator==(const Aabb2i& other) const; + bool operator!=(const Aabb2i& other) const; Aabb2i GetClamped(Aabb2i rhs) const; bool IsValid() const; diff --git a/Gems/Terrain/Code/Source/TerrainRenderer/ClipmapBounds.cpp b/Gems/Terrain/Code/Source/TerrainRenderer/ClipmapBounds.cpp new file mode 100644 index 0000000000..29cc853328 --- /dev/null +++ b/Gems/Terrain/Code/Source/TerrainRenderer/ClipmapBounds.cpp @@ -0,0 +1,278 @@ +/* + * Copyright (c) Contributors to the Open 3D Engine Project. + * For complete copyright and license terms please see the LICENSE at the root of this distribution. + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + * + */ + +#include +#include +#include + +namespace Terrain +{ + bool ClipmapBoundsRegion::operator==(const ClipmapBoundsRegion& other) const + { + return m_localAabb == other.m_localAabb && m_worldAabb.IsClose(other.m_worldAabb); + } + + bool ClipmapBoundsRegion::operator!=(const ClipmapBoundsRegion& other) const + { + return !(*this == other); + } + + ClipmapBounds::ClipmapBounds(const ClipmapBoundsDescriptor& desc) + : m_size(desc.m_size) + , m_halfSize(desc.m_size >> 1) + , m_clipmapUpdateMultiple(AZ::GetMax(desc.m_clipmapUpdateMultiple, 1)) + , m_scale(desc.m_clipToWorldScale) + , m_rcpScale(1.0f / desc.m_clipToWorldScale) + { + AZ_Error("ClipmapBounds", m_scale > 0.0f, "ClipmapBounds should have a scale that is greater than 0.0f."); + m_scale = AZ::GetMax(m_scale, AZ::Constants::FloatEpsilon); + + // recalculate m_center + m_center = GetSnappedCenter(GetClipSpaceVector(desc.m_worldSpaceCenter)); + } + + auto ClipmapBounds::UpdateCenter(const AZ::Vector2& newCenter, AZ::Aabb* untouchedRegion) -> ClipmapBoundsRegionList + { + return UpdateCenter(GetClipSpaceVector(newCenter), untouchedRegion); + } + + auto ClipmapBounds::UpdateCenter(const Vector2i& newCenter, AZ::Aabb* untouchedRegion) -> ClipmapBoundsRegionList + { + AZStd::vector updateRegions; + + // If the new snapped center isn't the same as the old, then generate update regions in clipmap space + Vector2i updatedCenter = GetSnappedCenter(newCenter); + + int32_t xDiff = updatedCenter.m_x - m_center.m_x; + int32_t updateWidth = AZStd::GetMin(abs(xDiff), m_size); + + /* + Calculate the update regions. In the common case, there will be two update regions that form either + an L or inverted L shape. To avoid double-counting the corner, it is always put in the vertical box: + _ + | | + | |____ + |_|____| + + */ + + // Calculate the vertical box + if (updatedCenter.m_x != m_center.m_x) + { + updateRegions.push_back(); + Aabb2i& updateRegion = updateRegions.back(); + + if (updatedCenter.m_x < m_center.m_x) + { + updateRegion.m_min.m_x = updatedCenter.m_x - m_halfSize; + updateRegion.m_max.m_x = updateRegion.m_min.m_x + updateWidth; + } + else + { + updateRegion.m_max.m_x = updatedCenter.m_x + m_halfSize; + updateRegion.m_min.m_x = updateRegion.m_max.m_x - updateWidth; + } + updateRegion.m_min.m_y = updatedCenter.m_y - m_halfSize; + updateRegion.m_max.m_y = updatedCenter.m_y + m_halfSize; + } + + // Calculate the horizontal box + if (updatedCenter.m_y != m_center.m_y && updateWidth < m_size) + { + updateRegions.push_back(); + Aabb2i& updateRegion = updateRegions.back(); + + uint32_t updateHeight = AZStd::GetMin(abs(updatedCenter.m_y - m_center.m_y), m_size); + if (updatedCenter.m_y < m_center.m_y) + { + updateRegion.m_min.m_y = updatedCenter.m_y - m_halfSize; + updateRegion.m_max.m_y = updateRegion.m_min.m_y + updateHeight; + } + else + { + updateRegion.m_max.m_y = updatedCenter.m_y + m_halfSize; + updateRegion.m_min.m_y = updateRegion.m_max.m_y - updateHeight; + } + + // If there was a vertical box, then don't double-count the corner of the update. + if (xDiff < 0) + { + updateRegion.m_min.m_x = updatedCenter.m_x - m_halfSize + updateWidth; + updateRegion.m_max.m_x = updatedCenter.m_x + m_halfSize; + } + else if (xDiff > 0) + { + updateRegion.m_min.m_x = updatedCenter.m_x - m_halfSize; + updateRegion.m_max.m_x = updatedCenter.m_x + m_halfSize - updateWidth; + } + } + + if (untouchedRegion) + { + // Default to the entire area being untouched. + AZ::Aabb worldBounds = GetWorldBounds(); + float maxX = worldBounds.GetMax().GetX(); + float minX = worldBounds.GetMin().GetX(); + float maxY = worldBounds.GetMax().GetY(); + float minY = worldBounds.GetMin().GetY(); + + if (updatedCenter.m_x < m_center.m_x) + { + maxX = (updatedCenter.m_x + m_halfSize) * m_rcpScale; + } + else if (updatedCenter.m_x > m_center.m_x) + { + minX = (updatedCenter.m_x - m_halfSize) * m_rcpScale; + } + if (updatedCenter.m_y < m_center.m_y) + { + maxY = (updatedCenter.m_y + m_halfSize) * m_rcpScale; + } + else if (updatedCenter.m_y > m_center.m_y) + { + minY = (updatedCenter.m_y - m_halfSize) * m_rcpScale; + } + + untouchedRegion->Set(AZ::Vector3(minX, minY, 0.0f), AZ::Vector3(maxX, maxY, 0.0f)); + } + + m_center = updatedCenter; + + m_modCenter.m_x = (m_size + (m_center.m_x % m_size)) % m_size; + m_modCenter.m_y = (m_size + (m_center.m_y % m_size)) % m_size; + + ClipmapBoundsRegionList boundsUpdate; + for (Aabb2i& updateRegion : updateRegions) + { + ClipmapBoundsRegionList update = TransformRegion(updateRegion); + boundsUpdate.insert(boundsUpdate.end(), update.begin(), update.end()); + } + + return boundsUpdate; + } + + auto ClipmapBounds::TransformRegion(AZ::Aabb worldSpaceRegion) -> ClipmapBoundsRegionList + { + AZ::Vector2 worldMin = AZ::Vector2(worldSpaceRegion.GetMin().GetX(), worldSpaceRegion.GetMin().GetY()); + AZ::Vector2 worldMax = AZ::Vector2(worldSpaceRegion.GetMax().GetX(), worldSpaceRegion.GetMax().GetY()); + + Aabb2i clipSpaceRegion; + clipSpaceRegion.m_min = GetClipSpaceVector(worldMin); + clipSpaceRegion.m_max = GetClipSpaceVector(worldMax); + + return TransformRegion(clipSpaceRegion); + } + + auto ClipmapBounds::TransformRegion(Aabb2i region) -> ClipmapBoundsRegionList + { + ClipmapBoundsRegionList transformedRegions; + + Aabb2i clampedRegion = region.GetClamped(GetLocalBounds()); + if (!clampedRegion.IsValid()) + { + // Early out if the region is outside the bounds + return transformedRegions; + } + + Vector2i minCorner = m_center - m_halfSize; + + Vector2i minBoundary; + minBoundary.m_x = (minCorner.m_x / m_size - (minCorner.m_x < 0 ? 1 : 0)) * m_size; + minBoundary.m_y = (minCorner.m_y / m_size - (minCorner.m_y < 0 ? 1 : 0)) * m_size; + + Aabb2i bottomLeftTile = Aabb2i(minBoundary, minBoundary + m_size); + + // For each of the 4 quadrants: + auto calculateQuadrant = [&](Aabb2i tile) + { + Aabb2i regionClampedToTile = clampedRegion.GetClamped(tile); + if (regionClampedToTile.IsValid()) + { + transformedRegions.push_back( + ClipmapBoundsRegion({ + GetWorldSpaceAabb(regionClampedToTile), + regionClampedToTile - tile.m_min + }) + ); + } + }; + + calculateQuadrant(bottomLeftTile); + calculateQuadrant(bottomLeftTile + Vector2i(m_size, 0)); + calculateQuadrant(bottomLeftTile + Vector2i(0, m_size)); + calculateQuadrant(bottomLeftTile + Vector2i(m_size, m_size)); + + return transformedRegions; + } + + AZ::Aabb ClipmapBounds::GetWorldBounds() const + { + Aabb2i localBounds = GetLocalBounds(); + + return AZ::Aabb::CreateFromMinMaxValues( + localBounds.m_min.m_x * m_scale, localBounds.m_min.m_y * m_scale, 0.0f, + localBounds.m_max.m_x * m_scale, localBounds.m_max.m_y * m_scale, 0.0f); + } + + float ClipmapBounds::GetWorldSpaceSafeDistance() const + { + return (m_halfSize - m_clipmapUpdateMultiple) * m_scale; + } + + Vector2i ClipmapBounds::GetSnappedCenter(const Vector2i& center) + { + Vector2i updatedCenter = m_center; + + // Update the snapped center if the new center has drifted beyond the margin + auto UpdateDim = [&](int32_t centerDim, int32_t& snappedCenterDim) -> void + { + int32_t diff = centerDim - snappedCenterDim; + + int32_t scaledCenterDim = (centerDim / m_clipmapUpdateMultiple); + if (centerDim < 0) + { + // Force rounding down for negatives + scaledCenterDim--; + } + + if (diff >= m_clipmapUpdateMultiple) + { + snappedCenterDim = scaledCenterDim * m_clipmapUpdateMultiple; + } + if (diff < -m_clipmapUpdateMultiple) + { + snappedCenterDim = (scaledCenterDim + 1) * m_clipmapUpdateMultiple; + } + }; + UpdateDim(center.m_x, updatedCenter.m_x); + UpdateDim(center.m_y, updatedCenter.m_y); + + return updatedCenter; + } + + Aabb2i ClipmapBounds::GetLocalBounds() const + { + return Aabb2i(m_center - m_halfSize, m_center + m_halfSize); + } + + Vector2i ClipmapBounds::GetClipSpaceVector(const AZ::Vector2& worldSpaceVector) const + { + // Get rounded integer x/y coords in clipmap space. + int32_t x = AZStd::lround(worldSpaceVector.GetX() * m_rcpScale); + int32_t y = AZStd::lround(worldSpaceVector.GetY() * m_rcpScale); + return Vector2i(x, y); + } + + AZ::Aabb ClipmapBounds::GetWorldSpaceAabb(const Aabb2i& clipSpaceAabb) const + { + return AZ::Aabb::CreateFromMinMaxValues( + clipSpaceAabb.m_min.m_x * m_scale, clipSpaceAabb.m_min.m_y * m_scale, 0.0f, + clipSpaceAabb.m_max.m_x * m_scale, clipSpaceAabb.m_max.m_y * m_scale, 0.0f + ); + } +} diff --git a/Gems/Terrain/Code/Source/TerrainRenderer/ClipmapBounds.h b/Gems/Terrain/Code/Source/TerrainRenderer/ClipmapBounds.h new file mode 100644 index 0000000000..1aac4d8c9e --- /dev/null +++ b/Gems/Terrain/Code/Source/TerrainRenderer/ClipmapBounds.h @@ -0,0 +1,159 @@ +/* + * Copyright (c) Contributors to the Open 3D Engine Project. + * For complete copyright and license terms please see the LICENSE at the root of this distribution. + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + * + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace Terrain +{ + + struct ClipmapBoundsDescriptor + { + //! Width and height of the clipmap in texels. + uint32_t m_size = 1024; + + //! Current center location of the clipmap in world space + AZ::Vector2 m_worldSpaceCenter = AZ::Vector2::CreateZero(); + + // Updates to the clipmap will be produced in multiples of this value. This + // allows for larger but less frequent updates, and gives some wiggle room + // for each movement before an update is triggered. + // Note: This also means that whatever uses this clipmap should only ever + // display m_size - (2 * m_clipmapUpdateMultiple) pixels from the clipmap. + // Use GetWorldSpaceSafeDistance() to get the safe distance from center. + uint32_t m_clipmapUpdateMultiple = 4; + + //! Scale of the clip map compared to the world. A scale of 0.5 means that + //! a clipmap of size 1024 would cover 512 meters. + float m_clipToWorldScale = 1.0f; + }; + + struct ClipmapBoundsRegion + { + //! The world bounds of the updated region. Z is ignored. + AZ::Aabb m_worldAabb; + + //! The clipmaps bounds of the updated region. Will always be between 0 and size. + //! Min inclusive, max exclusive. + Aabb2i m_localAabb; + + bool operator==(const ClipmapBoundsRegion& other) const; + bool operator!=(const ClipmapBoundsRegion& other) const; + }; + + // This class manages a single clipmap region. A clipmap is a virtual view into a much larger + // region, where the clipmap view is centered around a point like the current camera position. + // The clipmap texture wraps to form a repeating grid and never moves, but only data within + // the clipmap bounds is actually valid. This makes looking up data in the clipmap trivial + // since it's just the world coordinate scaled by some amount. This technique also allows for + // only the edge areas of the clipmap to be updated as the center point moves around the world. + // + // The edges of the clipmap bounds will typically run through the texture, dividing it into 4 + // regions, except in cases where the clipmap bounds happen to be aligned with the underlying + // grid. This means whenever some bounding box needs to be updated in the clipmap, it may actually + // translate to 4 different areas of the underlying texture - one for each quadrant. + // + // This class aids in figuring out which areas of a clipmap need to be updated as its center point + // moves around in the world, and can map a single region that needs to be updated into several + // separate regions for each quadrant. + + /* + ___________________________ + | | | | | Clipmap Clipmap + | | | | | Bounds Texture (Tiled) + |______|______|______|______| ______ ______ + | | _|____ | | | | | |____|_| + | | | | | | | |_|_*__| | | | + |______|____|_|_*__|_|______| |_|____| |_*__|_| + | | |_|____| | | + | | | | | + |______|______|______|______| + | | | | | + | | | | | + |______|______|______|______| + + */ + + class ClipmapBounds + { + public: + + explicit ClipmapBounds(const ClipmapBoundsDescriptor& desc); + ~ClipmapBounds() = default; + + using ClipmapBoundsRegionList = AZStd::vector; + + //! Updates the clipmap bounds using a world coordinate center position and returns + //! 0-2 regions that need to be updated due to moving beyond the margins. These update + //! regions will always be at least the size of the margin, and will represent horizontal + //! and/or vertical strips along the edges of the clipmap. An optional untouched region + //! aabb can be passed to this function to get an aabb of areas inside the bounds of the + //! clipmap but not updated by the center moving. This can be useful in cases where part + //! of the bounds of the clipmap is dirty, but areas that will already be updated due + //! to the center moving shouldn't be updated twice. + ClipmapBoundsRegionList UpdateCenter(const AZ::Vector2& newCenter, AZ::Aabb* untouchedRegion = nullptr); + + //! Updates the clipmap bounds using a position in clipmap space (no scaling) and returns + //! 0-2 regions that need to be updated due to moving beyond the margins. These update + //! regions will always be at least the size of the margin, and will represent horizontal + //! and/or vertical strips along the edges of the clipmap. An optional untouched region + //! aabb can be passed to this function to get an aabb of areas inside the bounds of the + //! clipmap but not updated by the center moving. This can be useful in cases where part + //! of the bounds of the clipmap is dirty, but areas that will already be updated due + //! to the center moving shouldn't be updated twice. + ClipmapBoundsRegionList UpdateCenter(const Vector2i& newCenter, AZ::Aabb* untouchedRegion = nullptr); + + //! Takes in a single world space region and transforms it into 0-4 regions in the clipmap clamped + //! to the bounds of the clipmap. + ClipmapBoundsRegionList TransformRegion(AZ::Aabb worldSpaceRegion); + + //! Takes in a single unscaled clipmap space region and transforms it into 0-4 regions in the clipmap clamped + //! to the bounds of the clipmap. + ClipmapBoundsRegionList TransformRegion(Aabb2i clipSpaceRegion); + + //! Returns the bounds covered by this clipmap in world space. Z component is always 0. + AZ::Aabb GetWorldBounds() const; + + //! Returns the safe x and y distance from the center in world space. This is based on the scale, + //! clipmap size, and m_clipmapUpdateMultiple. For example, a clipmap size 1024 with scale + //! 0.25 and margin of 4 would have a safe distance of (1024 * 0.5 - 4) * 0.25 = 127.0f. + float GetWorldSpaceSafeDistance() const; + + private: + + //! Returns the center point snapped to a multiple of m_clipmapUpdateMultiple. This isn't + //! a simple rounding operation. The value returned will only be different from the curernt + //! center if the value passed in is greater than m_clipmapUpdateMultiple away from the center. + Vector2i GetSnappedCenter(const Vector2i& center); + + //! Returns the bounds covered by the clipmap in local space + Aabb2i GetLocalBounds() const; + + //! Applies scale and averages a world space vector to get a clip space vector. + Vector2i GetClipSpaceVector(const AZ::Vector2& worldSpaceVector) const; + + //! Applies inverse scale to get a world aabb from clip space aabb. + AZ::Aabb GetWorldSpaceAabb(const Aabb2i& clipSpaceAabb) const; + + Vector2i m_center; + Vector2i m_modCenter; + int32_t m_size; + int32_t m_halfSize; + int32_t m_clipmapUpdateMultiple; + float m_scale; + float m_rcpScale; + + }; + +} diff --git a/Gems/Terrain/Code/Source/TerrainRenderer/Vector2i.cpp b/Gems/Terrain/Code/Source/TerrainRenderer/Vector2i.cpp index 1df945b88c..058b9ae76f 100644 --- a/Gems/Terrain/Code/Source/TerrainRenderer/Vector2i.cpp +++ b/Gems/Terrain/Code/Source/TerrainRenderer/Vector2i.cpp @@ -7,35 +7,90 @@ */ #include +#include namespace Terrain { - auto Vector2i::operator+(const Vector2i& rhs) const -> Vector2i + + Vector2i::Vector2i(int32_t x, int32_t y) + : m_x(x) + , m_y(y) + {} + + Vector2i::Vector2i(uint32_t value) + : m_x(aznumeric_cast(value)) + , m_y(aznumeric_cast(value)) + {} + + Vector2i::Vector2i(int32_t value) + : m_x(value) + , m_y(value) + {} + + Vector2i Vector2i::operator+(const Vector2i& rhs) const { - Vector2i offsetPoint = *this; - offsetPoint += rhs; - return offsetPoint; + Vector2i returnPoint = *this; + returnPoint += rhs; + return returnPoint; } - auto Vector2i::operator+=(const Vector2i& rhs) -> Vector2i& + Vector2i& Vector2i::operator+=(const Vector2i& rhs) { m_x += rhs.m_x; m_y += rhs.m_y; return *this; } - auto Vector2i::operator-(const Vector2i& rhs) const -> Vector2i + Vector2i Vector2i::operator-(const Vector2i& rhs) const { return *this + -rhs; } - auto Vector2i::operator-=(const Vector2i& rhs) -> Vector2i& + Vector2i& Vector2i::operator-=(const Vector2i& rhs) { return *this += -rhs; } - auto Vector2i::operator-() const -> Vector2i + Vector2i Vector2i::operator-() const { return {-m_x, -m_y}; } + + Vector2i Vector2i::operator*(const Vector2i& rhs) const + { + Vector2i returnPoint = *this; + returnPoint *= rhs; + return returnPoint; + } + + Vector2i& Vector2i::operator*=(const Vector2i& rhs) + { + m_x *= rhs.m_x; + m_y *= rhs.m_y; + return *this; + } + + Vector2i Vector2i::operator/(const Vector2i& rhs) const + { + Vector2i returnPoint = *this; + returnPoint /= rhs; + return returnPoint; + } + + Vector2i& Vector2i::operator/=(const Vector2i& rhs) + { + m_x /= rhs.m_x; + m_y /= rhs.m_y; + return *this; + } + + bool Vector2i::operator==(const Vector2i& rhs) const + { + return rhs.m_x == m_x && rhs.m_y == m_y; + } + + bool Vector2i::operator!=(const Vector2i& rhs) const + { + return !(*this == rhs); + } } diff --git a/Gems/Terrain/Code/Source/TerrainRenderer/Vector2i.h b/Gems/Terrain/Code/Source/TerrainRenderer/Vector2i.h index 1244972fe9..a7c598a91f 100644 --- a/Gems/Terrain/Code/Source/TerrainRenderer/Vector2i.h +++ b/Gems/Terrain/Code/Source/TerrainRenderer/Vector2i.h @@ -16,14 +16,26 @@ namespace Terrain { public: + Vector2i() = default; + Vector2i(int32_t x, int32_t y); + Vector2i(uint32_t value); + Vector2i(int32_t value); + Vector2i operator+(const Vector2i& rhs) const; Vector2i& operator+=(const Vector2i& rhs); Vector2i operator-(const Vector2i& rhs) const; Vector2i& operator-=(const Vector2i& rhs); Vector2i operator-() const; + Vector2i operator*(const Vector2i& rhs) const; + Vector2i& operator*=(const Vector2i& rhs); + Vector2i operator/(const Vector2i& rhs) const; + Vector2i& operator/=(const Vector2i& rhs); + + bool operator==(const Vector2i& rhs) const; + bool operator!=(const Vector2i& rhs) const; + int32_t m_x{ 0 }; int32_t m_y{ 0 }; - }; } diff --git a/Gems/Terrain/Code/Tests/ClipmapBoundsTests.cpp b/Gems/Terrain/Code/Tests/ClipmapBoundsTests.cpp new file mode 100644 index 0000000000..9b09deb8f5 --- /dev/null +++ b/Gems/Terrain/Code/Tests/ClipmapBoundsTests.cpp @@ -0,0 +1,336 @@ +/* + * Copyright (c) Contributors to the Open 3D Engine Project. + * For complete copyright and license terms please see the LICENSE at the root of this distribution. + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + * + */ + +#include +#include + +#include +#include + +#include +#include + +namespace UnitTest +{ + class ClipmapBoundsTests + : public UnitTest::AllocatorsTestFixture + { + public: + void CheckTransformRegionFullBounds(const Terrain::ClipmapBoundsDescriptor& desc); + }; + + void ClipmapBoundsTests::CheckTransformRegionFullBounds(const Terrain::ClipmapBoundsDescriptor& desc) + { + Terrain::ClipmapBounds bounds(desc); + + AZ::Aabb worldBounds = bounds.GetWorldBounds(); + float worldBoundsSize = worldBounds.GetXExtent(); + + auto output = bounds.TransformRegion(worldBounds); + ASSERT_EQ(output.size(), 4); + + AZ::Vector2 boundary = AZ::Vector2( + floorf(worldBounds.GetMax().GetX() / worldBoundsSize), + floorf(worldBounds.GetMax().GetY() / worldBoundsSize) + ) * worldBoundsSize; + + Terrain::Vector2i localMax = { + aznumeric_cast(AZStd::lround(desc.m_worldSpaceCenter.GetX() / desc.m_clipToWorldScale)), + aznumeric_cast(AZStd::lround(desc.m_worldSpaceCenter.GetY() / desc.m_clipToWorldScale)) + }; + localMax += aznumeric_cast(desc.m_size / 2ul); + + int32_t intSize = int32_t(desc.m_size); + Terrain::Vector2i localBoundary = { + ((localMax.m_x % intSize) + intSize) % intSize, + ((localMax.m_y % intSize) + intSize) % intSize + }; + + // Check each quadrant returned + AZStd::vector expected; + expected.resize(4); + + expected.at(0).m_localAabb = Terrain::Aabb2i({localBoundary.m_x, localBoundary.m_y}, {intSize, intSize}); + expected.at(0).m_worldAabb = AZ::Aabb::CreateFromMinMaxValues( + worldBounds.GetMin().GetX(), worldBounds.GetMin().GetY(), 0.0f, + boundary.GetX(), boundary.GetY(), 0.0f); + + expected.at(1).m_localAabb = Terrain::Aabb2i({0, localBoundary.m_y}, {localBoundary.m_x, intSize}); + expected.at(1).m_worldAabb = AZ::Aabb::CreateFromMinMaxValues( + boundary.GetX(), worldBounds.GetMin().GetY(), 0.0f, + worldBounds.GetMax().GetX(), boundary.GetY(), 0.0f); + + expected.at(2).m_localAabb = Terrain::Aabb2i({localBoundary.m_x, 0}, {intSize, localBoundary.m_y}); + expected.at(2).m_worldAabb = AZ::Aabb::CreateFromMinMaxValues( + worldBounds.GetMin().GetX(), boundary.GetY(), 0.0f, + boundary.GetX(), worldBounds.GetMax().GetY(), 0.0f); + + expected.at(3).m_localAabb = Terrain::Aabb2i({ 0, 0 }, { localBoundary.m_x, localBoundary.m_y }); + expected.at(3).m_worldAabb = AZ::Aabb::CreateFromMinMaxValues( + boundary.GetX(), boundary.GetY(), 0.0f, + worldBounds.GetMax().GetX(), worldBounds.GetMax().GetY(), 0.0f); + + EXPECT_THAT(output, ::testing::UnorderedElementsAreArray(expected)); + } + + TEST_F(ClipmapBoundsTests, Construction) + { + Terrain::ClipmapBoundsDescriptor desc; + Terrain::ClipmapBounds bounds(desc); + } + + TEST_F(ClipmapBoundsTests, BasicTransform) + { + // Create clipmap around 0.0, so it's perfectly divided into 4 quadrants + Terrain::ClipmapBoundsDescriptor desc; + desc.m_worldSpaceCenter = AZ::Vector2(0.0f, 0.0f); + desc.m_clipmapUpdateMultiple = 0; + desc.m_clipToWorldScale = 1.0f; + desc.m_size = 1024; + Terrain::ClipmapBounds bounds(desc); + + auto output = bounds.TransformRegion(AZ::Aabb::CreateFromMinMaxValues(-512.0f, -512.0f, 0.0f, 512.0f, 512.0f, 0.0f)); + + ASSERT_EQ(output.size(), 4); + + // Check each quadrant returned + EXPECT_EQ(output.at(0).m_localAabb, Terrain::Aabb2i({512, 512}, {1024, 1024})); + EXPECT_TRUE(output.at(0).m_worldAabb.IsClose(AZ::Aabb::CreateFromMinMaxValues(-512.0f, -512.0f, 0.0f, 0.0f, 0.0f, 0.0f))); + EXPECT_EQ(output.at(1).m_localAabb, Terrain::Aabb2i({0, 512}, {512, 1024})); + EXPECT_TRUE(output.at(1).m_worldAabb.IsClose(AZ::Aabb::CreateFromMinMaxValues(0.0f, -512.0f, 0.0f, 512.0f, 0.0f, 0.0f))); + EXPECT_EQ(output.at(2).m_localAabb, Terrain::Aabb2i({512, 0}, {1024, 512})); + EXPECT_TRUE(output.at(2).m_worldAabb.IsClose(AZ::Aabb::CreateFromMinMaxValues(-512.0f, 0.0f, 0.0f, 0.0f, 512.0f, 0.0f))); + EXPECT_EQ(output.at(3).m_localAabb, Terrain::Aabb2i({0, 0}, {512, 512})); + EXPECT_TRUE(output.at(3).m_worldAabb.IsClose(AZ::Aabb::CreateFromMinMaxValues(0.0f, 0.0f, 0.0f, 512.0f, 512.0f, 0.0f))); + } + + TEST_F(ClipmapBoundsTests, ScaledTransform) + { + // Create clipmap around 0.0, so it's perfectly divided into 4 quadrants, but half-scale + Terrain::ClipmapBoundsDescriptor desc; + desc.m_worldSpaceCenter = AZ::Vector2(0.0f, 0.0f); + desc.m_clipmapUpdateMultiple = 0; + desc.m_clipToWorldScale = 0.5f; + desc.m_size = 1024; + Terrain::ClipmapBounds bounds(desc); + + auto output = bounds.TransformRegion(AZ::Aabb::CreateFromMinMaxValues(-256.0f, -256.0f, 0.0f, 256.0f, 256.0f, 0.0f)); + + ASSERT_EQ(output.size(), 4); + + // Check each quadrant returned + EXPECT_EQ(output.at(0).m_localAabb, Terrain::Aabb2i({512, 512}, {1024, 1024})); + EXPECT_TRUE(output.at(0).m_worldAabb.IsClose(AZ::Aabb::CreateFromMinMaxValues(-256.0f, -256.0f, 0.0f, 0.0f, 0.0f, 0.0f))); + EXPECT_EQ(output.at(1).m_localAabb, Terrain::Aabb2i({0, 512}, {512, 1024})); + EXPECT_TRUE(output.at(1).m_worldAabb.IsClose(AZ::Aabb::CreateFromMinMaxValues(0.0f, -256.0f, 0.0f, 256.0f, 0.0f, 0.0f))); + EXPECT_EQ(output.at(2).m_localAabb, Terrain::Aabb2i({512, 0}, {1024, 512})); + EXPECT_TRUE(output.at(2).m_worldAabb.IsClose(AZ::Aabb::CreateFromMinMaxValues(-256.0f, 0.0f, 0.0f, 0.0f, 256.0f, 0.0f))); + EXPECT_EQ(output.at(3).m_localAabb, Terrain::Aabb2i({0, 0}, {512, 512})); + EXPECT_TRUE(output.at(3).m_worldAabb.IsClose(AZ::Aabb::CreateFromMinMaxValues(0.0f, 0.0f, 0.0f, 256.0f, 256.0f, 0.0f))); + } + + TEST_F(ClipmapBoundsTests, ComplexTransformsFullBounds) + { + // Check 4 different clipmaps - one in completely positive space, one in negative space, and two straddling the axis + + // Clipmap in negative space + { + Terrain::ClipmapBoundsDescriptor desc; + desc.m_worldSpaceCenter = AZ::Vector2(-1234.0f, -5432.0f); + desc.m_clipmapUpdateMultiple = 0; + desc.m_clipToWorldScale = 0.75f; + desc.m_size = 512; + CheckTransformRegionFullBounds(desc); + } + + // Clipmap in positive space + { + Terrain::ClipmapBoundsDescriptor desc; + desc.m_worldSpaceCenter = AZ::Vector2(1234.0f, 5432.0f); + desc.m_clipmapUpdateMultiple = 0; + desc.m_clipToWorldScale = 1.25f; + desc.m_size = 1024; + CheckTransformRegionFullBounds(desc); + } + + // Clipmap on x axis + { + Terrain::ClipmapBoundsDescriptor desc; + desc.m_worldSpaceCenter = AZ::Vector2(1234.0f, -100.0f); + desc.m_clipmapUpdateMultiple = 0; + desc.m_clipToWorldScale = 1.5f; + desc.m_size = 256; + CheckTransformRegionFullBounds(desc); + } + // Clipmap on y axis + { + Terrain::ClipmapBoundsDescriptor desc; + desc.m_worldSpaceCenter = AZ::Vector2(-100.0f, 5432.0f); + desc.m_clipmapUpdateMultiple = 0; + desc.m_clipToWorldScale = 1.0f; + desc.m_size = 2048; + CheckTransformRegionFullBounds(desc); + } + } + + TEST_F(ClipmapBoundsTests, TransformSmallBounds) + { + // Create clipmap around 0.0, so it's perfectly divided into 4 quadrants + Terrain::ClipmapBoundsDescriptor desc; + desc.m_worldSpaceCenter = AZ::Vector2(0.0f, 0.0f); + desc.m_clipmapUpdateMultiple = 0; + desc.m_clipToWorldScale = 1.0f; + desc.m_size = 1024; + Terrain::ClipmapBounds bounds(desc); + + { + // Single quadrant positive + + AZ::Aabb smallArea = AZ::Aabb::CreateFromMinMaxValues( + 10.0f, 10.0f, 0.0f, 50.0f, 50.0f, 0.0f + ); + + auto output = bounds.TransformRegion(smallArea); + + ASSERT_EQ(output.size(), 1); + EXPECT_EQ(output.at(0).m_localAabb, Terrain::Aabb2i({10, 10}, {50, 50})); + EXPECT_TRUE(output.at(0).m_worldAabb.IsClose(AZ::Aabb::CreateFromMinMaxValues(10.0f, 10.0f, 0.0f, 50.0f, 50.0f, 0.0f))); + } + + { + // Single quadrant negative + + AZ::Aabb smallArea = AZ::Aabb::CreateFromMinMaxValues( + -50.0f, -50.0f, 0.0f, -10.0f, -10.0f, 0.0f + ); + + auto output = bounds.TransformRegion(smallArea); + + ASSERT_EQ(output.size(), 1); + EXPECT_EQ(output.at(0).m_localAabb, Terrain::Aabb2i({974, 974}, {1014, 1014})); + EXPECT_TRUE(output.at(0).m_worldAabb.IsClose(AZ::Aabb::CreateFromMinMaxValues(-50.0f, -50.0f, 0.0f, -10.0f, -10.0f, 0.0f))); + } + + { + // 2 quadrant positive + + AZ::Aabb smallArea = AZ::Aabb::CreateFromMinMaxValues( + 10.0f, -10.0f, 0.0f, 50.0f, 50.0f, 0.0f + ); + + auto output = bounds.TransformRegion(smallArea); + + ASSERT_EQ(output.size(), 2); + EXPECT_EQ(output.at(0).m_localAabb, Terrain::Aabb2i({10, 1014}, {50, 1024})); + EXPECT_TRUE(output.at(0).m_worldAabb.IsClose(AZ::Aabb::CreateFromMinMaxValues(10.0f, -10.0f, 0.0f, 50.0f, 0.0f, 0.0f))); + EXPECT_EQ(output.at(1).m_localAabb, Terrain::Aabb2i({10, 0}, {50, 50})); + EXPECT_TRUE(output.at(1).m_worldAabb.IsClose(AZ::Aabb::CreateFromMinMaxValues(10.0f, 0.0f, 0.0f, 50.0f, 50.0f, 0.0f))); + } + } + + TEST_F(ClipmapBoundsTests, MarginReducesUpdates) + { + // With a margin defined, the bounds should only trigger updates when the camera moves outside the margins + + // Create clipmap around 0.0, so it's perfectly divided into 4 quadrants + Terrain::ClipmapBoundsDescriptor desc; + desc.m_worldSpaceCenter = AZ::Vector2(0.0f, 0.0f); + desc.m_clipmapUpdateMultiple = 16; + desc.m_clipToWorldScale = 1.0f; + desc.m_size = 1024; + Terrain::ClipmapBounds bounds(desc); + + // center moved forward to 10, still within margin + auto output1 = bounds.UpdateCenter(AZ::Vector2(10.0f, 10.0f)); + EXPECT_EQ(output1.size(), 0); + // center moved forwrd to 20, beyond margin, triggers update + auto output2 = bounds.UpdateCenter(AZ::Vector2(20.0f, 20.0f)); + EXPECT_GT(output2.size(), 0); + // center moved back to 10, still within margin + auto output3 = bounds.UpdateCenter(AZ::Vector2(10.0f, 10.0f)); + EXPECT_EQ(output3.size(), 0); + // center moved back to 0, still within margin (on edge) + auto output4 = bounds.UpdateCenter(AZ::Vector2(0.0f, 0.0f)); + EXPECT_EQ(output4.size(), 0); + // center moved back to -10, beyond margin, triggers update + auto output5 = bounds.UpdateCenter(AZ::Vector2(-10.0f, -10.0f)); + EXPECT_GT(output5.size(), 0); + } + + TEST_F(ClipmapBoundsTests, CenterMovementUpdates) + { + // Create clipmap around 0.0, so it's perfectly divided into 4 quadrants + Terrain::ClipmapBoundsDescriptor desc; + desc.m_worldSpaceCenter = AZ::Vector2(0.0f, 0.0f); + desc.m_clipmapUpdateMultiple = 16; + desc.m_clipToWorldScale = 1.0f; + desc.m_size = 1024; + Terrain::ClipmapBounds bounds(desc); + + { + AZ::Aabb untouchedRegion = AZ::Aabb::CreateNull(); + auto output = bounds.UpdateCenter(AZ::Vector2(20.0f, 20.0f), &untouchedRegion); + ASSERT_EQ(output.size(), 4); + + // Instead of checking bounds directly, do several checks to make sure the bounds are appropriate. Since + // the center moved just outside the margin along the diagonal, we should expect two edges to be updated + // that are the width of the margin. + + // 1. The number of pixels updated in the bounds should be two sides of margin width + float pixelsCovered = 0; + for (auto& region : output) + { + // Note: GetSurfaceArea() returns the area of all 6 sides of the aabb. With a Z extent of 0, that + // means that only the top and bottom will be counted, so we need to multiply by 0.5. + pixelsCovered += region.m_worldAabb.GetSurfaceArea() * 0.5f; + } + + // Two edges of margin * size, minus the overlap in the corner. + const uint32_t updateMultiple = desc.m_clipmapUpdateMultiple; + float expectedCoverage = updateMultiple * desc.m_size * 2.0f - updateMultiple * updateMultiple; + EXPECT_NEAR(pixelsCovered, expectedCoverage, 0.0001f); + + // 2. The untouched region area should match what's expected + float untouchedRegionArea = untouchedRegion.GetSurfaceArea() * 0.5f; + float expectedUntouchedRegionSide = aznumeric_cast(desc.m_size - desc.m_clipmapUpdateMultiple); + float expectedUntouchedRegionArea = expectedUntouchedRegionSide * expectedUntouchedRegionSide; + EXPECT_NEAR(untouchedRegionArea, expectedUntouchedRegionArea, 0.0001f); + + // 3. All of the update regions should be inside the world bounds of the clipmap + AZ::Aabb worldBounds = bounds.GetWorldBounds(); + for (auto& region : output) + { + EXPECT_EQ(region.m_worldAabb.GetClamped(worldBounds), region.m_worldAabb); + } + + // 4. The untouched region should also be inside the world bounds of the clipmap; + EXPECT_EQ(untouchedRegion.GetClamped(worldBounds), untouchedRegion); + + // 5. None of the update regions should overlap each other or the untouched region + + // push the untouched region on the vector to make comparisons easier + output.push_back(Terrain::ClipmapBoundsRegion({untouchedRegion, Terrain::Aabb2i({}) })); + for (uint32_t i = 0; i < output.size(); ++i) + { + const AZ::Aabb boundsToCheck = output.at(i).m_worldAabb; + for (uint32_t j = i + 1; j < output.size(); ++j) + { + // AZ::Aabb::Overlaps() counts touching edges as overlapping, so we need a strict version + auto strictOverlaps = [](const AZ::Aabb& aabb1, const AZ::Aabb& aabb2) -> bool + { + return aabb1.GetMin().IsLessThan(aabb2.GetMax()) && + aabb1.GetMax().IsGreaterThan(aabb2.GetMin()); + }; + EXPECT_FALSE(strictOverlaps(boundsToCheck, output.at(j).m_worldAabb)); + } + } + + } + + } +} diff --git a/Gems/Terrain/Code/terrain_files.cmake b/Gems/Terrain/Code/terrain_files.cmake index 4b994c2587..53a7c6ac6d 100644 --- a/Gems/Terrain/Code/terrain_files.cmake +++ b/Gems/Terrain/Code/terrain_files.cmake @@ -35,6 +35,10 @@ set(FILES Source/TerrainRenderer/Components/TerrainMacroMaterialComponent.h Source/TerrainRenderer/Aabb2i.cpp Source/TerrainRenderer/Aabb2i.h + Source/TerrainRenderer/BindlessImageArrayHandler.cpp + Source/TerrainRenderer/BindlessImageArrayHandler.h + Source/TerrainRenderer/ClipmapBounds.cpp + Source/TerrainRenderer/ClipmapBounds.h Source/TerrainRenderer/TerrainFeatureProcessor.cpp Source/TerrainRenderer/TerrainFeatureProcessor.h Source/TerrainRenderer/TerrainDetailMaterialManager.cpp @@ -43,8 +47,6 @@ set(FILES Source/TerrainRenderer/TerrainMacroMaterialManager.h Source/TerrainRenderer/TerrainMeshManager.cpp Source/TerrainRenderer/TerrainMeshManager.h - Source/TerrainRenderer/BindlessImageArrayHandler.cpp - Source/TerrainRenderer/BindlessImageArrayHandler.h Source/TerrainRenderer/TerrainAreaMaterialRequestBus.h Source/TerrainRenderer/TerrainMacroMaterialBus.cpp Source/TerrainRenderer/TerrainMacroMaterialBus.h diff --git a/Gems/Terrain/Code/terrain_tests_files.cmake b/Gems/Terrain/Code/terrain_tests_files.cmake index 1a46fdd203..39cbe0861b 100644 --- a/Gems/Terrain/Code/terrain_tests_files.cmake +++ b/Gems/Terrain/Code/terrain_tests_files.cmake @@ -7,14 +7,15 @@ # set(FILES - Tests/TerrainTest.cpp - Tests/TerrainSystemTest.cpp + Tests/ClipmapBoundsTests.cpp Tests/LayerSpawnerTests.cpp - Tests/TerrainPhysicsColliderTests.cpp - Tests/SurfaceMaterialsListTest.cpp Tests/MockAxisAlignedBoxShapeComponent.h Tests/TerrainHeightGradientListTests.cpp Tests/TerrainMacroMaterialTests.cpp + Tests/SurfaceMaterialsListTest.cpp + Tests/TerrainPhysicsColliderTests.cpp Tests/TerrainSurfaceGradientListTests.cpp Tests/TerrainSystemBenchmarks.cpp + Tests/TerrainSystemTest.cpp + Tests/TerrainTest.cpp ) diff --git a/cmake/FileUtil.cmake b/cmake/FileUtil.cmake index 5721cf452c..aa63b97c9d 100644 --- a/cmake/FileUtil.cmake +++ b/cmake/FileUtil.cmake @@ -153,6 +153,36 @@ function(ly_get_last_path_segment_concat_sha256 absolute_path output_path) set(${output_path} ${last_path_segment_sha256_path} PARENT_SCOPE) endfunction() +#! ly_get_root_subdirectory_which_is_parent: Locates the root source directory added the input directory +# as a subdirectory of the build, which an actual prefix of the input directory +# This is done by recursing through the PARENT_DIRECTORY "DIRECTORY" property +# The use for this is to locate the top most directory which called add_subdirectory from any input path +# i.e Given an +# LY_ROOT_FOLDER = D:\o3de +# EXTERNAL_SUBDIRS = [D:\TestGem, D:\o3de\Gems\MyGem] +# The LY_ROOT_FOLDER is responsible for invoking add_subdirectory on the external subdirectories +# so it in the PARENT_DIRECTORY property, of the subdirectory, though it might not be an actual "parent" +# If the input path to this function is D:\TestGem\Code, then the return value is D:\TestGem +# If the input path to this function is D:\o3de\Gems\MyGem, then the return value is D:\o3de + +# \arg:absolute_path - directory to locate top most parent "subdirectory", which is an "parent" of the input +# \return:output_path- top most parent subdirectory, which is actual parent(i.e a prefix) +function(ly_get_root_subdirectory_which_is_parent absolute_path output_path) + # Walk up the parent add_subdirectory calls until a parent directory which is not a prefix of the target directory + # is found + cmake_path(SET candidate_path ${absolute_path}) + get_property(parent_subdir DIRECTORY ${candidate_path} PROPERTY PARENT_DIRECTORY) + cmake_path(IS_PREFIX parent_subdir ${candidate_path} is_parent_subdir) + while(parent_subdir AND is_parent_subdir) + cmake_path(SET candidate_path "${parent_subdir}") + get_property(parent_subdir DIRECTORY ${candidate_path} PROPERTY PARENT_DIRECTORY) + cmake_path(IS_PREFIX parent_subdir ${candidate_path} is_parent_subdir) + endwhile() + + message(DEBUG "Root subdirectory of path \"${absolute_path}\" is \"${candidate_path}\"") + set(${output_path} ${candidate_path} PARENT_SCOPE) +endfunction() + #! ly_get_engine_relative_source_dir: Attempts to form a path relative to the BASE_DIRECTORY. # If that fails the last path segment of the absolute_target_source_dir concatenated with a SHA256 hash to form a target directory # \arg:BASE_DIRECTORY - Directory to base relative path against. Defaults to LY_ROOT_FOLDER @@ -167,14 +197,16 @@ function(ly_get_engine_relative_source_dir absolute_target_source_dir output_sou endif() # Get a relative target source directory to the LY root folder if possible - # Otherwise use the final component name + # Otherwise use the top most source directory which led to calling add_subdirectory on the input directory + ly_get_root_subdirectory_which_is_parent(${absolute_target_source_dir} root_subdir_of_target) + cmake_path(RELATIVE_PATH absolute_target_source_dir BASE_DIRECTORY ${root_subdir_of_target} OUTPUT_VARIABLE relative_target_source_dir) + cmake_path(IS_PREFIX LY_ROOT_FOLDER ${absolute_target_source_dir} is_target_source_dir_subdirectory_of_engine) - if(is_target_source_dir_subdirectory_of_engine) - cmake_path(RELATIVE_PATH absolute_target_source_dir BASE_DIRECTORY ${LY_ROOT_FOLDER} OUTPUT_VARIABLE relative_target_source_dir) - else() - ly_get_last_path_segment_concat_sha256(${absolute_target_source_dir} target_source_dir_last_path_segment) + if(NOT is_target_source_dir_subdirectory_of_engine) + cmake_path(GET root_subdir_of_target FILENAME root_subdir_dirname) + set(relative_subdir ${relative_target_source_dir}) unset(relative_target_source_dir) - cmake_path(APPEND relative_target_source_dir "External" ${target_source_dir_last_path_segment}) + cmake_path(APPEND relative_target_source_dir "External" ${root_subdir_dirname} ${relative_subdir}) endif() set(${output_source_dir} ${relative_target_source_dir} PARENT_SCOPE) diff --git a/cmake/PAL.cmake b/cmake/PAL.cmake index a43e601c74..b76fb8ac81 100644 --- a/cmake/PAL.cmake +++ b/cmake/PAL.cmake @@ -51,49 +51,21 @@ function(o3de_read_manifest o3de_manifest_json_data) endif() endfunction() -#! o3de_recurse_gems: returns the gem paths -# -# \arg:object json path -# \arg:gems returns the gems from the external subdirectory elements from the manifest -function(o3de_recurse_gems object_json_path gems) - get_filename_component(object_json_parent_path ${object_json_path} DIRECTORY) - ly_file_read(${object_json_path} json_data) - string(JSON external_subdirectories_count ERROR_VARIABLE json_error LENGTH ${json_data} "external_subdirectories") - if(NOT json_error) - if(external_subdirectories_count GREATER 0) - math(EXPR external_subdirectories_range "${external_subdirectories_count}-1") - foreach(external_subdirectories_index RANGE ${external_subdirectories_range}) - string(JSON external_subdirectories_entry ERROR_VARIABLE json_error GET ${json_data} "external_subdirectories" "${external_subdirectories_index}") - cmake_path(IS_RELATIVE external_subdirectories_entry is_relative) - if(${is_relative}) - cmake_path(ABSOLUTE_PATH external_subdirectories_entry BASE_DIRECTORY ${object_json_parent_path} NORMALIZE OUTPUT_VARIABLE external_subdirectories_entry) - endif() - if(EXISTS ${external_subdirectories_entry}/gem.json) - list(APPEND gem_entries ${external_subdirectories_entry}) - o3de_recurse_gems(${external_subdirectories_entry}/gem.json gem_entries) - endif() - endforeach() - endif() - endif() - set(${gems} ${gem_entries} PARENT_SCOPE) -endfunction() #! o3de_find_gem: returns the gem path # # \arg:gem_name the gem name to find # \arg:the path of the gem function(o3de_find_gem gem_name gem_path) - o3de_get_manifest_path(manifest_path) - if(EXISTS ${manifest_path}) - o3de_recurse_gems(${manifest_path} gems) - endif() - o3de_recurse_gems(${LY_ROOT_FOLDER}/engine.json gems) - foreach(gem ${gems}) - ly_file_read(${gem}/gem.json json_data) - string(JSON gem_json_name ERROR_VARIABLE json_error GET ${json_data} "gem_name") - if(gem_json_name STREQUAL gem_name) - set(${gem_path} ${gem} PARENT_SCOPE) - return() + get_all_external_subdirectories(all_external_subdirs) + foreach(external_subdir IN LISTS all_external_subdirs) + set(candidate_gem_path ${external_subdir}/gem.json) + if(EXISTS ${candidate_gem_path}) + o3de_read_json_key(gem_json_name ${candidate_gem_path} "gem_name") + if(gem_json_name STREQUAL gem_name) + set(${gem_path} ${external_subdir} PARENT_SCOPE) + return() + endif() endif() endforeach() endfunction() diff --git a/cmake/Platform/Common/Install_common.cmake b/cmake/Platform/Common/Install_common.cmake index e42664c107..5d92b9944e 100644 --- a/cmake/Platform/Common/Install_common.cmake +++ b/cmake/Platform/Common/Install_common.cmake @@ -448,9 +448,27 @@ function(ly_setup_cmake_install) # Transform the LY_EXTERNAL_SUBDIRS global property list into a json array set(indent " ") get_property(external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS) + list(REMOVE_DUPLICATES external_subdirs) foreach(external_subdir ${external_subdirs}) - cmake_path(RELATIVE_PATH external_subdir BASE_DIRECTORY ${LY_ROOT_FOLDER} OUTPUT_VARIABLE engine_rel_external_subdir) - list(APPEND relative_external_subdirs "\"${engine_rel_external_subdir}\"") + # If an external subdirectory is not a subdirectory of the engine root, then + # prepend "External" to its subdirectory root + ly_get_root_subdirectory_which_is_parent(${external_subdir} root_subdir_of_external_subdir) + cmake_path(RELATIVE_PATH external_subdir BASE_DIRECTORY ${root_subdir_of_external_subdir} OUTPUT_VARIABLE engine_rel_external_subdir) + + cmake_path(IS_PREFIX LY_ROOT_FOLDER ${external_subdir} is_subdirectory_of_engine) + if(NOT is_subdirectory_of_engine) + cmake_path(GET root_subdir_of_external_subdir FILENAME root_subdir_dirname) + set(relative_subdir ${engine_rel_external_subdir}) + unset(engine_rel_external_subdir) + cmake_path(APPEND engine_rel_external_subdir "External" ${root_subdir_dirname} ${relative_subdir}) + endif() + + set(quoted_engine_rel_external_subdir "\"${engine_rel_external_subdir}\"") + if (quoted_engine_rel_external_subdir IN_LIST relative_external_subdirs) + message(WARNING "An external subdirectory \"${external_subdir}\" has been found twice when generating the engine.json for the install layout") + else() + list(APPEND relative_external_subdirs "\"${engine_rel_external_subdir}\"") + endif() endforeach() list(JOIN relative_external_subdirs ",\n${indent}" LY_INSTALL_EXTERNAL_SUBDIRS) @@ -507,7 +525,17 @@ function(ly_setup_cmake_install) # Add to find_subdirectories all directories in which ly_add_target were called in get_property(all_subdirectories GLOBAL PROPERTY LY_ALL_TARGET_DIRECTORIES) foreach(target_subdirectory IN LISTS all_subdirectories) - cmake_path(RELATIVE_PATH target_subdirectory BASE_DIRECTORY ${LY_ROOT_FOLDER} OUTPUT_VARIABLE relative_target_subdirectory) + ly_get_root_subdirectory_which_is_parent(${target_subdirectory} root_subdir_of_target) + cmake_path(RELATIVE_PATH target_subdirectory BASE_DIRECTORY ${root_subdir_of_target} OUTPUT_VARIABLE relative_target_subdirectory) + + cmake_path(IS_PREFIX LY_ROOT_FOLDER ${target_subdirectory} is_subdirectory_of_engine) + if(NOT is_subdirectory_of_engine) + cmake_path(GET root_subdir_of_target FILENAME root_subdir_dirname) + set(relative_subdir ${relative_target_subdirectory}) + unset(relative_target_subdirectory) + cmake_path(APPEND relative_target_subdirectory "External" ${root_subdir_dirname} ${relative_subdir}) + endif() + string(APPEND find_subdirectories "add_subdirectory(${relative_target_subdirectory})\n") endforeach() set(permutation_find_subdirectories ${CMAKE_CURRENT_BINARY_DIR}/cmake/Platform/${PAL_PLATFORM_NAME}/${LY_BUILD_PERMUTATION}/o3de_subdirectories_${PAL_PLATFORM_NAME_LOWERCASE}.cmake) @@ -657,12 +685,12 @@ function(ly_setup_assets) set_property(GLOBAL APPEND PROPERTY global_gem_candidate_dirs_prop ${gem_candidate_dir}) endforeach() - # Iterate over each gem candidate directories and read populate a directory property + # Iterate over each gem candidate directories and populate a directory property # containing the files to copy over get_property(gem_candidate_dirs GLOBAL PROPERTY global_gem_candidate_dirs_prop) foreach(gem_candidate_dir IN LISTS gem_candidate_dirs) get_property(filtered_asset_paths DIRECTORY ${gem_candidate_dir} PROPERTY directory_filtered_asset_paths) - ly_get_last_path_segment_concat_sha256(${gem_candidate_dir} last_gem_root_path_segment) + # Check if the gem is a subdirectory of the engine cmake_path(IS_PREFIX LY_ROOT_FOLDER ${gem_candidate_dir} is_gem_subdirectory_of_engine) @@ -697,15 +725,16 @@ function(ly_setup_assets) # gem directories and files to install get_property(gems_assets_paths DIRECTORY ${gem_candidate_dir} PROPERTY gems_assets_paths) foreach(gem_absolute_path IN LISTS gems_assets_paths) - if(is_gem_subdirectory_of_engine) - cmake_path(RELATIVE_PATH gem_absolute_path BASE_DIRECTORY ${LY_ROOT_FOLDER} OUTPUT_VARIABLE gem_install_dest_dir) - else() - # The gem resides outside of the LY_ROOT_FOLDER, so the destination is made relative to the - # gem candidate directory and placed under the "External" directory" - # directory - cmake_path(RELATIVE_PATH gem_absolute_path BASE_DIRECTORY ${gem_candidate_dir} OUTPUT_VARIABLE gem_relative_path) + # If an external subdirectory is not a subdirectory of the engine root, then + # prepend "External" to its subdirectory root + ly_get_root_subdirectory_which_is_parent(${gem_candidate_dir} root_subdir_of_gem) + cmake_path(RELATIVE_PATH gem_absolute_path BASE_DIRECTORY ${root_subdir_of_gem} OUTPUT_VARIABLE gem_install_dest_dir) + + if(NOT is_gem_subdirectory_of_engine) + cmake_path(GET root_subdir_of_gem FILENAME root_subdir_dirname) + set(relative_subdir ${gem_install_dest_dir}) unset(gem_install_dest_dir) - cmake_path(APPEND gem_install_dest_dir "External" ${last_gem_root_path_segment} ${gem_relative_path}) + cmake_path(APPEND gem_install_dest_dir "External" ${root_subdir_dirname} ${relative_subdir}) endif() cmake_path(GET gem_install_dest_dir PARENT_PATH gem_install_dest_dir) diff --git a/cmake/Projects.cmake b/cmake/Projects.cmake index c6aa6c382a..acfa921ef1 100644 --- a/cmake/Projects.cmake +++ b/cmake/Projects.cmake @@ -118,30 +118,6 @@ function(ly_generate_project_build_path_setreg project_real_path) file(GENERATE OUTPUT ${project_user_build_path_setreg_file} CONTENT ${project_build_path_setreg_content}) endfunction() -function(add_gem_json_external_subdirectories gem_path) - set(gem_json_path ${gem_path}/gem.json) - if(EXISTS ${gem_json_path}) - read_json_external_subdirs(gem_external_subdirs ${gem_path}/gem.json) - foreach(gem_external_subdir ${gem_external_subdirs}) - file(REAL_PATH ${gem_external_subdir} real_external_subdir BASE_DIRECTORY ${gem_path}) - set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS ${real_external_subdir}) - add_gem_json_external_subdirectories(${real_external_subdir}) - endforeach() - endif() -endfunction() - -function(add_project_json_external_subdirectories project_path) - set(project_json_path ${project_path}/project.json) - if(EXISTS ${project_json_path}) - read_json_external_subdirs(project_external_subdirs ${project_path}/project.json) - foreach(project_external_subdir ${project_external_subdirs}) - file(REAL_PATH ${project_external_subdir} real_external_subdir BASE_DIRECTORY ${project_path}) - set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS ${real_external_subdir}) - add_gem_json_external_subdirectories(${real_external_subdir}) - endforeach() - endif() -endfunction() - function(install_project_asset_artifacts project_real_path) # The cmake tar command has a bit of a flaw # Any paths within the archive files it creates are relative to the current working directory. @@ -212,16 +188,16 @@ foreach(project ${LY_PROJECTS}) # when the external subdirectory contains relative paths of significant length string(SUBSTRING ${full_directory_hash} 0 8 full_directory_hash) - get_filename_component(project_folder_name ${project} NAME) + cmake_path(GET project FILENAME project_folder_name ) list(APPEND LY_PROJECTS_FOLDER_NAME ${project_folder_name}) add_subdirectory(${project} "${project_folder_name}-${full_directory_hash}") ly_generate_project_build_path_setreg(${full_directory_path}) - add_project_json_external_subdirectories(${full_directory_path}) # Get project name o3de_read_json_key(project_name ${full_directory_path}/project.json "project_name") + add_project_json_external_subdirectories(${full_directory_path} "${project_name}") - install_project_asset_artifacts(${full_directory_path}) + install_project_asset_artifacts(${full_directory_path}) endforeach() diff --git a/cmake/Subdirectories.cmake b/cmake/Subdirectories.cmake new file mode 100644 index 0000000000..3580710c82 --- /dev/null +++ b/cmake/Subdirectories.cmake @@ -0,0 +1,197 @@ +# +# Copyright (c) Contributors to the Open 3D Engine Project. +# For complete copyright and license terms please see the LICENSE at the root of this distribution. +# +# SPDX-License-Identifier: Apache-2.0 OR MIT +# +# + +include_guard() + +################################################################################ +# Subdirectory processing +################################################################################ + +# this function is building up the LY_EXTERNAL_SUBDIRS global property +function(add_engine_gem_json_external_subdirectories gem_path) + set(gem_json_path ${gem_path}/gem.json) + if(EXISTS ${gem_json_path}) + read_json_external_subdirs(gem_external_subdirs ${gem_path}/gem.json) + foreach(gem_external_subdir ${gem_external_subdirs}) + file(REAL_PATH ${gem_external_subdir} real_external_subdir BASE_DIRECTORY ${gem_path}) + + # Append external subdirectory if it is not in global property + get_property(current_external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS) + if(NOT real_external_subdir IN_LIST current_external_subdirs) + set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS ${real_external_subdir}) + # Also append the project external subdirectores to the LY_EXTERNAL_SUBDIRS_ENGINE property + set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS_ENGINE ${real_external_subdir}) + add_engine_gem_json_external_subdirectories(${real_external_subdir}) + endif() + endforeach() + endif() +endfunction() + +function(add_engine_json_external_subdirectories) + set(engine_json_path ${LY_ROOT_FOLDER}/engine.json) + if(EXISTS ${engine_json_path}) + read_json_external_subdirs(engine_external_subdirs ${engine_json_path}) + foreach(engine_external_subdir ${engine_external_subdirs}) + file(REAL_PATH ${engine_external_subdir} real_external_subdir BASE_DIRECTORY ${LY_ROOT_FOLDER}) + + # Append external subdirectory if it is not in global property + get_property(current_external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS) + if(NOT real_external_subdir IN_LIST current_external_subdirs) + set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS ${real_external_subdir}) + # Also append the project external subdirectores to the LY_EXTERNAL_SUBDIRS_ENGINE property + set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS_ENGINE ${real_external_subdir}) + add_engine_gem_json_external_subdirectories(${real_external_subdir}) + endif() + endforeach() + endif() +endfunction() + + +function(add_project_gem_json_external_subdirectories gem_path project_name) + set(gem_json_path ${gem_path}/gem.json) + if(EXISTS ${gem_json_path}) + read_json_external_subdirs(gem_external_subdirs ${gem_path}/gem.json) + foreach(gem_external_subdir ${gem_external_subdirs}) + file(REAL_PATH ${gem_external_subdir} real_external_subdir BASE_DIRECTORY ${gem_path}) + + # Append external subdirectory if it is not in global property + get_property(current_external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS) + if(NOT real_external_subdir IN_LIST current_external_subdirs) + set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS ${real_external_subdir}) + # Also append the project external subdirectores to the LY_EXTERNAL_SUBDIRS_${project_name} property + set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS_${project_name} ${real_external_subdir}) + add_project_gem_json_external_subdirectories(${real_external_subdir} "${project_name}") + endif() + endforeach() + endif() +endfunction() + +function(add_project_json_external_subdirectories project_path project_name) + set(project_json_path ${project_path}/project.json) + if(EXISTS ${project_json_path}) + read_json_external_subdirs(project_external_subdirs ${project_path}/project.json) + foreach(project_external_subdir ${project_external_subdirs}) + file(REAL_PATH ${project_external_subdir} real_external_subdir BASE_DIRECTORY ${project_path}) + + # Append external subdirectory if it is not in global property + get_property(current_external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS) + if(NOT real_external_subdir IN_LIST current_external_subdirs) + set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS ${real_external_subdir}) + # Also append the project external subdirectores to the LY_EXTERNAL_SUBDIRS_${project_name} property + set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS_${project_name} ${real_external_subdir}) + add_project_gem_json_external_subdirectories(${real_external_subdir} "${project_name}") + endif() + endforeach() + endif() +endfunction() + + +#! add_o3de_manifest_gem_json_external_subdirectories : Recurses through external subdirectories +#! originally found in the add_o3de_manifest_json_external_subdirectories command +function(add_o3de_manifest_gem_json_external_subdirectories gem_path) + set(gem_json_path ${gem_path}/gem.json) + if(EXISTS ${gem_json_path}) + read_json_external_subdirs(gem_external_subdirs ${gem_path}/gem.json) + foreach(gem_external_subdir ${gem_external_subdirs}) + file(REAL_PATH ${gem_external_subdir} real_external_subdir BASE_DIRECTORY ${gem_path}) + + # Append external subdirectory ONLY to LY_EXTERNAL_SUBDIRS_O3DE_MANIFEST PROPERTY + # It is not appended to LY_EXTERNAL_SUBDIRS unless that gem is used by the project + get_property(current_external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS_O3DE_MANIFEST) + if(NOT real_external_subdir IN_LIST current_external_subdirs) + set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS_O3DE_MANIFEST ${real_external_subdir}) + add_o3de_manifest_gem_json_external_subdirectories(${real_external_subdir}) + endif() + endforeach() + endif() +endfunction() + +#! add_o3de_manifest_json_external_subdirectories : Adds the list of external_subdirectories +#! in the user o3de_manifest.json to the LY_EXTERNAL_SUBDIRS_O3DE_MANIFEST property +function(add_o3de_manifest_json_external_subdirectories) + o3de_get_manifest_path(manifest_path) + if(EXISTS ${manifest_path}) + read_json_external_subdirs(o3de_manifest_external_subdirs ${manifest_path}) + foreach(manifest_external_subdir ${o3de_manifest_external_subdirs}) + file(REAL_PATH ${manifest_external_subdir} real_external_subdir BASE_DIRECTORY ${LY_ROOT_FOLDER}) + + # Append external subdirectory ONLY to LY_EXTERNAL_SUBDIRS_O3DE_MANIFEST PROPERTY + # It is not appended to LY_EXTERNAL_SUBDIRS unless that gem is used by the project + get_property(current_external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS_O3DE_MANIFEST) + if(NOT real_external_subdir IN_LIST current_external_subdirs) + set_property(GLOBAL APPEND PROPERTY LY_EXTERNAL_SUBDIRS_O3DE_MANIFEST ${real_external_subdir}) + add_o3de_manifest_gem_json_external_subdirectories(${real_external_subdir}) + endif() + endforeach() + endif() +endfunction() + +#! Gather unique_list of all external subdirectories that is union +#! of the engine.json, project.json, o3de_manifest.json and any gem.json files found visiting +function(get_all_external_subdirectories output_subdirs) + get_property(all_external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS) + get_property(manifest_external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS_O3DE_MANIFEST) + list(APPEND all_external_subdirs ${manifest_external_subdirs}) + list(REMOVE_DUPLICATES all_external_subdirs) + set(${output_subdirs} ${all_external_subdirs} PARENT_SCOPE) +endfunction() + +#! add_registered_gems_to_external_subdirs: +#! Accepts a list of gem_names (which can be read from the project.json or engine.json) +#! and cross checks them against union of all external subdirectories to determine the gem path. +#! If that gem exist it is appended to LY_EXTERNAL_SUBDIRS so that that the build generator +#! adds to the generated build project. +#! Otherwise a fatal error is logged indicating that is not gem could not be found in the list of external subdirectories +function(add_registered_gems_to_external_subdirs gem_names) + if (gem_names) + get_all_external_subdirectories(all_external_subdirs) + foreach(gem_name IN LISTS gem_names) + unset(gem_path) + o3de_find_gem(${gem_name} gem_path) + if (gem_path) + set_property(GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS ${gem_path} APPEND) + else() + list(JOIN all_external_subdirs "\n" external_subdirs_formatted) + message(SEND_ERROR "The gem \"${gem_name}\" from the \"gem_names\" field in the engine.json/project.json " + " could not be found in any gem.json from the following list of registered external subdirectories:\n" + "${external_subdirs_formatted}") + break() + endif() + endforeach() + endif() +endfunction() + +function(add_subdirectory_on_external_subdirs) + # Lookup the paths of "gem_names" array all project.json files and engine.json + # and append them to the LY_EXTERNAL_SUBDIRS property + foreach(project ${LY_PROJECTS}) + file(REAL_PATH ${project} full_directory_path BASE_DIRECTORY ${CMAKE_SOURCE_DIR}) + o3de_read_json_array(gem_names ${full_directory_path}/project.json "gem_names") + add_registered_gems_to_external_subdirs("${gem_names}") + endforeach() + o3de_read_json_array(gem_names ${LY_ROOT_FOLDER}/engine.json "gem_names") + add_registered_gems_to_external_subdirs("${gem_names}") + + get_property(external_subdirs GLOBAL PROPERTY LY_EXTERNAL_SUBDIRS) + list(APPEND LY_EXTERNAL_SUBDIRS ${external_subdirs}) + list(REMOVE_DUPLICATES LY_EXTERNAL_SUBDIRS) + # Loop over the additional external subdirectories and invoke add_subdirectory on them + foreach(external_directory ${LY_EXTERNAL_SUBDIRS}) + # Hash the external_directory name and append it to the Binary Directory section of add_subdirectory + # This is to deal with potential situations where multiple external directories has the same last directory name + # For example if D:/Company1/RayTracingGem and F:/Company2/Path/RayTracingGem were both added as a subdirectory + file(REAL_PATH ${external_directory} full_directory_path) + string(SHA256 full_directory_hash ${full_directory_path}) + # Truncate the full_directory_hash down to 8 characters to avoid hitting the Windows 260 character path limit + # when the external subdirectory contains relative paths of significant length + string(SUBSTRING ${full_directory_hash} 0 8 full_directory_hash) + # Use the last directory as the suffix path to use for the Binary Directory + cmake_path(GET external_directory FILENAME directory_name) + add_subdirectory(${external_directory} ${CMAKE_BINARY_DIR}/External/${directory_name}-${full_directory_hash}) + endforeach() +endfunction() diff --git a/cmake/cmake_files.cmake b/cmake/cmake_files.cmake index caeab9d12e..90f1e9a2d8 100644 --- a/cmake/cmake_files.cmake +++ b/cmake/cmake_files.cmake @@ -35,6 +35,7 @@ set(FILES Projects.cmake RuntimeDependencies.cmake SettingsRegistry.cmake + Subdirectories.cmake UnitTest.cmake Version.cmake ) diff --git a/engine.json b/engine.json index 51b6738001..5c64eed308 100644 --- a/engine.json +++ b/engine.json @@ -54,7 +54,7 @@ "Gems/NvCloth", "Gems/PhysX", "Gems/PhysXDebug", - "Gems/Prefab", + "Gems/Prefab/PrefabBuilder", "Gems/Presence", "Gems/PrimitiveAssets", "Gems/Profiler", diff --git a/scripts/build/Jenkins/Jenkinsfile b/scripts/build/Jenkins/Jenkinsfile index b55689b871..e4957992a5 100644 --- a/scripts/build/Jenkins/Jenkinsfile +++ b/scripts/build/Jenkins/Jenkinsfile @@ -9,6 +9,7 @@ import groovy.json.JsonOutput PIPELINE_CONFIG_FILE = 'scripts/build/Jenkins/lumberyard.json' INCREMENTAL_BUILD_SCRIPT_PATH = 'scripts/build/bootstrap/incremental_build_util.py' +EBS_SNAPSHOT_SCRIPT_PATH = 'scripts/build/tools/ebs_snapshot.py' PIPELINE_RETRY_ATTEMPTS = 3 EMPTY_JSON = readJSON text: '{}' @@ -206,7 +207,8 @@ def CheckoutBootstrapScripts(String branchName) { [$class: 'SparseCheckoutPaths', sparseCheckoutPaths: [ [ $class: 'SparseCheckoutPath', path: 'scripts/build/Jenkins/' ], [ $class: 'SparseCheckoutPath', path: 'scripts/build/bootstrap/' ], - [ $class: 'SparseCheckoutPath', path: 'scripts/build/Platform' ] + [ $class: 'SparseCheckoutPath', path: 'scripts/build/Platform' ], + [ $class: 'SparseCheckoutPath', path: 'scripts/build/tools/' ] ]], // Shallow checkouts break changelog computation. Do not enable. [$class: 'CloneOption', noTags: false, reference: '', shallow: false] @@ -495,6 +497,19 @@ def PostBuildCommonSteps(String workspace, Map params, boolean mount = true) { } } +def HandleDriveSnapshots(String repositoryName, String projectName, String pipeline, String branchName, String platform, String buildType) { + unstash name: 'ebs_snapshot_script' + + catchError(message: "Error snapshotting volume (this won't fail the build)", buildResult: 'UNSTABLE', stageResult: 'FAILURE') { + def pythonCmd = 'python3 -u ' + + mountName = "Name:${repositoryName}_${projectName}_${pipeline}_${branchName}_${platform}_${buildType}" + mountName = mountName.replace('/', '_').replace('\\', '_') + palSh("${pythonCmd} ${EBS_SNAPSHOT_SCRIPT_PATH} --action create --tags ${mountName} --execute", "Starting volume snapshots", true) + palSh("${pythonCmd} ${EBS_SNAPSHOT_SCRIPT_PATH} --action delete --tags ${mountName} --retention ${env.SNAP_RETENTION} --execute", "Cleaning up old snapshots", true) + } +} + def CreateSetupStage(Map pipelineConfig, String snapshot, String repositoryName, String projectName, String pipelineName, String branchName, String platformName, String jobName, Map environmentVars, boolean onlyMountEBSVolume = false) { return { stage('Setup') { @@ -564,6 +579,14 @@ def CreateTeardownStage(Map environmentVars, Map params) { } } +def CreateSnapshotStage(String repositoryName, String projectName, String pipelineName, String branchName, String platformName, String buildType, String jobName) { + return{ + stage("${jobName}_snapshot_ebs_volume") { + HandleDriveSnapshots(repositoryName, projectName, pipelineName, branchName, platformName, buildType) + } + } +} + def CreateSingleNode(Map pipelineConfig, def platform, def build_job, Map envVars, String branchName, String pipelineName, String repositoryName, String projectName, boolean onlyMountEBSVolume = false) { def nodeLabel = envVars['NODE_LABEL'] return { @@ -632,6 +655,9 @@ def CreateSingleNode(Map pipelineConfig, def platform, def build_job, Map envVar CreateExportTestScreenshotsStage(pipelineConfig, branchName, platform.key, build_job_name, envVars, params).call() } CreateTeardownStage(envVars, params).call() + if (envVars['CREATE_SNAPSHOT']?.toBoolean()) { + CreateSnapshotStage(repositoryName, projectName, pipelineName, branchName, platform.key, build_job.key, build_job_name).call() + } } } } @@ -816,9 +842,11 @@ try { pipelineProperties.add(parameters(pipelineParameters.unique())) properties(pipelineProperties) - // Stash the INCREMENTAL_BUILD_SCRIPT_PATH since all nodes will use it + // Stash the INCREMENTAL_BUILD_SCRIPT_PATH and EBS_SNAPSHOT_SCRIPT_PATH since all nodes will use it stash name: 'incremental_build_script', includes: INCREMENTAL_BUILD_SCRIPT_PATH + stash name: 'ebs_snapshot_script', + includes: EBS_SNAPSHOT_SCRIPT_PATH } } } diff --git a/scripts/build/Platform/Android/build_config.json b/scripts/build/Platform/Android/build_config.json index da538b22ec..e31aa4d5a0 100644 --- a/scripts/build/Platform/Android/build_config.json +++ b/scripts/build/Platform/Android/build_config.json @@ -9,7 +9,8 @@ }, "profile_pipe": { "TAGS": [ - "default" + "default", + "snapshot" ], "steps": [ "profile" @@ -77,7 +78,8 @@ "default", "weekly-build-metrics", "nightly-incremental", - "nightly-clean" + "nightly-clean", + "snapshot" ], "COMMAND":"../Windows/build_asset_windows.cmd", "PARAMETERS": { @@ -127,7 +129,8 @@ "gradle": { "TAGS":[ "default", - "weekly-build-metrics" + "weekly-build-metrics", + "snapshot" ], "COMMAND":"gradle_windows.cmd", "PARAMETERS": { diff --git a/scripts/build/Platform/Android/pipeline.json b/scripts/build/Platform/Android/pipeline.json index 2e426c7891..ca3a9f998e 100644 --- a/scripts/build/Platform/Android/pipeline.json +++ b/scripts/build/Platform/Android/pipeline.json @@ -14,6 +14,10 @@ }, "nightly-clean": { "CLEAN_WORKSPACE": true + }, + "snapshot": { + "CLEAN_WORKSPACE": true, + "CREATE_SNAPSHOT": true } } -} \ No newline at end of file +} diff --git a/scripts/build/Platform/Linux/build_config.json b/scripts/build/Platform/Linux/build_config.json index c8f8752609..c7c20dca28 100644 --- a/scripts/build/Platform/Linux/build_config.json +++ b/scripts/build/Platform/Linux/build_config.json @@ -9,7 +9,8 @@ }, "profile_nounity_pipe": { "TAGS": [ - "default" + "default", + "snapshot" ], "steps": [ "profile_nounity", diff --git a/scripts/build/Platform/Linux/pipeline.json b/scripts/build/Platform/Linux/pipeline.json index 605adf1837..e12d41294b 100644 --- a/scripts/build/Platform/Linux/pipeline.json +++ b/scripts/build/Platform/Linux/pipeline.json @@ -12,6 +12,10 @@ }, "nightly-clean": { "CLEAN_WORKSPACE": true + }, + "snapshot": { + "CLEAN_WORKSPACE": true, + "CREATE_SNAPSHOT": true } }, "PIPELINE_JENKINS_PARAMETERS": { diff --git a/scripts/build/Platform/Windows/build_config.json b/scripts/build/Platform/Windows/build_config.json index e3d5a4a3fc..017f56bb68 100644 --- a/scripts/build/Platform/Windows/build_config.json +++ b/scripts/build/Platform/Windows/build_config.json @@ -9,7 +9,8 @@ }, "validation_pipe": { "TAGS": [ - "default" + "default", + "snapshot" ], "steps": [ "validation" @@ -27,7 +28,8 @@ }, "profile_pipe": { "TAGS": [ - "default" + "default", + "snapshot" ], "steps": [ "profile", @@ -288,7 +290,8 @@ "default", "nightly-incremental", "nightly-clean", - "weekly-build-metrics" + "weekly-build-metrics", + "snapshot" ], "COMMAND": "build_windows.cmd", "PARAMETERS": { diff --git a/scripts/build/Platform/Windows/pipeline.json b/scripts/build/Platform/Windows/pipeline.json index 47e6af2d10..3b93da9713 100644 --- a/scripts/build/Platform/Windows/pipeline.json +++ b/scripts/build/Platform/Windows/pipeline.json @@ -12,6 +12,10 @@ }, "nightly-clean": { "CLEAN_WORKSPACE": true + }, + "snapshot": { + "CLEAN_WORKSPACE": true, + "CREATE_SNAPSHOT": true } }, "PIPELINE_JENKINS_PARAMETERS": { @@ -62,4 +66,4 @@ } ] } -} \ No newline at end of file +} diff --git a/scripts/build/build_node/Platform/Common/requirements.txt b/scripts/build/build_node/Platform/Common/requirements.txt index 1429c33731..74ae31350c 100644 --- a/scripts/build/build_node/Platform/Common/requirements.txt +++ b/scripts/build/build_node/Platform/Common/requirements.txt @@ -40,9 +40,9 @@ idna==2.10 \ --hash=sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6 \ --hash=sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0 \ # via requests -jinja2==2.11.2 \ - --hash=sha256:89aab215427ef59c34ad58735269eb58b1a5808103067f7bb9d5836c651b3bb0 \ - --hash=sha256:f0a4641d3cf955324a89c04f3d94663aa4d638abe8f733ecd3582848e1c37035 \ +jinja2==2.11.3 \ + --hash=sha256:03e47ad063331dd6a3f04a43eddca8a966a26ba0c5b7207a9a9e4e08f1b29419 \ + --hash=sha256:a6d58433de0ae800347cab1fa3043cebbabe8baa9d29e668f1c768cb87a333c6 \ # via -r requirements.txt jmespath==0.10.0 \ --hash=sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9 \ @@ -131,19 +131,17 @@ pytz==2020.4 \ --hash=sha256:3e6b7dd2d1e0a59084bcee14a17af60c5c562cdc16d828e8eba2e683d3a7e268 \ --hash=sha256:5c55e189b682d420be27c6995ba6edce0c0a77dd67bfbe2ae6607134d5851ffd \ # via -r requirements.txt -pywin32==228 \ - --hash=sha256:00eaf43dbd05ba6a9b0080c77e161e0b7a601f9a3f660727a952e40140537de7 \ - --hash=sha256:11cb6610efc2f078c9e6d8f5d0f957620c333f4b23466931a247fb945ed35e89 \ - --hash=sha256:1f45db18af5d36195447b2cffacd182fe2d296849ba0aecdab24d3852fbf3f80 \ - --hash=sha256:37dc9935f6a383cc744315ae0c2882ba1768d9b06700a70f35dc1ce73cd4ba9c \ - --hash=sha256:6e38c44097a834a4707c1b63efa9c2435f5a42afabff634a17f563bc478dfcc8 \ - --hash=sha256:8319bafdcd90b7202c50d6014efdfe4fde9311b3ff15fd6f893a45c0868de203 \ - --hash=sha256:9b3466083f8271e1a5eb0329f4e0d61925d46b40b195a33413e0905dccb285e8 \ - --hash=sha256:a60d795c6590a5b6baeacd16c583d91cce8038f959bd80c53bd9a68f40130f2d \ - --hash=sha256:af40887b6fc200eafe4d7742c48417529a8702dcc1a60bf89eee152d1d11209f \ - --hash=sha256:ec16d44b49b5f34e99eb97cf270806fdc560dff6f84d281eb2fcb89a014a56a9 \ - --hash=sha256:ed74b72d8059a6606f64842e7917aeee99159ebd6b8d6261c518d002837be298 \ - --hash=sha256:fa6ba028909cfc64ce9e24bcf22f588b14871980d9787f1e2002c99af8f1850c \ +pywin32==301 \ + --hash=sha256:93367c96e3a76dfe5003d8291ae16454ca7d84bb24d721e0b74a07610b7be4a7 \ + --hash=sha256:9635df6998a70282bd36e7ac2a5cef9ead1627b0a63b17c731312c7a0daebb72 \ + --hash=sha256:c866f04a182a8cb9b7855de065113bbd2e40524f570db73ef1ee99ff0a5cc2f0 \ + --hash=sha256:dafa18e95bf2a92f298fe9c582b0e205aca45c55f989937c52c454ce65b93c78 \ + --hash=sha256:98f62a3f60aa64894a290fb7494bfa0bfa0a199e9e052e1ac293b2ad3cd2818b \ + --hash=sha256:fb3b4933e0382ba49305cc6cd3fb18525df7fd96aa434de19ce0878133bf8e4a \ + --hash=sha256:88981dd3cfb07432625b180f49bf4e179fb8cbb5704cd512e38dd63636af7a17 \ + --hash=sha256:8c9d33968aa7fcddf44e47750e18f3d034c3e443a707688a008a2e52bbef7e96 \ + --hash=sha256:595d397df65f1b2e0beaca63a883ae6d8b6df1cdea85c16ae85f6d2e648133fe \ + --hash=sha256:87604a4087434cd814ad8973bd47d6524bd1fa9e971ce428e76b62a5e0860fdf \ # via -r requirements.txt pyxb==1.2.6 \ --hash=sha256:2a00f38dd1d87b88f92d79bc5a09718d730419b88e814545f472bbd5a3bf27b4 \ @@ -175,9 +173,9 @@ requests==2.25.0 \ --hash=sha256:7f1a0b932f4a60a1a65caa4263921bb7d9ee911957e0ae4a23a6dd08185ad5f8 \ --hash=sha256:e786fa28d8c9154e6a4de5d46a1d921b8749f8b74e28bde23768e5e16eece998 \ # via -r requirements.txt -rsa==4.5 \ - --hash=sha256:35c5b5f6675ac02120036d97cf96f1fde4d49670543db2822ba5015e21a18032 \ - --hash=sha256:4d409f5a7d78530a4a2062574c7bd80311bc3af29b364e293aa9b03eea77714f \ +rsa==4.7 \ + --hash=sha256:a8774e55b59fd9fc893b0d05e9bfc6f47081f46ff5b46f39ccf24631b7be356b \ + --hash=sha256:69805d6b69f56eb05b62daea3a7dbd7aa44324ad1306445e05da8060232d00f4 \ # via -r requirements.txt s3transfer==0.3.3 \ --hash=sha256:2482b4259524933a022d59da830f51bd746db62f047d6eb213f2f8855dcb8a13 \ diff --git a/scripts/build/tools/ebs_snapshot.py b/scripts/build/tools/ebs_snapshot.py new file mode 100644 index 0000000000..483bceb988 --- /dev/null +++ b/scripts/build/tools/ebs_snapshot.py @@ -0,0 +1,197 @@ +# +# Copyright (c) Contributors to the Open 3D Engine Project. +# For complete copyright and license terms please see the LICENSE at the root of this distribution. +# +# SPDX-License-Identifier: Apache-2.0 OR MIT +# +# + +import argparse +import boto3 +import logging +import sys +from botocore.config import Config + +log = logging.getLogger(__name__) +log.setLevel(logging.INFO) +log.addHandler(logging.StreamHandler()) + +DEFAULT_REGION = 'us-west-2' +DEFAULT_SNAPSHOT_RETAIN = 2 +DEFAULT_SNAPSHOT_DESCRIPTION = 'Created for Build Artifact Snapshots' +DEFAULT_DRYRUN = True + +def _kv_to_dict(kv_string): + """ + Simple splitting of a key value string to dictionary in "Name: , Values: []" form + + :param kv_string: String in the form of "key:value" + :return Dictionary of values + """ + dict = {} + + if ":" not in kv_string: + log.error(f'Keyvalue parameter not in the form of "key:value"') + raise ValueError + + kv = kv_string.split(':') + dict['Name'] = f'tag:{kv[0]}' + dict['Values'] = [kv[1]] + + return dict + +def _format_tags(tag_keyvalue): + """ + Format tags in list form + + :param tag_keyvalue: String of comma separated key value pairs + :return List of dictionary values + """ + tag_filter = [] + + for keyvalue in tag_keyvalue: + tag_filter.append(_kv_to_dict(keyvalue)) + + return tag_filter + +def get_ec2_resource(): + """ + Get the AWS EC2 resource object, with appropriate region + + :return The EC2 resource object + """ + session = boto3.session.Session() + region = session.region_name + if region is None: + region = DEFAULT_REGION + + resource_config = Config( + region_name=region, + retries={ + 'mode': 'standard' + } + ) + resource = boto3.resource('ec2', config=resource_config) + return resource + +def create_snapshot(ec2_resource, tag_keyvalue, snap_description, snap_dryrun=DEFAULT_DRYRUN): + """ + Find and snapshot all EBS volumes that have a matching tag value. Injects all volume tags into the snapshot, + including the name and adds a description + + :param ec2_resource: The EC2 resource object + :param tag_keyvalue: List of Strings with tag keyvalues in the form of "key:value" + :param snap_description: String with the snapshot description to write + :param snap_dryrun: Boolean to dryrun the action. Set to true by default (always dryrun) + :return: Number of EBS volumes that are snapshotted successfully, number of EBS volumes that failed to be snapshotted + """ + success = 0 + failure = 0 + + tags = _format_tags(tag_keyvalue) + + for tag in tags: + response = ec2_resource.volumes.filter(Filters=[tag]) + log.info(f'Snapshotting EBS volumes with tags that match {tag}...') + for volume in response: + try: + log.info(f'Snapshotting volume {volume.volume_id}') + volume.create_snapshot(Description=snap_description, TagSpecifications=[{'ResourceType': 'snapshot', 'Tags': volume.tags}], DryRun=snap_dryrun) + success += 1 + except Exception as e: + log.error(f'Failed to snapshot volume {volume.volume_id}.') + log.error(e) + failure += 1 + + return success, failure + +def delete_snapshot(ec2_resource, tag_keyvalue, snap_description, snap_retention, snap_dryrun=DEFAULT_DRYRUN): + """ + Find all EBS snapshots that have a matching tag value AND description. If the number of snapshots exceeds a retention amount, + delete the oldest snapshot until retention is achived. + + :param ec2_resource: The EC2 resource object + :param tag_keyvalue: List of Strings with tag keyvalues in the form of "key:value" + :param snap_description: String with the snapshot description to search + :param snap_retention: Integer with the number of snapshots to retain + :param snap_dryrun: Boolean to dryrun the action. Set to true by default (always dryrun) + :return: Number of EBS snapshots deleted successfully, number of EBS snapshots that failed to be deleted + """ + success = 0 + failure = 0 + description_filter = {"Name": "description", "Values": [snap_description]} + + tags = _format_tags(tag_keyvalue) + + for tag in tags: + response = list(ec2_resource.snapshots.filter(Filters=[tag,description_filter])) + log.info(f'Getting snapshots with tags that match {tag}...') + num_snaps = len(response) + log.info(f'Tag {tag} has {num_snaps} snapshots') + + if num_snaps > snap_retention: + log.info(f'Deleting oldest snapshots to keep retention of {snap_retention}') + snap_list = sorted(response, key=lambda k: k.start_time) # Get a sorted list of snapshots by start time in descending order + diff_snap = num_snaps - snap_retention + for n in range(diff_snap): + try: + log.info(f'Deleting snapshot {snap_list[n].snapshot_id}') + snap_list[n].delete(DryRun=snap_dryrun) + success += 1 + except Exception as e: + log.error(f'Failed to delete snapshot {snap_list[n].snapshot_id}.') + log.error(e) + failure += 1 + + return success, failure + +def list_snapshot(ec2_resource, tag_keyvalue, snap_description): + """ + Find all EBS snapshots that have a matching tag value AND description. Prints snap id, description, tags, and start time. + + :param ec2_resource: The EC2 resource object + :param tag_keyvalue: List of Strings with tag keyvalues in the form of "key:value" + :param snap_description: String with the snapshot description to search + :return: None + """ + + description_filter = {"Name": "description", "Values": [snap_description]} + + tags = _format_tags(tag_keyvalue) + + for tag in tags: + response = ec2_resource.snapshots.filter(Filters=[tag,description_filter]) + log.info(f'Getting snapshots with tags that match {tag}...') + num_snaps = len(list(response)) + log.info(f'Tag {tag} has {num_snaps} snapshots') + snap_list = sorted(response, key=lambda k: k.start_time) + for n in range(num_snaps): + print(f'Snap ID: {snap_list[n].snapshot_id} \n Description: {snap_list[n].description} \n Tags: {snap_list[n].tags} \n Start Time: {snap_list[n].start_time}') + + return None + +def parse_args(): + parser = argparse.ArgumentParser(description='Script to manage EBS snapshots for build artifacts') + parser.add_argument('--action', '-a', type=str, help='(create|delete|list) Creates, deletes, or lists EBS snapshots based on tag. Requires --tags argument') + parser.add_argument('--tags', '-t', type=str, required=True, help='Comma separated key value tags to search for in the form of "key:value", for example, "PipelineAndBranch:default_development","PipelineAndBranch:default_development"') + parser.add_argument('--description', '-d', default=DEFAULT_SNAPSHOT_DESCRIPTION, help=f'Snapshot description to write or search for. Defaults to "{DEFAULT_SNAPSHOT_DESCRIPTION}"') + parser.add_argument('--retention', '-r', nargs="?", const=DEFAULT_SNAPSHOT_RETAIN, type=int, help=f'Integer with the number of snapshots to retain. Defaults to {DEFAULT_SNAPSHOT_RETAIN}') + parser.add_argument('--execute', '-e', action='store_false', help=f'Execute the snapshot commands. This needs to be set, otherwise it will always dryrun') + return parser.parse_args() + +def main(): + args = parse_args() + tag_list = args.tags.split(",") + ec2_resource = get_ec2_resource() + if 'create' in args.action: + ret = create_snapshot(ec2_resource, tag_list, args.description, args.execute) + log.info(f'{ret[0]} snapshots created, {ret[1]} snapshots failed') + elif 'delete' in args.action: + ret = delete_snapshot(ec2_resource, tag_list, args.description, args.retention, args.execute) + log.info(f'{ret[0]} snapshots deleted, {ret[1]} snapshot deletions failed') + elif 'list' in args.action: + ret = list_snapshot(ec2_resource, tag_list, args.description) + +if __name__ == "__main__": + sys.exit(main()) + \ No newline at end of file diff --git a/scripts/o3de/o3de/cmake.py b/scripts/o3de/o3de/cmake.py index 7a820a9677..163d77f2f4 100644 --- a/scripts/o3de/o3de/cmake.py +++ b/scripts/o3de/o3de/cmake.py @@ -150,7 +150,10 @@ def remove_gem_dependency(cmake_file: pathlib.Path, # If the in_gem_list was flipped to false, that means the currently parsed line contained the # line end marker, so append that to the result_line result_line += enable_gem_end_marker if not in_gem_list else '' - t_data.append(result_line + '\n') + # Strip of trailing whitespace. This also strips result lines which are empty of the indent + result_line = result_line.rstrip() + if result_line: + t_data.append(result_line + '\n') else: t_data.append(line) @@ -165,11 +168,6 @@ def remove_gem_dependency(cmake_file: pathlib.Path, return 0 -def get_project_gems(project_path: pathlib.Path, - platform: str = 'Common') -> set: - return get_gems_from_cmake_file(get_enabled_gem_cmake_file(project_path=project_path, platform=platform)) - - def get_enabled_gems(cmake_file: pathlib.Path) -> set: """ Gets a list of enabled gems from the cmake file @@ -206,15 +204,6 @@ def get_enabled_gems(cmake_file: pathlib.Path) -> set: return gem_target_set -def get_project_gem_paths(project_path: pathlib.Path, - platform: str = 'Common') -> set: - gem_names = get_project_gems(project_path, platform) - gem_paths = set() - for gem_name in gem_names: - gem_paths.add(manifest.get_registered(gem_name=gem_name, project_path=project_path)) - return gem_paths - - def get_enabled_gem_cmake_file(project_name: str = None, project_path: str or pathlib.Path = None, platform: str = 'Common') -> pathlib.Path or None: diff --git a/scripts/o3de/o3de/disable_gem.py b/scripts/o3de/o3de/disable_gem.py index eecc765d32..617ad8ba41 100644 --- a/scripts/o3de/o3de/disable_gem.py +++ b/scripts/o3de/o3de/disable_gem.py @@ -15,7 +15,7 @@ import os import pathlib import sys -from o3de import cmake, manifest, utils +from o3de import cmake, manifest, project_properties, utils logger = logging.getLogger('o3de.disable_gem') logging.basicConfig(format=utils.LOG_FORMAT) @@ -68,8 +68,8 @@ def disable_gem_in_project(gem_name: str = None, f' {project_path / "project.json"}, engine.json') return 1 gem_path = pathlib.Path(gem_path).resolve() - # make sure this gem already exists if we're adding. We can always remove a gem. - if not gem_path.exists(): + # make sure the gem path is a directory + if not gem_path.is_dir(): logger.error(f'Gem Path {gem_path} does not exist.') return 1 @@ -79,9 +79,6 @@ def disable_gem_in_project(gem_name: str = None, logger.error(f'Could not read gem.json content under {gem_path}.') return 1 - # when removing we will try to do as much as possible even with failures so ret_val will be the last error code - ret_val = 0 - if not enabled_gem_file: enabled_gem_file = cmake.get_enabled_gem_cmake_file(project_path=project_path) @@ -89,10 +86,14 @@ def disable_gem_in_project(gem_name: str = None, if not enabled_gem_file.is_file(): logger.error(f'Enabled gem file {enabled_gem_file} is not present.') return 1 + # remove the gem error_code = cmake.remove_gem_dependency(enabled_gem_file, gem_json_data['gem_name']) - if error_code: - ret_val = error_code + + # Remove the name of the gem from the project.json "gem_names" field if the gem is neither + # registered with the project.json nor engine.json + ret_val = project_properties.edit_project_props(project_path, + delete_gem_names=gem_json_data['gem_name']) or error_code return ret_val diff --git a/scripts/o3de/o3de/enable_gem.py b/scripts/o3de/o3de/enable_gem.py index ea79f6c73f..bcfcd7157a 100644 --- a/scripts/o3de/o3de/enable_gem.py +++ b/scripts/o3de/o3de/enable_gem.py @@ -16,7 +16,7 @@ import os import pathlib import sys -from o3de import cmake, manifest, register, validation, utils +from o3de import cmake, manifest, project_properties, register, validation, utils logger = logging.getLogger('o3de.enable_gem') logging.basicConfig(format=utils.LOG_FORMAT) @@ -33,7 +33,7 @@ def enable_gem_in_project(gem_name: str = None, :param gem_path: path to the gem to add :param project_name: name of to the project to add the gem to :param project_path: path to the project to add the gem to - :param enabled_gem_file_file: if this dependency goes/is in a specific file + :param enabled_gem_file: if this dependency goes/is in a specific file :return: 0 for success or non 0 failure code """ # we need either a project name or path @@ -80,8 +80,6 @@ def enable_gem_in_project(gem_name: str = None, logger.error(f'Could not read gem.json content under {gem_path}.') return 1 - - ret_val = 0 if enabled_gem_file: # make sure this is a project has an enabled gems file if not enabled_gem_file.is_file(): @@ -96,17 +94,16 @@ def enable_gem_in_project(gem_name: str = None, if not project_enabled_gem_file.is_file(): project_enabled_gem_file.touch() - # Before adding the gem_dependency check if the project is registered in either the project or engine - # manifest + # Before adding the gem_dependency check if the project is registered in either the project or engine manifest buildable_gems = manifest.get_engine_gems() buildable_gems.extend(manifest.get_project_gems(project_path)) - # Convert each path to pathlib.Path object and filter out duplictes using dict.fromkeys + # Convert each path to pathlib.Path object and filter out duplicates using dict.fromkeys buildable_gems = list(dict.fromkeys(map(lambda gem_path_string: pathlib.Path(gem_path_string), buildable_gems))) ret_val = 0 - # If the gem is not part of buildable set, it needs to be registered - if not gem_path in buildable_gems: - ret_val = register.register(gem_path=gem_path, external_subdir_project_path=project_path) + # If the gem is not part of buildable set, it's gem_name should be registered to the "gem_names" field + if gem_path not in buildable_gems: + ret_val = project_properties.edit_project_props(project_path, new_gem_names=gem_json_data['gem_name']) # add the gem if it is registered in either the project.json or engine.json ret_val = ret_val or cmake.add_gem_dependency(project_enabled_gem_file, gem_json_data['gem_name']) diff --git a/scripts/o3de/o3de/engine_properties.py b/scripts/o3de/o3de/engine_properties.py index 636e32704e..06bff88d64 100644 --- a/scripts/o3de/o3de/engine_properties.py +++ b/scripts/o3de/o3de/engine_properties.py @@ -18,11 +18,35 @@ from o3de import manifest, utils logger = logging.getLogger('o3de.engine_properties') logging.basicConfig(format=utils.LOG_FORMAT) +def _edit_gem_names(engine_json: dict, + new_gem_names: str or list = None, + delete_gem_names: str or list = None, + replace_gem_names: str or list = None): + if new_gem_names: + tag_list = new_gem_names.split() if isinstance(new_gem_names, str) else new_gem_names + engine_json.setdefault('gem_names', []).extend(tag_list) + if delete_gem_names: + removal_list = delete_gem_names.split() if isinstance(delete_gem_names, str) else delete_gem_names + if 'gem_names' in engine_json: + for tag in removal_list: + if tag in engine_json['gem_names']: + engine_json['gem_names'].remove(tag) + if replace_gem_names: + tag_list = replace_gem_names.split() if isinstance(replace_gem_names, str) else replace_gem_names + engine_json['gem_names'] = tag_list + + # Remove duplicates from list + engine_json['gem_names'] = list(dict.fromkeys(engine_json.get('gem_names', []))) + def edit_engine_props(engine_path: pathlib.Path = None, engine_name: str = None, new_name: str = None, - new_version: str = None) -> int: + new_version: str = None, + new_gem_names: str or list = None, + delete_gem_names: str or list = None, + replace_gem_names: str or list = None + ) -> int: if not engine_path and not engine_name: logger.error(f'Either a engine path or a engine name must be supplied to lookup engine.json') return 1 @@ -51,13 +75,20 @@ def edit_engine_props(engine_path: pathlib.Path = None, if new_version: engine_json_data['O3DEVersion'] = new_version + # Update the gem_names field in the engine.json + _edit_gem_names(engine_json_data, new_gem_names, delete_gem_names, replace_gem_names) + return 0 if manifest.save_o3de_manifest(engine_json_data, pathlib.Path(engine_path) / 'engine.json') else 1 def _edit_engine_props(args: argparse) -> int: return edit_engine_props(args.engine_path, - args.engine_name, - args.engine_new_name, - args.engine_version) + args.engine_name, + args.engine_new_name, + args.engine_version, + args.add_gem_names, + args.delete_gem_names, + args.replace_gem_names + ) def add_parser_args(parser): group = parser.add_mutually_exclusive_group(required=True) @@ -70,6 +101,13 @@ def add_parser_args(parser): help='Sets the name for the engine.') group.add_argument('-ev', '--engine-version', type=str, required=False, help='Sets the version for the engine.') + group = parser.add_mutually_exclusive_group(required=False) + group.add_argument('-agn', '--add-gem-names', type=str, nargs='*', required=False, + help='Adds gem name(s) to gem_names field. Space delimited list (ex. -at A B C)') + group.add_argument('-dgn', '--delete-gem-names', type=str, nargs='*', required=False, + help='Removes gem name(s) from the gem_names field. Space delimited list (ex. -dt A B C') + group.add_argument('-rgn', '--replace-gem-names', type=str, nargs='*', required=False, + help='Replace entirety of gem_names field with space delimited list of values') parser.set_defaults(func=_edit_engine_props) def add_args(subparsers) -> None: diff --git a/scripts/o3de/o3de/project_properties.py b/scripts/o3de/o3de/project_properties.py index 9ee55773b5..f809bc8153 100644 --- a/scripts/o3de/o3de/project_properties.py +++ b/scripts/o3de/o3de/project_properties.py @@ -28,6 +28,27 @@ def get_project_props(name: str = None, path: pathlib.Path = None) -> dict: return proj_json +def _edit_gem_names(proj_json: dict, + new_gem_names: str or list = None, + delete_gem_names: str or list = None, + replace_gem_names: str or list = None): + if new_gem_names: + tag_list = new_gem_names.split() if isinstance(new_gem_names, str) else new_gem_names + proj_json.setdefault('gem_names', []).extend(tag_list) + if delete_gem_names: + removal_list = delete_gem_names.split() if isinstance(delete_gem_names, str) else delete_gem_names + if 'gem_names' in proj_json: + for tag in removal_list: + if tag in proj_json['gem_names']: + proj_json['gem_names'].remove(tag) + if replace_gem_names: + tag_list = replace_gem_names.split() if isinstance(replace_gem_names, str) else replace_gem_names + proj_json['gem_names'] = tag_list + + # Remove duplicates from list + proj_json['gem_names'] = list(dict.fromkeys(proj_json.get('gem_names', []))) + + def edit_project_props(proj_path: pathlib.Path = None, proj_name: str = None, new_name: str = None, @@ -38,7 +59,11 @@ def edit_project_props(proj_path: pathlib.Path = None, new_icon: str = None, new_tags: str or list = None, delete_tags: str or list = None, - replace_tags: str or list = None) -> int: + replace_tags: str or list = None, + new_gem_names: str or list = None, + delete_gem_names: str or list = None, + replace_gem_names: str or list = None + ) -> int: proj_json = get_project_props(proj_name, proj_path) if not proj_json: @@ -74,6 +99,8 @@ def edit_project_props(proj_path: pathlib.Path = None, if replace_tags: tag_list = replace_tags.split() if isinstance(replace_tags, str) else replace_tags proj_json['user_tags'] = tag_list + # Update the gem_names field in the project.json + _edit_gem_names(proj_json, new_gem_names, delete_gem_names, replace_gem_names) return 0 if manifest.save_o3de_manifest(proj_json, pathlib.Path(proj_path) / 'project.json') else 1 @@ -89,7 +116,10 @@ def _edit_project_props(args: argparse) -> int: args.project_icon, args.add_tags, args.delete_tags, - args.replace_tags) + args.replace_tags, + args.add_gem_names, + args.delete_gem_names, + args.replace_gem_names) def add_parser_args(parser): @@ -118,6 +148,13 @@ def add_parser_args(parser): help='Removes tag(s) from the user_tags property. Space delimited list (ex. -dt A B C') group.add_argument('-rt', '--replace-tags', type=str, nargs ='*', required=False, help='Replace entirety of user_tags property with space delimited list of values') + group = parser.add_mutually_exclusive_group(required=False) + group.add_argument('-agn', '--add-gem-names', type=str, nargs='*', required=False, + help='Adds gem name(s) to gem_names field. Space delimited list (ex. -at A B C)') + group.add_argument('-dgn', '--delete-gem-names', type=str, nargs='*', required=False, + help='Removes gem name(s) from the gem_names field. Space delimited list (ex. -dt A B C') + group.add_argument('-rgn', '--replace-gem-names', type=str, nargs='*', required=False, + help='Replace entirety of gem_names field with space delimited list of values') parser.set_defaults(func=_edit_project_props) diff --git a/scripts/o3de/tests/CMakeLists.txt b/scripts/o3de/tests/CMakeLists.txt index de8e9e4974..4c75e5e7df 100644 --- a/scripts/o3de/tests/CMakeLists.txt +++ b/scripts/o3de/tests/CMakeLists.txt @@ -13,70 +13,77 @@ endif() # Add a test to test out the o3de package `o3de.py register` command ly_add_pytest( NAME o3de_register - PATH ${CMAKE_CURRENT_LIST_DIR}/unit_test_register.py + PATH ${CMAKE_CURRENT_LIST_DIR}/test_register.py TEST_SUITE smoke EXCLUDE_TEST_RUN_TARGET_FROM_IDE ) ly_add_pytest( NAME o3de_cmake - PATH ${CMAKE_CURRENT_LIST_DIR}/unit_test_cmake.py + PATH ${CMAKE_CURRENT_LIST_DIR}/test_cmake.py + TEST_SUITE smoke + EXCLUDE_TEST_RUN_TARGET_FROM_IDE +) + +ly_add_pytest( + NAME o3de_disable_gem + PATH ${CMAKE_CURRENT_LIST_DIR}/test_disable_gem.py TEST_SUITE smoke EXCLUDE_TEST_RUN_TARGET_FROM_IDE ) ly_add_pytest( NAME o3de_enable_gem - PATH ${CMAKE_CURRENT_LIST_DIR}/unit_test_enable_gem.py + PATH ${CMAKE_CURRENT_LIST_DIR}/test_enable_gem.py TEST_SUITE smoke EXCLUDE_TEST_RUN_TARGET_FROM_IDE ) ly_add_pytest( NAME o3de_global_project - PATH ${CMAKE_CURRENT_LIST_DIR}/unit_test_global_project.py + PATH ${CMAKE_CURRENT_LIST_DIR}/test_global_project.py TEST_SUITE smoke EXCLUDE_TEST_RUN_TARGET_FROM_IDE ) ly_add_pytest( NAME o3de_manifest - PATH ${CMAKE_CURRENT_LIST_DIR}/unit_test_manifest.py + PATH ${CMAKE_CURRENT_LIST_DIR}/test_manifest.py TEST_SUITE smoke EXCLUDE_TEST_RUN_TARGET_FROM_IDE ) ly_add_pytest( NAME o3de_engine_properties - PATH ${CMAKE_CURRENT_LIST_DIR}/unit_test_engine_properties.py + PATH ${CMAKE_CURRENT_LIST_DIR}/test_engine_properties.py TEST_SUITE smoke EXCLUDE_TEST_RUN_TARGET_FROM_IDE ) ly_add_pytest( NAME o3de_project_properties - PATH ${CMAKE_CURRENT_LIST_DIR}/unit_test_project_properties.py + PATH ${CMAKE_CURRENT_LIST_DIR}/test_project_properties.py TEST_SUITE smoke EXCLUDE_TEST_RUN_TARGET_FROM_IDE ) ly_add_pytest( NAME o3de_gem_properties - PATH ${CMAKE_CURRENT_LIST_DIR}/unit_test_gem_properties.py + PATH ${CMAKE_CURRENT_LIST_DIR}/test_gem_properties.py TEST_SUITE smoke EXCLUDE_TEST_RUN_TARGET_FROM_IDE ) ly_add_pytest( NAME o3de_template - PATH ${CMAKE_CURRENT_LIST_DIR}/unit_test_engine_template.py + PATH ${CMAKE_CURRENT_LIST_DIR}/test_engine_template.py TEST_SUITE smoke EXCLUDE_TEST_RUN_TARGET_FROM_IDE ) ly_add_pytest( NAME o3de_register_show - PATH ${CMAKE_CURRENT_LIST_DIR}/unit_test_print_registration.py + PATH ${CMAKE_CURRENT_LIST_DIR}/test_print_registration.py TEST_SUITE smoke EXCLUDE_TEST_RUN_TARGET_FROM_IDE ) diff --git a/scripts/o3de/tests/unit_test_cmake.py b/scripts/o3de/tests/test_cmake.py similarity index 100% rename from scripts/o3de/tests/unit_test_cmake.py rename to scripts/o3de/tests/test_cmake.py diff --git a/scripts/o3de/tests/test_disable_gem.py b/scripts/o3de/tests/test_disable_gem.py new file mode 100644 index 0000000000..18e0402185 --- /dev/null +++ b/scripts/o3de/tests/test_disable_gem.py @@ -0,0 +1,200 @@ +# +# Copyright (c) Contributors to the Open 3D Engine Project. +# For complete copyright and license terms please see the LICENSE at the root of this distribution. +# +# SPDX-License-Identifier: Apache-2.0 OR MIT +# +# + +import json + +import pytest +import pathlib +from unittest.mock import patch + +from o3de import cmake, disable_gem, enable_gem + + +TEST_PROJECT_JSON_PAYLOAD = ''' +{ + "project_name": "TestProject", + "origin": "The primary repo for TestProject goes here: i.e. http://www.mydomain.com", + "license": "What license TestProject uses goes here: i.e. https://opensource.org/licenses/MIT", + "display_name": "TestProject", + "summary": "A short description of TestProject.", + "canonical_tags": [ + "Project" + ], + "user_tags": [ + "TestProject" + ], + "icon_path": "preview.png", + "engine": "o3de-install", + "restricted_name": "projects", + "external_subdirectories": [ + ] +} +''' + +TEST_GEM_JSON_PAYLOAD = ''' +{ + "gem_name": "TestGem", + "display_name": "TestGem", + "license": "Apache-2.0 Or MIT", + "license_url": "https://github.com/o3de/o3de/blob/development/LICENSE.txt", + "origin": "Open 3D Engine - o3de.org", + "origin_url": "https://github.com/o3de/o3de", + "type": "Code", + "summary": "A short description of TestGem.", + "canonical_tags": [ + "Gem" + ], + "user_tags": [ + "TestGem" + ], + "icon_path": "preview.png", + "requirements": "Any requirement goes here.", + "documentation_url": "The link to the documentation goes here.", + "dependencies": [ + ] +} +''' + +TEST_O3DE_MANIFEST_JSON_PAYLOAD = ''' +{ + "o3de_manifest_name": "testuser", + "origin": "C:/Users/testuser/.o3de", + "default_engines_folder": "C:/Users/testuser/.o3de/Engines", + "default_projects_folder": "C:/Users/testuser/.o3de/Projects", + "default_gems_folder": "C:/Users/testuser/.o3de/Gems", + "default_templates_folder": "C:/Users/testuser/.o3de/Templates", + "default_restricted_folder": "C:/Users/testuser/.o3de/Restricted", + "default_third_party_folder": "C:/Users/testuser/.o3de/3rdParty", + "projects": [ + "D:/MinimalProject" + ], + "external_subdirectories": [], + "templates": [], + "restricted": [], + "repos": [], + "engines": [ + "D:/o3de/o3de" + ], + "engines_path": { + "o3de": "D:/o3de/o3de" + } +} +''' + +@pytest.fixture(scope='class') +def init_disable_gem_data(request): + class DisableGemData: + def __init__(self): + self.project_data = json.loads(TEST_PROJECT_JSON_PAYLOAD) + self.gem_data = json.loads(TEST_GEM_JSON_PAYLOAD) + request.cls.disable_gem = DisableGemData() + + +@pytest.mark.usefixtures('init_disable_gem_data') +class TestDisableGemCommand: + @pytest.mark.parametrize("gem_path, project_path, gem_registered_with_project, gem_registered_with_engine," + "expected_result", [ + pytest.param(pathlib.PurePath('TestProject/TestGem'), pathlib.PurePath('TestProject'), False, True, 0), + pytest.param(pathlib.PurePath('TestProject/TestGem'), pathlib.PurePath('TestProject'), False, False, 0), + pytest.param(pathlib.PurePath('TestProject/TestGem'), pathlib.PurePath('TestProject'), True, False, 0), + pytest.param(pathlib.PurePath('TestGem'), pathlib.PurePath('TestProject'), False, False, 0), + ] + ) + def test_disable_gem_registers_gem_name_with_project_json(self, gem_path, project_path, gem_registered_with_project, + gem_registered_with_engine, expected_result): + + project_gem_dependencies = [] + + def get_registered_path(project_name: str = None, gem_name: str = None) -> pathlib.Path or None: + if project_name: + return project_path + elif gem_name: + return gem_path + return None + + def save_o3de_manifest(new_project_data: dict, manifest_path: pathlib.Path = None) -> bool: + if manifest_path == project_path / 'project.json': + self.disable_gem.project_data = new_project_data + return True + + def load_o3de_manifest(manifest_path: pathlib.Path = None) -> dict or None: + if not manifest_path: + return json.loads(TEST_O3DE_MANIFEST_JSON_PAYLOAD) + return None + + def get_project_json_data(project_name: str = None, project_path: pathlib.Path = None): + return self.disable_gem.project_data + + def get_gem_json_data(gem_path: pathlib.Path, project_path: pathlib.Path): + return self.disable_gem.gem_data + + def get_project_gems(project_path: pathlib.Path): + return [pathlib.Path(gem_path).resolve()] if gem_registered_with_project else [] + + def get_engine_gems(): + return [pathlib.Path(gem_path).resolve()] if gem_registered_with_engine else [] + + def add_gem_dependency(enable_gem_cmake_file: pathlib.Path, gem_name: str): + project_gem_dependencies.append(gem_name) + return 0 + + def remove_gem_dependency(enable_gem_cmake_file: pathlib.Path, gem_name: str): + project_gem_dependencies.remove(gem_name) + return 0 + + def get_enabled_gems(enable_gem_cmake_file: pathlib.Path) -> list: + return project_gem_dependencies + + + with patch('pathlib.Path.is_dir', return_value=True) as pathlib_is_dir_patch,\ + patch('pathlib.Path.is_file', return_value=True) as pathlib_is_file_patch, \ + patch('o3de.manifest.load_o3de_manifest', side_effect=load_o3de_manifest) as load_o3de_manifest_patch, \ + patch('o3de.manifest.save_o3de_manifest', side_effect=save_o3de_manifest) as save_o3de_manifest_patch,\ + patch('o3de.manifest.get_registered', side_effect=get_registered_path) as get_registered_patch,\ + patch('o3de.manifest.get_gem_json_data', side_effect=get_gem_json_data) as get_gem_json_data_patch,\ + patch('o3de.manifest.get_project_json_data', side_effect=get_project_json_data) as get_gem_json_data_patch,\ + patch('o3de.manifest.get_project_gems', side_effect=get_project_gems) as get_project_gems_patch,\ + patch('o3de.manifest.get_engine_gems', side_effect=get_engine_gems) as get_engine_gems_patch,\ + patch('o3de.cmake.add_gem_dependency', side_effect=add_gem_dependency) as add_gem_dependency_patch, \ + patch('o3de.cmake.remove_gem_dependency', + side_effect=remove_gem_dependency) as remove_gem_dependency_patch, \ + patch('o3de.cmake.get_enabled_gems', + side_effect=get_enabled_gems) as get_enabled_gems, \ + patch('o3de.validation.valid_o3de_gem_json', return_value=True) as valid_gem_json_patch: + + # Clear out any "gem_names" from the previous iterations + self.disable_gem.project_data.pop('gem_names', None) + + # First enable the gem + assert enable_gem.enable_gem_in_project(gem_path=gem_path, project_path=project_path) == 0 + + # Check that the gem is enabled + gem_json = get_gem_json_data(gem_path, project_path) + project_json = get_project_json_data(project_path=project_path) + enabled_gems_list = cmake.get_enabled_gems(project_path / "Gem/enabled_gems.cmake") + assert gem_json.get('gem_name', '') in enabled_gems_list + + # If the gem that is neither registered in the project.json nor engine.json, + # then it must appear in the "gem_names" field. + if not gem_registered_with_engine and not gem_registered_with_project: + assert gem_json.get('gem_name', '') in project_json.get('gem_names', []) + else: + assert gem_json.get('gem_name', '') not in project_json.get('gem_names', []) + + # Now disable the gem + result = disable_gem.disable_gem_in_project(gem_path=gem_path, project_path=project_path) + assert result == expected_result + + # Refresh the enabled_gems list and check for removal of the gem + gem_json = get_gem_json_data(gem_path, project_path) + project_json = get_project_json_data(project_path=project_path) + enabled_gems_list = cmake.get_enabled_gems(project_path / "Gem/enabled_gems.cmake") + assert gem_json.get('gem_name', '') not in enabled_gems_list + + # If gem name should no longer appear in the "gem_names" field + assert gem_json.get('gem_name', '') not in project_json.get('gem_names', []) diff --git a/scripts/o3de/tests/unit_test_enable_gem.py b/scripts/o3de/tests/test_enable_gem.py similarity index 82% rename from scripts/o3de/tests/unit_test_enable_gem.py rename to scripts/o3de/tests/test_enable_gem.py index 8067fc329d..9e9fe0a713 100644 --- a/scripts/o3de/tests/unit_test_enable_gem.py +++ b/scripts/o3de/tests/test_enable_gem.py @@ -104,10 +104,11 @@ class TestEnableGemCommand: pytest.param(pathlib.PurePath('TestProject/TestGem'), pathlib.PurePath('TestProject'), False, True, 0), pytest.param(pathlib.PurePath('TestProject/TestGem'), pathlib.PurePath('TestProject'), False, False, 0), pytest.param(pathlib.PurePath('TestProject/TestGem'), pathlib.PurePath('TestProject'), True, False, 0), + pytest.param(pathlib.PurePath('TestGem'), pathlib.PurePath('TestProject'), False, False, 0), ] ) - def test_enable_gem_registers_gem_as_well(self, gem_path, project_path, gem_registered_with_project, gem_registered_with_engine, - expected_result): + def test_enable_gem_registers_gem_name_with_project_json(self, gem_path, project_path, gem_registered_with_project, + gem_registered_with_engine, expected_result): def get_registered_path(project_name: str = None, gem_name: str = None) -> pathlib.Path: if project_name: @@ -116,11 +117,8 @@ class TestEnableGemCommand: return gem_path return None - def get_registered_gem_path(gem_name: str) -> pathlib.Path: - return gem_path - def save_o3de_manifest(new_project_data: dict, manifest_path: pathlib.Path = None) -> bool: - if manifest_path == project_path: + if manifest_path == project_path / 'project.json': self.enable_gem.project_data = new_project_data return True @@ -129,17 +127,17 @@ class TestEnableGemCommand: return json.loads(TEST_O3DE_MANIFEST_JSON_PAYLOAD) return None - def get_project_json_data(project_path: pathlib.Path): + def get_project_json_data(project_name: str = None, project_path: pathlib.Path = None): return self.enable_gem.project_data def get_gem_json_data(gem_path: pathlib.Path, project_path: pathlib.Path): return self.enable_gem.gem_data def get_project_gems(project_path: pathlib.Path): - return [gem_path] if gem_registered_with_project else [] + return [pathlib.Path(gem_path).resolve()] if gem_registered_with_project else [] def get_engine_gems(): - return [gem_path] if gem_registered_with_engine else [] + return [pathlib.Path(gem_path).resolve()] if gem_registered_with_engine else [] def add_gem_dependency(enable_gem_cmake_file: pathlib.Path, gem_name: str): return 0 @@ -155,11 +153,14 @@ class TestEnableGemCommand: patch('o3de.manifest.get_engine_gems', side_effect=get_engine_gems) as get_engine_gems_patch,\ patch('o3de.cmake.add_gem_dependency', side_effect=add_gem_dependency) as add_gem_dependency_patch,\ patch('o3de.validation.valid_o3de_gem_json', return_value=True) as valid_gem_json_patch: + + self.enable_gem.project_data.pop('gem_names', None) result = enable_gem.enable_gem_in_project(gem_path=gem_path, project_path=project_path) assert result == expected_result - # If the gem isn't registered with the engine or project already it should now be registered with the project - if not gem_registered_with_engine and gem_registered_with_project: - # Prepend the project path to each external subdirectory - project_relative_subdirs = map(lambda subdir: (pathlib.Path(project_path) / subdir).as_posix(), - self.enable_gem.project_data.get('external_subdirectories', [])) - assert gem_path.as_posix() in project_relative_subdirs + + gem_json = get_gem_json_data(gem_path, project_path) + project_json = get_project_json_data(project_path=project_path) + if not gem_registered_with_engine and not gem_registered_with_project: + assert gem_json.get('gem_name', '') in project_json.get('gem_names', []) + else: + assert gem_json.get('gem_name', '') not in project_json.get('gem_names', []) diff --git a/scripts/o3de/tests/unit_test_engine_properties.py b/scripts/o3de/tests/test_engine_properties.py similarity index 100% rename from scripts/o3de/tests/unit_test_engine_properties.py rename to scripts/o3de/tests/test_engine_properties.py diff --git a/scripts/o3de/tests/unit_test_engine_template.py b/scripts/o3de/tests/test_engine_template.py similarity index 100% rename from scripts/o3de/tests/unit_test_engine_template.py rename to scripts/o3de/tests/test_engine_template.py diff --git a/scripts/o3de/tests/unit_test_gem_properties.py b/scripts/o3de/tests/test_gem_properties.py similarity index 100% rename from scripts/o3de/tests/unit_test_gem_properties.py rename to scripts/o3de/tests/test_gem_properties.py diff --git a/scripts/o3de/tests/unit_test_global_project.py b/scripts/o3de/tests/test_global_project.py similarity index 100% rename from scripts/o3de/tests/unit_test_global_project.py rename to scripts/o3de/tests/test_global_project.py diff --git a/scripts/o3de/tests/unit_test_manifest.py b/scripts/o3de/tests/test_manifest.py similarity index 100% rename from scripts/o3de/tests/unit_test_manifest.py rename to scripts/o3de/tests/test_manifest.py diff --git a/scripts/o3de/tests/unit_test_print_registration.py b/scripts/o3de/tests/test_print_registration.py similarity index 100% rename from scripts/o3de/tests/unit_test_print_registration.py rename to scripts/o3de/tests/test_print_registration.py diff --git a/scripts/o3de/tests/unit_test_project_properties.py b/scripts/o3de/tests/test_project_properties.py similarity index 100% rename from scripts/o3de/tests/unit_test_project_properties.py rename to scripts/o3de/tests/test_project_properties.py diff --git a/scripts/o3de/tests/unit_test_register.py b/scripts/o3de/tests/test_register.py similarity index 100% rename from scripts/o3de/tests/unit_test_register.py rename to scripts/o3de/tests/test_register.py diff --git a/scripts/o3de/tests/unit_test_utils.py b/scripts/o3de/tests/test_utils.py similarity index 100% rename from scripts/o3de/tests/unit_test_utils.py rename to scripts/o3de/tests/test_utils.py