From 443e8071e76441759032f9b278b3593bd0895d42 Mon Sep 17 00:00:00 2001 From: AMZN-Phil Date: Fri, 9 Jul 2021 09:32:40 -0700 Subject: [PATCH 01/33] Show link to log while building for currently building build. Signed-off-by: AMZN-Phil --- .../Source/ProjectBuilderController.cpp | 1 + .../ProjectManager/Source/ProjectButtonWidget.cpp | 12 ++++++++++++ .../ProjectManager/Source/ProjectButtonWidget.h | 1 + 3 files changed, 14 insertions(+) diff --git a/Code/Tools/ProjectManager/Source/ProjectBuilderController.cpp b/Code/Tools/ProjectManager/Source/ProjectBuilderController.cpp index 33defeec8b..115bec3d08 100644 --- a/Code/Tools/ProjectManager/Source/ProjectBuilderController.cpp +++ b/Code/Tools/ProjectManager/Source/ProjectBuilderController.cpp @@ -72,6 +72,7 @@ namespace O3DE::ProjectManager { m_projectButton->SetButtonOverlayText(QString("%1 (%2%)\n\n").arg(tr("Building Project..."), QString::number(progress))); m_projectButton->SetProgressBarValue(progress); + m_projectButton->ShowBuildLogsLink(true, m_worker->GetLogFilePath()); } } diff --git a/Code/Tools/ProjectManager/Source/ProjectButtonWidget.cpp b/Code/Tools/ProjectManager/Source/ProjectButtonWidget.cpp index cce9da5734..540305057f 100644 --- a/Code/Tools/ProjectManager/Source/ProjectButtonWidget.cpp +++ b/Code/Tools/ProjectManager/Source/ProjectButtonWidget.cpp @@ -266,6 +266,18 @@ namespace O3DE::ProjectManager SetProjectButtonAction(tr("Build Project"), [this]() { emit BuildProject(m_projectInfo); }); } + void ProjectButton::ShowBuildLogsLink(bool show, const QUrl& logUrl) + { + if (!logUrl.isEmpty()) + { + m_projectImageLabel->GetWarningLabel()->setText(tr("Click to view logs.")); + } + + m_projectImageLabel->GetWarningLabel()->setTextInteractionFlags(Qt::LinksAccessibleByMouse); + m_projectImageLabel->GetWarningLabel()->setVisible(show); + m_projectImageLabel->SetLogUrl(logUrl); + } + void ProjectButton::ShowBuildFailed(bool show, const QUrl& logUrl) { if (!logUrl.isEmpty()) diff --git a/Code/Tools/ProjectManager/Source/ProjectButtonWidget.h b/Code/Tools/ProjectManager/Source/ProjectButtonWidget.h index a99984c12f..747fd89a31 100644 --- a/Code/Tools/ProjectManager/Source/ProjectButtonWidget.h +++ b/Code/Tools/ProjectManager/Source/ProjectButtonWidget.h @@ -77,6 +77,7 @@ namespace O3DE::ProjectManager void SetProjectButtonAction(const QString& text, AZStd::function lambda); void SetProjectBuildButtonAction(); + void ShowBuildLogsLink(bool show, const QUrl& logUrl); void ShowBuildFailed(bool show, const QUrl& logUrl); void SetLaunchButtonEnabled(bool enabled); From b92848ed88a0441bf2f91b49943dfb6edc5f4d16 Mon Sep 17 00:00:00 2001 From: AMZN-Phil Date: Fri, 9 Jul 2021 14:12:31 -0700 Subject: [PATCH 02/33] Move the show log text to be part of building label. Signed-off-by: AMZN-Phil --- .../Source/ProjectBuilderController.cpp | 4 ++-- .../ProjectManager/Source/ProjectButtonWidget.cpp | 11 +++-------- .../Tools/ProjectManager/Source/ProjectButtonWidget.h | 2 +- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/Code/Tools/ProjectManager/Source/ProjectBuilderController.cpp b/Code/Tools/ProjectManager/Source/ProjectBuilderController.cpp index 115bec3d08..007d4a72e3 100644 --- a/Code/Tools/ProjectManager/Source/ProjectBuilderController.cpp +++ b/Code/Tools/ProjectManager/Source/ProjectBuilderController.cpp @@ -70,9 +70,9 @@ namespace O3DE::ProjectManager m_lastProgress = progress; if (m_projectButton) { - m_projectButton->SetButtonOverlayText(QString("%1 (%2%)\n\n").arg(tr("Building Project..."), QString::number(progress))); + m_projectButton->SetButtonOverlayText(QString("%1 (%2%)
%3
").arg(tr("Building Project..."), QString::number(progress), tr("Click to view logs."))); m_projectButton->SetProgressBarValue(progress); - m_projectButton->ShowBuildLogsLink(true, m_worker->GetLogFilePath()); + m_projectButton->SetBuildLogsLink(m_worker->GetLogFilePath()); } } diff --git a/Code/Tools/ProjectManager/Source/ProjectButtonWidget.cpp b/Code/Tools/ProjectManager/Source/ProjectButtonWidget.cpp index 540305057f..af9a1a7bd9 100644 --- a/Code/Tools/ProjectManager/Source/ProjectButtonWidget.cpp +++ b/Code/Tools/ProjectManager/Source/ProjectButtonWidget.cpp @@ -39,7 +39,9 @@ namespace O3DE::ProjectManager m_overlayLabel->setObjectName("labelButtonOverlay"); m_overlayLabel->setWordWrap(true); m_overlayLabel->setAlignment(Qt::AlignCenter); + m_overlayLabel->setTextInteractionFlags(Qt::LinksAccessibleByMouse); m_overlayLabel->setVisible(false); + connect(m_overlayLabel, &QLabel::linkActivated, this, &LabelButton::OnLinkActivated); vLayout->addWidget(m_overlayLabel); m_buildOverlayLayout = new QVBoxLayout(); @@ -266,15 +268,8 @@ namespace O3DE::ProjectManager SetProjectButtonAction(tr("Build Project"), [this]() { emit BuildProject(m_projectInfo); }); } - void ProjectButton::ShowBuildLogsLink(bool show, const QUrl& logUrl) + void ProjectButton::SetBuildLogsLink(const QUrl& logUrl) { - if (!logUrl.isEmpty()) - { - m_projectImageLabel->GetWarningLabel()->setText(tr("Click to view logs.")); - } - - m_projectImageLabel->GetWarningLabel()->setTextInteractionFlags(Qt::LinksAccessibleByMouse); - m_projectImageLabel->GetWarningLabel()->setVisible(show); m_projectImageLabel->SetLogUrl(logUrl); } diff --git a/Code/Tools/ProjectManager/Source/ProjectButtonWidget.h b/Code/Tools/ProjectManager/Source/ProjectButtonWidget.h index 747fd89a31..27559b325e 100644 --- a/Code/Tools/ProjectManager/Source/ProjectButtonWidget.h +++ b/Code/Tools/ProjectManager/Source/ProjectButtonWidget.h @@ -77,7 +77,7 @@ namespace O3DE::ProjectManager void SetProjectButtonAction(const QString& text, AZStd::function lambda); void SetProjectBuildButtonAction(); - void ShowBuildLogsLink(bool show, const QUrl& logUrl); + void SetBuildLogsLink(const QUrl& logUrl); void ShowBuildFailed(bool show, const QUrl& logUrl); void SetLaunchButtonEnabled(bool enabled); From b0bc076febcde313313a59464168a770a83878d1 Mon Sep 17 00:00:00 2001 From: abrmich Date: Thu, 1 Jul 2021 18:01:25 -0700 Subject: [PATCH 03/33] Removed unused bus connection Signed-off-by: abrmich --- Gems/LyShine/Code/Source/LyShineSystemComponent.cpp | 2 -- Gems/LyShine/Code/Source/LyShineSystemComponent.h | 1 - 2 files changed, 3 deletions(-) diff --git a/Gems/LyShine/Code/Source/LyShineSystemComponent.cpp b/Gems/LyShine/Code/Source/LyShineSystemComponent.cpp index 7521c8b6c1..b420e551b5 100644 --- a/Gems/LyShine/Code/Source/LyShineSystemComponent.cpp +++ b/Gems/LyShine/Code/Source/LyShineSystemComponent.cpp @@ -148,7 +148,6 @@ namespace LyShine { LyShineAllocatorScope::ActivateAllocators(); - LyShineRequestBus::Handler::BusConnect(); UiSystemBus::Handler::BusConnect(); UiSystemToolsBus::Handler::BusConnect(); UiFrameworkBus::Handler::BusConnect(); @@ -196,7 +195,6 @@ namespace LyShine UiSystemBus::Handler::BusDisconnect(); UiSystemToolsBus::Handler::BusDisconnect(); UiFrameworkBus::Handler::BusDisconnect(); - LyShineRequestBus::Handler::BusDisconnect(); CrySystemEventBus::Handler::BusDisconnect(); LyShineAllocatorScope::DeactivateAllocators(); diff --git a/Gems/LyShine/Code/Source/LyShineSystemComponent.h b/Gems/LyShine/Code/Source/LyShineSystemComponent.h index 5b455715ee..2737ba8667 100644 --- a/Gems/LyShine/Code/Source/LyShineSystemComponent.h +++ b/Gems/LyShine/Code/Source/LyShineSystemComponent.h @@ -28,7 +28,6 @@ namespace LyShine class LyShineSystemComponent : public AZ::Component - , protected LyShineRequestBus::Handler , protected UiSystemBus::Handler , protected UiSystemToolsBus::Handler , protected LyShineAllocatorScope From 5fe980edf19b8ac26dd75be0f0c5d4982ca29a4d Mon Sep 17 00:00:00 2001 From: abrmich Date: Fri, 9 Jul 2021 17:58:14 -0700 Subject: [PATCH 04/33] Remove unused header Signed-off-by: abrmich --- Gems/LyShine/Code/Source/LyShineSystemComponent.h | 1 - 1 file changed, 1 deletion(-) diff --git a/Gems/LyShine/Code/Source/LyShineSystemComponent.h b/Gems/LyShine/Code/Source/LyShineSystemComponent.h index 2737ba8667..70e7eb5fe5 100644 --- a/Gems/LyShine/Code/Source/LyShineSystemComponent.h +++ b/Gems/LyShine/Code/Source/LyShineSystemComponent.h @@ -13,7 +13,6 @@ #include -#include #include #include #include From a8435ec982bfcee020eba7ca964e18b52ccce384 Mon Sep 17 00:00:00 2001 From: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> Date: Mon, 12 Jul 2021 10:23:48 -0700 Subject: [PATCH 05/33] Issues/2045 3rdParty runtime dependencies copied multiple times (#2058) * 3rdParty runtime dependencies copied multiple times Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> * 3rdParty to update timestamps when uncompressing to provoke copy of runtime dependencies Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> * typo fix Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> --- cmake/3rdPartyPackages.cmake | 6 ++++++ cmake/Platform/Common/runtime_dependencies_common.cmake.in | 1 + cmake/Platform/Mac/runtime_dependencies_mac.cmake.in | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/cmake/3rdPartyPackages.cmake b/cmake/3rdPartyPackages.cmake index 9a632798e8..0d1667f0e1 100644 --- a/cmake/3rdPartyPackages.cmake +++ b/cmake/3rdPartyPackages.cmake @@ -529,6 +529,12 @@ function(ly_force_download_package package_name) execute_process(COMMAND ${CMAKE_COMMAND} -E tar xf ${temp_download_target} WORKING_DIRECTORY ${final_folder} COMMAND_ECHO STDOUT OUTPUT_VARIABLE unpack_result) + # For the runtime dependencies cases, we need the timestamps of the files coming from 3rdParty to be newer than the ones + # from the output so the new versions get copied over. The untar from the previous step preserves timestamps so they + # can produce binaries with older timestamps to the ones that are in the build output. + file(GLOB_RECURSE package_files LIST_DIRECTORIES false ${final_folder}/*) + file(TOUCH_NOCREATE ${package_files}) + if (NOT ${unpack_result} EQUAL 0) message(SEND_ERROR "ly_package: required package {package_name} could not be unpacked. Compile may fail! Enable LY_PACKAGE_DEBUG to debug.") return() diff --git a/cmake/Platform/Common/runtime_dependencies_common.cmake.in b/cmake/Platform/Common/runtime_dependencies_common.cmake.in index 9d23a73a26..104c20205c 100644 --- a/cmake/Platform/Common/runtime_dependencies_common.cmake.in +++ b/cmake/Platform/Common/runtime_dependencies_common.cmake.in @@ -15,6 +15,7 @@ function(ly_copy source_file target_directory) if("${source_file}" IS_NEWER_THAN "${target_directory}/${target_filename}") message(STATUS "Copying \"${source_file}\" to \"${target_directory}\"...") file(COPY "${source_file}" DESTINATION "${target_directory}" FILE_PERMISSIONS @LY_COPY_PERMISSIONS@ FOLLOW_SYMLINK_CHAIN) + file(TOUCH_NOCREATE ${target_directory}/${target_filename}) endif() endif() endfunction() diff --git a/cmake/Platform/Mac/runtime_dependencies_mac.cmake.in b/cmake/Platform/Mac/runtime_dependencies_mac.cmake.in index 9f9006c4c0..9869871f85 100644 --- a/cmake/Platform/Mac/runtime_dependencies_mac.cmake.in +++ b/cmake/Platform/Mac/runtime_dependencies_mac.cmake.in @@ -125,7 +125,7 @@ function(ly_copy source_file target_directory) file(LOCK ${target_directory}/${target_filename}.lock GUARD FUNCTION TIMEOUT 300) endif() file(COPY "${source_file}" DESTINATION "${target_directory}" FILE_PERMISSIONS @LY_COPY_PERMISSIONS@ FOLLOW_SYMLINK_CHAIN) - file(TOUCH ${target_directory}/${target_filename}) + file(TOUCH_NOCREATE ${target_directory}/${target_filename}) set(anything_new TRUE PARENT_SCOPE) endif() endif() From f83439e6c4d2fdd16b0433488855bab540154a95 Mon Sep 17 00:00:00 2001 From: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> Date: Mon, 12 Jul 2021 10:24:45 -0700 Subject: [PATCH 06/33] Add Vcpkg settings to map configs to what vcpkg expects (this prevents a message from being issue when vcpkg is installed) (#2063) Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> --- cmake/Platform/Common/Directory.Build.props | 1 + cmake/Platform/Common/VisualStudio_common.cmake | 12 +++++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/cmake/Platform/Common/Directory.Build.props b/cmake/Platform/Common/Directory.Build.props index 29dbcd8584..9389a008ad 100644 --- a/cmake/Platform/Common/Directory.Build.props +++ b/cmake/Platform/Common/Directory.Build.props @@ -9,6 +9,7 @@ SPDX-License-Identifier: Apache-2.0 OR MIT true true +@VCPKG_CONFIGURATION_MAPPING@ diff --git a/cmake/Platform/Common/VisualStudio_common.cmake b/cmake/Platform/Common/VisualStudio_common.cmake index 345214274b..91817bef6f 100644 --- a/cmake/Platform/Common/VisualStudio_common.cmake +++ b/cmake/Platform/Common/VisualStudio_common.cmake @@ -6,5 +6,15 @@ # if(CMAKE_GENERATOR MATCHES "Visual Studio 16") - configure_file("${CMAKE_CURRENT_LIST_DIR}/Directory.Build.props" "${CMAKE_BINARY_DIR}/Directory.Build.props" COPYONLY) + + foreach(conf IN LISTS CMAKE_CONFIGURATION_TYPES) + if(conf STREQUAL debug) + string(APPEND VCPKG_CONFIGURATION_MAPPING " Debug\n") + else() + string(APPEND VCPKG_CONFIGURATION_MAPPING " Release\n") + endif() + endforeach() + + configure_file("${CMAKE_CURRENT_LIST_DIR}/Directory.Build.props" "${CMAKE_BINARY_DIR}/Directory.Build.props" @ONLY) + endif() \ No newline at end of file From a7751de2715fb3febb7372fa00a226010dd57468 Mon Sep 17 00:00:00 2001 From: AMZN-Phil Date: Mon, 12 Jul 2021 11:07:09 -0700 Subject: [PATCH 07/33] Fix feature category spellings Signed-off-by: AMZN-Phil --- Gems/AssetMemoryAnalyzer/gem.json | 2 +- Gems/AudioEngineWwise/gem.json | 2 +- Gems/AudioSystem/gem.json | 2 +- Gems/ExpressionEvaluation/gem.json | 2 +- Gems/GameStateSamples/gem.json | 2 +- Gems/ScriptCanvas/gem.json | 2 +- Gems/SurfaceData/gem.json | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Gems/AssetMemoryAnalyzer/gem.json b/Gems/AssetMemoryAnalyzer/gem.json index 8c63d51e1a..45610e417c 100644 --- a/Gems/AssetMemoryAnalyzer/gem.json +++ b/Gems/AssetMemoryAnalyzer/gem.json @@ -6,7 +6,7 @@ "type": "Code", "summary": "The Asset Memory Analyzer Gem provides tools to profile asset memory usage in Open 3D Engine through ImGUI (Immediate Mode Graphical User Interface).", "canonical_tags": ["Gem"], - "user_tags": ["Debug", "Utillity", "Tools"], + "user_tags": ["Debug", "Utility", "Tools"], "icon_path": "preview.png", "requirements": "" } diff --git a/Gems/AudioEngineWwise/gem.json b/Gems/AudioEngineWwise/gem.json index 8c6b5767b0..dc5f968bc7 100644 --- a/Gems/AudioEngineWwise/gem.json +++ b/Gems/AudioEngineWwise/gem.json @@ -6,7 +6,7 @@ "type": "Code", "summary": "The Wwise Audio Engine Gem provides support for Audiokinetic Wave Works Interactive Sound Engine (Wwise).", "canonical_tags": ["Gem"], - "user_tags": ["Audio", "Utiltity", "Tools"], + "user_tags": ["Audio", "Utility", "Tools"], "icon_path": "preview.png", "requirements": "Users will need to download WWise from the AudioKinetic web site: https://www.audiokinetic.com/download/" } diff --git a/Gems/AudioSystem/gem.json b/Gems/AudioSystem/gem.json index a1dbe9406f..a9058cb42e 100644 --- a/Gems/AudioSystem/gem.json +++ b/Gems/AudioSystem/gem.json @@ -6,7 +6,7 @@ "type": "Code", "summary": "The Audio System Gem provides the Audio Translation Layer (ATL) and Audio Controls Editor, which add support for audio in Open 3D Engine.", "canonical_tags": ["Gem"], - "user_tags": ["Audio", "Utiltity", "Tools"], + "user_tags": ["Audio", "Utility", "Tools"], "icon_path": "preview.png", "requirements": "" } diff --git a/Gems/ExpressionEvaluation/gem.json b/Gems/ExpressionEvaluation/gem.json index 9302af152a..92d5963891 100644 --- a/Gems/ExpressionEvaluation/gem.json +++ b/Gems/ExpressionEvaluation/gem.json @@ -6,7 +6,7 @@ "type": "Code", "summary": "The Expression Evaluation Gem provides a method for parsing and executing string expressions in Open 3D Engine.", "canonical_tags": ["Gem"], - "user_tags": ["Scripting", "Utiltity"], + "user_tags": ["Scripting", "Utility"], "icon_path": "preview.png", "requirements": "" } diff --git a/Gems/GameStateSamples/gem.json b/Gems/GameStateSamples/gem.json index ce0fcbd868..0241a8a1b7 100644 --- a/Gems/GameStateSamples/gem.json +++ b/Gems/GameStateSamples/gem.json @@ -6,7 +6,7 @@ "type": "Code", "summary": "The Game State Samples Gem provides a set of sample game states (built on top of the Game State Gem), including primary user selection, main menu, level loading, level running, and level paused.", "canonical_tags": ["Gem"], - "user_tags": ["Gameplay", "Samples", "Assets"], + "user_tags": ["Gameplay", "Sample", "Assets"], "icon_path": "preview.png", "requirements": "" } diff --git a/Gems/ScriptCanvas/gem.json b/Gems/ScriptCanvas/gem.json index 680ce6ef1f..413e58ef7e 100644 --- a/Gems/ScriptCanvas/gem.json +++ b/Gems/ScriptCanvas/gem.json @@ -6,7 +6,7 @@ "type": "Tool", "summary": "The Script Canvas Gem provides Open 3D Engine's visual scripting environment, Script Canvas.", "canonical_tags": ["Gem"], - "user_tags": ["Scripting", "Tools", "Utiltiy"], + "user_tags": ["Scripting", "Tools", "Utility"], "icon_path": "preview.png", "requirements": "" } diff --git a/Gems/SurfaceData/gem.json b/Gems/SurfaceData/gem.json index 1fb521ef58..24f7f1cb21 100644 --- a/Gems/SurfaceData/gem.json +++ b/Gems/SurfaceData/gem.json @@ -6,7 +6,7 @@ "type": "Code", "summary": "The Surface Data Gem provides functionality to emit signals or tags from surfaces such as meshes and terrain.", "canonical_tags": ["Gem"], - "user_tags": ["Environment", "Utiltiy", "Design"], + "user_tags": ["Environment", "Utility", "Design"], "icon_path": "preview.png", "requirements": "" } From 41e864de95a04ba8767773f9b6eb0a1e07351ca1 Mon Sep 17 00:00:00 2001 From: brianherrera Date: Fri, 9 Jul 2021 15:48:37 -0700 Subject: [PATCH 08/33] Remove separate git lfs steps. The steps to remove git lfs hooks and inject creds are no longer required with the public repo. Signed-off-by: brianherrera --- scripts/build/Jenkins/Jenkinsfile | 9 --------- 1 file changed, 9 deletions(-) diff --git a/scripts/build/Jenkins/Jenkinsfile b/scripts/build/Jenkins/Jenkinsfile index a7a796f3b7..70618fea78 100644 --- a/scripts/build/Jenkins/Jenkinsfile +++ b/scripts/build/Jenkins/Jenkinsfile @@ -204,8 +204,6 @@ def CheckoutRepo(boolean disableSubmodules = false) { if (!fileExists(ENGINE_REPOSITORY_NAME)) { palMkdir(ENGINE_REPOSITORY_NAME) } - - palSh('git lfs uninstall', 'Git LFS Uninstall') // Prevent git from pulling lfs objects during checkout if(fileExists('.git')) { // If the repository after checkout is locked, likely we took a snapshot while git was running, @@ -239,13 +237,6 @@ def CheckoutRepo(boolean disableSubmodules = false) { ] } - // Run lfs in a separate step. Jenkins is unable to load the credentials for the custom LFS endpoint - withCredentials([usernamePassword(credentialsId: "${env.GITHUB_USER}", passwordVariable: 'accesstoken', usernameVariable: 'username')]) { - palSh("git config -f .lfsconfig lfs.url https://${username}:${accesstoken}@${env.LFS_URL}", 'Set credentials', false) - } - palSh('git lfs install', 'Git LFS Install') - palSh('git lfs pull', 'Git LFS Pull') - // CHANGE_ID is used by some scripts to identify uniquely the current change (usually metric jobs) palSh('git rev-parse HEAD > commitid', 'Getting commit id') env.CHANGE_ID = readFile file: 'commitid' From 585caabd950dc7abdee2efe3e29f0dbebb8ac020 Mon Sep 17 00:00:00 2001 From: AMZN-ScottR <24445312+AMZN-ScottR@users.noreply.github.com> Date: Mon, 12 Jul 2021 11:38:05 -0700 Subject: [PATCH 09/33] [installer/2106-3p-license-fix] replaced 3rd party license url arg with auto generated one from version string Signed-off-by: AMZN-ScottR <24445312+AMZN-ScottR@users.noreply.github.com> --- cmake/Packaging.cmake | 55 ++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 30 deletions(-) diff --git a/cmake/Packaging.cmake b/cmake/Packaging.cmake index d77b5a30f8..a07e1ec1d3 100644 --- a/cmake/Packaging.cmake +++ b/cmake/Packaging.cmake @@ -125,37 +125,32 @@ install(FILES ${_cmake_package_dest} DESTINATION ./Tools/Redistributables/CMake ) -# temporary workaround for acquiring the 3rd party SPDX license manifest, the desired location is from -# another git repository that's private. once it's public, only how the URL is formed should change -set(LY_INSTALLER_3RD_PARTY_LICENSE_URL "" CACHE STRING "URL to the 3rd party SPDX license manifest file for inclusion in packaging.") - -if(${LY_VERSION_STRING} VERSION_GREATER "0.0.0.0" AND NOT LY_INSTALLER_3RD_PARTY_LICENSE_URL) - message(FATAL_ERROR "Missing required URL for the 3rd party SPDX license manifest file. " - "Please specifiy where to acquire the file via LY_INSTALLER_3RD_PARTY_LICENSE_URL") -endif() - -string(REPLACE "/" ";" _url_components ${LY_INSTALLER_3RD_PARTY_LICENSE_URL}) -list(POP_BACK _url_components _3rd_party_license_filename) - -set(_3rd_party_license_dest ${CPACK_BINARY_DIR}/${_3rd_party_license_filename}) - -# use the plain file downloader as we don't have the file hash available and using a dummy will -# delete the file once it fails hash verification -file(DOWNLOAD - ${LY_INSTALLER_3RD_PARTY_LICENSE_URL} - ${_3rd_party_license_dest} - STATUS _status - TLS_VERIFY ON -) -list(POP_FRONT _status _status_code) - -if (${_status_code} EQUAL 0 AND EXISTS ${_3rd_party_license_dest}) - install(FILES ${_3rd_party_license_dest} - DESTINATION . +# the version string and git tags are intended to be synchronized so it should be safe to use that instead +# of directly calling into git which could get messy in certain scenarios +if(${CPACK_PACKAGE_VERSION} VERSION_GREATER "0.0.0.0") + set(_3rd_party_license_filename SPDX-Licenses.txt) + + set(_3rd_party_license_url "https://raw.githubusercontent.com/o3de/3p-package-source/${CPACK_PACKAGE_VERSION}/${_3rd_party_license_filename}") + set(_3rd_party_license_dest ${CPACK_BINARY_DIR}/${_3rd_party_license_filename}) + + # use the plain file downloader as we don't have the file hash available and using a dummy will + # delete the file once it fails hash verification + file(DOWNLOAD + ${_3rd_party_license_url} + ${_3rd_party_license_dest} + STATUS _status + TLS_VERIFY ON ) -else() - file(REMOVE ${_3rd_party_license_dest}) - message(FATAL_ERROR "Failed to acquire the 3rd Party license manifest file. Error: ${_status}") + list(POP_FRONT _status _status_code) + + if (${_status_code} EQUAL 0 AND EXISTS ${_3rd_party_license_dest}) + install(FILES ${_3rd_party_license_dest} + DESTINATION . + ) + else() + file(REMOVE ${_3rd_party_license_dest}) + message(FATAL_ERROR "Failed to acquire the 3rd Party license manifest file at ${_3rd_party_license_url}. Error: ${_status}") + endif() endif() # checks for and removes trailing slash From 2fad7f37db565815d12bcc2f053e605f02e5384d Mon Sep 17 00:00:00 2001 From: AMZN-nggieber <52797929+AMZN-nggieber@users.noreply.github.com> Date: Mon, 12 Jul 2021 11:54:36 -0700 Subject: [PATCH 10/33] Added a Warning When VS2019 is not Installed with Link to Download it (#2042) * Added a warning when VS2019 is not installed with link to download it Signed-off-by: nggieber * Widden VSWarning dialog Signed-off-by: nggieber * Fix issue with checking for Visual Studio Signed-off-by: nggieber * PALify compiler detection Signed-off-by: nggieber * Changed windows compiler check to waitForFinished instead of waitForReadyRead Signed-off-by: nggieber --- .../Platform/Linux/PAL_linux_files.cmake | 1 + .../Platform/Linux/ProjectUtils_linux.cpp | 21 +++++++ .../Platform/Mac/PAL_mac_files.cmake | 1 + .../Platform/Mac/ProjectUtils_mac.cpp | 21 +++++++ .../Platform/Windows/PAL_windows_files.cmake | 1 + .../Platform/Windows/ProjectUtils_windows.cpp | 60 +++++++++++++++++++ .../Source/CreateProjectCtrl.cpp | 47 ++++++++------- .../ProjectManager/Source/ProjectUtils.cpp | 53 ++++++---------- .../ProjectManager/Source/ProjectUtils.h | 4 +- .../ProjectManager/Source/ProjectsScreen.cpp | 2 +- 10 files changed, 153 insertions(+), 58 deletions(-) create mode 100644 Code/Tools/ProjectManager/Platform/Linux/ProjectUtils_linux.cpp create mode 100644 Code/Tools/ProjectManager/Platform/Mac/ProjectUtils_mac.cpp create mode 100644 Code/Tools/ProjectManager/Platform/Windows/ProjectUtils_windows.cpp diff --git a/Code/Tools/ProjectManager/Platform/Linux/PAL_linux_files.cmake b/Code/Tools/ProjectManager/Platform/Linux/PAL_linux_files.cmake index e8085de555..4125861f2b 100644 --- a/Code/Tools/ProjectManager/Platform/Linux/PAL_linux_files.cmake +++ b/Code/Tools/ProjectManager/Platform/Linux/PAL_linux_files.cmake @@ -8,4 +8,5 @@ set(FILES Python_linux.cpp ProjectBuilderWorker_linux.cpp + ProjectUtils_linux.cpp ) diff --git a/Code/Tools/ProjectManager/Platform/Linux/ProjectUtils_linux.cpp b/Code/Tools/ProjectManager/Platform/Linux/ProjectUtils_linux.cpp new file mode 100644 index 0000000000..64d18ec605 --- /dev/null +++ b/Code/Tools/ProjectManager/Platform/Linux/ProjectUtils_linux.cpp @@ -0,0 +1,21 @@ +/* + * Copyright (c) Contributors to the Open 3D Engine Project. For complete copyright and license terms please see the LICENSE at the root of this distribution. + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + * + */ + +#include + +namespace O3DE::ProjectManager +{ + namespace ProjectUtils + { + AZ::Outcome FindSupportedCompilerForPlatform() + { + // Compiler detection not supported on platform + return AZ::Success(); + } + + } // namespace ProjectUtils +} // namespace O3DE::ProjectManager diff --git a/Code/Tools/ProjectManager/Platform/Mac/PAL_mac_files.cmake b/Code/Tools/ProjectManager/Platform/Mac/PAL_mac_files.cmake index 01d6b5f112..a0d3add840 100644 --- a/Code/Tools/ProjectManager/Platform/Mac/PAL_mac_files.cmake +++ b/Code/Tools/ProjectManager/Platform/Mac/PAL_mac_files.cmake @@ -8,4 +8,5 @@ set(FILES Python_mac.cpp ProjectBuilderWorker_mac.cpp + ProjectUtils_mac.cpp ) diff --git a/Code/Tools/ProjectManager/Platform/Mac/ProjectUtils_mac.cpp b/Code/Tools/ProjectManager/Platform/Mac/ProjectUtils_mac.cpp new file mode 100644 index 0000000000..64d18ec605 --- /dev/null +++ b/Code/Tools/ProjectManager/Platform/Mac/ProjectUtils_mac.cpp @@ -0,0 +1,21 @@ +/* + * Copyright (c) Contributors to the Open 3D Engine Project. For complete copyright and license terms please see the LICENSE at the root of this distribution. + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + * + */ + +#include + +namespace O3DE::ProjectManager +{ + namespace ProjectUtils + { + AZ::Outcome FindSupportedCompilerForPlatform() + { + // Compiler detection not supported on platform + return AZ::Success(); + } + + } // namespace ProjectUtils +} // namespace O3DE::ProjectManager diff --git a/Code/Tools/ProjectManager/Platform/Windows/PAL_windows_files.cmake b/Code/Tools/ProjectManager/Platform/Windows/PAL_windows_files.cmake index eb45c61807..8af7638696 100644 --- a/Code/Tools/ProjectManager/Platform/Windows/PAL_windows_files.cmake +++ b/Code/Tools/ProjectManager/Platform/Windows/PAL_windows_files.cmake @@ -8,4 +8,5 @@ set(FILES Python_windows.cpp ProjectBuilderWorker_windows.cpp + ProjectUtils_windows.cpp ) diff --git a/Code/Tools/ProjectManager/Platform/Windows/ProjectUtils_windows.cpp b/Code/Tools/ProjectManager/Platform/Windows/ProjectUtils_windows.cpp new file mode 100644 index 0000000000..c0d977a5f5 --- /dev/null +++ b/Code/Tools/ProjectManager/Platform/Windows/ProjectUtils_windows.cpp @@ -0,0 +1,60 @@ +/* + * Copyright (c) Contributors to the Open 3D Engine Project. For complete copyright and license terms please see the LICENSE at the root of this distribution. + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + * + */ + +#include + +#include +#include +#include +#include + +namespace O3DE::ProjectManager +{ + namespace ProjectUtils + { + AZ::Outcome FindSupportedCompilerForPlatform() + { + QProcessEnvironment environment = QProcessEnvironment::systemEnvironment(); + QString programFilesPath = environment.value("ProgramFiles(x86)"); + QString vsWherePath = QDir(programFilesPath).filePath("Microsoft Visual Studio/Installer/vswhere.exe"); + + QFileInfo vsWhereFile(vsWherePath); + if (vsWhereFile.exists() && vsWhereFile.isFile()) + { + QProcess vsWhereProcess; + vsWhereProcess.setProcessChannelMode(QProcess::MergedChannels); + + vsWhereProcess.start( + vsWherePath, + QStringList{ + "-version", + "16.0", + "-latest", + "-requires", + "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", + "-property", + "isComplete" + }); + + if (vsWhereProcess.waitForStarted() && vsWhereProcess.waitForFinished()) + { + QString vsWhereOutput(vsWhereProcess.readAllStandardOutput()); + if (vsWhereOutput.startsWith("1")) + { + return AZ::Success(); + } + } + } + + return AZ::Failure(QObject::tr("Visual Studio 2019 not found.\n\n" + "Visual Studio 2019 is required to build this project." + " Install any edition of Visual Studio 2019" + " before proceeding to the next step.")); + } + + } // namespace ProjectUtils +} // namespace O3DE::ProjectManager diff --git a/Code/Tools/ProjectManager/Source/CreateProjectCtrl.cpp b/Code/Tools/ProjectManager/Source/CreateProjectCtrl.cpp index 9c0b4726ed..b2b656dbae 100644 --- a/Code/Tools/ProjectManager/Source/CreateProjectCtrl.cpp +++ b/Code/Tools/ProjectManager/Source/CreateProjectCtrl.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -222,38 +223,42 @@ namespace O3DE::ProjectManager void CreateProjectCtrl::CreateProject() { - if (m_newProjectSettingsScreen->Validate()) + if (ProjectUtils::FindSupportedCompiler(this)) { - ProjectInfo projectInfo = m_newProjectSettingsScreen->GetProjectInfo(); - QString projectTemplatePath = m_newProjectSettingsScreen->GetProjectTemplatePath(); - - auto result = PythonBindingsInterface::Get()->CreateProject(projectTemplatePath, projectInfo); - if (result.IsSuccess()) + if (m_newProjectSettingsScreen->Validate()) { - // automatically register the project - PythonBindingsInterface::Get()->AddProject(projectInfo.m_path); + ProjectInfo projectInfo = m_newProjectSettingsScreen->GetProjectInfo(); + QString projectTemplatePath = m_newProjectSettingsScreen->GetProjectTemplatePath(); -#ifdef TEMPLATE_GEM_CONFIGURATION_ENABLED - if (!m_gemCatalogScreen->EnableDisableGemsForProject(projectInfo.m_path)) + auto result = PythonBindingsInterface::Get()->CreateProject(projectTemplatePath, projectInfo); + if (result.IsSuccess()) { - QMessageBox::critical(this, tr("Failed to configure gems"), tr("Failed to configure gems for template.")); - return; - } + // automatically register the project + PythonBindingsInterface::Get()->AddProject(projectInfo.m_path); + +#ifdef TEMPLATE_GEM_CONFIGURATION_ENABLED + if (!m_gemCatalogScreen->EnableDisableGemsForProject(projectInfo.m_path)) + { + QMessageBox::critical(this, tr("Failed to configure gems"), tr("Failed to configure gems for template.")); + return; + } #endif // TEMPLATE_GEM_CONFIGURATION_ENABLED - projectInfo.m_needsBuild = true; - emit NotifyBuildProject(projectInfo); - emit ChangeScreenRequest(ProjectManagerScreen::Projects); + projectInfo.m_needsBuild = true; + emit NotifyBuildProject(projectInfo); + emit ChangeScreenRequest(ProjectManagerScreen::Projects); + } + else + { + QMessageBox::critical(this, tr("Project creation failed"), tr("Failed to create project.")); + } } else { - QMessageBox::critical(this, tr("Project creation failed"), tr("Failed to create project.")); + QMessageBox::warning( + this, tr("Invalid project settings"), tr("Please correct the indicated project settings and try again.")); } } - else - { - QMessageBox::warning(this, tr("Invalid project settings"), tr("Please correct the indicated project settings and try again.")); - } } void CreateProjectCtrl::ReinitGemCatalogForSelectedTemplate() diff --git a/Code/Tools/ProjectManager/Source/ProjectUtils.cpp b/Code/Tools/ProjectManager/Source/ProjectUtils.cpp index 83e7546670..827973cbec 100644 --- a/Code/Tools/ProjectManager/Source/ProjectUtils.cpp +++ b/Code/Tools/ProjectManager/Source/ProjectUtils.cpp @@ -17,6 +17,8 @@ #include #include #include +#include +#include namespace O3DE::ProjectManager { @@ -374,46 +376,27 @@ namespace O3DE::ProjectManager return true; } - static bool IsVS2019Installed_internal() + bool FindSupportedCompiler(QWidget* parent) { - QProcessEnvironment environment = QProcessEnvironment::systemEnvironment(); - QString programFilesPath = environment.value("ProgramFiles(x86)"); - QString vsWherePath = programFilesPath + "\\Microsoft Visual Studio\\Installer\\vswhere.exe"; + auto findCompilerResult = FindSupportedCompilerForPlatform(); - QFileInfo vsWhereFile(vsWherePath); - if (vsWhereFile.exists() && vsWhereFile.isFile()) + if (!findCompilerResult.IsSuccess()) { - QProcess vsWhereProcess; - vsWhereProcess.setProcessChannelMode(QProcess::MergedChannels); - - vsWhereProcess.start( - vsWherePath, - QStringList{ "-version", "16.0", "-latest", "-requires", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", - "-property", "isComplete" }); - - if (!vsWhereProcess.waitForStarted()) - { - return false; - } - - while (vsWhereProcess.waitForReadyRead()) - { - } - - QString vsWhereOutput(vsWhereProcess.readAllStandardOutput()); - if (vsWhereOutput.startsWith("1")) - { - return true; - } + QMessageBox vsWarningMessage(parent); + vsWarningMessage.setIcon(QMessageBox::Warning); + vsWarningMessage.setWindowTitle(QObject::tr("Create Project")); + // Makes link clickable + vsWarningMessage.setTextFormat(Qt::RichText); + vsWarningMessage.setText(findCompilerResult.GetError()); + vsWarningMessage.setStandardButtons(QMessageBox::Close); + + QSpacerItem* horizontalSpacer = new QSpacerItem(600, 0, QSizePolicy::Minimum, QSizePolicy::Expanding); + QGridLayout* layout = reinterpret_cast(vsWarningMessage.layout()); + layout->addItem(horizontalSpacer, layout->rowCount(), 0, 1, layout->columnCount()); + vsWarningMessage.exec(); } - return false; - } - - bool IsVS2019Installed() - { - static bool vs2019Installed = IsVS2019Installed_internal(); - return vs2019Installed; + return findCompilerResult.IsSuccess(); } ProjectManagerScreen GetProjectManagerScreen(const QString& screen) diff --git a/Code/Tools/ProjectManager/Source/ProjectUtils.h b/Code/Tools/ProjectManager/Source/ProjectUtils.h index 2829a45180..1035ceea65 100644 --- a/Code/Tools/ProjectManager/Source/ProjectUtils.h +++ b/Code/Tools/ProjectManager/Source/ProjectUtils.h @@ -8,6 +8,7 @@ #include #include +#include namespace O3DE::ProjectManager { @@ -23,7 +24,8 @@ namespace O3DE::ProjectManager bool ReplaceFile(const QString& origFile, const QString& newFile, QWidget* parent = nullptr, bool interactive = true); - bool IsVS2019Installed(); + bool FindSupportedCompiler(QWidget* parent = nullptr); + AZ::Outcome FindSupportedCompilerForPlatform(); ProjectManagerScreen GetProjectManagerScreen(const QString& screen); } // namespace ProjectUtils diff --git a/Code/Tools/ProjectManager/Source/ProjectsScreen.cpp b/Code/Tools/ProjectManager/Source/ProjectsScreen.cpp index a31cd8b263..54fdc370d7 100644 --- a/Code/Tools/ProjectManager/Source/ProjectsScreen.cpp +++ b/Code/Tools/ProjectManager/Source/ProjectsScreen.cpp @@ -516,7 +516,7 @@ namespace O3DE::ProjectManager bool ProjectsScreen::StartProjectBuild(const ProjectInfo& projectInfo) { - if (ProjectUtils::IsVS2019Installed()) + if (ProjectUtils::FindSupportedCompiler(this)) { QMessageBox::StandardButton buildProject = QMessageBox::information( this, From aeae1555febd5a8117f06f56d01f4c4d8c9ea184 Mon Sep 17 00:00:00 2001 From: AMZN-ScottR <24445312+AMZN-ScottR@users.noreply.github.com> Date: Mon, 12 Jul 2021 12:29:27 -0700 Subject: [PATCH 11/33] [installer/2106-3p-license-fix] fixed issue with CrashHandler requiring 4-component version strings Signed-off-by: AMZN-ScottR <24445312+AMZN-ScottR@users.noreply.github.com> --- Code/Tools/CrashHandler/CMakeLists.txt | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/Code/Tools/CrashHandler/CMakeLists.txt b/Code/Tools/CrashHandler/CMakeLists.txt index 22d37b2c5d..86e030bd69 100644 --- a/Code/Tools/CrashHandler/CMakeLists.txt +++ b/Code/Tools/CrashHandler/CMakeLists.txt @@ -38,8 +38,19 @@ ly_add_target( string(REPLACE "." ";" version_list "${LY_VERSION_STRING}") list(GET version_list 0 EXE_VERSION_INFO_0) list(GET version_list 1 EXE_VERSION_INFO_1) -list(GET version_list 2 EXE_VERSION_INFO_2) -list(GET version_list 3 EXE_VERSION_INFO_3) + +list(LENGTH version_list version_component_count) +if(${version_component_count} GREATER_EQUAL 3) + list(GET version_list 2 EXE_VERSION_INFO_2) +else() + set(EXE_VERSION_INFO_2 0) +endif() + +if(${version_component_count} GREATER_EQUAL 4) + list(GET version_list 3 EXE_VERSION_INFO_3) +else() + set(EXE_VERSION_INFO_3 0) +endif() ly_add_source_properties( SOURCES Shared/CrashHandler.cpp From afe661cacd58f20bb730ffde0775c9deb4bfa5cf Mon Sep 17 00:00:00 2001 From: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> Date: Mon, 12 Jul 2021 13:39:30 -0700 Subject: [PATCH 12/33] Add support for VS2022 (#2065) * Add generation code for VS, cleanup Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> * Remove unused code and fixed comments of what is supported Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> * Code review comments Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> --- .../Common/MSVC/Configurations_msvc.cmake | 41 ++++--------------- .../Platform/Common/VisualStudio_common.cmake | 21 ++++------ 2 files changed, 18 insertions(+), 44 deletions(-) diff --git a/cmake/Platform/Common/MSVC/Configurations_msvc.cmake b/cmake/Platform/Common/MSVC/Configurations_msvc.cmake index 1850bec114..8b3a1eee49 100644 --- a/cmake/Platform/Common/MSVC/Configurations_msvc.cmake +++ b/cmake/Platform/Common/MSVC/Configurations_msvc.cmake @@ -8,39 +8,16 @@ include(cmake/Platform/Common/Configurations_common.cmake) include(cmake/Platform/Common/VisualStudio_common.cmake) -set(LY_MSVC_SUPPORTED_GENERATORS - "Visual Studio 15" - "Visual Studio 16" -) -set(FOUND_SUPPORTED_GENERATOR) -foreach(supported_generator ${LY_MSVC_SUPPORTED_GENERATORS}) - if(CMAKE_GENERATOR MATCHES ${supported_generator}) - set(FOUND_SUPPORTED_GENERATOR TRUE) - break() - endif() -endforeach() -# VS2017's checks since it defaults the toolchain and target architecture to x86 -if(CMAKE_GENERATOR MATCHES "Visual Studio 15") - if(CMAKE_VS_PLATFORM_NAME AND CMAKE_VS_PLATFORM_NAME STREQUAL "Win32") # VS2017 has Win32 as the default architecture - message(FATAL_ERROR "Win32 architecture not supported, specify \"-A x64\" when invoking cmake") - endif() - if(NOT CMAKE_VS_PLATFORM_TOOLSET_HOST_ARCHITECTURE STREQUAL "x64") # There is at least one library (EditorLib) that make the x86 linker to run out of memory - message(FATAL_ERROR "x86 toolset not supported, specify \"-T host=x64\" when invoking cmake") - endif() -else() - # For the other cases, verify that it wasn't invoked with an unsupported architecture. defaults to x86 architecture - if(SUPPORTED_VS_PLATFORM_NAME_OVERRIDE) - set(SUPPORTED_VS_PLATFORM_NAME ${SUPPORTED_VS_PLATFORM_NAME_OVERRIDE}) - else() - set(SUPPORTED_VS_PLATFORM_NAME x64) - endif() +if(NOT CMAKE_GENERATOR MATCHES "Visual Studio 1[6-7]") + message(FATAL_ERROR "Generator ${CMAKE_GENERATOR} not supported") +endif() - if(CMAKE_VS_PLATFORM_NAME AND NOT CMAKE_VS_PLATFORM_NAME STREQUAL "${SUPPORTED_VS_PLATFORM_NAME}") - message(FATAL_ERROR "${CMAKE_VS_PLATFORM_NAME} architecture not supported") - endif() - if(CMAKE_VS_PLATFORM_TOOLSET_HOST_ARCHITECTURE AND NOT CMAKE_VS_PLATFORM_TOOLSET_HOST_ARCHITECTURE STREQUAL "x64") - message(FATAL_ERROR "${CMAKE_VS_PLATFORM_TOOLSET_HOST_ARCHITECTURE} toolset not supported") - endif() +# Verify that it wasn't invoked with an unsupported target/host architecture. Currently only supports x64/x64 +if(CMAKE_VS_PLATFORM_NAME AND NOT CMAKE_VS_PLATFORM_NAME STREQUAL "x64") + message(FATAL_ERROR "${CMAKE_VS_PLATFORM_NAME} target architecture is not supported, it must be 'x64'") +endif() +if(CMAKE_VS_PLATFORM_TOOLSET_HOST_ARCHITECTURE AND NOT CMAKE_VS_PLATFORM_TOOLSET_HOST_ARCHITECTURE STREQUAL "x64") + message(FATAL_ERROR "${CMAKE_VS_PLATFORM_TOOLSET_HOST_ARCHITECTURE} host toolset is not supported, it must be 'x64'") endif() ly_append_configurations_options( diff --git a/cmake/Platform/Common/VisualStudio_common.cmake b/cmake/Platform/Common/VisualStudio_common.cmake index 91817bef6f..982d32f558 100644 --- a/cmake/Platform/Common/VisualStudio_common.cmake +++ b/cmake/Platform/Common/VisualStudio_common.cmake @@ -5,16 +5,13 @@ # # -if(CMAKE_GENERATOR MATCHES "Visual Studio 16") +foreach(conf IN LISTS CMAKE_CONFIGURATION_TYPES) + if(conf STREQUAL debug) + string(APPEND VCPKG_CONFIGURATION_MAPPING " Debug\n") + else() + string(APPEND VCPKG_CONFIGURATION_MAPPING " Release\n") + endif() +endforeach() + +configure_file("${CMAKE_CURRENT_LIST_DIR}/Directory.Build.props" "${CMAKE_BINARY_DIR}/Directory.Build.props" @ONLY) - foreach(conf IN LISTS CMAKE_CONFIGURATION_TYPES) - if(conf STREQUAL debug) - string(APPEND VCPKG_CONFIGURATION_MAPPING " Debug\n") - else() - string(APPEND VCPKG_CONFIGURATION_MAPPING " Release\n") - endif() - endforeach() - - configure_file("${CMAKE_CURRENT_LIST_DIR}/Directory.Build.props" "${CMAKE_BINARY_DIR}/Directory.Build.props" @ONLY) - -endif() \ No newline at end of file From 23f690cd55c74b73661955176b444971fbf52e95 Mon Sep 17 00:00:00 2001 From: Shirang Jia Date: Mon, 12 Jul 2021 16:58:33 -0700 Subject: [PATCH 13/33] Cleanup incremental build scripts (#2004) * Cleanup incremental build scripts Signed-off-by: shiranj * Fix timeout function since signal.SIGALRM is not available on Windows Signed-off-by: shiranj * Convert missing fstrings Signed-off-by: shiranj * Add PipelineAndBranch back to EBS volume tag Signed-off-by: shiranj --- .../build/bootstrap/incremental_build_util.py | 308 +++++++++--------- 1 file changed, 162 insertions(+), 146 deletions(-) diff --git a/scripts/build/bootstrap/incremental_build_util.py b/scripts/build/bootstrap/incremental_build_util.py index 32a6b0f526..296fbb5424 100755 --- a/scripts/build/bootstrap/incremental_build_util.py +++ b/scripts/build/bootstrap/incremental_build_util.py @@ -4,18 +4,18 @@ # import argparse -import ast import boto3 import datetime import urllib.request, urllib.error, urllib.parse import os import psutil import time -import requests import subprocess import sys import tempfile -import traceback +from contextlib import contextmanager +import threading +import _thread DEFAULT_REGION = 'us-west-2' DEFAULT_DISK_SIZE = 300 @@ -42,14 +42,18 @@ if os.name == 'nt': kernel32 = ctypes.WinDLL('kernel32', use_last_error=True) kernel32.GetDiskFreeSpaceExW.argtypes = (ctypes.c_wchar_p,) + (PULARGE_INTEGER,) * 3 + class UsageTuple(collections.namedtuple('UsageTuple', 'total, used, free')): def __str__(self): # Add thousands separator to numbers displayed return self.__class__.__name__ + '(total={:n}, used={:n}, free={:n})'.format(*self) + def is_dir_symlink(path): FILE_ATTRIBUTE_REPARSE_POINT = 0x0400 - return os.path.isdir(path) and (ctypes.windll.kernel32.GetFileAttributesW(str(path)) & FILE_ATTRIBUTE_REPARSE_POINT) + return os.path.isdir(path) and ( + ctypes.windll.kernel32.GetFileAttributesW(str(path)) & FILE_ATTRIBUTE_REPARSE_POINT) + def get_free_space_mb(path): if sys.version_info < (3,): # Python 2? @@ -77,16 +81,39 @@ if os.name == 'nt': used = total.value - free.value - return free.value / 1024 / 1024#for now + return free.value / 1024 / 1024 # for now else: def get_free_space_mb(dirname): st = os.statvfs(dirname) return st.f_bavail * st.f_frsize / 1024 / 1024 + def error(message): print(message) exit(1) + +@contextmanager +def timeout(duration, timeout_message): + timer = threading.Timer(duration, lambda: _thread.interrupt_main()) + timer.start() + try: + yield + except KeyboardInterrupt: + print(timeout_message) + raise TimeoutError + finally: + # If the action ends in specified time, timer is canceled + timer.cancel() + + +def print_drives(): + if os.name == 'nt': + drives_before = win32api.GetLogicalDriveStrings() + drives_before = drives_before.split('\000')[:-1] + print(drives_before) + + def parse_args(): parser = argparse.ArgumentParser() parser.add_argument('-a', '--action', dest="action", help="Action (mount|unmount|delete)") @@ -96,8 +123,10 @@ def parse_args(): parser.add_argument('-b', '--branch', dest="branch", help="Branch") parser.add_argument('-plat', '--platform', dest="platform", help="Platform") parser.add_argument('-c', '--build_type', dest="build_type", help="Build type") - parser.add_argument('-ds', '--disk_size', dest="disk_size", help="Disk size in Gigabytes (defaults to {})".format(DEFAULT_DISK_SIZE), default=DEFAULT_DISK_SIZE) - parser.add_argument('-dt', '--disk_type', dest="disk_type", help="Disk type (defaults to {})".format(DEFAULT_DISK_TYPE), default=DEFAULT_DISK_TYPE) + parser.add_argument('-ds', '--disk_size', dest="disk_size", + help=f"Disk size in Gigabytes (defaults to {DEFAULT_DISK_SIZE})", default=DEFAULT_DISK_SIZE) + parser.add_argument('-dt', '--disk_type', dest="disk_type", help=f"Disk type (defaults to {DEFAULT_DISK_TYPE})", + default=DEFAULT_DISK_TYPE) args = parser.parse_args() # Input validation @@ -117,19 +146,30 @@ def parse_args(): error('No platform specified') if args.build_type is None: error('No build_type specified') - + return args + def get_mount_name(repository_name, project, pipeline, branch, platform, build_type): - mount_name = "{}_{}_{}_{}_{}_{}".format(repository_name, project, pipeline, branch, platform, build_type) - mount_name = mount_name.replace('/','_').replace('\\','_') + mount_name = f"{repository_name}_{project}_{pipeline}_{branch}_{platform}_{build_type}" + mount_name = mount_name.replace('/', '_').replace('\\', '_') return mount_name + def get_pipeline_and_branch(pipeline, branch): - pipeline_and_branch = "{}_{}".format(pipeline, branch) - pipeline_and_branch = pipeline_and_branch.replace('/','_').replace('\\','_') + pipeline_and_branch = f"{pipeline}_{branch}" + pipeline_and_branch = pipeline_and_branch.replace('/', '_').replace('\\', '_') return pipeline_and_branch + +def get_region_name(): + session = boto3.session.Session() + region = session.region_name + if region is None: + region = DEFAULT_REGION + return region + + def get_ec2_client(region): client = boto3.client('ec2', region_name=region) return client @@ -140,48 +180,51 @@ def get_ec2_instance_id(): instance_id = urllib.request.urlopen('http://169.254.169.254/latest/meta-data/instance-id').read() return instance_id.decode("utf-8") except Exception as e: - print(e.message) + print(e) error('No EC2 metadata! Check if you are running this script on an EC2 instance.') def get_availability_zone(): try: - availability_zone = urllib.request.urlopen('http://169.254.169.254/latest/meta-data/placement/availability-zone').read() + availability_zone = urllib.request.urlopen( + 'http://169.254.169.254/latest/meta-data/placement/availability-zone').read() return availability_zone.decode("utf-8") except Exception as e: - print(e.message) + print(e) error('No EC2 metadata! Check if you are running this script on an EC2 instance.') def kill_processes(workspace='/dev/'): - ''' + """ Kills all processes that have open file paths associated with the workspace. Uses PSUtil for cross-platform compatibility - ''' + """ print('Checking for any stuck processes...') for proc in psutil.process_iter(): try: if workspace in str(proc.open_files()): - print("{} has open files in {}. Terminating".format(proc.name(), proc.open_files())) + print(f"{proc.name()} has open files in {proc.open_files()}. Terminating") proc.kill() - time.sleep(1) # Just to make sure a parent process has time to close + time.sleep(1) # Just to make sure a parent process has time to close except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess): continue def delete_volume(ec2_client, volume_id): response = ec2_client.delete_volume(VolumeId=volume_id) - print('Volume {} deleted'.format(volume_id)) + print(f'Volume {volume_id} deleted') + def find_snapshot_id(ec2_client, repository_name, project, pipeline, platform, build_type, disk_size): - mount_name = get_mount_name(repository_name, project, pipeline, 'stabilization_2106', platform, build_type) # we take snapshots out of stabilization_2106 - response = ec2_client.describe_snapshots(Filters= [{ + mount_name = get_mount_name(repository_name, project, pipeline, 'stabilization_2106', platform, + build_type) # we take snapshots out of stabilization_2106 + response = ec2_client.describe_snapshots(Filters=[{ 'Name': 'tag:Name', 'Values': [mount_name] }]) snapshot_id = None if 'Snapshots' in response and len(response['Snapshots']) > 0: - snapshot_start_time_max = None # find the latest snapshot + snapshot_start_time_max = None # find the latest snapshot for snapshot in response['Snapshots']: if snapshot['State'] == 'completed' and snapshot['VolumeSize'] == disk_size: snapshot_start_time = snapshot['StartTime'] @@ -190,28 +233,33 @@ def find_snapshot_id(ec2_client, repository_name, project, pipeline, platform, b snapshot_id = snapshot['SnapshotId'] return snapshot_id -def create_volume(ec2_client, availability_zone, repository_name, project, pipeline, branch, platform, build_type, disk_size, disk_type): - # The actual EBS default calculation for IOps is a floating point number, the closest approxmiation is 4x of the disk size for simplicity + +def create_volume(ec2_client, availability_zone, repository_name, project, pipeline, branch, platform, build_type, + disk_size, disk_type): mount_name = get_mount_name(repository_name, project, pipeline, branch, platform, build_type) - pipeline_and_branch = get_pipeline_and_branch(pipeline, branch) + pipeline_and_branch = get_pipeline_and_branch(pipeline, branch) parameters = dict( - AvailabilityZone = availability_zone, + AvailabilityZone=availability_zone, VolumeType=disk_type, - TagSpecifications= [{ + TagSpecifications=[{ 'ResourceType': 'volume', 'Tags': [ - { 'Key': 'Name', 'Value': mount_name }, - { 'Key': 'RepositoryName', 'Value': repository_name}, - { 'Key': 'Project', 'Value': project }, - { 'Key': 'Pipeline', 'Value': pipeline }, - { 'Key': 'BranchName', 'Value': branch }, - { 'Key': 'Platform', 'Value': platform }, - { 'Key': 'BuildType', 'Value': build_type }, - { 'Key': 'PipelineAndBranch', 'Value': pipeline_and_branch }, # used so the snapshoting easily identifies which volumes to snapshot + {'Key': 'Name', 'Value': mount_name}, + {'Key': 'RepositoryName', 'Value': repository_name}, + {'Key': 'Project', 'Value': project}, + {'Key': 'Pipeline', 'Value': pipeline}, + {'Key': 'BranchName', 'Value': branch}, + {'Key': 'Platform', 'Value': platform}, + {'Key': 'BuildType', 'Value': build_type}, + # Used so the snapshoting easily identifies which volumes to snapshot + {'Key': 'PipelineAndBranch', 'Value': pipeline_and_branch}, + ] }] ) - if 'io1' in disk_type.lower(): + # The actual EBS default calculation for IOps is a floating point number, + # the closest approxmiation is 4x of the disk size for simplicity + if 'io1' in disk_type.lower(): parameters['Iops'] = (4 * disk_size) snapshot_id = find_snapshot_id(ec2_client, repository_name, project, pipeline, platform, build_type, disk_size) @@ -230,16 +278,17 @@ def create_volume(ec2_client, availability_zone, repository_name, project, pipel time.sleep(1) response = ec2_client.describe_volumes(VolumeIds=[volume_id, ]) - while (response['Volumes'][0]['State'] != 'available'): - time.sleep(1) - response = ec2_client.describe_volumes(VolumeIds=[volume_id, ]) + with timeout(DEFAULT_TIMEOUT, 'ERROR: Timeout reached trying to create EBS.'): + while response['Volumes'][0]['State'] != 'available': + time.sleep(1) + response = ec2_client.describe_volumes(VolumeIds=[volume_id, ]) - print(("Volume {} created\n\tSnapshot: {}\n\tRepository {}\n\tProject {}\n\tPipeline {}\n\tBranch {}\n\tPlatform: {}\n\tBuild type: {}" - .format(volume_id, snapshot_id, repository_name, project, pipeline, branch, platform, build_type))) + print(f"Volume {volume_id} created\n\tSnapshot: {snapshot_id}\n\tRepository {repository_name}\n\t" + f"Project {project}\n\tPipeline {pipeline}\n\tBranch {branch}\n\tPlatform: {platform}\n\tBuild type: {build_type}") return volume_id, created -def mount_volume(created): +def mount_volume_to_device(created): print('Mounting volume...') if os.name == 'nt': f = tempfile.NamedTemporaryFile(delete=False) @@ -247,7 +296,7 @@ def mount_volume(created): select disk 1 online disk attribute disk clear readonly - """.encode('utf-8')) # assume disk # for now + """.encode('utf-8')) # assume disk # for now if created: print('Creating filesystem on new volume') @@ -259,18 +308,12 @@ def mount_volume(created): """.encode('utf-8')) f.close() - + subprocess.call(['diskpart', '/s', f.name]) time.sleep(5) - drives_after = win32api.GetLogicalDriveStrings() - drives_after = drives_after.split('\000')[:-1] - - print(drives_after) - - #drive_letter = next(item for item in drives_after if item not in drives_before) - drive_letter = MOUNT_PATH + print_drives() os.unlink(f.name) @@ -283,8 +326,8 @@ def mount_volume(created): subprocess.call(['mount', '/dev/xvdf', MOUNT_PATH]) -def attach_volume(volume, volume_id, instance_id, timeout=DEFAULT_TIMEOUT): - print('Attaching volume {} to instance {}'.format(volume_id, instance_id)) +def attach_volume_to_ec2_instance(volume, volume_id, instance_id, timeout_duration=DEFAULT_TIMEOUT): + print(f'Attaching volume {volume_id} to instance {instance_id}') volume.attach_to_instance(Device='xvdf', InstanceId=instance_id, VolumeId=volume_id) @@ -292,13 +335,10 @@ def attach_volume(volume, volume_id, instance_id, timeout=DEFAULT_TIMEOUT): time.sleep(2) # reload the volume just in case volume.load() - timeout_init = time.clock() - while (len(volume.attachments) and volume.attachments[0]['State'] != 'attached'): - time.sleep(1) - volume.load() - if (time.clock() - timeout_init) > timeout: - print('ERROR: Timeout reached trying to mount EBS') - exit(1) + with timeout(timeout_duration, 'ERROR: Timeout reached trying to mount EBS.'): + while len(volume.attachments) and volume.attachments[0]['State'] != 'attached': + time.sleep(1) + volume.load() volume.create_tags( Tags=[ { @@ -307,11 +347,11 @@ def attach_volume(volume, volume_id, instance_id, timeout=DEFAULT_TIMEOUT): }, ] ) - print('Volume {} has been attached to instance {}'.format(volume_id, instance_id)) + print(f'Volume {volume_id} has been attached to instance {instance_id}') -def unmount_volume(): - print('Umounting volume...') +def unmount_volume_from_device(): + print('Unmounting EBS volume from device...') if os.name == 'nt': kill_processes(MOUNT_PATH + 'workspace') f = tempfile.NamedTemporaryFile(delete=False) @@ -327,50 +367,31 @@ def unmount_volume(): subprocess.call(['umount', '-f', MOUNT_PATH]) -def detach_volume(volume, ec2_instance_id, force, timeout=DEFAULT_TIMEOUT): - print('Detaching volume {} from instance {}'.format(volume.volume_id, ec2_instance_id)) +def detach_volume_from_ec2_instance(volume, ec2_instance_id, force, timeout_duration=DEFAULT_TIMEOUT): + print(f'Detaching volume {volume.volume_id} from instance {ec2_instance_id}') volume.detach_from_instance(Device='xvdf', Force=force, InstanceId=ec2_instance_id, VolumeId=volume.volume_id) - timeout_init = time.clock() - while len(volume.attachments) and volume.attachments[0]['State'] != 'detached': - time.sleep(1) - volume.load() - if (time.clock() - timeout_init) > timeout: - print('ERROR: Timeout reached trying to unmount EBS.') - volume.detach_from_instance(Device='xvdf',Force=True,InstanceId=ec2_instance_id,VolumeId=volume.volume_id) - exit(1) - - print('Volume {} has been detached from instance {}'.format(volume.volume_id, ec2_instance_id)) + try: + with timeout(timeout_duration, 'ERROR: Timeout reached trying to unmount EBS.'): + while len(volume.attachments) and volume.attachments[0]['State'] != 'detached': + time.sleep(1) + volume.load() + except TimeoutError: + print('Force detaching EBS.') + volume.detach_from_instance(Device='xvdf', Force=True, InstanceId=ec2_instance_id, VolumeId=volume.volume_id) + + print(f'Volume {volume.volume_id} has been detached from instance {ec2_instance_id}') volume.load() if len(volume.attachments): print('Volume still has attachments') for attachment in volume.attachments: - print('Volume {} {} to instance {}'.format(attachment['VolumeId'], attachment['State'], attachment['InstanceId'])) - - -def attach_ebs_and_create_partition_with_retry(volume, volume_id, ec2_instance_id, created): - attach_volume(volume, volume_id, ec2_instance_id) - mount_volume(created) - attempt = 1 - while attempt <= MAX_EBS_MOUNTING_ATTEMPT: - if os.name == 'nt': - drives_after = win32api.GetLogicalDriveStrings() - drives_after = drives_after.split('\000')[:-1] - if MOUNT_PATH not in drives_after: - print('Disk partitioning failed, retrying...') - unmount_volume() - detach_volume(volume, ec2_instance_id, False) - attach_volume(volume, volume_id, ec2_instance_id) - mount_volume(created) - attempt += 1 + print(f"Volume {attachment['VolumeId']} {attachment['State']} to instance {attachment['InstanceId']}") + def mount_ebs(repository_name, project, pipeline, branch, platform, build_type, disk_size, disk_type): - session = boto3.session.Session() - region = session.region_name - if region is None: - region = DEFAULT_REGION + region = get_region_name() ec2_client = get_ec2_client(region) ec2_instance_id = get_ec2_instance_id() ec2_availability_zone = get_availability_zone() @@ -379,70 +400,70 @@ def mount_ebs(repository_name, project, pipeline, branch, platform, build_type, for volume in ec2_instance.volumes.all(): for attachment in volume.attachments: - print('attachment device: {}'.format(attachment['Device'])) + print(f"attachment device: {attachment['Device']}") if 'xvdf' in attachment['Device'] and attachment['State'] != 'detached': - print('A device is already attached to xvdf. This likely means a previous build failed to detach its ' \ + print('A device is already attached to xvdf. This likely means a previous build failed to detach its ' 'build volume. This volume is considered orphaned and will be detached from this instance.') - unmount_volume() - detach_volume(volume, ec2_instance_id, False) # Force unmounts should not be used, as that will cause the EBS block device driver to fail the remount + unmount_volume_from_device() + detach_volume_from_ec2_instance(volume, ec2_instance_id, + False) # Force unmounts should not be used, as that will cause the EBS block device driver to fail the remount mount_name = get_mount_name(repository_name, project, pipeline, branch, platform, build_type) response = ec2_client.describe_volumes(Filters=[{ 'Name': 'tag:Name', 'Values': [mount_name] - }]) + }]) created = False if 'Volumes' in response and not len(response['Volumes']): - print('Volume for {} doesn\'t exist creating it...'.format(mount_name)) + print(f'Volume for {mount_name} doesn\'t exist creating it...') # volume doesn't exist, create it - volume_id, created = create_volume(ec2_client, ec2_availability_zone, repository_name, project, pipeline, branch, platform, build_type, disk_size, disk_type) + volume_id, created = create_volume(ec2_client, ec2_availability_zone, repository_name, project, pipeline, + branch, platform, build_type, disk_size, disk_type) else: volume = response['Volumes'][0] volume_id = volume['VolumeId'] - print('Current volume {} is a {} GB {}'.format(volume_id, volume['Size'], volume['VolumeType'])) - if (volume['Size'] != disk_size or volume['VolumeType'] != disk_type): - print('Override disk attributes does not match the existing volume, deleting {} and replacing the volume'.format(volume_id)) + print(f"Current volume {volume_id} is a {volume['Size']} GB {volume['VolumeType']}") + if volume['Size'] != disk_size or volume['VolumeType'] != disk_type: + print( + f'Override disk attributes does not match the existing volume, deleting {volume_id} and replacing the volume') delete_volume(ec2_client, volume_id) - volume_id, created = create_volume(ec2_client, ec2_availability_zone, repository_name, project, pipeline, branch, platform, build_type, disk_size, disk_type) + volume_id, created = create_volume(ec2_client, ec2_availability_zone, repository_name, project, pipeline, + branch, platform, build_type, disk_size, disk_type) if len(volume['Attachments']): # this is bad we shouldn't be attached, we should have detached at the end of a build attachment = volume['Attachments'][0] - print(('Volume already has attachment {}, detaching...'.format(attachment))) - detach_volume(ec2_resource.Volume(volume_id), attachment['InstanceId'], True) + print(f'Volume already has attachment {attachment}, detaching...') + detach_volume_from_ec2_instance(ec2_resource.Volume(volume_id), attachment['InstanceId'], True) volume = ec2_resource.Volume(volume_id) - if os.name == 'nt': - drives_before = win32api.GetLogicalDriveStrings() - drives_before = drives_before.split('\000')[:-1] - - print(drives_before) - - attach_ebs_and_create_partition_with_retry(volume, volume_id, ec2_instance_id, created) + print_drives() + attach_volume_to_ec2_instance(volume, volume_id, ec2_instance_id) + mount_volume_to_device(created) + print_drives() free_space_mb = get_free_space_mb(MOUNT_PATH) - print('Free disk space {}MB'.format(free_space_mb)) - + print(f'Free disk space {free_space_mb}MB') + if free_space_mb < LOW_EBS_DISK_SPACE_LIMIT: - print('Volume is running below EBS free disk space treshhold {}MB. Recreating volume and running clean build.'.format(LOW_EBS_DISK_SPACE_LIMIT)) - unmount_volume() - detach_volume(volume, ec2_instance_id, False) + print(f'Volume is running below EBS free disk space treshhold {LOW_EBS_DISK_SPACE_LIMIT}MB. Recreating volume and running clean build.') + unmount_volume_from_device() + detach_volume_from_ec2_instance(volume, ec2_instance_id, False) delete_volume(ec2_client, volume_id) new_disk_size = int(volume.size * 1.25) if new_disk_size > MAX_EBS_DISK_SIZE: - print('Error: EBS disk size reached to the allowed maximum disk size {}MB, please contact ly-infra@ and ly-build@ to investigate.'.format(MAX_EBS_DISK_SIZE)) + print(f'Error: EBS disk size reached to the allowed maximum disk size {MAX_EBS_DISK_SIZE}MB, please contact ly-infra@ and ly-build@ to investigate.') exit(1) - print('Recreating the EBS with disk size {}'.format(new_disk_size)) - volume_id, created = create_volume(ec2_client, ec2_availability_zone, repository_name, project, pipeline, branch, platform, build_type, new_disk_size, disk_type) + print(f'Recreating the EBS with disk size {new_disk_size}') + volume_id, created = create_volume(ec2_client, ec2_availability_zone, repository_name, project, pipeline, + branch, platform, build_type, new_disk_size, disk_type) volume = ec2_resource.Volume(volume_id) - attach_ebs_and_create_partition_with_retry(volume, volume_id, ec2_instance_id, created) + attach_volume_to_ec2_instance(volume, volume_id, ec2_instance_id) + mount_volume_to_device(created) + def unmount_ebs(): - session = boto3.session.Session() - region = session.region_name - if region is None: - region = DEFAULT_REGION - ec2_client = get_ec2_client(region) + region = get_region_name() ec2_instance_id = get_ec2_instance_id() ec2_resource = boto3.resource('ec2', region_name=region) ec2_instance = ec2_resource.Instance(ec2_instance_id) @@ -454,7 +475,7 @@ def unmount_ebs(): for attached_volume in ec2_instance.volumes.all(): for attachment in attached_volume.attachments: - print('attachment device: {}'.format(attachment['Device'])) + print(f"attachment device: {attachment['Device']}") if attachment['Device'] == 'xvdf': volume = attached_volume @@ -462,24 +483,18 @@ def unmount_ebs(): # volume is not mounted print('Volume is not mounted') else: - unmount_volume() - detach_volume(volume, ec2_instance_id, False) + unmount_volume_from_device() + detach_volume_from_ec2_instance(volume, ec2_instance_id, False) + def delete_ebs(repository_name, project, pipeline, branch, platform, build_type): unmount_ebs() - - session = boto3.session.Session() - region = session.region_name - if region is None: - region = DEFAULT_REGION + region = get_region_name() ec2_client = get_ec2_client(region) - ec2_instance_id = get_ec2_instance_id() - ec2_resource = boto3.resource('ec2', region_name=region) - ec2_instance = ec2_resource.Instance(ec2_instance_id) mount_name = get_mount_name(repository_name, project, pipeline, branch, platform, build_type) response = ec2_client.describe_volumes(Filters=[ - { 'Name': 'tag:Name', 'Values': [mount_name] } + {'Name': 'tag:Name', 'Values': [mount_name]} ]) if 'Volumes' in response and len(response['Volumes']): @@ -496,7 +511,8 @@ def main(action, repository_name, project, pipeline, branch, platform, build_typ elif action == 'delete': delete_ebs(repository_name, project, pipeline, branch, platform, build_type) + if __name__ == "__main__": args = parse_args() - ret = main(args.action, args.repository_name, args.project, args.pipeline, args.branch, args.platform, args.build_type, args.disk_size, args.disk_type) - sys.exit(ret) \ No newline at end of file + main(args.action, args.repository_name, args.project, args.pipeline, args.branch, args.platform, + args.build_type, args.disk_size, args.disk_type) From 158e25bd23e4aa5618cb31c712b81b50913083ba Mon Sep 17 00:00:00 2001 From: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com> Date: Mon, 12 Jul 2021 20:22:32 -0500 Subject: [PATCH 14/33] ly_create_alias() now adds the directory it was called from to the (#2120) LY_ALL_TARGET_DIRECTORIES property if that directory has not already been added This addresses an issue where if a CMakeLists.txt contains a call to ly_create_alias(), but NOT a call to ly_add_target, it would not be added to the generated CMakeLists.txt for the install layout Signed-off-by: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com> --- Gems/Atom/CMakeLists.txt | 8 -------- .../AtomBridge/Code/CMakeLists.txt | 15 +++++++++++++++ Gems/AtomLyIntegration/CMakeLists.txt | 8 -------- cmake/Gems.cmake | 7 +++++++ 4 files changed, 22 insertions(+), 16 deletions(-) diff --git a/Gems/Atom/CMakeLists.txt b/Gems/Atom/CMakeLists.txt index 8dc38ec7ad..3c7a6951f6 100644 --- a/Gems/Atom/CMakeLists.txt +++ b/Gems/Atom/CMakeLists.txt @@ -14,11 +14,3 @@ add_subdirectory(RPI) add_subdirectory(Tools) add_subdirectory(Utils) -# The "Atom" Gem will alias the real Atom_AtomBridge target variants -# allows the enabling and disabling the "Atom" Gem to build the pre-requisite dependencies -ly_create_alias(NAME Atom.Clients NAMESPACE Gem TARGETS Gem::Atom_AtomBridge.Clients) -ly_create_alias(NAME Atom.Servers NAMESPACE Gem TARGETS Gem::Atom_AtomBridge.Servers) -if(PAL_TRAIT_BUILD_HOST_TOOLS) - ly_create_alias(NAME Atom.Builders NAMESPACE Gem TARGETS Gem::Atom_AtomBridge.Builders) - ly_create_alias(NAME Atom.Tools NAMESPACE Gem TARGETS Gem::Atom_AtomBridge.Tools) -endif() diff --git a/Gems/AtomLyIntegration/AtomBridge/Code/CMakeLists.txt b/Gems/AtomLyIntegration/AtomBridge/Code/CMakeLists.txt index bd535dee72..db14ca1751 100644 --- a/Gems/AtomLyIntegration/AtomBridge/Code/CMakeLists.txt +++ b/Gems/AtomLyIntegration/AtomBridge/Code/CMakeLists.txt @@ -115,3 +115,18 @@ if(PAL_TRAIT_BUILD_HOST_TOOLS) ly_create_alias(NAME Atom_AtomBridge.Builders NAMESPACE Gem TARGETS Gem::Atom_AtomBridge.Editor) ly_create_alias(NAME Atom_AtomBridge.Tools NAMESPACE Gem TARGETS Gem::Atom_AtomBridge.Editor) endif() + +# The "Atom" Gem will alias the real Atom_AtomBridge target variants +# allows the enabling and disabling the "Atom" Gem to build the pre-requisite dependencies +# The "AtomLyIntegration" Gem will also alias the real Atom_AtomBridge target variants +# The Atom Gem does the same at the moment. +ly_create_alias(NAME Atom.Clients NAMESPACE Gem TARGETS Gem::Atom_AtomBridge.Clients) +ly_create_alias(NAME Atom.Servers NAMESPACE Gem TARGETS Gem::Atom_AtomBridge.Servers) +ly_create_alias(NAME AtomLyIntegration.Clients NAMESPACE Gem TARGETS Gem::Atom_AtomBridge.Clients) +ly_create_alias(NAME AtomLyIntegration.Servers NAMESPACE Gem TARGETS Gem::Atom_AtomBridge.Servers) +if(PAL_TRAIT_BUILD_HOST_TOOLS) + ly_create_alias(NAME Atom.Builders NAMESPACE Gem TARGETS Gem::Atom_AtomBridge.Builders) + ly_create_alias(NAME Atom.Tools NAMESPACE Gem TARGETS Gem::Atom_AtomBridge.Tools) + ly_create_alias(NAME AtomLyIntegration.Builders NAMESPACE Gem TARGETS Gem::Atom_AtomBridge.Builders) + ly_create_alias(NAME AtomLyIntegration.Tools NAMESPACE Gem TARGETS Gem::Atom_AtomBridge.Tools) +endif() diff --git a/Gems/AtomLyIntegration/CMakeLists.txt b/Gems/AtomLyIntegration/CMakeLists.txt index 35d4d388a6..3620d5152a 100644 --- a/Gems/AtomLyIntegration/CMakeLists.txt +++ b/Gems/AtomLyIntegration/CMakeLists.txt @@ -15,11 +15,3 @@ add_subdirectory(AtomBridge) add_subdirectory(AtomViewportDisplayInfo) add_subdirectory(AtomViewportDisplayIcons) -# The "AtomLyIntegration" Gem will also alias the real Atom_AtomBridge target variants -# The Atom Gem does the same at the moment. -ly_create_alias(NAME AtomLyIntegration.Clients NAMESPACE Gem TARGETS Gem::Atom_AtomBridge.Clients) -ly_create_alias(NAME AtomLyIntegration.Servers NAMESPACE Gem TARGETS Gem::Atom_AtomBridge.Servers) -if(PAL_TRAIT_BUILD_HOST_TOOLS) - ly_create_alias(NAME AtomLyIntegration.Builders NAMESPACE Gem TARGETS Gem::Atom_AtomBridge.Builders) - ly_create_alias(NAME AtomLyIntegration.Tools NAMESPACE Gem TARGETS Gem::Atom_AtomBridge.Tools) -endif() diff --git a/cmake/Gems.cmake b/cmake/Gems.cmake index b120c28db3..b5fc0d158c 100644 --- a/cmake/Gems.cmake +++ b/cmake/Gems.cmake @@ -90,6 +90,13 @@ function(ly_create_alias) # Replace the CMake list separator with a space to replicate the space separated TARGETS arguments string(REPLACE ";" " " create_alias_args "${ly_create_alias_NAME},${ly_create_alias_NAMESPACE},${ly_create_alias_TARGETS}") set_property(DIRECTORY APPEND PROPERTY LY_CREATE_ALIAS_ARGUMENTS "${create_alias_args}") + + # Store the directory path in the GLOBAL property so that it can be accessed + # in the layout install logic. Skip if the directory has already been added + get_property(ly_all_target_directories GLOBAL PROPERTY LY_ALL_TARGET_DIRECTORIES) + if(NOT CMAKE_CURRENT_SOURCE_DIR IN_LIST ly_all_target_directories) + set_property(GLOBAL APPEND PROPERTY LY_ALL_TARGET_DIRECTORIES ${CMAKE_CURRENT_SOURCE_DIR}) + endif() endfunction() # ly_enable_gems From 91de09cff532b066938d34c832afd3306c1188c2 Mon Sep 17 00:00:00 2001 From: Benjamin Jillich <43751992+amzn-jillich@users.noreply.github.com> Date: Mon, 12 Jul 2021 23:32:35 -0700 Subject: [PATCH 15/33] EMotion FX: Root bone not initialized without opening Scene Settings (#2088) * User provided model was exporting correctly with an .assetinfo while it was not when just placing the .fbx file in the project folder. * Turned out that the best matching root bone was set by opening the Scene Settings for the first time, so after saving it again it worked correctly. * We're now chosing the best matching root bone when initializing the actor group, which fixes the issue. Signed-off-by: Benjamin Jillich --- .../Behaviors/ActorGroupBehavior.cpp | 1 + .../SceneAPIExt/Groups/ActorGroup.cpp | 20 +++++++++++++++++++ .../Pipeline/SceneAPIExt/Groups/ActorGroup.h | 2 +- .../Pipeline/SceneAPIExt/Groups/IActorGroup.h | 6 ++++++ 4 files changed, 28 insertions(+), 1 deletion(-) diff --git a/Gems/EMotionFX/Code/EMotionFX/Pipeline/SceneAPIExt/Behaviors/ActorGroupBehavior.cpp b/Gems/EMotionFX/Code/EMotionFX/Pipeline/SceneAPIExt/Behaviors/ActorGroupBehavior.cpp index c3202398b4..1e9c14d2e0 100644 --- a/Gems/EMotionFX/Code/EMotionFX/Pipeline/SceneAPIExt/Behaviors/ActorGroupBehavior.cpp +++ b/Gems/EMotionFX/Code/EMotionFX/Pipeline/SceneAPIExt/Behaviors/ActorGroupBehavior.cpp @@ -143,6 +143,7 @@ namespace EMotionFX Group::ActorGroup* group = azrtti_cast(&target); group->SetName(AZ::SceneAPI::DataTypes::Utilities::CreateUniqueName(scene.GetName(), scene.GetManifest())); + group->SetBestMatchingRootBone(scene.GetGraph()); // LOD Rule need to be built first in the actor, so we know which mesh and bone belongs to LOD. // After this call, LOD rule will be populated with all the LOD bones diff --git a/Gems/EMotionFX/Code/EMotionFX/Pipeline/SceneAPIExt/Groups/ActorGroup.cpp b/Gems/EMotionFX/Code/EMotionFX/Pipeline/SceneAPIExt/Groups/ActorGroup.cpp index e4065f58e0..b623086bc1 100644 --- a/Gems/EMotionFX/Code/EMotionFX/Pipeline/SceneAPIExt/Groups/ActorGroup.cpp +++ b/Gems/EMotionFX/Code/EMotionFX/Pipeline/SceneAPIExt/Groups/ActorGroup.cpp @@ -9,10 +9,14 @@ #include #include #include +#include +#include +#include #include #include #include #include +#include #include #include #include @@ -76,6 +80,22 @@ namespace EMotionFX m_selectedRootBone = selectedRootBone; } + void ActorGroup::SetBestMatchingRootBone(const AZ::SceneAPI::Containers::SceneGraph& sceneGraph) + { + auto nameContentView = AZ::SceneAPI::Containers::Views::MakePairView(sceneGraph.GetNameStorage(), sceneGraph.GetContentStorage()); + auto graphDownwardsView = AZ::SceneAPI::Containers::Views::MakeSceneGraphDownwardsView( + sceneGraph, sceneGraph.GetRoot(), nameContentView.begin(), true); + + for (auto it = graphDownwardsView.begin(); it != graphDownwardsView.end(); ++it) + { + if (it->second && it->second->RTTI_IsTypeOf(AZ::SceneData::GraphData::RootBoneData::TYPEINFO_Uuid())) + { + SetSelectedRootBone(it->first.GetPath()); + return; + } + } + } + void ActorGroup::Reflect(AZ::ReflectContext* context) { AZ::SerializeContext* serializeContext = azrtti_cast(context); diff --git a/Gems/EMotionFX/Code/EMotionFX/Pipeline/SceneAPIExt/Groups/ActorGroup.h b/Gems/EMotionFX/Code/EMotionFX/Pipeline/SceneAPIExt/Groups/ActorGroup.h index fddb0d6afc..0725b3e88b 100644 --- a/Gems/EMotionFX/Code/EMotionFX/Pipeline/SceneAPIExt/Groups/ActorGroup.h +++ b/Gems/EMotionFX/Code/EMotionFX/Pipeline/SceneAPIExt/Groups/ActorGroup.h @@ -45,8 +45,8 @@ namespace EMotionFX // IActorGroup overrides const AZStd::string& GetSelectedRootBone() const override; - void SetSelectedRootBone(const AZStd::string& selectedRootBone) override; + void SetBestMatchingRootBone(const AZ::SceneAPI::Containers::SceneGraph& sceneGraph) override; static void Reflect(AZ::ReflectContext* context); static bool IActorGroupVersionConverter(AZ::SerializeContext& context, AZ::SerializeContext::DataElementNode& classElement); diff --git a/Gems/EMotionFX/Code/EMotionFX/Pipeline/SceneAPIExt/Groups/IActorGroup.h b/Gems/EMotionFX/Code/EMotionFX/Pipeline/SceneAPIExt/Groups/IActorGroup.h index 398ec2c559..b57a9dfb13 100644 --- a/Gems/EMotionFX/Code/EMotionFX/Pipeline/SceneAPIExt/Groups/IActorGroup.h +++ b/Gems/EMotionFX/Code/EMotionFX/Pipeline/SceneAPIExt/Groups/IActorGroup.h @@ -10,6 +10,11 @@ #include #include +namespace AZ::SceneAPI::Containers +{ + class SceneGraph; +} + namespace EMotionFX { namespace Pipeline @@ -26,6 +31,7 @@ namespace EMotionFX virtual const AZStd::string& GetSelectedRootBone() const = 0; virtual void SetSelectedRootBone(const AZStd::string& selectedRootBone) = 0; + virtual void SetBestMatchingRootBone(const AZ::SceneAPI::Containers::SceneGraph& sceneGraph) = 0; }; } } From fcef9333a7b0bd4ff92cda7e3348623024779c71 Mon Sep 17 00:00:00 2001 From: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com> Date: Wed, 14 Jul 2021 13:16:06 -0500 Subject: [PATCH 16/33] Fixed creation of paths in the install layout for external source paths (#2152) * Fixed creation of paths in the install layout for external source paths Any source path that is not within the engine root is now copied to an External directory within the engine root Updated the gem copy regex to include the preview.png file and Editor/Scripts directory Added an ly_add_install_paths function for installing a list of paths to the install layout Signed-off-by: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com> * Moved the ly_get_last_path_segment_concat_sha256 and ly_get_engine_relative_source_dir functions to cmake/FileUtil.cmake Signed-off-by: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com> --- cmake/FileUtil.cmake | 39 +++++ cmake/Platform/Common/Install_common.cmake | 174 +++++++++++++-------- 2 files changed, 147 insertions(+), 66 deletions(-) diff --git a/cmake/FileUtil.cmake b/cmake/FileUtil.cmake index 77291d8dbb..4f9cdc346a 100644 --- a/cmake/FileUtil.cmake +++ b/cmake/FileUtil.cmake @@ -124,3 +124,42 @@ function(ly_file_read path content) set(${content} ${file_content} PARENT_SCOPE) set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${path}) endfunction() + + +#! ly_get_last_path_segment_concat_sha256 : Concatenates the last path segment of the absolute path +# with the first 8 characters of the absolute path SHA256 hash to make a unique relative path segment +function(ly_get_last_path_segment_concat_sha256 absolute_path output_path) + string(SHA256 target_source_hash ${absolute_path}) + string(SUBSTRING ${target_source_hash} 0 8 target_source_hash) + cmake_path(GET absolute_path FILENAME last_path_segment) + cmake_path(SET last_path_segment_sha256_path "${last_path_segment}-${target_source_hash}") + + set(${output_path} ${last_path_segment_sha256_path} PARENT_SCOPE) +endfunction() + +#! ly_get_engine_relative_source_dir: Attempts to form a path relative to the BASE_DIRECTORY. +# If that fails the last path segment of the absolute_target_source_dir concatenated with a SHA256 hash to form a target directory +# \arg:BASE_DIRECTORY - Directory to base relative path against. Defaults to LY_ROOT_FOLDER +function(ly_get_engine_relative_source_dir absolute_target_source_dir output_source_dir) + + set(options) + set(oneValueArgs BASE_DIRECTORY) + set(multiValueArgs) + cmake_parse_arguments(ly_get_engine_relative_source_dir "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + if(NOT ly_get_engine_relative_source_dir_BASE_DIRECTORY) + set(ly_get_engine_relative_source_dir_BASE_DIRECTORY ${LY_ROOT_FOLDER}) + endif() + + # Get a relative target source directory to the LY root folder if possible + # Otherwise use the final component name + cmake_path(IS_PREFIX LY_ROOT_FOLDER ${absolute_target_source_dir} is_target_source_dir_subdirectory_of_engine) + if(is_target_source_dir_subdirectory_of_engine) + cmake_path(RELATIVE_PATH absolute_target_source_dir BASE_DIRECTORY ${LY_ROOT_FOLDER} OUTPUT_VARIABLE relative_target_source_dir) + else() + ly_get_last_path_segment_concat_sha256(${absolute_target_source_dir} target_source_dir_last_path_segment) + unset(relative_target_source_dir) + cmake_path(APPEND relative_target_source_dir "External" ${target_source_dir_last_path_segment}) + endif() + + set(${output_source_dir} ${relative_target_source_dir} PARENT_SCOPE) +endfunction() diff --git a/cmake/Platform/Common/Install_common.cmake b/cmake/Platform/Common/Install_common.cmake index 7ab6006220..5b476667e2 100644 --- a/cmake/Platform/Common/Install_common.cmake +++ b/cmake/Platform/Common/Install_common.cmake @@ -5,6 +5,8 @@ # # +include(cmake/FileUtil.cmake) + set(CMAKE_INSTALL_MESSAGE NEVER) # Simplify messages to reduce output noise ly_set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME Core) @@ -18,26 +20,6 @@ cmake_path(RELATIVE_PATH CMAKE_LIBRARY_OUTPUT_DIRECTORY BASE_DIRECTORY ${CMAKE_B set(install_output_folder "\${CMAKE_INSTALL_PREFIX}/${runtime_output_directory}/${PAL_PLATFORM_NAME}/$") -function(ly_get_engine_relative_source_dir absolute_target_source_dir output_source_dir) - # Get a relative target source directory to the LY root folder if possible - # Otherwise use the final component name - cmake_path(IS_PREFIX LY_ROOT_FOLDER ${absolute_target_source_dir} is_target_prefix_of_engine_root) - if(is_target_prefix_of_engine_root) - cmake_path(RELATIVE_PATH absolute_target_source_dir BASE_DIRECTORY ${LY_ROOT_FOLDER} OUTPUT_VARIABLE relative_target_source_dir) - else() - # In this case the target source directory is outside of the engine root of the target source directory and concatenate the first - # is used first 8 characters of the absolute path SHA256 hash to make a unique relative directory - # that can be used to install the generated CMakeLists.txt - # of a SHA256 hash - string(SHA256 target_source_hash ${absolute_target_source_dir}) - string(SUBSTRING ${target_source_hash} 0 8 target_source_hash) - cmake_path(GET absolute_target_source_dir FILENAME target_source_dirname) - cmake_path(SET relative_target_source_dir "${target_source_dirname}-${target_source_hash}") - endif() - - set(${output_source_dir} ${relative_target_source_dir} PARENT_SCOPE) -endfunction() - #! ly_setup_target: Setup the data needed to re-create the cmake target commands for a single target function(ly_setup_target OUTPUT_CONFIGURED_TARGET ALIAS_TARGET_NAME absolute_target_source_dir) # De-alias target name @@ -254,7 +236,6 @@ endfunction() #! ly_setup_subdirectory: setup all targets in the subdirectory function(ly_setup_subdirectory absolute_target_source_dir) - # Get the target source directory relative to the LY roo folder ly_get_engine_relative_source_dir(${absolute_target_source_dir} relative_target_source_dir) # The builtin BUILDSYSTEM_TARGETS property isn't being used here as that returns the de-alised @@ -540,56 +521,74 @@ function(ly_setup_others) ) # Exclude transient artifacts that shouldn't be copied to the install layout list(FILTER external_subdir_files EXCLUDE REGEX "/([Bb]uild|[Cc]ache|[Uu]ser)$") - list(APPEND filtered_asset_paths ${external_subdir_files}) + # Storing a "mapping" of gem candidate directories, to external_subdirectory files using + # a DIRECTORY property for the "value" and the GLOBAL property for the "key" + set_property(DIRECTORY ${gem_candidate_dir} APPEND PROPERTY directory_filtered_asset_paths "${external_subdir_files}") + set_property(GLOBAL APPEND PROPERTY global_gem_candidate_dirs_prop ${gem_candidate_dir}) endforeach() - # At this point the filtered_assets_paths contains the list of all directories and files - # that are non-excluded candidates that can be scanned for target directories and files - # to copy over to the install layout - foreach(filtered_asset_path IN LISTS filtered_asset_paths) - if(IS_DIRECTORY ${filtered_asset_path}) - file(GLOB_RECURSE - recurse_assets_paths - LIST_DIRECTORIES TRUE - "${filtered_asset_path}/*" - ) - set(gem_file_paths ${recurse_assets_paths}) - # Make sure to prepend the current path iteration to the gem_dirs_path to filter - set(gem_dir_paths ${filtered_asset_path} ${recurse_assets_paths}) - - # Gather directories to copy over - # Currently only the Assets, Registry and Config directories are copied over - list(FILTER gem_dir_paths INCLUDE REGEX "/(Assets|Registry|Config)$") - list(APPEND gems_assets_dir_path ${gem_dir_paths}) - else() - set(gem_file_paths ${filtered_asset_path}) - endif() + # Iterate over each gem candidate directories and read populate a directory property + # containing the files to copy over + get_property(gem_candidate_dirs GLOBAL PROPERTY global_gem_candidate_dirs_prop) + foreach(gem_candidate_dir IN LISTS gem_candidate_dirs) + get_property(filtered_asset_paths DIRECTORY ${gem_candidate_dir} PROPERTY directory_filtered_asset_paths) + ly_get_last_path_segment_concat_sha256(${gem_candidate_dir} last_gem_root_path_segment) + # Check if the gem is a subdirectory of the engine + cmake_path(IS_PREFIX LY_ROOT_FOLDER ${gem_candidate_dir} is_gem_subdirectory_of_engine) + + # At this point the filtered_assets_paths contains the list of all directories and files + # that are non-excluded candidates that can be scanned for target directories and files + # to copy over to the install layout + foreach(filtered_asset_path IN LISTS filtered_asset_paths) + if(IS_DIRECTORY ${filtered_asset_path}) + file(GLOB_RECURSE + recurse_assets_paths + LIST_DIRECTORIES TRUE + "${filtered_asset_path}/*" + ) + set(gem_file_paths ${recurse_assets_paths}) + # Make sure to prepend the current path iteration to the gem_dirs_path to filter + set(gem_dir_paths ${filtered_asset_path} ${recurse_assets_paths}) + + # Gather directories to copy over + # Currently only the Assets, Registry and Config directories are copied over + list(FILTER gem_dir_paths INCLUDE REGEX "/(Assets|Registry|Config|Editor/Scripts)$") + set_property(DIRECTORY ${gem_candidate_dir} APPEND PROPERTY gems_assets_paths ${gem_dir_paths}) + else() + set(gem_file_paths ${filtered_asset_path}) + endif() - # Gather files to copy over - # Currently only the gem.json file is copied over - list(FILTER gem_file_paths INCLUDE REGEX "/(gem.json)$") - list(APPEND gems_assets_file_path "${gem_file_paths}") - endforeach() + # Gather files to copy over + # Currently only the gem.json file is copied over + list(FILTER gem_file_paths INCLUDE REGEX "/(gem.json|preview.png)$") + set_property(DIRECTORY ${gem_candidate_dir} APPEND PROPERTY gems_assets_paths "${gem_file_paths}") + endforeach() - # gem directories to install - foreach(gem_absolute_dir_path ${gems_assets_dir_path}) - cmake_path(RELATIVE_PATH gem_absolute_dir_path BASE_DIRECTORY ${LY_ROOT_FOLDER} OUTPUT_VARIABLE gem_relative_dir_path) - if (EXISTS ${gem_absolute_dir_path}) - # The trailing slash is IMPORTANT here as that is needed to prevent - # the "Assets" folder from being copied underneath the /Assets folder - install(DIRECTORY "${gem_absolute_dir_path}/" - DESTINATION ${gem_relative_dir_path} - ) - endif() - endforeach() + # gem directories and files to install + get_property(gems_assets_dir_path DIRECTORY ${gem_candidate_dir} PROPERTY gems_assets_paths) + foreach(gem_absolute_path IN LISTS gems_assets_paths) + if(is_gem_subdirectory_of_engine) + cmake_path(RELATIVE_PATH gem_absolute_path BASE_DIRECTORY ${LY_ROOT_FOLDER} OUTPUT_VARIABLE gem_install_dest_dir) + else() + # The gem resides outside of the LY_ROOT_FOLDER, so the destination is made relative to the + # gem candidate directory and placed under the "External" directory" + # directory + cmake_path(RELATIVE_PATH gem_absolute_path BASE_DIRECTORY ${gem_candidate_dir} OUTPUT_VARIABLE gem_relative_path) + unset(gem_install_dest_dir) + cmake_path(APPEND gem_install_dest_dir "External" ${last_gem_root_path_segment} ${gem_relative_path}) + endif() + + cmake_path(GET gem_install_dest_dir PARENT_PATH gem_install_dest_dir) + if (NOT gem_install_dest_dir) + cmake_path(SET gem_install_dest_dir .) + endif() + if(IS_DIRECTORY ${gem_absolute_path}) + install(DIRECTORY "${gem_absolute_path}" DESTINATION ${gem_install_dest_dir}) + elseif (EXISTS ${gem_absolute_path}) + install(FILES ${gem_absolute_path} DESTINATION ${gem_install_dest_dir}) + endif() + endforeach() - # gem files to install - foreach(gem_absolute_file_path ${gems_assets_file_path}) - cmake_path(RELATIVE_PATH gem_absolute_file_path BASE_DIRECTORY ${LY_ROOT_FOLDER} OUTPUT_VARIABLE gem_relative_file_path) - cmake_path(GET gem_relative_file_path PARENT_PATH gem_relative_parent_dir) - install(FILES ${gem_absolute_file_path} - DESTINATION ${gem_relative_parent_dir} - ) endforeach() # Templates @@ -626,3 +625,46 @@ function(ly_setup_target_generator) ) endfunction() + +#! ly_add_install_paths: Adds the list of path to copy to the install layout relative to the same folder +# \arg:PATHS - Paths to copy over to the install layout. The DESTINATION sub argument is optional +# The INPUT sub-argument is required +# \arg:BASE_DIRECTORY(Optional) - Absolute path where a relative path from the each input path will be +# based off of. Defaults to LY_ROOT_FOLDER if not supplied +function(ly_add_install_paths) + set(options) + set(oneValueArgs BASE_DIRECTORY) + set(multiValueArgs PATHS) + cmake_parse_arguments(ly_add_install_paths "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + if(NOT ly_add_install_paths_PATHS) + message(FATAL_ERROR "ly_add_install_paths requires at least one input path to copy to the destination") + endif() + + # The default is the "." directory if not supplied + if(NOT ly_add_install_paths_BASE_DIRECTORY) + cmake_path(SET ly_add_install_paths_BASE_DIRECTORY ${LY_ROOT_FOLDER}) + endif() + + # Separate each path into an INPUT and DESTINATION parameter + set(options) + set(oneValueArgs INPUT DESTINATION) + set(multiValueArgs) + foreach(install_path IN LISTS ly_add_install_paths_PATHS) + string(REPLACE " " ";" install_path ${install_path}) + cmake_parse_arguments(install "${options}" "${oneValueArgs}" "${multiValueArgs}" ${install_path}) + if(NOT install_DESTINATION) + ly_get_engine_relative_source_dir(${install_INPUT} rel_to_root_input_path + BASE_DIRECTORY ${ly_add_install_paths_BASE_DIRECTORY}) + cmake_path(GET rel_to_root_input_path PARENT_PATH install_DESTINATION) + endif() + if(NOT install_DESTINATION) + cmake_path(SET install_DESTINATION .) + endif() + if(IS_DIRECTORY ${install_INPUT}) + install(DIRECTORY ${install_INPUT} DESTINATION ${install_DESTINATION}) + elseif(EXISTS ${install_INPUT}) + install(FILES ${install_INPUT} DESTINATION ${install_DESTINATION}) + endif() + endforeach() + +endfunction() From e6ddc29cb4b128ec7d05e8b30de25886cce83e84 Mon Sep 17 00:00:00 2001 From: Benjamin Jillich <43751992+amzn-jillich@users.noreply.github.com> Date: Thu, 15 Jul 2021 02:55:47 -0700 Subject: [PATCH 17/33] Data integrity issue, keyframe times need to be ascending (#2130) (#2154) * User provided animation resulted in a data integrity error. * The first track had 0.66 as max time while another had a keyframe more resulting in 0.69. * When updating the duration before fixing up the last keyframe for each of the tracks, the duration was returned early with 0.66 while there were other tracks having a keyframe more. * We're now crawling through all tracks to determine the maximum duration. * This results in a correct fixup of the last frame. Signed-off-by: Benjamin Jillich --- .../MotionData/NonUniformMotionData.cpp | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/Gems/EMotionFX/Code/EMotionFX/Source/MotionData/NonUniformMotionData.cpp b/Gems/EMotionFX/Code/EMotionFX/Source/MotionData/NonUniformMotionData.cpp index f9e0911d51..5c530e35e3 100644 --- a/Gems/EMotionFX/Code/EMotionFX/Source/MotionData/NonUniformMotionData.cpp +++ b/Gems/EMotionFX/Code/EMotionFX/Source/MotionData/NonUniformMotionData.cpp @@ -412,25 +412,24 @@ namespace EMotionFX void NonUniformMotionData::UpdateDuration() { + m_duration = 0.0f; + for (const JointData& jointData : m_jointData) { if (!jointData.m_positionTrack.m_times.empty()) { - m_duration = jointData.m_positionTrack.m_times.back(); - return; + m_duration = AZ::GetMax(m_duration, jointData.m_positionTrack.m_times.back()); } if (!jointData.m_rotationTrack.m_times.empty()) { - m_duration = jointData.m_rotationTrack.m_times.back(); - return; + m_duration = AZ::GetMax(m_duration, jointData.m_rotationTrack.m_times.back()); } #ifndef EMFX_SCALE_DISABLED if (!jointData.m_scaleTrack.m_times.empty()) { - m_duration = jointData.m_scaleTrack.m_times.back(); - return; + m_duration = AZ::GetMax(m_duration, jointData.m_scaleTrack.m_times.back()); } #endif } @@ -439,8 +438,7 @@ namespace EMotionFX { if (!morphData.m_track.m_times.empty()) { - m_duration = morphData.m_track.m_times.back(); - return; + m_duration = AZ::GetMax(m_duration, morphData.m_track.m_times.back()); } } @@ -448,12 +446,9 @@ namespace EMotionFX { if (!floatData.m_track.m_times.empty()) { - m_duration = floatData.m_track.m_times.back(); - return; + m_duration = AZ::GetMax(m_duration, floatData.m_track.m_times.back()); } } - - m_duration = 0.0f; } void NonUniformMotionData::AllocateJointPositionSamples(size_t jointDataIndex, size_t numSamples) From 69312a5e56976d2c3640c5706d52a98525b95137 Mon Sep 17 00:00:00 2001 From: AMZN-nggieber <52797929+AMZN-nggieber@users.noreply.github.com> Date: Thu, 15 Jul 2021 17:48:04 -0700 Subject: [PATCH 18/33] Clear Build and Cache Directory After Project Move or Copy (#2184) * Clear build and Cache directories when copying or moving projects Signed-off-by: nggieber * Updated tests for moving projects and Added tests for copying projects and skipping build files Signed-off-by: nggieber * Address PR feedback and fix a few minor UX issues, and request project be rebuild after copying or moving Signed-off-by: nggieber * Include ProjectInfo.h in ProjectUtils.h so linux will compile Signed-off-by: nggieber --- .../Platform/Linux/PAL_linux_files.cmake | 1 + .../Linux/ProjectManagerDefs_linux.cpp | 15 ++ .../Platform/Mac/PAL_mac_files.cmake | 1 + .../Platform/Mac/ProjectManagerDefs_mac.cpp | 15 ++ .../Platform/Windows/PAL_windows_files.cmake | 1 + .../Windows/ProjectBuilderWorker_windows.cpp | 25 ++- .../Windows/ProjectManagerDefs_windows.cpp | 14 ++ .../Source/ProjectBuilderWorker.cpp | 2 - .../Source/ProjectBuilderWorker.h | 2 +- .../Source/ProjectButtonWidget.cpp | 2 +- .../Source/ProjectButtonWidget.h | 2 +- .../Source/ProjectManagerDefs.h | 4 +- .../ProjectManager/Source/ProjectUtils.cpp | 141 ++++++++++++++--- .../ProjectManager/Source/ProjectUtils.h | 8 +- .../ProjectManager/Source/ProjectsScreen.cpp | 9 +- .../ProjectManager/Source/ProjectsScreen.h | 2 +- .../Source/UpdateProjectCtrl.cpp | 4 +- .../Tools/ProjectManager/tests/UtilsTests.cpp | 148 +++++++++++++++--- 18 files changed, 333 insertions(+), 63 deletions(-) create mode 100644 Code/Tools/ProjectManager/Platform/Linux/ProjectManagerDefs_linux.cpp create mode 100644 Code/Tools/ProjectManager/Platform/Mac/ProjectManagerDefs_mac.cpp create mode 100644 Code/Tools/ProjectManager/Platform/Windows/ProjectManagerDefs_windows.cpp diff --git a/Code/Tools/ProjectManager/Platform/Linux/PAL_linux_files.cmake b/Code/Tools/ProjectManager/Platform/Linux/PAL_linux_files.cmake index 4125861f2b..22f8e785f2 100644 --- a/Code/Tools/ProjectManager/Platform/Linux/PAL_linux_files.cmake +++ b/Code/Tools/ProjectManager/Platform/Linux/PAL_linux_files.cmake @@ -9,4 +9,5 @@ set(FILES Python_linux.cpp ProjectBuilderWorker_linux.cpp ProjectUtils_linux.cpp + ProjectManagerDefs_linux.cpp ) diff --git a/Code/Tools/ProjectManager/Platform/Linux/ProjectManagerDefs_linux.cpp b/Code/Tools/ProjectManager/Platform/Linux/ProjectManagerDefs_linux.cpp new file mode 100644 index 0000000000..5ccfbf8eff --- /dev/null +++ b/Code/Tools/ProjectManager/Platform/Linux/ProjectManagerDefs_linux.cpp @@ -0,0 +1,15 @@ +/* + * Copyright (c) Contributors to the Open 3D Engine Project. For complete copyright and license terms please see the LICENSE at the root of this distribution. + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + * + */ + +#include + + +namespace O3DE::ProjectManager +{ + const QString ProjectBuildPathPostfix = ProjectBuildDirectoryName + "/linux"; + +} // namespace O3DE::ProjectManager diff --git a/Code/Tools/ProjectManager/Platform/Mac/PAL_mac_files.cmake b/Code/Tools/ProjectManager/Platform/Mac/PAL_mac_files.cmake index a0d3add840..550698af0b 100644 --- a/Code/Tools/ProjectManager/Platform/Mac/PAL_mac_files.cmake +++ b/Code/Tools/ProjectManager/Platform/Mac/PAL_mac_files.cmake @@ -9,4 +9,5 @@ set(FILES Python_mac.cpp ProjectBuilderWorker_mac.cpp ProjectUtils_mac.cpp + ProjectManagerDefs_mac.cpp ) diff --git a/Code/Tools/ProjectManager/Platform/Mac/ProjectManagerDefs_mac.cpp b/Code/Tools/ProjectManager/Platform/Mac/ProjectManagerDefs_mac.cpp new file mode 100644 index 0000000000..01a7f9e375 --- /dev/null +++ b/Code/Tools/ProjectManager/Platform/Mac/ProjectManagerDefs_mac.cpp @@ -0,0 +1,15 @@ +/* + * Copyright (c) Contributors to the Open 3D Engine Project. For complete copyright and license terms please see the LICENSE at the root of this distribution. + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + * + */ + +#include + + +namespace O3DE::ProjectManager +{ + const QString ProjectBuildPathPostfix = ProjectBuildDirectoryName + "/mac_xcode"; + +} // namespace O3DE::ProjectManager diff --git a/Code/Tools/ProjectManager/Platform/Windows/PAL_windows_files.cmake b/Code/Tools/ProjectManager/Platform/Windows/PAL_windows_files.cmake index 8af7638696..e0a3fc6bcd 100644 --- a/Code/Tools/ProjectManager/Platform/Windows/PAL_windows_files.cmake +++ b/Code/Tools/ProjectManager/Platform/Windows/PAL_windows_files.cmake @@ -9,4 +9,5 @@ set(FILES Python_windows.cpp ProjectBuilderWorker_windows.cpp ProjectUtils_windows.cpp + ProjectManagerDefs_windows.cpp ) diff --git a/Code/Tools/ProjectManager/Platform/Windows/ProjectBuilderWorker_windows.cpp b/Code/Tools/ProjectManager/Platform/Windows/ProjectBuilderWorker_windows.cpp index 87a8a6c0ec..eca7b0b9a8 100644 --- a/Code/Tools/ProjectManager/Platform/Windows/ProjectBuilderWorker_windows.cpp +++ b/Code/Tools/ProjectManager/Platform/Windows/ProjectBuilderWorker_windows.cpp @@ -75,8 +75,17 @@ namespace O3DE::ProjectManager m_configProjectProcess->start( "cmake", - QStringList{ "-B", QDir(m_projectInfo.m_path).filePath(ProjectBuildPathPostfix), "-S", m_projectInfo.m_path, "-G", - "Visual Studio 16", "-DLY_3RDPARTY_PATH=" + engineInfo.m_thirdPartyPath }); + QStringList + { + "-B", + QDir(m_projectInfo.m_path).filePath(ProjectBuildPathPostfix), + "-S", + m_projectInfo.m_path, + "-G", + "Visual Studio 16", + "-DLY_3RDPARTY_PATH=" + engineInfo.m_thirdPartyPath, + "-DLY_UNITY_BUILD=1" + }); if (!m_configProjectProcess->waitForStarted()) { @@ -124,8 +133,16 @@ namespace O3DE::ProjectManager m_buildProjectProcess->start( "cmake", - QStringList{ "--build", QDir(m_projectInfo.m_path).filePath(ProjectBuildPathPostfix), "--target", - m_projectInfo.m_projectName + ".GameLauncher", "Editor", "--config", "profile" }); + QStringList + { + "--build", + QDir(m_projectInfo.m_path).filePath(ProjectBuildPathPostfix), + "--target", + m_projectInfo.m_projectName + ".GameLauncher", + "Editor", + "--config", + "profile" + }); if (!m_buildProjectProcess->waitForStarted()) { diff --git a/Code/Tools/ProjectManager/Platform/Windows/ProjectManagerDefs_windows.cpp b/Code/Tools/ProjectManager/Platform/Windows/ProjectManagerDefs_windows.cpp new file mode 100644 index 0000000000..6bd2194967 --- /dev/null +++ b/Code/Tools/ProjectManager/Platform/Windows/ProjectManagerDefs_windows.cpp @@ -0,0 +1,14 @@ +/* + * Copyright (c) Contributors to the Open 3D Engine Project. For complete copyright and license terms please see the LICENSE at the root of this distribution. + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + * + */ + +#include + +namespace O3DE::ProjectManager +{ + const QString ProjectBuildPathPostfix = ProjectBuildDirectoryName + "/windows_vs2019"; + +} // namespace O3DE::ProjectManager diff --git a/Code/Tools/ProjectManager/Source/ProjectBuilderWorker.cpp b/Code/Tools/ProjectManager/Source/ProjectBuilderWorker.cpp index 63899035e0..5aed95e531 100644 --- a/Code/Tools/ProjectManager/Source/ProjectBuilderWorker.cpp +++ b/Code/Tools/ProjectManager/Source/ProjectBuilderWorker.cpp @@ -14,8 +14,6 @@ namespace O3DE::ProjectManager { - const QString ProjectBuilderWorker::BuildCancelled = QObject::tr("Build Cancelled."); - ProjectBuilderWorker::ProjectBuilderWorker(const ProjectInfo& projectInfo) : QObject() , m_projectInfo(projectInfo) diff --git a/Code/Tools/ProjectManager/Source/ProjectBuilderWorker.h b/Code/Tools/ProjectManager/Source/ProjectBuilderWorker.h index 0557b21831..3219632098 100644 --- a/Code/Tools/ProjectManager/Source/ProjectBuilderWorker.h +++ b/Code/Tools/ProjectManager/Source/ProjectBuilderWorker.h @@ -22,7 +22,7 @@ namespace O3DE::ProjectManager // QProcess::waitForFinished uses -1 to indicate that the process should not timeout static constexpr int MaxBuildTimeMSecs = -1; // Build was cancelled - static const QString BuildCancelled; + inline static const QString BuildCancelled = QObject::tr("Build Cancelled."); Q_OBJECT diff --git a/Code/Tools/ProjectManager/Source/ProjectButtonWidget.cpp b/Code/Tools/ProjectManager/Source/ProjectButtonWidget.cpp index af9a1a7bd9..8ac5f096fb 100644 --- a/Code/Tools/ProjectManager/Source/ProjectButtonWidget.cpp +++ b/Code/Tools/ProjectManager/Source/ProjectButtonWidget.cpp @@ -233,7 +233,7 @@ namespace O3DE::ProjectManager AzQtComponents::ShowFileOnDesktop(m_projectInfo.m_path); }); menu->addSeparator(); - menu->addAction(tr("Duplicate"), this, [this]() { emit CopyProject(m_projectInfo.m_path); }); + menu->addAction(tr("Duplicate"), this, [this]() { emit CopyProject(m_projectInfo); }); menu->addSeparator(); menu->addAction(tr("Remove from O3DE"), this, [this]() { emit RemoveProject(m_projectInfo.m_path); }); menu->addAction(tr("Delete this Project"), this, [this]() { emit DeleteProject(m_projectInfo.m_path); }); diff --git a/Code/Tools/ProjectManager/Source/ProjectButtonWidget.h b/Code/Tools/ProjectManager/Source/ProjectButtonWidget.h index 27559b325e..fe1ce9ad05 100644 --- a/Code/Tools/ProjectManager/Source/ProjectButtonWidget.h +++ b/Code/Tools/ProjectManager/Source/ProjectButtonWidget.h @@ -88,7 +88,7 @@ namespace O3DE::ProjectManager signals: void OpenProject(const QString& projectName); void EditProject(const QString& projectName); - void CopyProject(const QString& projectName); + void CopyProject(const ProjectInfo& projectInfo); void RemoveProject(const QString& projectName); void DeleteProject(const QString& projectName); void BuildProject(const ProjectInfo& projectInfo); diff --git a/Code/Tools/ProjectManager/Source/ProjectManagerDefs.h b/Code/Tools/ProjectManager/Source/ProjectManagerDefs.h index 79ad04c43a..7b49b89be1 100644 --- a/Code/Tools/ProjectManager/Source/ProjectManagerDefs.h +++ b/Code/Tools/ProjectManager/Source/ProjectManagerDefs.h @@ -14,8 +14,10 @@ namespace O3DE::ProjectManager inline constexpr static int ProjectPreviewImageHeight = 280; inline constexpr static int ProjectTemplateImageWidth = 92; - static const QString ProjectBuildPathPostfix = "build/windows_vs2019"; + static const QString ProjectBuildDirectoryName = "build"; + extern const QString ProjectBuildPathPostfix; static const QString ProjectBuildPathCmakeFiles = "CMakeFiles"; static const QString ProjectBuildErrorLogName = "CMakeProjectBuildError.log"; + static const QString ProjectCacheDirectoryName = "Cache"; static const QString ProjectPreviewImagePath = "preview.png"; } // namespace O3DE::ProjectManager diff --git a/Code/Tools/ProjectManager/Source/ProjectUtils.cpp b/Code/Tools/ProjectManager/Source/ProjectUtils.cpp index 827973cbec..6b3750b6bb 100644 --- a/Code/Tools/ProjectManager/Source/ProjectUtils.cpp +++ b/Code/Tools/ProjectManager/Source/ProjectUtils.cpp @@ -6,6 +6,7 @@ */ #include +#include #include #include @@ -59,29 +60,63 @@ namespace O3DE::ProjectManager return false; } + static bool SkipFilePaths(const QString& curPath, QStringList& skippedPaths, QStringList& deeperSkippedPaths) + { + bool skip = false; + for (const QString& skippedPath : skippedPaths) + { + QString nativeSkippedPath = QDir::toNativeSeparators(skippedPath); + QString firstSectionSkippedPath = nativeSkippedPath.section(QDir::separator(), 0, 0); + if (curPath == firstSectionSkippedPath) + { + // We are at the end of the path to skip, so skip it + if (nativeSkippedPath == firstSectionSkippedPath) + { + skippedPaths.removeAll(skippedPath); + skip = true; + break; + } + // Append the next section of the skipped path + else + { + deeperSkippedPaths.append(nativeSkippedPath.section(QDir::separator(), 1)); + } + } + } + + return skip; + } + typedef AZStd::function StatusFunction; - static void RecursiveGetAllFiles(const QDir& directory, QStringList& outFileList, qint64& outTotalSizeInBytes, StatusFunction statusCallback) + static void RecursiveGetAllFiles(const QDir& directory, QStringList& skippedPaths, int& outFileCount, qint64& outTotalSizeInBytes, StatusFunction statusCallback) { const QStringList entries = directory.entryList(QDir::Dirs | QDir::Files | QDir::NoSymLinks | QDir::NoDotAndDotDot); for (const QString& entryPath : entries) { const QString filePath = QDir::toNativeSeparators(QString("%1/%2").arg(directory.path()).arg(entryPath)); + + QStringList deeperSkippedPaths; + if (SkipFilePaths(entryPath, skippedPaths, deeperSkippedPaths)) + { + continue; + } + QFileInfo fileInfo(filePath); if (fileInfo.isDir()) { QDir subDirectory(filePath); - RecursiveGetAllFiles(subDirectory, outFileList, outTotalSizeInBytes, statusCallback); + RecursiveGetAllFiles(subDirectory, deeperSkippedPaths, outFileCount, outTotalSizeInBytes, statusCallback); } else { - outFileList.push_back(filePath); + ++outFileCount; outTotalSizeInBytes += fileInfo.size(); const int updateStatusEvery = 64; - if (outFileList.size() % updateStatusEvery == 0) + if (outFileCount % updateStatusEvery == 0) { - statusCallback(outFileList.size(), outTotalSizeInBytes); + statusCallback(outFileCount, outTotalSizeInBytes); } } } @@ -90,7 +125,8 @@ namespace O3DE::ProjectManager static bool CopyDirectory(QProgressDialog* progressDialog, const QString& origPath, const QString& newPath, - QStringList& filesToCopy, + QStringList& skippedPaths, + int filesToCopyCount, int& outNumCopiedFiles, qint64 totalSizeToCopy, qint64& outCopiedFileSize, @@ -102,18 +138,24 @@ namespace O3DE::ProjectManager return false; } - for (QString directory : original.entryList(QDir::Dirs | QDir::NoDotAndDotDot)) + for (const QString& directory : original.entryList(QDir::Dirs | QDir::NoDotAndDotDot)) { if (progressDialog->wasCanceled()) { return false; } + QStringList deeperSkippedPaths; + if (SkipFilePaths(directory, skippedPaths, deeperSkippedPaths)) + { + continue; + } + QString newDirectoryPath = newPath + QDir::separator() + directory; original.mkpath(newDirectoryPath); - if (!CopyDirectory(progressDialog, origPath + QDir::separator() + directory, - newDirectoryPath, filesToCopy, outNumCopiedFiles, totalSizeToCopy, outCopiedFileSize, showIgnoreFileDialog)) + if (!CopyDirectory(progressDialog, origPath + QDir::separator() + directory, newDirectoryPath, deeperSkippedPaths, + filesToCopyCount, outNumCopiedFiles, totalSizeToCopy, outCopiedFileSize, showIgnoreFileDialog)) { return false; } @@ -121,18 +163,25 @@ namespace O3DE::ProjectManager QLocale locale; const float progressDialogRangeHalf = qFabs(progressDialog->maximum() - progressDialog->minimum()) * 0.5f; - for (QString file : original.entryList(QDir::Files)) + for (const QString& file : original.entryList(QDir::Files)) { if (progressDialog->wasCanceled()) { return false; } + // Unused by this function but neccesary to pass in to SkipFilePaths + QStringList deeperSkippedPaths; + if (SkipFilePaths(file, skippedPaths, deeperSkippedPaths)) + { + continue; + } + // Progress window update { // Weight in the number of already copied files as well as the copied bytes to get a better progress indication // for cases combining many small files and some really large files. - const float normalizedNumFiles = static_cast(outNumCopiedFiles) / filesToCopy.count(); + const float normalizedNumFiles = static_cast(outNumCopiedFiles) / filesToCopyCount; const float normalizedFileSize = static_cast(outCopiedFileSize) / totalSizeToCopy; const int progress = normalizedNumFiles * progressDialogRangeHalf + normalizedFileSize * progressDialogRangeHalf; progressDialog->setValue(progress); @@ -140,7 +189,7 @@ namespace O3DE::ProjectManager const QString copiedFileSizeString = locale.formattedDataSize(outCopiedFileSize); const QString totalFileSizeString = locale.formattedDataSize(totalSizeToCopy); progressDialog->setLabelText(QString("Coping file %1 of %2 (%3 of %4) ...").arg(QString::number(outNumCopiedFiles), - QString::number(filesToCopy.count()), + QString::number(filesToCopyCount), copiedFileSizeString, totalFileSizeString)); qApp->processEvents(QEventLoop::ExcludeUserInputEvents); @@ -194,6 +243,39 @@ namespace O3DE::ProjectManager return true; } + static bool ClearProjectBuildArtifactsAndCache(const QString& origPath, const QString& newPath, QWidget* parent) + { + QDir buildDirectory = QDir(newPath); + if ((!buildDirectory.cd(ProjectBuildDirectoryName) || !DeleteProjectFiles(buildDirectory.path(), true)) + && QDir(origPath).cd(ProjectBuildDirectoryName)) + { + QMessageBox::warning( + parent, + QObject::tr("Clear Build Artifacts"), + QObject::tr("Build artifacts failed to delete for moved project. Please manually delete build directory at \"%1\"") + .arg(buildDirectory.path()), + QMessageBox::Close); + + return false; + } + + QDir cacheDirectory = QDir(newPath); + if ((!cacheDirectory.cd(ProjectCacheDirectoryName) || !DeleteProjectFiles(cacheDirectory.path(), true)) + && QDir(origPath).cd(ProjectCacheDirectoryName)) + { + QMessageBox::warning( + parent, + QObject::tr("Clear Asset Cache"), + QObject::tr("Asset cache failed to delete for moved project. Please manually delete cache directory at \"%1\"") + .arg(cacheDirectory.path()), + QMessageBox::Close); + + return false; + } + + return false; + } + bool AddProjectDialog(QWidget* parent) { QString path = QDir::toNativeSeparators(QFileDialog::getExistingDirectory(parent, QObject::tr("Select Project Directory"))); @@ -215,7 +297,7 @@ namespace O3DE::ProjectManager return PythonBindingsInterface::Get()->RemoveProject(path); } - bool CopyProjectDialog(const QString& origPath, QWidget* parent) + bool CopyProjectDialog(const QString& origPath, ProjectInfo& newProjectInfo, QWidget* parent) { bool copyResult = false; @@ -225,6 +307,8 @@ namespace O3DE::ProjectManager QFileDialog::getExistingDirectory(parent, QObject::tr("Select New Project Directory"), parentOrigDir.path())); if (!newPath.isEmpty()) { + newProjectInfo.m_path = newPath; + if (!WarnDirectoryOverwrite(newPath, parent)) { return false; @@ -236,7 +320,7 @@ namespace O3DE::ProjectManager return copyResult; } - bool CopyProject(const QString& origPath, const QString& newPath, QWidget* parent) + bool CopyProject(const QString& origPath, const QString& newPath, QWidget* parent, bool skipRegister) { // Disallow copying from or into subdirectory if (IsDirectoryDescedent(origPath, newPath) || IsDirectoryDescedent(newPath, origPath)) @@ -244,8 +328,13 @@ namespace O3DE::ProjectManager return false; } - QStringList filesToCopy; + int filesToCopyCount = 0; qint64 totalSizeInBytes = 0; + QStringList skippedPaths + { + ProjectBuildDirectoryName, + ProjectCacheDirectoryName + }; QProgressDialog* progressDialog = new QProgressDialog(parent); progressDialog->setAutoClose(true); @@ -256,7 +345,8 @@ namespace O3DE::ProjectManager progressDialog->show(); QLocale locale; - RecursiveGetAllFiles(origPath, filesToCopy, totalSizeInBytes, [=](int fileCount, int sizeInBytes) + QStringList getFilesSkippedPaths(skippedPaths); + RecursiveGetAllFiles(origPath, getFilesSkippedPaths, filesToCopyCount, totalSizeInBytes, [=](int fileCount, int sizeInBytes) { // Create a human-readable version of the file size. const QString fileSizeString = locale.formattedDataSize(sizeInBytes); @@ -275,8 +365,10 @@ namespace O3DE::ProjectManager // Phase 1: Copy files bool showIgnoreFileDialog = true; - bool success = CopyDirectory(progressDialog, origPath, newPath, filesToCopy, numFilesCopied, totalSizeInBytes, copiedFileSize, showIgnoreFileDialog); - if (success) + QStringList copyFilesSkippedPaths(skippedPaths); + bool success = CopyDirectory(progressDialog, origPath, newPath, copyFilesSkippedPaths, filesToCopyCount, numFilesCopied, + totalSizeInBytes, copiedFileSize, showIgnoreFileDialog); + if (success && !skipRegister) { // Phase 2: Register project success = RegisterProject(newPath); @@ -299,7 +391,7 @@ namespace O3DE::ProjectManager QDir projectDirectory(path); if (projectDirectory.exists()) { - // Check if there is an actual project hereor just force it + // Check if there is an actual project here or just force it if (force || PythonBindingsInterface::Get()->GetProject(path).IsSuccess()) { return projectDirectory.removeRecursively(); @@ -309,12 +401,12 @@ namespace O3DE::ProjectManager return false; } - bool MoveProject(QString origPath, QString newPath, QWidget* parent, bool ignoreRegister) + bool MoveProject(QString origPath, QString newPath, QWidget* parent, bool skipRegister) { origPath = QDir::toNativeSeparators(origPath); newPath = QDir::toNativeSeparators(newPath); - if (!WarnDirectoryOverwrite(newPath, parent) || (!ignoreRegister && !UnregisterProject(origPath))) + if (!WarnDirectoryOverwrite(newPath, parent) || (!skipRegister && !UnregisterProject(origPath))) { return false; } @@ -334,8 +426,13 @@ namespace O3DE::ProjectManager DeleteProjectFiles(origPath, true); } + else + { + // If directoy rename succeeded then build and cache directories need to be deleted seperately + ClearProjectBuildArtifactsAndCache(origPath, newPath, parent); + } - if (!ignoreRegister && !RegisterProject(newPath)) + if (!skipRegister && !RegisterProject(newPath)) { return false; } diff --git a/Code/Tools/ProjectManager/Source/ProjectUtils.h b/Code/Tools/ProjectManager/Source/ProjectUtils.h index 1035ceea65..2c4e873b65 100644 --- a/Code/Tools/ProjectManager/Source/ProjectUtils.h +++ b/Code/Tools/ProjectManager/Source/ProjectUtils.h @@ -7,6 +7,8 @@ #pragma once #include +#include + #include #include @@ -17,10 +19,10 @@ namespace O3DE::ProjectManager bool AddProjectDialog(QWidget* parent = nullptr); bool RegisterProject(const QString& path); bool UnregisterProject(const QString& path); - bool CopyProjectDialog(const QString& origPath, QWidget* parent = nullptr); - bool CopyProject(const QString& origPath, const QString& newPath, QWidget* parent); + bool CopyProjectDialog(const QString& origPath, ProjectInfo& newProjectInfo, QWidget* parent = nullptr); + bool CopyProject(const QString& origPath, const QString& newPath, QWidget* parent, bool skipRegister = false); bool DeleteProjectFiles(const QString& path, bool force = false); - bool MoveProject(QString origPath, QString newPath, QWidget* parent = nullptr, bool ignoreRegister = false); + bool MoveProject(QString origPath, QString newPath, QWidget* parent, bool skipRegister = false); bool ReplaceFile(const QString& origFile, const QString& newFile, QWidget* parent = nullptr, bool interactive = true); diff --git a/Code/Tools/ProjectManager/Source/ProjectsScreen.cpp b/Code/Tools/ProjectManager/Source/ProjectsScreen.cpp index 54fdc370d7..974858f2b2 100644 --- a/Code/Tools/ProjectManager/Source/ProjectsScreen.cpp +++ b/Code/Tools/ProjectManager/Source/ProjectsScreen.cpp @@ -384,14 +384,17 @@ namespace O3DE::ProjectManager emit ChangeScreenRequest(ProjectManagerScreen::UpdateProject); } } - void ProjectsScreen::HandleCopyProject(const QString& projectPath) + void ProjectsScreen::HandleCopyProject(const ProjectInfo& projectInfo) { - if (!WarnIfInBuildQueue(projectPath)) + if (!WarnIfInBuildQueue(projectInfo.m_path)) { + ProjectInfo newProjectInfo(projectInfo); + // Open file dialog and choose location for copied project then register copy with O3DE - if (ProjectUtils::CopyProjectDialog(projectPath, this)) + if (ProjectUtils::CopyProjectDialog(projectInfo.m_path, newProjectInfo, this)) { ResetProjectsContent(); + emit NotifyBuildProject(newProjectInfo); emit ChangeScreenRequest(ProjectManagerScreen::Projects); } } diff --git a/Code/Tools/ProjectManager/Source/ProjectsScreen.h b/Code/Tools/ProjectManager/Source/ProjectsScreen.h index 787db16c59..a6e5c20c52 100644 --- a/Code/Tools/ProjectManager/Source/ProjectsScreen.h +++ b/Code/Tools/ProjectManager/Source/ProjectsScreen.h @@ -44,7 +44,7 @@ namespace O3DE::ProjectManager void HandleAddProjectButton(); void HandleOpenProject(const QString& projectPath); void HandleEditProject(const QString& projectPath); - void HandleCopyProject(const QString& projectPath); + void HandleCopyProject(const ProjectInfo& projectInfo); void HandleRemoveProject(const QString& projectPath); void HandleDeleteProject(const QString& projectPath); diff --git a/Code/Tools/ProjectManager/Source/UpdateProjectCtrl.cpp b/Code/Tools/ProjectManager/Source/UpdateProjectCtrl.cpp index 5bd6cc4552..89555d2e72 100644 --- a/Code/Tools/ProjectManager/Source/UpdateProjectCtrl.cpp +++ b/Code/Tools/ProjectManager/Source/UpdateProjectCtrl.cpp @@ -219,11 +219,13 @@ namespace O3DE::ProjectManager // Move project first to avoid trying to update settings at the new location before it has been moved there if (newProjectSettings.m_path != m_projectInfo.m_path) { - if (!ProjectUtils::MoveProject(m_projectInfo.m_path, newProjectSettings.m_path)) + if (!ProjectUtils::MoveProject(m_projectInfo.m_path, newProjectSettings.m_path, this)) { QMessageBox::critical(this, tr("Project move failed"), tr("Failed to move project.")); return false; } + + emit NotifyBuildProject(newProjectSettings); } // Update project if settings changed diff --git a/Code/Tools/ProjectManager/tests/UtilsTests.cpp b/Code/Tools/ProjectManager/tests/UtilsTests.cpp index 51f7162278..a02a9cb796 100644 --- a/Code/Tools/ProjectManager/tests/UtilsTests.cpp +++ b/Code/Tools/ProjectManager/tests/UtilsTests.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -25,16 +26,31 @@ namespace O3DE::ProjectManager : public ::UnitTest::ScopedAllocatorSetupFixture { public: + static inline QString ReplaceFirstAWithB(const QString& originalString) + { + QString bString(originalString); + return bString.replace(bString.indexOf('A'), 1, 'B'); + } + ProjectManagerUtilsTests() { m_application = AZStd::make_unique(); m_application->Init(false); + m_projectAPath = "ProjectA"; + + // Replaces first 'A' with 'B' + m_projectBPath = ReplaceFirstAWithB(m_projectAPath); + m_projectABuildPath = QString("%1%2%3").arg(m_projectAPath, QDir::separator(), ProjectBuildDirectoryName); + m_projectBBuildPath = ReplaceFirstAWithB(m_projectABuildPath); + QDir dir; - dir.mkdir("ProjectA"); - dir.mkdir("ProjectB"); + dir.mkpath(m_projectABuildPath); + dir.mkdir(m_projectBPath); - QFile origFile("ProjectA/origFile.txt"); + m_projectAOrigFilePath = QString("%1%2%3").arg(m_projectAPath, QDir::separator(), "origFile.txt"); + m_projectBOrigFilePath = ReplaceFirstAWithB(m_projectAOrigFilePath); + QFile origFile(m_projectAOrigFilePath); if (origFile.open(QIODevice::ReadWrite)) { QTextStream stream(&origFile); @@ -42,63 +58,153 @@ namespace O3DE::ProjectManager origFile.close(); } - QFile replaceFile("ProjectA/replaceFile.txt"); + m_projectAReplaceFilePath = QString("%1%2%3").arg(m_projectAPath, QDir::separator(), "replaceFile.txt"); + m_projectBReplaceFilePath = ReplaceFirstAWithB(m_projectAReplaceFilePath); + QFile replaceFile(m_projectAReplaceFilePath); if (replaceFile.open(QIODevice::ReadWrite)) { QTextStream stream(&replaceFile); stream << "replace" << Qt::endl; replaceFile.close(); } + + m_projectABuildFilePath = QString("%1%2%3").arg(m_projectABuildPath, QDir::separator(), "build.obj"); + m_projectBBuildFilePath = ReplaceFirstAWithB(m_projectABuildFilePath); + QFile buildFile(m_projectABuildFilePath); + if (buildFile.open(QIODevice::ReadWrite)) + { + QTextStream stream(&buildFile); + stream << "x0FFFFFFFF" << Qt::endl; + buildFile.close(); + } } ~ProjectManagerUtilsTests() { - QDir dirA("ProjectA"); + QDir dirA(m_projectAPath); dirA.removeRecursively(); - QDir dirB("ProjectB"); + QDir dirB(m_projectBPath); dirB.removeRecursively(); m_application.reset(); } AZStd::unique_ptr m_application; + + QString m_projectAPath; + QString m_projectAOrigFilePath; + QString m_projectAReplaceFilePath; + QString m_projectABuildPath; + QString m_projectABuildFilePath; + QString m_projectBPath; + QString m_projectBOrigFilePath; + QString m_projectBReplaceFilePath; + QString m_projectBBuildPath; + QString m_projectBBuildFilePath; + }; #if AZ_TRAIT_DISABLE_FAILED_PROJECT_MANAGER_TESTS - TEST_F(ProjectManagerUtilsTests, DISABLED_MoveProject_Succeeds) + TEST_F(ProjectManagerUtilsTests, DISABLED_MoveProject_MovesExpectedFiles) +#else + TEST_F(ProjectManagerUtilsTests, MoveProject_MovesExpectedFiles) +#endif // !AZ_TRAIT_DISABLE_FAILED_PROJECT_MANAGER_TESTS + { + EXPECT_TRUE(MoveProject( + QDir::currentPath() + QDir::separator() + m_projectAPath, + QDir::currentPath() + QDir::separator() + m_projectBPath, + nullptr, true)); + + QFileInfo origFile(m_projectAOrigFilePath); + EXPECT_FALSE(origFile.exists()); + + QFileInfo replaceFile(m_projectAReplaceFilePath); + EXPECT_FALSE(replaceFile.exists()); + + QFileInfo origFileMoved(m_projectBOrigFilePath); + EXPECT_TRUE(origFileMoved.exists() && origFileMoved.isFile()); + + QFileInfo replaceFileMoved(m_projectBReplaceFilePath); + EXPECT_TRUE(replaceFileMoved.exists() && replaceFileMoved.isFile()); + } + +#if AZ_TRAIT_DISABLE_FAILED_PROJECT_MANAGER_TESTS + TEST_F(ProjectManagerUtilsTests, DISABLED_MoveProject_DoesntMoveBuild) #else - TEST_F(ProjectManagerUtilsTests, MoveProject_Succeeds) + TEST_F(ProjectManagerUtilsTests, MoveProject_DoesntMoveBuild) #endif // !AZ_TRAIT_DISABLE_FAILED_PROJECT_MANAGER_TESTS { EXPECT_TRUE(MoveProject( - QDir::currentPath() + QDir::separator() + "ProjectA", - QDir::currentPath() + QDir::separator() + "ProjectB", + QDir::currentPath() + QDir::separator() + m_projectAPath, + QDir::currentPath() + QDir::separator() + m_projectBPath, nullptr, true)); - QFileInfo origFile("ProjectA/origFile.txt"); - EXPECT_TRUE(!origFile.exists()); + QFileInfo origFile(m_projectAOrigFilePath); + EXPECT_FALSE(origFile.exists()); - QFileInfo replaceFile("ProjectA/replaceFile.txt"); - EXPECT_TRUE(!replaceFile.exists()); + QFileInfo origFileMoved(m_projectBOrigFilePath); + EXPECT_TRUE(origFileMoved.exists() && origFileMoved.isFile()); + + QDir buildDir(m_projectBBuildPath); + EXPECT_FALSE(buildDir.exists()); + } + +#if AZ_TRAIT_DISABLE_FAILED_PROJECT_MANAGER_TESTS + TEST_F(ProjectManagerUtilsTests, DISABLED_CopyProject_CopiesExpectedFiles) +#else + TEST_F(ProjectManagerUtilsTests, CopyProject_CopiesExpectedFiles) +#endif // !AZ_TRAIT_DISABLE_FAILED_PROJECT_MANAGER_TESTS + { + EXPECT_TRUE(CopyProject( + QDir::currentPath() + QDir::separator() + m_projectAPath, + QDir::currentPath() + QDir::separator() + m_projectBPath, + nullptr, true)); - QFileInfo origFileMoved("ProjectB/origFile.txt"); + QFileInfo origFile(m_projectAOrigFilePath); + EXPECT_TRUE(origFile.exists()); + + QFileInfo replaceFile(m_projectAReplaceFilePath); + EXPECT_TRUE(replaceFile.exists()); + + QFileInfo origFileMoved(m_projectBOrigFilePath); EXPECT_TRUE(origFileMoved.exists() && origFileMoved.isFile()); - QFileInfo replaceFileMoved("ProjectB/replaceFile.txt"); + QFileInfo replaceFileMoved(m_projectBReplaceFilePath); EXPECT_TRUE(replaceFileMoved.exists() && replaceFileMoved.isFile()); } +#if AZ_TRAIT_DISABLE_FAILED_PROJECT_MANAGER_TESTS + TEST_F(ProjectManagerUtilsTests, DISABLED_CopyProject_DoesntCopyBuild) +#else + TEST_F(ProjectManagerUtilsTests, CopyProject_DoesntCopyBuild) +#endif // !AZ_TRAIT_DISABLE_FAILED_PROJECT_MANAGER_TESTS + { + EXPECT_TRUE(CopyProject( + QDir::currentPath() + QDir::separator() + m_projectAPath, + QDir::currentPath() + QDir::separator() + m_projectBPath, + nullptr, true)); + + QFileInfo origFile(m_projectAOrigFilePath); + EXPECT_TRUE(origFile.exists()); + + QFileInfo origFileMoved(m_projectBOrigFilePath); + EXPECT_TRUE(origFileMoved.exists() && origFileMoved.isFile()); + + QDir buildDir(m_projectBBuildPath); + EXPECT_FALSE(buildDir.exists()); + } + #if AZ_TRAIT_DISABLE_FAILED_PROJECT_MANAGER_TESTS TEST_F(ProjectManagerUtilsTests, DISABLED_ReplaceFile_Succeeds) #else TEST_F(ProjectManagerUtilsTests, ReplaceFile_Succeeds) #endif // !AZ_TRAIT_DISABLE_FAILED_PROJECT_MANAGER_TESTS { - EXPECT_TRUE(ReplaceFile("ProjectA/origFile.txt", "ProjectA/replaceFile.txt", nullptr, false)); + EXPECT_TRUE(ReplaceFile(m_projectAOrigFilePath, m_projectAReplaceFilePath, nullptr, false)); - QFile origFile("ProjectA/origFile.txt"); - if (origFile.open(QIODevice::ReadOnly)) + QFile origFile(m_projectAOrigFilePath); + EXPECT_TRUE(origFile.open(QIODevice::ReadOnly)); { QTextStream stream(&origFile); QString line = stream.readLine(); @@ -106,10 +212,6 @@ namespace O3DE::ProjectManager origFile.close(); } - else - { - FAIL(); - } } } // namespace ProjectUtils } // namespace O3DE::ProjectManager From 6b747cf1e06066d29eca2dc26619f576d89e4660 Mon Sep 17 00:00:00 2001 From: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com> Date: Fri, 16 Jul 2021 11:20:43 -0500 Subject: [PATCH 19/33] Added missing EMotionFX.Editor target as a runtime dependency of the EMotionFXAtom.Editor target to fix launching of the Editor when the EMotionFX gem isn't explictly enabled. (#2228) Filled the dependencies of the AtomContent.Builders gem with the AtomConent_ReferenceMaterial and AtomContent_Sponza gem Signed-off-by: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com> --- Gems/AtomContent/CMakeLists.txt | 4 ++++ Gems/AtomLyIntegration/EMotionFXAtom/Code/CMakeLists.txt | 2 ++ 2 files changed, 6 insertions(+) diff --git a/Gems/AtomContent/CMakeLists.txt b/Gems/AtomContent/CMakeLists.txt index f16436c159..bf3ce03fee 100644 --- a/Gems/AtomContent/CMakeLists.txt +++ b/Gems/AtomContent/CMakeLists.txt @@ -6,3 +6,7 @@ # add_subdirectory(ReferenceMaterials) add_subdirectory(Sponza) + +if(PAL_TRAIT_BUILD_HOST_TOOLS) + ly_create_alias(NAME AtomContent.Builders NAMESPACE Gem TARGETS Gem::AtomContent_ReferenceMaterials.Builders Gem::AtomContent_Sponza.Builders) +endif() diff --git a/Gems/AtomLyIntegration/EMotionFXAtom/Code/CMakeLists.txt b/Gems/AtomLyIntegration/EMotionFXAtom/Code/CMakeLists.txt index d42a283eea..3b936f7644 100644 --- a/Gems/AtomLyIntegration/EMotionFXAtom/Code/CMakeLists.txt +++ b/Gems/AtomLyIntegration/EMotionFXAtom/Code/CMakeLists.txt @@ -57,5 +57,7 @@ if(PAL_TRAIT_BUILD_HOST_TOOLS) PRIVATE AZ::AzCore Gem::EMotionFX_Atom.Static + RUNTIME_DEPENDENCIES + Gem::EMotionFX.Editor ) endif() From 67c37762a36904c5c5ba1f55fa7418e48a3ddba5 Mon Sep 17 00:00:00 2001 From: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com> Date: Fri, 16 Jul 2021 11:22:21 -0500 Subject: [PATCH 20/33] Fixed iteration of the gem_assets_paths variable in the (#2222) Install_common.cmake to make sure the the Gem assets are copied to the install layout Signed-off-by: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com> --- cmake/Platform/Common/Install_common.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/Platform/Common/Install_common.cmake b/cmake/Platform/Common/Install_common.cmake index 5b476667e2..cb02e71cc8 100644 --- a/cmake/Platform/Common/Install_common.cmake +++ b/cmake/Platform/Common/Install_common.cmake @@ -565,7 +565,7 @@ function(ly_setup_others) endforeach() # gem directories and files to install - get_property(gems_assets_dir_path DIRECTORY ${gem_candidate_dir} PROPERTY gems_assets_paths) + get_property(gems_assets_paths DIRECTORY ${gem_candidate_dir} PROPERTY gems_assets_paths) foreach(gem_absolute_path IN LISTS gems_assets_paths) if(is_gem_subdirectory_of_engine) cmake_path(RELATIVE_PATH gem_absolute_path BASE_DIRECTORY ${LY_ROOT_FOLDER} OUTPUT_VARIABLE gem_install_dest_dir) From 448469025b3ce69049a22789be6fcf34f5a3526a Mon Sep 17 00:00:00 2001 From: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com> Date: Fri, 16 Jul 2021 15:13:47 -0500 Subject: [PATCH 21/33] Updated the AZ Path Code to perform case-insensitive hashing of path (#2226) segments when using the Windows Path Separator Signed-off-by: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com> --- Code/Framework/AzCore/AzCore/IO/Path/Path.h | 4 +-- Code/Framework/AzCore/AzCore/IO/Path/Path.inl | 27 +++++++++++++---- .../AzCore/Tests/IO/Path/PathTests.cpp | 30 +++++++++++++++++++ 3 files changed, 53 insertions(+), 8 deletions(-) diff --git a/Code/Framework/AzCore/AzCore/IO/Path/Path.h b/Code/Framework/AzCore/AzCore/IO/Path/Path.h index 3a0ab4b59a..4e86e5ecce 100644 --- a/Code/Framework/AzCore/AzCore/IO/Path/Path.h +++ b/Code/Framework/AzCore/AzCore/IO/Path/Path.h @@ -277,7 +277,7 @@ namespace AZ::IO //! then their hash values are also equal //! For example : path "a//b" equals "a/b", the //! hash value of "a//b" would also equal the hash value of "a/b" - constexpr size_t hash_value(const PathView& pathToHash) noexcept; + size_t hash_value(const PathView& pathToHash) noexcept; // path.comparison constexpr bool operator==(const PathView& lhs, const PathView& rhs) noexcept; @@ -622,7 +622,7 @@ namespace AZ::IO //! For example : path "a//b" equals "a/b", the //! hash value of "a//b" would also equal the hash value of "a/b" template - constexpr size_t hash_value(const BasicPath& pathToHash); + size_t hash_value(const BasicPath& pathToHash); // path.append template diff --git a/Code/Framework/AzCore/AzCore/IO/Path/Path.inl b/Code/Framework/AzCore/AzCore/IO/Path/Path.inl index f899522916..b2e5051446 100644 --- a/Code/Framework/AzCore/AzCore/IO/Path/Path.inl +++ b/Code/Framework/AzCore/AzCore/IO/Path/Path.inl @@ -1951,7 +1951,7 @@ namespace AZ::IO } template - constexpr size_t hash_value(const BasicPath& pathToHash) + inline size_t hash_value(const BasicPath& pathToHash) { return AZStd::hash>{}(pathToHash); } @@ -2082,13 +2082,28 @@ namespace AZStd template <> struct hash { - constexpr size_t operator()(const AZ::IO::PathView& pathToHash) noexcept + /// Path is using FNV-1a algorithm 64 bit version. + static size_t hash_path(AZStd::string_view pathSegment, const char pathSeparator) + { + size_t hash = 14695981039346656037ULL; + constexpr size_t fnvPrime = 1099511628211ULL; + + for (const char first : pathSegment) + { + hash ^= static_cast((pathSeparator == AZ::IO::PosixPathSeparator) + ? first : tolower(first)); + hash *= fnvPrime; + } + return hash; + } + + size_t operator()(const AZ::IO::PathView& pathToHash) noexcept { auto pathParser = AZ::IO::parser::PathParser::CreateBegin(pathToHash.Native(), pathToHash.m_preferred_separator); size_t hash_value = 0; while (pathParser) { - AZStd::hash_combine(hash_value, AZStd::hash{}(*pathParser)); + AZStd::hash_combine(hash_value, hash_path(*pathParser, pathToHash.m_preferred_separator)); ++pathParser; } return hash_value; @@ -2097,7 +2112,7 @@ namespace AZStd template struct hash> { - constexpr size_t operator()(const AZ::IO::BasicPath& pathToHash) noexcept + const size_t operator()(const AZ::IO::BasicPath& pathToHash) noexcept { return AZStd::hash{}(pathToHash); } @@ -2108,11 +2123,11 @@ namespace AZStd template struct hash; } -// Explicit instantations of our support Path classes +// Explicit instantiations of our support Path classes namespace AZ::IO { // PathView hash - constexpr size_t hash_value(const PathView& pathToHash) noexcept + inline size_t hash_value(const PathView& pathToHash) noexcept { return AZStd::hash{}(pathToHash); } diff --git a/Code/Framework/AzCore/Tests/IO/Path/PathTests.cpp b/Code/Framework/AzCore/Tests/IO/Path/PathTests.cpp index 84136e024c..a81bbe1f44 100644 --- a/Code/Framework/AzCore/Tests/IO/Path/PathTests.cpp +++ b/Code/Framework/AzCore/Tests/IO/Path/PathTests.cpp @@ -182,6 +182,36 @@ namespace UnitTest AZStd::tuple(R"(foO/Bar)", "foo/bar") )); + using PathHashParamFixture = PathParamFixture; + TEST_P(PathHashParamFixture, HashOperator_HashesCaseInsensitiveForWindowsPaths) + { + AZ::IO::Path path1{ AZStd::get<0>(GetParam()), AZ::IO::WindowsPathSeparator }; + AZ::IO::Path path2{ AZStd::get<1>(GetParam()), AZ::IO::WindowsPathSeparator }; + size_t path1Hash = AZStd::hash{}(path1); + size_t path2Hash = AZStd::hash{}(path2); + EXPECT_EQ(path1Hash, path2Hash) << AZStd::string::format(R"(path1 "%s" should hash to path2 "%s"\n)", + path1.c_str(), path2.c_str()).c_str(); + } + + TEST_P(PathHashParamFixture, HashOperator_HashesCaseSensitiveForPosixPaths) + { + AZ::IO::Path path1{ AZStd::get<0>(GetParam()), AZ::IO::PosixPathSeparator }; + AZ::IO::Path path2{ AZStd::get<1>(GetParam()), AZ::IO::PosixPathSeparator }; + size_t path1Hash = AZStd::hash{}(path1); + size_t path2Hash = AZStd::hash{}(path2); + EXPECT_NE(path1Hash, path2Hash) << AZStd::string::format(R"(path1 "%s" should NOT hash to path2 "%s"\n)", + path1.c_str(), path2.c_str()).c_str(); + } + + INSTANTIATE_TEST_CASE_P( + HashPaths, + PathHashParamFixture, + ::testing::Values( + AZStd::tuple("C:/test/foo", R"(c:\test/foo)"), + AZStd::tuple(R"(D:\test/bar/baz//foo)", "d:/test/bar/baz\\\\\\foo"), + AZStd::tuple(R"(foO/Bar)", "foo/bar") + )); + class PathSingleParamFixture : public ScopedAllocatorSetupFixture , public ::testing::WithParamInterface> From ae60bd0167483bad342bcaae9fa2638ee5df91d9 Mon Sep 17 00:00:00 2001 From: AMZN-ScottR <24445312+AMZN-ScottR@users.noreply.github.com> Date: Fri, 16 Jul 2021 14:33:39 -0700 Subject: [PATCH 22/33] [installer/2106-font] revert back to default install bootstrapper font Signed-off-by: AMZN-ScottR <24445312+AMZN-ScottR@users.noreply.github.com> --- .../Windows/Packaging/BootstrapperTheme.xml.in | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/Platform/Windows/Packaging/BootstrapperTheme.xml.in b/cmake/Platform/Windows/Packaging/BootstrapperTheme.xml.in index 8d8416539e..c670ef3d9a 100644 --- a/cmake/Platform/Windows/Packaging/BootstrapperTheme.xml.in +++ b/cmake/Platform/Windows/Packaging/BootstrapperTheme.xml.in @@ -4,15 +4,15 @@ #(loc.WindowTitle) - Open Sans + Segoe UI - Open Sans + Segoe UI - Open Sans + Segoe UI - Open Sans + Segoe UI - Open Sans + Segoe UI From 679ecf480197d234cc86fd123c5535a33c25d474 Mon Sep 17 00:00:00 2001 From: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com> Date: Fri, 16 Jul 2021 17:40:27 -0500 Subject: [PATCH 23/33] External Gem and Projects now appear in Editor AssetBrowser (#2227) * External Gem and Projects now appear in Editor AssetBrowser Optimized logic populating the FolderAssetBrowserEntries in the Editor AssetBrowser. Added an "[External]" tag to scan folders which reside outside of the Engine Root Signed-off-by: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com> * Fixed conversion of fixed_string to string when normalizing an added AssetBrowser file path. Signed-off-by: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com> * RootAssetBrowserEntry.cpp Linux conditional expression fix Because an AZ::IO::PathView is convertible to an AZ::IO::FixedMaxPath and vice-versa, the conversion of the second part of the ternary expression absolutePathView needs to explicitly convert to an AZ::IO::FixedMaxPath Signed-off-by: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com> --- .../Entries/AssetBrowserEntry.cpp | 4 +- .../AssetBrowser/Entries/AssetBrowserEntry.h | 5 +- .../Entries/FolderAssetBrowserEntry.cpp | 4 +- .../Entries/RootAssetBrowserEntry.cpp | 193 ++++++++---------- .../Entries/RootAssetBrowserEntry.h | 12 +- .../Entries/SourceAssetBrowserEntry.cpp | 4 +- 6 files changed, 101 insertions(+), 121 deletions(-) diff --git a/Code/Framework/AzToolsFramework/AzToolsFramework/AssetBrowser/Entries/AssetBrowserEntry.cpp b/Code/Framework/AzToolsFramework/AzToolsFramework/AssetBrowser/Entries/AssetBrowserEntry.cpp index 17a20a9d19..7eae7621d2 100644 --- a/Code/Framework/AzToolsFramework/AzToolsFramework/AssetBrowser/Entries/AssetBrowserEntry.cpp +++ b/Code/Framework/AzToolsFramework/AzToolsFramework/AssetBrowser/Entries/AssetBrowserEntry.cpp @@ -206,12 +206,12 @@ namespace AzToolsFramework const AZStd::string& AssetBrowserEntry::GetRelativePath() const { - return m_relativePath; + return m_relativePath.Native(); } const AZStd::string& AssetBrowserEntry::GetFullPath() const { - return m_fullPath; + return m_fullPath.Native(); } const AssetBrowserEntry* AssetBrowserEntry::GetChild(int index) const diff --git a/Code/Framework/AzToolsFramework/AzToolsFramework/AssetBrowser/Entries/AssetBrowserEntry.h b/Code/Framework/AzToolsFramework/AzToolsFramework/AssetBrowser/Entries/AssetBrowserEntry.h index e4a7b6b4cb..24f037a623 100644 --- a/Code/Framework/AzToolsFramework/AzToolsFramework/AssetBrowser/Entries/AssetBrowserEntry.h +++ b/Code/Framework/AzToolsFramework/AzToolsFramework/AssetBrowser/Entries/AssetBrowserEntry.h @@ -9,6 +9,7 @@ #if !defined(Q_MOC_RUN) #include #include +#include #include #include @@ -130,8 +131,8 @@ namespace AzToolsFramework protected: AZStd::string m_name; QString m_displayName; - AZStd::string m_relativePath; - AZStd::string m_fullPath; + AZ::IO::Path m_relativePath; + AZ::IO::Path m_fullPath; AZStd::vector m_children; AssetBrowserEntry* m_parentAssetEntry = nullptr; diff --git a/Code/Framework/AzToolsFramework/AzToolsFramework/AssetBrowser/Entries/FolderAssetBrowserEntry.cpp b/Code/Framework/AzToolsFramework/AzToolsFramework/AssetBrowser/Entries/FolderAssetBrowserEntry.cpp index 3c5afab0d0..febd406f23 100644 --- a/Code/Framework/AzToolsFramework/AzToolsFramework/AssetBrowser/Entries/FolderAssetBrowserEntry.cpp +++ b/Code/Framework/AzToolsFramework/AzToolsFramework/AssetBrowser/Entries/FolderAssetBrowserEntry.cpp @@ -37,8 +37,8 @@ namespace AzToolsFramework void FolderAssetBrowserEntry::UpdateChildPaths(AssetBrowserEntry* child) const { - child->m_relativePath = m_relativePath + AZ_CORRECT_DATABASE_SEPARATOR + child->m_name; - child->m_fullPath = m_fullPath + AZ_CORRECT_DATABASE_SEPARATOR + child->m_name; + child->m_relativePath = m_relativePath / child->m_name; + child->m_fullPath = m_fullPath / child->m_name; AssetBrowserEntry::UpdateChildPaths(child); } diff --git a/Code/Framework/AzToolsFramework/AzToolsFramework/AssetBrowser/Entries/RootAssetBrowserEntry.cpp b/Code/Framework/AzToolsFramework/AzToolsFramework/AssetBrowser/Entries/RootAssetBrowserEntry.cpp index 090ad0d6ca..0e132d2581 100644 --- a/Code/Framework/AzToolsFramework/AzToolsFramework/AssetBrowser/Entries/RootAssetBrowserEntry.cpp +++ b/Code/Framework/AzToolsFramework/AzToolsFramework/AssetBrowser/Entries/RootAssetBrowserEntry.cpp @@ -10,6 +10,7 @@ #include #include +#include #include #include @@ -24,8 +25,6 @@ namespace AzToolsFramework { namespace AssetBrowser { - const char* GEMS_FOLDER_NAME = "Gems"; - RootAssetBrowserEntry::RootAssetBrowserEntry() : AssetBrowserEntry() { @@ -53,13 +52,7 @@ namespace AzToolsFramework EntryCache::GetInstance()->Clear(); m_enginePath = enginePath; - - // there is no "Gems" scan folder registered in db, create one manually - auto gemFolder = aznew FolderAssetBrowserEntry(); - gemFolder->m_name = m_enginePath + AZ_CORRECT_DATABASE_SEPARATOR + GEMS_FOLDER_NAME; - gemFolder->m_displayName = GEMS_FOLDER_NAME; - gemFolder->m_isGemsFolder = true; - AddChild(gemFolder); + m_fullPath = enginePath; } bool RootAssetBrowserEntry::IsInitialUpdate() const @@ -80,8 +73,17 @@ namespace AzToolsFramework if (AZ::IO::FileIOBase::GetInstance()->IsDirectory(scanFolderDatabaseEntry.m_scanFolder.c_str())) { - const auto scanFolder = CreateFolders(scanFolderDatabaseEntry.m_scanFolder.c_str(), this); - scanFolder->m_displayName = QString::fromUtf8(scanFolderDatabaseEntry.m_displayName.c_str()); + const auto scanFolder = CreateFolders(scanFolderDatabaseEntry.m_scanFolder, this); + // Append an "[External]" to the display if the Scan Folder is NOT relative to the Engine Root path + if (!AZ::IO::PathView(scanFolderDatabaseEntry.m_scanFolder).IsRelativeTo(m_enginePath)) + { + scanFolder->m_displayName += " [External]"; + } + else + { + scanFolder->m_displayName = QString::fromUtf8(scanFolderDatabaseEntry.m_displayName.c_str()); + } + EntryCache::GetInstance()->m_scanFolderIdMap[scanFolderDatabaseEntry.m_scanFolderID] = scanFolder; } } @@ -121,38 +123,34 @@ namespace AzToolsFramework return; } - const char* filePath = fileDatabaseEntry.m_fileName.c_str(); + AZ::IO::FixedMaxPath absoluteFilePath = AZ::IO::FixedMaxPath(AZStd::string_view{ scanFolder->GetFullPath() }) + / fileDatabaseEntry.m_fileName.c_str(); AssetBrowserEntry* file; // file can be either folder or actual file if (fileDatabaseEntry.m_isFolder) { - file = CreateFolders(filePath, scanFolder); + file = CreateFolders(absoluteFilePath.Native(), scanFolder); } else { - AZStd::string sourcePath; - AZStd::string sourceName; - AZStd::string sourceExtension; - StringFunc::Path::Split(filePath, nullptr, &sourcePath, &sourceName, &sourceExtension); // if missing create folders leading to file's location and get immediate parent // (we don't need to have fileIds for any folders created yet, they will be added later) - auto parent = CreateFolders(sourcePath.c_str(), scanFolder); + auto parent = CreateFolders(absoluteFilePath.ParentPath().Native(), scanFolder); // for simplicity in AB, files are represented as sources, but they are missing SourceDatabaseEntry-specific information such as SourceUuid auto source = aznew SourceAssetBrowserEntry(); - source->m_name = (sourceName + sourceExtension).c_str(); + source->m_name = absoluteFilePath.Filename().Native(); source->m_fileId = fileDatabaseEntry.m_fileID; source->m_displayName = QString::fromUtf8(source->m_name.c_str()); source->m_scanFolderId = fileDatabaseEntry.m_scanFolderPK; - source->m_extension = sourceExtension.c_str(); + source->m_extension = absoluteFilePath.Extension().Native(); parent->AddChild(source); file = source; } EntryCache::GetInstance()->m_fileIdMap[fileDatabaseEntry.m_fileID] = file; - AZStd::string fullPath = file->m_fullPath; - AzFramework::StringFunc::Path::Normalize(fullPath); - EntryCache::GetInstance()->m_absolutePathToFileId[fullPath] = fileDatabaseEntry.m_fileID; + AZStd::string filePath = AZ::IO::PathView(file->m_fullPath).LexicallyNormal().String(); + EntryCache::GetInstance()->m_absolutePathToFileId[filePath] = fileDatabaseEntry.m_fileID; } bool RootAssetBrowserEntry::RemoveFile(const AZ::s64& fileId) const @@ -305,116 +303,95 @@ namespace AzToolsFramework } } - FolderAssetBrowserEntry* RootAssetBrowserEntry::CreateFolder(const char* folderName, AssetBrowserEntry* parent) + AssetBrowserEntry* RootAssetBrowserEntry::GetNearestAncestor(AZ::IO::PathView absolutePathView, AssetBrowserEntry* parent, + AZStd::unordered_set& visitedSet) { - auto it = AZStd::find_if(parent->m_children.begin(), parent->m_children.end(), [folderName](AssetBrowserEntry* entry) - { - if (!azrtti_istypeof(entry)) - { - return false; - } - return AzFramework::StringFunc::Equal(entry->m_name.c_str(), folderName); - }); - if (it != parent->m_children.end()) + auto IsPathRelativeToEntry = [absolutePathView](AssetBrowserEntry* assetBrowserEntry) { - return azrtti_cast(*it); + auto& childPath = assetBrowserEntry->m_fullPath; + return absolutePathView.IsRelativeTo(AZ::IO::PathView(childPath)); + }; + + if (visitedSet.contains(parent)) + { + return {}; } - const auto folder = aznew FolderAssetBrowserEntry(); - folder->m_name = folderName; - folder->m_displayName = folderName; - parent->AddChild(folder); - return folder; - } - AssetBrowserEntry* RootAssetBrowserEntry::CreateFolders(const char* relativePath, AssetBrowserEntry* parent) - { - auto children(parent->m_children); - int n = 0; + visitedSet.insert(parent); - // check if folder with the same name already exists - // step through every character in relativePath and compare to each child's relative path of suggested parent - // if a character @n in child's rel path mismatches character at n in relativePath, remove that child from further search - while (!children.empty() && relativePath[n]) + AssetBrowserEntry* nearestAncestor{}; + for (AssetBrowserEntry* childBrowserEntry : parent->m_children) { - AZStd::vector toRemove; - for (auto child : children) + if (IsPathRelativeToEntry(childBrowserEntry)) { - auto& childPath = azrtti_istypeof(parent) ? child->m_fullPath : child->m_relativePath; - - // child's path mismatched, remove it from search candidates - if (childPath.length() == n || childPath[n] != relativePath[n]) + // Walk the AssetBrowserEntry Tree looking for a nearer ancestor to the absolute path + // If one is not found in the recursive call to GetNearestAncestor, then the childBrowserEntry + // is the current best candidate + AssetBrowserEntry* candidateAncestor = GetNearestAncestor(absolutePathView, childBrowserEntry, visitedSet); + candidateAncestor = candidateAncestor != nullptr ? candidateAncestor : childBrowserEntry; + AZ::IO::PathView candidatePathView(candidateAncestor->m_fullPath); + // If the candidate is relative to the current nearest ancestor, then it is even nearer to the path + if (!nearestAncestor || candidatePathView.IsRelativeTo(nearestAncestor->m_fullPath)) { - toRemove.push_back(child); - - // it is possible that child may be a closer parent, substitute it as new potential parent - // e.g. child->m_relativePath = 'Gems', relativePath = 'Gems/Assets', old parent = root, new parent = Gems - if (childPath.length() == n && relativePath[n] == AZ_CORRECT_DATABASE_SEPARATOR) + nearestAncestor = candidateAncestor; + // If the full path compares equal to the AssetBrowserEntry path, then no need to proceed any further + if (AZ::IO::PathView(nearestAncestor->m_fullPath) == absolutePathView) { - parent = child; - relativePath += n; // advance relative path n characters since the parent has changed - n = 0; // Once the relative path pointer is advanced, reset n + break; } } } - for (auto entry : toRemove) - { - children.erase(AZStd::remove(children.begin(), children.end(), entry), children.end()); - } - n++; } - // filter out the remaining children that don't end with '/' or '\0' - // for example if folderName = "foo", while children may still remain with names like "foo123", - // which is not the same folder - AZStd::vector toRemove; - for (auto child : children) + return nearestAncestor; + } + + FolderAssetBrowserEntry* RootAssetBrowserEntry::CreateFolder(AZStd::string_view folderName, AssetBrowserEntry* parent) + { + auto it = AZStd::find_if(parent->m_children.begin(), parent->m_children.end(), [folderName](AssetBrowserEntry* entry) { - auto& childPath = azrtti_istypeof(parent) ? child->m_fullPath : child->m_relativePath; - // check if there are non-null characters remaining @n - if (childPath.length() > n) + if (!azrtti_istypeof(entry)) { - toRemove.push_back(child); + return false; } - } - for (auto entry : toRemove) + return AZ::IO::PathView(entry->m_name) == AZ::IO::PathView(folderName); + }); + if (it != parent->m_children.end()) { - children.erase(AZStd::remove(children.begin(), children.end(), entry), children.end()); + return azrtti_cast(*it); } + const auto folder = aznew FolderAssetBrowserEntry(); + folder->m_name = folderName; + folder->m_displayName = QString::fromUtf8(folderName.data(), aznumeric_caster(folderName.size())); + parent->AddChild(folder); + return folder; + } + + AssetBrowserEntry* RootAssetBrowserEntry::CreateFolders(AZStd::string_view absolutePath, AssetBrowserEntry* parent) + { + AZ::IO::PathView absolutePathView(absolutePath); + // Find the nearest ancestor path to the absolutePath + AZStd::unordered_set visitedSet; - // at least one child remains, this means the folder with this name already exists, return it - if (!children.empty()) + if (AssetBrowserEntry* nearestAncestor = GetNearestAncestor(absolutePathView, parent, visitedSet); + nearestAncestor != nullptr) { - parent = children.front(); + parent = nearestAncestor; } - // if it's a scanfolder, then do not create folders leading to it - // e.g. instead of 'C:\dev\SampleProject' just create 'SampleProject' - else if (parent->GetEntryType() == AssetEntryType::Root) + + // If the nearest ancestor is the absolutePath, then it is already crated + if (absolutePathView == AZ::IO::PathView(parent->GetFullPath())) { - AZStd::string folderName; - AzFramework::StringFunc::Path::Split(relativePath, nullptr, nullptr, &folderName); - parent = CreateFolder(folderName.c_str(), parent); - parent->m_fullPath = relativePath; + return parent; } - // otherwise create all missing folders - else + + // create all missing folders + auto proximateToPath = absolutePathView.IsRelativeTo(parent->m_fullPath) + ? absolutePathView.LexicallyProximate(parent->m_fullPath) + : AZ::IO::FixedMaxPath(absolutePathView); + for (AZ::IO::FixedMaxPath scanFolderSegment : proximateToPath) { - n = 0; - AZStd::string folderName(strlen(relativePath) + 1, '\0'); - // iterate through relativePath until the first '/' - while (relativePath[n] && relativePath[n] != AZ_CORRECT_DATABASE_SEPARATOR) - { - folderName[n] = relativePath[n]; - n++; - } - if (n > 0) - { - parent = CreateFolder(folderName.c_str(), parent); - } - // n+1 also skips the '/' character - if (relativePath[n] && relativePath[n + 1]) - { - parent = CreateFolders(relativePath + n + 1, parent); - } + parent = CreateFolder(scanFolderSegment.c_str(), parent); } return parent; } diff --git a/Code/Framework/AzToolsFramework/AzToolsFramework/AssetBrowser/Entries/RootAssetBrowserEntry.h b/Code/Framework/AzToolsFramework/AzToolsFramework/AssetBrowser/Entries/RootAssetBrowserEntry.h index 883f9dec01..62fbc7a39d 100644 --- a/Code/Framework/AzToolsFramework/AzToolsFramework/AssetBrowser/Entries/RootAssetBrowserEntry.h +++ b/Code/Framework/AzToolsFramework/AzToolsFramework/AssetBrowser/Entries/RootAssetBrowserEntry.h @@ -8,6 +8,7 @@ #include #include +#include #include #include @@ -77,12 +78,15 @@ namespace AzToolsFramework private: AZ_DISABLE_COPY_MOVE(RootAssetBrowserEntry); - AZStd::string m_enginePath; + AZ::IO::Path m_enginePath; //! Create folder entry child - FolderAssetBrowserEntry* CreateFolder(const char* folderName, AssetBrowserEntry* parent); - //! Recursively create folder structure leading to relative path from parent - AssetBrowserEntry* CreateFolders(const char* relativePath, AssetBrowserEntry* parent); + FolderAssetBrowserEntry* CreateFolder(AZStd::string_view folderName, AssetBrowserEntry* parent); + //! Recursively create folder structure leading to path from parent + AssetBrowserEntry* CreateFolders(AZStd::string_view absolutePath, AssetBrowserEntry* parent); + // Retrieves the nearest ancestor AssetBrowserEntry from the absolutePath + static AssetBrowserEntry* GetNearestAncestor(AZ::IO::PathView absolutePath, AssetBrowserEntry* parent, + AZStd::unordered_set& visitedSet); bool m_isInitialUpdate = false; }; diff --git a/Code/Framework/AzToolsFramework/AzToolsFramework/AssetBrowser/Entries/SourceAssetBrowserEntry.cpp b/Code/Framework/AzToolsFramework/AzToolsFramework/AssetBrowser/Entries/SourceAssetBrowserEntry.cpp index e45c234c92..1eea909f7c 100644 --- a/Code/Framework/AzToolsFramework/AzToolsFramework/AssetBrowser/Entries/SourceAssetBrowserEntry.cpp +++ b/Code/Framework/AzToolsFramework/AzToolsFramework/AssetBrowser/Entries/SourceAssetBrowserEntry.cpp @@ -26,9 +26,7 @@ namespace AzToolsFramework if (EntryCache* cache = EntryCache::GetInstance()) { cache->m_fileIdMap.erase(m_fileId); - AZStd::string fullPath = m_fullPath; - AzFramework::StringFunc::Path::Normalize(fullPath); - cache->m_absolutePathToFileId.erase(fullPath); + cache->m_absolutePathToFileId.erase(m_fullPath.LexicallyNormal().Native()); if (m_sourceId != -1) { From 173126f5a5f1b3a1efc65e5a902c48d63b2a5ad9 Mon Sep 17 00:00:00 2001 From: Brian Herrera Date: Mon, 19 Jul 2021 16:09:47 -0700 Subject: [PATCH 24/33] Add script for license scanner (#2267) * Add script for license scanner This script will scan the source tree for license files and generate a file with the contents of all the licenses. Signed-off-by: brianherrera --- scripts/license_scanner/license_scanner.py | 129 ++++++++++++++++++++ scripts/license_scanner/scanner_config.json | 12 ++ 2 files changed, 141 insertions(+) create mode 100644 scripts/license_scanner/license_scanner.py create mode 100644 scripts/license_scanner/scanner_config.json diff --git a/scripts/license_scanner/license_scanner.py b/scripts/license_scanner/license_scanner.py new file mode 100644 index 0000000000..c0e3c1f1ba --- /dev/null +++ b/scripts/license_scanner/license_scanner.py @@ -0,0 +1,129 @@ +# +# Copyright (c) Contributors to the Open 3D Engine Project. For complete copyright and license terms please see the LICENSE at the root of this distribution. +# +# SPDX-License-Identifier: Apache-2.0 OR MIT +# +# + +import argparse +import fnmatch +import json +import os +import pathlib +import re +import sys + + +class LicenseScanner: + """Class to contain license scanner. + + Scans source tree for license files using provided filename patterns and generates a file + with the contents of all the licenses. + + :param config_file: Config file with license patterns and scanner settings + """ + + DEFAULT_CONFIG_FILE = 'scanner_config.json' + + def __init__(self, config_file=None): + self.config_file = config_file + self.config_data = self._load_config() + self.license_regex = self._load_license_regex() + + def _load_config(self): + """Load config from the provided file. Sets default file if one is not provided.""" + if self.config_file is None: + script_directory = os.path.dirname(os.path.abspath(__file__)) # Default file expected in same dir as script + self.config_file = os.path.join(script_directory, self.DEFAULT_CONFIG_FILE) + + try: + with open(self.config_file) as f: + return json.load(f) + except FileNotFoundError: + print('Config file cannot be found') + raise + + def _load_license_regex(self): + """Returns regex object with case-insensitive matching from the list of filename patterns.""" + regex_patterns = [] + for pattern in self.config_data['license_patterns']: + regex_patterns.append(fnmatch.translate(pattern)) + return re.compile('|'.join(regex_patterns), re.IGNORECASE) + + def scan(self, path=os.curdir): + """Scan directory tree for filenames matching license_regex. + + :param path: Path of the directory to run scanner + :return: Package paths and their corresponding license file contents + :rtype: dict + """ + licenses = 0 + license_files = {} + + for dirpath, dirnames, filenames in os.walk(path): + for file in filenames: + if self.license_regex.match(file): + license_file_content = self._get_license_file_contents(os.path.join(dirpath, file)) + rel_dirpath = os.path.relpath(dirpath, path) # Limit path inside scanned directory + license_files[rel_dirpath] = license_file_content + licenses += 1 + print(f'License file: {os.path.join(dirpath, file)}') + + # Remove directories that should not be scanned + for dir in self.config_data['excluded_directories']: + if dir in dirnames: + dirnames.remove(dir) + print(f'{licenses} license files found.') + return license_files + + def _get_license_file_contents(self, filepath): + try: + with open(filepath, encoding='utf8') as f: + return f.read() + except UnicodeDecodeError: + print(f'Unable to read license file: {filepath}') + pass + + def create_license_file(self, licenses, filepath='NOTICES.txt'): + """Creates file with all the provided license file contents. + + :param licenses: Dict with package paths and their corresponding license file contents + :param filepath: Path to write the file + """ + package_separator = '------------------------------------' + with open(filepath, 'w', encoding='utf8') as f: + for directory, license in licenses.items(): + license_output = '\n\n'.join([ + f'{package_separator}', + f'Package path: {directory}', + 'License:', + f'{license}\n' + ]) + f.write(license_output) + return None + + +def parse_args(): + parser = argparse.ArgumentParser( + description='Script to run LicenseScanner and generate license file') + parser.add_argument('--config-file', '-c', type=pathlib.Path, help='Config file for LicenseScanner') + parser.add_argument('--license-file-path', '-l', type=pathlib.Path, help='Create license file in the provided path') + parser.add_argument('--scan-path', '-s', default=os.curdir, type=pathlib.Path, help='Path to scan') + return parser.parse_args() + + +def main(): + try: + args = parse_args() + ls = LicenseScanner(args.config_file) + licenses = ls.scan(args.scan_path) + + if args.license_file_path: + ls.create_license_file(licenses, args.license_file_path) + except FileNotFoundError as e: + print(f'Type: {type(e).__name__}, Error: {e}') + return 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/license_scanner/scanner_config.json b/scripts/license_scanner/scanner_config.json new file mode 100644 index 0000000000..b5863a7d31 --- /dev/null +++ b/scripts/license_scanner/scanner_config.json @@ -0,0 +1,12 @@ +{ + "excluded_directories": [ + ".git", + ".venv", + "build", + "license_scanner" + ], + "license_patterns": [ + "LICENSE*", + "COPYING*" + ] +} From afb57c9cf679592a209dbccdd5f38126e03d6b59 Mon Sep 17 00:00:00 2001 From: Scott Romero <24445312+AMZN-ScottR@users.noreply.github.com> Date: Mon, 19 Jul 2021 18:17:55 -0700 Subject: [PATCH 25/33] [stabliziation/2106] update 3rd party license file name to be pulled into the installer package (#2281) License file name was changed in #2267 Signed-off-by: AMZN-ScottR 24445312+AMZN-ScottR@users.noreply.github.com --- cmake/Packaging.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/Packaging.cmake b/cmake/Packaging.cmake index a07e1ec1d3..2240ff472c 100644 --- a/cmake/Packaging.cmake +++ b/cmake/Packaging.cmake @@ -128,7 +128,7 @@ install(FILES ${_cmake_package_dest} # the version string and git tags are intended to be synchronized so it should be safe to use that instead # of directly calling into git which could get messy in certain scenarios if(${CPACK_PACKAGE_VERSION} VERSION_GREATER "0.0.0.0") - set(_3rd_party_license_filename SPDX-Licenses.txt) + set(_3rd_party_license_filename NOTICES.txt) set(_3rd_party_license_url "https://raw.githubusercontent.com/o3de/3p-package-source/${CPACK_PACKAGE_VERSION}/${_3rd_party_license_filename}") set(_3rd_party_license_dest ${CPACK_BINARY_DIR}/${_3rd_party_license_filename}) From abce54b3cd0a574fff1a8612be838779ab597a73 Mon Sep 17 00:00:00 2001 From: Axel Nana Date: Tue, 20 Jul 2021 01:05:09 +0100 Subject: [PATCH 26/33] Add missing `*.hxx` pattern Signed-off-by: Axel Nana --- cmake/Platform/Common/Install_common.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/Platform/Common/Install_common.cmake b/cmake/Platform/Common/Install_common.cmake index 5b476667e2..0a84d3ceb2 100644 --- a/cmake/Platform/Common/Install_common.cmake +++ b/cmake/Platform/Common/Install_common.cmake @@ -64,6 +64,7 @@ function(ly_setup_target OUTPUT_CONFIGURED_TARGET ALIAS_TARGET_NAME absolute_tar PATTERN *.h PATTERN *.hpp PATTERN *.inl + PATTERN *.hxx ) endif() endforeach() From b58286d9b1022c3e3051c3225b7000935066fd4c Mon Sep 17 00:00:00 2001 From: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com> Date: Tue, 20 Jul 2021 20:40:04 -0500 Subject: [PATCH 27/33] Fixed RootAssetBrowserEntry setting of child AssetBrowserEntries The issue is due to RootAssetBrowserEntry::UpdateChildPaths not taking the RootAssetBrowserEntry fullpath into account when appending the child path entry Signed-off-by: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com> --- .../AssetBrowser/Entries/RootAssetBrowserEntry.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Code/Framework/AzToolsFramework/AzToolsFramework/AssetBrowser/Entries/RootAssetBrowserEntry.cpp b/Code/Framework/AzToolsFramework/AzToolsFramework/AssetBrowser/Entries/RootAssetBrowserEntry.cpp index 0e132d2581..eec9970ca3 100644 --- a/Code/Framework/AzToolsFramework/AzToolsFramework/AssetBrowser/Entries/RootAssetBrowserEntry.cpp +++ b/Code/Framework/AzToolsFramework/AzToolsFramework/AssetBrowser/Entries/RootAssetBrowserEntry.cpp @@ -399,7 +399,7 @@ namespace AzToolsFramework void RootAssetBrowserEntry::UpdateChildPaths(AssetBrowserEntry* child) const { child->m_relativePath = child->m_name; - child->m_fullPath = child->m_name; + child->m_fullPath = m_fullPath / child->m_name; AssetBrowserEntry::UpdateChildPaths(child); } From 3050a4db479fb2dacf59942f7b0b42d7d82fcdcd Mon Sep 17 00:00:00 2001 From: jckand-amzn Date: Wed, 21 Jul 2021 10:18:18 -0500 Subject: [PATCH 28/33] Removing obsolete Vegetation Debugger test Signed-off-by: jckand-amzn --- .../EditorScripts/Debugger_DebugCVarsWorks.py | 55 ------------------ .../largeworlds/dyn_veg/test_Debugger.py | 56 ------------------- 2 files changed, 111 deletions(-) delete mode 100755 AutomatedTesting/Gem/PythonTests/largeworlds/dyn_veg/EditorScripts/Debugger_DebugCVarsWorks.py delete mode 100755 AutomatedTesting/Gem/PythonTests/largeworlds/dyn_veg/test_Debugger.py diff --git a/AutomatedTesting/Gem/PythonTests/largeworlds/dyn_veg/EditorScripts/Debugger_DebugCVarsWorks.py b/AutomatedTesting/Gem/PythonTests/largeworlds/dyn_veg/EditorScripts/Debugger_DebugCVarsWorks.py deleted file mode 100755 index 40fd67d957..0000000000 --- a/AutomatedTesting/Gem/PythonTests/largeworlds/dyn_veg/EditorScripts/Debugger_DebugCVarsWorks.py +++ /dev/null @@ -1,55 +0,0 @@ -""" -Copyright (c) Contributors to the Open 3D Engine Project. For complete copyright and license terms please see the LICENSE at the root of this distribution. - -SPDX-License-Identifier: Apache-2.0 OR MIT -""" - -import os -import sys - -import azlmbr.legacy.general as general -import azlmbr.paths - -sys.path.append(os.path.join(azlmbr.paths.devroot, "AutomatedTesting", "Gem", "PythonTests")) -import editor_python_test_tools.hydra_editor_utils as hydra -from editor_python_test_tools.editor_test_helper import EditorTestHelper - - -class TestDebuggerDebugCVarsWorks(EditorTestHelper): - def __init__(self): - EditorTestHelper.__init__(self, log_prefix="Debugger_DebugCVarsWorks", args=["level"]) - - def run_test(self): - """ - Summary: - C2789148 Vegetation Debug CVars are enabled when the Debugger component is present - - Expected Result: - The following commands are available in the Editor only when the Vegetation Debugger Level component is present: - veg_debugDumpReport (Command) - veg_debugRefreshAllAreas (Command) - - :return: None - """ - - # Create empty level - self.test_success = self.create_level( - self.args["level"], - heightmap_resolution=1024, - heightmap_meters_per_pixel=1, - terrain_texture_resolution=4096, - use_terrain=False, - ) - - # Initially run the command in console without Debugger component - general.run_console("veg_debugDumpReport") - - # Add the Vegetation Debugger component to the Level Inspector - hydra.add_level_component("Vegetation Debugger") - - # Run a command again after adding the Vegetation debugger - general.run_console("veg_debugRefreshAllAreas") - - -test = TestDebuggerDebugCVarsWorks() -test.run() diff --git a/AutomatedTesting/Gem/PythonTests/largeworlds/dyn_veg/test_Debugger.py b/AutomatedTesting/Gem/PythonTests/largeworlds/dyn_veg/test_Debugger.py deleted file mode 100755 index ed979136e0..0000000000 --- a/AutomatedTesting/Gem/PythonTests/largeworlds/dyn_veg/test_Debugger.py +++ /dev/null @@ -1,56 +0,0 @@ -""" -Copyright (c) Contributors to the Open 3D Engine Project. For complete copyright and license terms please see the LICENSE at the root of this distribution. - -SPDX-License-Identifier: Apache-2.0 OR MIT -""" - -import os -import pytest -import logging - -# Bail on the test if ly_test_tools doesn't exist. -pytest.importorskip("ly_test_tools") -import ly_test_tools.environment.file_system as file_system -import editor_python_test_tools.hydra_test_utils as hydra - -logger = logging.getLogger(__name__) -test_directory = os.path.join(os.path.dirname(__file__), "EditorScripts") - - -@pytest.mark.parametrize("project", ["AutomatedTesting"]) -@pytest.mark.parametrize("level", ["tmp_level"]) -@pytest.mark.usefixtures("automatic_process_killer") -@pytest.mark.parametrize("launcher_platform", ['windows_editor']) -class TestDebugger(object): - @pytest.fixture(autouse=True) - def setup_teardown(self, request, workspace, project, level): - # Cleanup our temp level - file_system.delete([os.path.join(workspace.paths.engine_root(), project, "Levels", level)], True, True) - - def teardown(): - # Cleanup our temp level - file_system.delete([os.path.join(workspace.paths.engine_root(), project, "Levels", level)], True, True) - - request.addfinalizer(teardown) - - @pytest.mark.test_case_id("C2789148") - @pytest.mark.SUITE_periodic - @pytest.mark.dynveg_misc - def test_Debugger_DebugCVarsWork(self, request, editor, level, workspace, launcher_platform): - cfg_args = [level] - - expected_lines = [ - "Debugger_DebugCVarsWorks: test started", - "[Warning] Unknown command: veg_debugDumpReport", - "[CONSOLE] Executing console command 'veg_debugRefreshAllAreas'", - "Debugger_DebugCVarsWorks: result=SUCCESS", - ] - - hydra.launch_and_validate_results( - request, - test_directory, - editor, - "Debugger_DebugCVarsWorks.py", - expected_lines=expected_lines, - cfg_args=cfg_args - ) From 7ebdde0be01477ba036bad7f5a1e36ae50e2d879 Mon Sep 17 00:00:00 2001 From: jckand-amzn Date: Wed, 21 Jul 2021 10:25:48 -0500 Subject: [PATCH 29/33] Adding xfail mark to test_LandscapeCanvas_GraphClosed_OnEntityDelete Signed-off-by: jckand-amzn --- .../landscape_canvas/test_GeneralGraphFunctionality.py | 1 + 1 file changed, 1 insertion(+) diff --git a/AutomatedTesting/Gem/PythonTests/largeworlds/landscape_canvas/test_GeneralGraphFunctionality.py b/AutomatedTesting/Gem/PythonTests/largeworlds/landscape_canvas/test_GeneralGraphFunctionality.py index bbca71461c..e2aa222c3f 100755 --- a/AutomatedTesting/Gem/PythonTests/largeworlds/landscape_canvas/test_GeneralGraphFunctionality.py +++ b/AutomatedTesting/Gem/PythonTests/largeworlds/landscape_canvas/test_GeneralGraphFunctionality.py @@ -104,6 +104,7 @@ class TestGeneralGraphFunctionality(object): @pytest.mark.test_case_id("C17488412") @pytest.mark.SUITE_periodic + @pytest.mark.xfail(reason="https://github.com/o3de/o3de/issues/2201") def test_LandscapeCanvas_GraphClosed_OnEntityDelete(self, request, editor, level, launcher_platform): cfg_args = [level] From fc1ccb9e8574a1d997bbc791fb46a6a1325ee3a2 Mon Sep 17 00:00:00 2001 From: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> Date: Wed, 21 Jul 2021 17:17:09 -0700 Subject: [PATCH 30/33] Issues/2126 PhysX Gem can't be used as build dependency in engine SDK Part (#2337) * Applying GENEX_EVAL to 2 cases where the genex can produce another genex Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> * Missing case when the folder is an external one to the engine/project (e.g. external gems) Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> --- Code/LauncherUnified/launcher_generator.cmake | 4 ++-- cmake/LYWrappers.cmake | 15 ++++++++++----- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/Code/LauncherUnified/launcher_generator.cmake b/Code/LauncherUnified/launcher_generator.cmake index 628b5fffac..28fddad1eb 100644 --- a/Code/LauncherUnified/launcher_generator.cmake +++ b/Code/LauncherUnified/launcher_generator.cmake @@ -40,9 +40,9 @@ foreach(project_name project_path IN ZIP_LISTS LY_PROJECTS_TARGET_NAME LY_PROJEC add_custom_target(${project_name}.Assets COMMENT "Processing ${project_name} assets..." COMMAND "${CMAKE_COMMAND}" - -DLY_LOCK_FILE=$/project_assets.lock + -DLY_LOCK_FILE=$>/project_assets.lock -P ${LY_ROOT_FOLDER}/cmake/CommandExecution.cmake - EXEC_COMMAND $ + EXEC_COMMAND $> --zeroAnalysisMode --project-path=${project_real_path} --platforms=${LY_ASSET_DEPLOY_ASSET_TYPE} diff --git a/cmake/LYWrappers.cmake b/cmake/LYWrappers.cmake index 40bf6ef541..cfe6df5c81 100644 --- a/cmake/LYWrappers.cmake +++ b/cmake/LYWrappers.cmake @@ -391,10 +391,10 @@ function(ly_delayed_target_link_libraries) endif() if(item_type STREQUAL MODULE_LIBRARY) - target_include_directories(${target} ${visibility} $) - target_link_libraries(${target} ${visibility} $) - target_compile_definitions(${target} ${visibility} $) - target_compile_options(${target} ${visibility} $) + target_include_directories(${target} ${visibility} $>) + target_link_libraries(${target} ${visibility} $>) + target_compile_definitions(${target} ${visibility} $>) + target_compile_options(${target} ${visibility} $>) else() ly_parse_third_party_dependencies(${item}) target_link_libraries(${target} ${visibility} ${item}) @@ -659,7 +659,12 @@ function(ly_get_vs_folder_directory absolute_target_source_dir output_source_dir if(is_target_prefix_of_engine_root) cmake_path(RELATIVE_PATH absolute_target_source_dir BASE_DIRECTORY ${LY_ROOT_FOLDER} OUTPUT_VARIABLE relative_target_source_dir) else() - cmake_path(GET absolute_target_source_dir RELATIVE_PART relative_target_source_dir) + cmake_path(IS_PREFIX CMAKE_SOURCE_DIR ${absolute_target_source_dir} is_target_prefix_of_source_dir) + if(is_target_prefix_of_source_dir) + cmake_path(RELATIVE_PATH absolute_target_source_dir BASE_DIRECTORY ${CMAKE_SOURCE_DIR} OUTPUT_VARIABLE relative_target_source_dir) + else() + cmake_path(GET absolute_target_source_dir RELATIVE_PART relative_target_source_dir) + endif() endif() set(${output_source_dir} ${relative_target_source_dir} PARENT_SCOPE) From 26c9853ff9916b7216a5e01eef6b9f7be8bb4df1 Mon Sep 17 00:00:00 2001 From: moraaar Date: Tue, 27 Jul 2021 09:11:28 +0100 Subject: [PATCH 31/33] Fixed cloth tangent generation (#2440) - The output vectors were not properly filled with zeros when they already had the expected size. - The tolerance was too large and was causing patches while computing tangents and bitangents. - The handedness was inverted to what is expected in the shader (which always inverts tangent's w). Signed-off-by: moraaar --- .../ClothComponentMesh/ClothComponentMesh.cpp | 2 +- .../Code/Source/System/TangentSpaceHelper.cpp | 20 ++++++++++++------- .../ClothConstraintsTest.cpp | 12 ++++++++--- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/Gems/NvCloth/Code/Source/Components/ClothComponentMesh/ClothComponentMesh.cpp b/Gems/NvCloth/Code/Source/Components/ClothComponentMesh/ClothComponentMesh.cpp index b791e8a543..7e0ac2f230 100644 --- a/Gems/NvCloth/Code/Source/Components/ClothComponentMesh/ClothComponentMesh.cpp +++ b/Gems/NvCloth/Code/Source/Components/ClothComponentMesh/ClothComponentMesh.cpp @@ -577,7 +577,7 @@ namespace NvCloth const AZ::Vector3& renderTangent = renderTangents[renderVertexIndex]; destTangentsBuffer[index].Set( renderTangent, - 1.0f); + -1.0f); // Shader function ConstructTBN inverts w to change bitangent sign, but the bitangents passed are already corrected, so passing -1.0 to counteract. } if (destBitangentsBuffer) diff --git a/Gems/NvCloth/Code/Source/System/TangentSpaceHelper.cpp b/Gems/NvCloth/Code/Source/System/TangentSpaceHelper.cpp index b43ad1923b..e388643c00 100644 --- a/Gems/NvCloth/Code/Source/System/TangentSpaceHelper.cpp +++ b/Gems/NvCloth/Code/Source/System/TangentSpaceHelper.cpp @@ -11,7 +11,7 @@ namespace NvCloth { namespace { - const float Tolerance = 0.0001f; + const float Tolerance = 1e-7f; } bool TangentSpaceHelper::CalculateNormals( @@ -33,7 +33,8 @@ namespace NvCloth const size_t vertexCount = vertices.size(); // Reset results - outNormals.resize(vertexCount, AZ::Vector3::CreateZero()); + outNormals.resize(vertexCount); + AZStd::fill(outNormals.begin(), outNormals.end(), AZ::Vector3::CreateZero()); // calculate the normals per triangle for (size_t i = 0; i < triangleCount; ++i) @@ -114,8 +115,10 @@ namespace NvCloth const size_t vertexCount = vertices.size(); // Reset results - outTangents.resize(vertexCount, AZ::Vector3::CreateZero()); - outBitangents.resize(vertexCount, AZ::Vector3::CreateZero()); + outTangents.resize(vertexCount); + outBitangents.resize(vertexCount); + AZStd::fill(outTangents.begin(), outTangents.end(), AZ::Vector3::CreateZero()); + AZStd::fill(outBitangents.begin(), outBitangents.end(), AZ::Vector3::CreateZero()); // calculate the base vectors per triangle for (size_t i = 0; i < triangleCount; ++i) @@ -192,9 +195,12 @@ namespace NvCloth const size_t vertexCount = vertices.size(); // Reset results - outTangents.resize(vertexCount, AZ::Vector3::CreateZero()); - outBitangents.resize(vertexCount, AZ::Vector3::CreateZero()); - outNormals.resize(vertexCount, AZ::Vector3::CreateZero()); + outTangents.resize(vertexCount); + outBitangents.resize(vertexCount); + outNormals.resize(vertexCount); + AZStd::fill(outTangents.begin(), outTangents.end(), AZ::Vector3::CreateZero()); + AZStd::fill(outBitangents.begin(), outBitangents.end(), AZ::Vector3::CreateZero()); + AZStd::fill(outNormals.begin(), outNormals.end(), AZ::Vector3::CreateZero()); // calculate the base vectors per triangle for (size_t i = 0; i < triangleCount; ++i) diff --git a/Gems/NvCloth/Code/Tests/Components/ClothComponentMesh/ClothConstraintsTest.cpp b/Gems/NvCloth/Code/Tests/Components/ClothComponentMesh/ClothConstraintsTest.cpp index 5f390ed45d..50ca5c17cd 100644 --- a/Gems/NvCloth/Code/Tests/Components/ClothComponentMesh/ClothConstraintsTest.cpp +++ b/Gems/NvCloth/Code/Tests/Components/ClothComponentMesh/ClothConstraintsTest.cpp @@ -124,6 +124,9 @@ namespace UnitTest const AZStd::vector& motionConstraints = clothConstraints->GetMotionConstraints(); EXPECT_TRUE(motionConstraints.size() == SimulationParticles.size()); + EXPECT_THAT(motionConstraints[0].GetAsVector3(), IsCloseTolerance(SimulationParticles[0].GetAsVector3(), Tolerance)); + EXPECT_THAT(motionConstraints[1].GetAsVector3(), IsCloseTolerance(SimulationParticles[1].GetAsVector3(), Tolerance)); + EXPECT_THAT(motionConstraints[2].GetAsVector3(), IsCloseTolerance(SimulationParticles[2].GetAsVector3(), Tolerance)); EXPECT_NEAR(motionConstraints[0].GetW(), 6.0f, Tolerance); EXPECT_NEAR(motionConstraints[1].GetW(), 0.0f, Tolerance); EXPECT_NEAR(motionConstraints[2].GetW(), 0.0f, Tolerance); @@ -277,6 +280,9 @@ namespace UnitTest const AZStd::vector& separationConstraints = clothConstraints->GetSeparationConstraints(); EXPECT_TRUE(motionConstraints.size() == newParticles.size()); + EXPECT_THAT(motionConstraints[0].GetAsVector3(), IsCloseTolerance(newParticles[0].GetAsVector3(), Tolerance)); + EXPECT_THAT(motionConstraints[1].GetAsVector3(), IsCloseTolerance(newParticles[1].GetAsVector3(), Tolerance)); + EXPECT_THAT(motionConstraints[2].GetAsVector3(), IsCloseTolerance(newParticles[2].GetAsVector3(), Tolerance)); EXPECT_NEAR(motionConstraints[0].GetW(), 3.0f, Tolerance); EXPECT_NEAR(motionConstraints[1].GetW(), 1.5f, Tolerance); EXPECT_NEAR(motionConstraints[2].GetW(), 0.0f, Tolerance); @@ -285,8 +291,8 @@ namespace UnitTest EXPECT_NEAR(separationConstraints[0].GetW(), 3.0f, Tolerance); EXPECT_NEAR(separationConstraints[1].GetW(), 1.5f, Tolerance); EXPECT_NEAR(separationConstraints[2].GetW(), 0.3f, Tolerance); - EXPECT_THAT(separationConstraints[0].GetAsVector3(), IsCloseTolerance(AZ::Vector3(-3.03902f, 2.80752f, 3.80752f), Tolerance)); - EXPECT_THAT(separationConstraints[1].GetAsVector3(), IsCloseTolerance(AZ::Vector3(-1.41659f, 0.651243f, -0.348757f), Tolerance)); - EXPECT_THAT(separationConstraints[2].GetAsVector3(), IsCloseTolerance(AZ::Vector3(6.15313f, -0.876132f, 0.123868f), Tolerance)); + EXPECT_THAT(separationConstraints[0].GetAsVector3(), IsCloseTolerance(AZ::Vector3(0.0f, 3.53553f, 4.53553f), Tolerance)); + EXPECT_THAT(separationConstraints[1].GetAsVector3(), IsCloseTolerance(AZ::Vector3(0.0f, 2.06066f, 1.06066f), Tolerance)); + EXPECT_THAT(separationConstraints[2].GetAsVector3(), IsCloseTolerance(AZ::Vector3(1.0f, -3.74767f, -2.74767f), Tolerance)); } } // namespace UnitTest From 902bdeb6d6092d0b88066fbb873eaa043c3b788a Mon Sep 17 00:00:00 2001 From: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> Date: Thu, 29 Jul 2021 16:50:50 -0700 Subject: [PATCH 32/33] Create RUN target as helpers for the project-centric workflow (#2520) (#2635) * Create RUN target as helpers for the project-centric workflow Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> * typo fix Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> * rename target as ".Imported" and create "" as the metatarget that is used for debugging and building in o3de Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> --- Code/Editor/CMakeLists.txt | 2 + cmake/Platform/Common/Install_common.cmake | 43 ++++++++++++++++++---- cmake/SettingsRegistry.cmake | 7 +--- cmake/install/InstalledTarget.in | 2 + 4 files changed, 42 insertions(+), 12 deletions(-) diff --git a/Code/Editor/CMakeLists.txt b/Code/Editor/CMakeLists.txt index 5a846a267a..db6b8e958e 100644 --- a/Code/Editor/CMakeLists.txt +++ b/Code/Editor/CMakeLists.txt @@ -163,6 +163,8 @@ ly_add_target( editor_files.cmake PLATFORM_INCLUDE_FILES Platform/${PAL_PLATFORM_NAME}/editor_${PAL_PLATFORM_NAME_LOWERCASE}.cmake + TARGET_PROPERTIES + LY_INSTALL_GENERATE_RUN_TARGET TRUE BUILD_DEPENDENCIES PRIVATE 3rdParty::Qt::Core diff --git a/cmake/Platform/Common/Install_common.cmake b/cmake/Platform/Common/Install_common.cmake index 1ac59ee83f..8058fb5908 100644 --- a/cmake/Platform/Common/Install_common.cmake +++ b/cmake/Platform/Common/Install_common.cmake @@ -9,6 +9,15 @@ include(cmake/FileUtil.cmake) set(CMAKE_INSTALL_MESSAGE NEVER) # Simplify messages to reduce output noise +define_property(TARGET PROPERTY LY_INSTALL_GENERATE_RUN_TARGET + BRIEF_DOCS "Defines if a \"RUN\" targets should be created when installing this target Gem" + FULL_DOCS [[ + Property which is set on targets that should generate a "RUN" + target when installed. This \"RUN\" target helps to run the + binary from the installed location directly from the IDE. + ]] +) + ly_set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME Core) cmake_path(RELATIVE_PATH CMAKE_RUNTIME_OUTPUT_DIRECTORY BASE_DIRECTORY ${CMAKE_BINARY_DIR} OUTPUT_VARIABLE runtime_output_directory) @@ -105,15 +114,19 @@ function(ly_setup_target OUTPUT_CONFIGURED_TARGET ALIAS_TARGET_NAME absolute_tar set(NAMESPACE_PLACEHOLDER "") set(NAME_PLACEHOLDER ${TARGET_NAME}) endif() + get_target_property(should_create_helper ${TARGET_NAME} LY_INSTALL_GENERATE_RUN_TARGET) + if(should_create_helper) + set(NAME_PLACEHOLDER ${NAME_PLACEHOLDER}.Imported) + endif() set(TARGET_TYPE_PLACEHOLDER "") - get_target_property(target_type ${NAME_PLACEHOLDER} TYPE) + get_target_property(target_type ${TARGET_NAME} TYPE) # Remove the _LIBRARY since we dont need to pass that to ly_add_targets string(REPLACE "_LIBRARY" "" TARGET_TYPE_PLACEHOLDER ${target_type}) # For HEADER_ONLY libs we end up generating "INTERFACE" libraries, need to specify HEADERONLY instead string(REPLACE "INTERFACE" "HEADERONLY" TARGET_TYPE_PLACEHOLDER ${TARGET_TYPE_PLACEHOLDER}) if(TARGET_TYPE_PLACEHOLDER STREQUAL "MODULE") - get_target_property(gem_module ${NAME_PLACEHOLDER} GEM_MODULE) + get_target_property(gem_module ${TARGET_NAME} GEM_MODULE) if(gem_module) set(TARGET_TYPE_PLACEHOLDER "GEM_MODULE") endif() @@ -146,7 +159,6 @@ function(ly_setup_target OUTPUT_CONFIGURED_TARGET ALIAS_TARGET_NAME absolute_tar unset(RUNTIME_DEPENDENCIES_PLACEHOLDER) endif() - get_target_property(inteface_build_dependencies_props ${TARGET_NAME} INTERFACE_LINK_LIBRARIES) unset(INTERFACE_BUILD_DEPENDENCIES_PLACEHOLDER) if(inteface_build_dependencies_props) @@ -170,6 +182,23 @@ function(ly_setup_target OUTPUT_CONFIGURED_TARGET ALIAS_TARGET_NAME absolute_tar list(REMOVE_DUPLICATES INTERFACE_BUILD_DEPENDENCIES_PLACEHOLDER) string(REPLACE ";" "\n" INTERFACE_BUILD_DEPENDENCIES_PLACEHOLDER "${INTERFACE_BUILD_DEPENDENCIES_PLACEHOLDER}") + # If the target is an executable/application, add a custom target so we can debug the target in project-centric workflow + if(should_create_helper) + string(REPLACE ".Imported" "" RUN_TARGET_NAME ${NAME_PLACEHOLDER}) + set(target_types_with_debugging_helper EXECUTABLE APPLICATION) + if(NOT target_type IN_LIST target_types_with_debugging_helper) + message(FATAL_ERROR "Cannot generate a RUN target for ${TARGET_NAME}, type is ${target_type}") + endif() + set(TARGET_RUN_HELPER +"add_custom_target(${RUN_TARGET_NAME}) +set_target_properties(${RUN_TARGET_NAME} PROPERTIES + FOLDER \"CMakePredefinedTargets/SDK\" + VS_DEBUGGER_COMMAND \$> + VS_DEBUGGER_COMMAND_ARGUMENTS \"--project-path=\${LY_DEFAULT_PROJECT_PATH}\" +)" +) + endif() + # Config file set(target_file_contents "# Generated by O3DE install\n\n") if(NOT target_type STREQUAL INTERFACE_LIBRARY) @@ -182,13 +211,13 @@ function(ly_setup_target OUTPUT_CONFIGURED_TARGET ALIAS_TARGET_NAME absolute_tar set(target_location "\${LY_ROOT_FOLDER}/${library_output_directory}/${PAL_PLATFORM_NAME}/$/${target_library_output_subdirectory}/$") elseif(target_type STREQUAL SHARED_LIBRARY) string(APPEND target_file_contents -"set_property(TARGET ${TARGET_NAME} +"set_property(TARGET ${NAME_PLACEHOLDER} APPEND_STRING PROPERTY IMPORTED_IMPLIB $<$$:\"\${LY_ROOT_FOLDER}/${archive_output_directory}/${PAL_PLATFORM_NAME}/$/$\"$ ) ") string(APPEND target_file_contents -"set_property(TARGET ${TARGET_NAME} +"set_property(TARGET ${NAME_PLACEHOLDER} PROPERTY IMPORTED_IMPLIB_$> \"\${LY_ROOT_FOLDER}/${archive_output_directory}/${PAL_PLATFORM_NAME}/$/$\" ) @@ -200,11 +229,11 @@ function(ly_setup_target OUTPUT_CONFIGURED_TARGET ALIAS_TARGET_NAME absolute_tar if(target_location) string(APPEND target_file_contents -"set_property(TARGET ${TARGET_NAME} +"set_property(TARGET ${NAME_PLACEHOLDER} APPEND_STRING PROPERTY IMPORTED_LOCATION $<$$:${target_location}$ ) -set_property(TARGET ${TARGET_NAME} +set_property(TARGET ${NAME_PLACEHOLDER} PROPERTY IMPORTED_LOCATION_$> ${target_location} ) diff --git a/cmake/SettingsRegistry.cmake b/cmake/SettingsRegistry.cmake index 934bae4e30..a1e6de73f8 100644 --- a/cmake/SettingsRegistry.cmake +++ b/cmake/SettingsRegistry.cmake @@ -158,10 +158,6 @@ function(ly_delayed_generate_settings_registry) message(FATAL_ERROR "Dependency ${gem_target} from ${target} does not exist") endif() - get_property(has_manually_added_dependencies TARGET ${gem_target} PROPERTY MANUALLY_ADDED_DEPENDENCIES SET) - get_target_property(target_type ${gem_target} TYPE) - - ly_get_gem_module_root(gem_module_root ${gem_target}) file(RELATIVE_PATH gem_module_root_relative_to_engine_root ${LY_ROOT_FOLDER} ${gem_module_root}) @@ -179,7 +175,8 @@ function(ly_delayed_generate_settings_registry) list(JOIN target_gem_dependencies_names ",\n" target_gem_dependencies_names) string(CONFIGURE ${gems_json_template} gem_json @ONLY) get_target_property(is_imported ${target} IMPORTED) - if(is_imported) + get_target_property(target_type ${target} TYPE) + if(is_imported OR target_type STREQUAL UTILITY) unset(target_dir) foreach(conf IN LISTS CMAKE_CONFIGURATION_TYPES) string(TOUPPER ${conf} UCONF) diff --git a/cmake/install/InstalledTarget.in b/cmake/install/InstalledTarget.in index 0503fd5f2b..a4f4fa4763 100644 --- a/cmake/install/InstalledTarget.in +++ b/cmake/install/InstalledTarget.in @@ -17,6 +17,8 @@ ly_add_target( @RUNTIME_DEPENDENCIES_PLACEHOLDER@ ) +@TARGET_RUN_HELPER@ + set(configs @CMAKE_CONFIGURATION_TYPES@) foreach(config ${configs}) include("@NAME_PLACEHOLDER@_${config}.cmake" OPTIONAL) From 4b817a6483560691a5f95a34355c87c8972b05ab Mon Sep 17 00:00:00 2001 From: Shirang Jia Date: Wed, 4 Aug 2021 17:57:47 -0700 Subject: [PATCH 33/33] Include build failure root cause in email notification (#2491) Signed-off-by: shiranj --- scripts/build/Jenkins/Jenkinsfile | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/scripts/build/Jenkins/Jenkinsfile b/scripts/build/Jenkins/Jenkinsfile index 70618fea78..e693168af0 100644 --- a/scripts/build/Jenkins/Jenkinsfile +++ b/scripts/build/Jenkins/Jenkinsfile @@ -577,13 +577,19 @@ finally { ) } node('controller') { - step([ - $class: 'Mailer', - notifyEveryUnstableBuild: true, - recipients: emailextrecipients([ + if("${currentBuild.currentResult}" == "SUCCESS") { + emailBody = "${BUILD_URL}\nSuccess!" + } else { + buildFailure = tm('${BUILD_FAILURE_ANALYZER}') + emailBody = "${BUILD_URL}\n${buildFailure}!" + } + emailext ( + body: "${emailBody}", + subject: "${currentBuild.currentResult}: ${JOB_NAME} - Build # ${BUILD_NUMBER}", + recipientProviders: [ [$class: 'RequesterRecipientProvider'] - ]) - ]) + ] + ) } } catch(Exception e) { }