diff --git a/Code/Framework/AzCore/AzCore/IO/Path/Path.h b/Code/Framework/AzCore/AzCore/IO/Path/Path.h index 59ab36648c..6f7e995a74 100644 --- a/Code/Framework/AzCore/AzCore/IO/Path/Path.h +++ b/Code/Framework/AzCore/AzCore/IO/Path/Path.h @@ -278,7 +278,7 @@ namespace AZ::IO //! then their hash values are also equal //! For example : path "a//b" equals "a/b", the //! hash value of "a//b" would also equal the hash value of "a/b" - constexpr size_t hash_value(const PathView& pathToHash) noexcept; + size_t hash_value(const PathView& pathToHash) noexcept; // path.comparison constexpr bool operator==(const PathView& lhs, const PathView& rhs) noexcept; @@ -623,7 +623,7 @@ namespace AZ::IO //! For example : path "a//b" equals "a/b", the //! hash value of "a//b" would also equal the hash value of "a/b" template - constexpr size_t hash_value(const BasicPath& pathToHash); + size_t hash_value(const BasicPath& pathToHash); // path.append template diff --git a/Code/Framework/AzCore/AzCore/IO/Path/Path.inl b/Code/Framework/AzCore/AzCore/IO/Path/Path.inl index d9747c0c73..4e8356b436 100644 --- a/Code/Framework/AzCore/AzCore/IO/Path/Path.inl +++ b/Code/Framework/AzCore/AzCore/IO/Path/Path.inl @@ -1952,7 +1952,7 @@ namespace AZ::IO } template - constexpr size_t hash_value(const BasicPath& pathToHash) + inline size_t hash_value(const BasicPath& pathToHash) { return AZStd::hash>{}(pathToHash); } @@ -2083,13 +2083,28 @@ namespace AZStd template <> struct hash { - constexpr size_t operator()(const AZ::IO::PathView& pathToHash) noexcept + /// Path is using FNV-1a algorithm 64 bit version. + static size_t hash_path(AZStd::string_view pathSegment, const char pathSeparator) + { + size_t hash = 14695981039346656037ULL; + constexpr size_t fnvPrime = 1099511628211ULL; + + for (const char first : pathSegment) + { + hash ^= static_cast((pathSeparator == AZ::IO::PosixPathSeparator) + ? first : tolower(first)); + hash *= fnvPrime; + } + return hash; + } + + size_t operator()(const AZ::IO::PathView& pathToHash) noexcept { auto pathParser = AZ::IO::parser::PathParser::CreateBegin(pathToHash.Native(), pathToHash.m_preferred_separator); size_t hash_value = 0; while (pathParser) { - AZStd::hash_combine(hash_value, AZStd::hash{}(*pathParser)); + AZStd::hash_combine(hash_value, hash_path(*pathParser, pathToHash.m_preferred_separator)); ++pathParser; } return hash_value; @@ -2098,7 +2113,7 @@ namespace AZStd template struct hash> { - constexpr size_t operator()(const AZ::IO::BasicPath& pathToHash) noexcept + const size_t operator()(const AZ::IO::BasicPath& pathToHash) noexcept { return AZStd::hash{}(pathToHash); } @@ -2109,11 +2124,11 @@ namespace AZStd template struct hash; } -// Explicit instantations of our support Path classes +// Explicit instantiations of our support Path classes namespace AZ::IO { // PathView hash - constexpr size_t hash_value(const PathView& pathToHash) noexcept + inline size_t hash_value(const PathView& pathToHash) noexcept { return AZStd::hash{}(pathToHash); } diff --git a/Code/Framework/AzCore/Tests/IO/Path/PathTests.cpp b/Code/Framework/AzCore/Tests/IO/Path/PathTests.cpp index 92d1bcf9f5..79d9f8c302 100644 --- a/Code/Framework/AzCore/Tests/IO/Path/PathTests.cpp +++ b/Code/Framework/AzCore/Tests/IO/Path/PathTests.cpp @@ -183,6 +183,36 @@ namespace UnitTest AZStd::tuple(R"(foO/Bar)", "foo/bar") )); + using PathHashParamFixture = PathParamFixture; + TEST_P(PathHashParamFixture, HashOperator_HashesCaseInsensitiveForWindowsPaths) + { + AZ::IO::Path path1{ AZStd::get<0>(GetParam()), AZ::IO::WindowsPathSeparator }; + AZ::IO::Path path2{ AZStd::get<1>(GetParam()), AZ::IO::WindowsPathSeparator }; + size_t path1Hash = AZStd::hash{}(path1); + size_t path2Hash = AZStd::hash{}(path2); + EXPECT_EQ(path1Hash, path2Hash) << AZStd::string::format(R"(path1 "%s" should hash to path2 "%s"\n)", + path1.c_str(), path2.c_str()).c_str(); + } + + TEST_P(PathHashParamFixture, HashOperator_HashesCaseSensitiveForPosixPaths) + { + AZ::IO::Path path1{ AZStd::get<0>(GetParam()), AZ::IO::PosixPathSeparator }; + AZ::IO::Path path2{ AZStd::get<1>(GetParam()), AZ::IO::PosixPathSeparator }; + size_t path1Hash = AZStd::hash{}(path1); + size_t path2Hash = AZStd::hash{}(path2); + EXPECT_NE(path1Hash, path2Hash) << AZStd::string::format(R"(path1 "%s" should NOT hash to path2 "%s"\n)", + path1.c_str(), path2.c_str()).c_str(); + } + + INSTANTIATE_TEST_CASE_P( + HashPaths, + PathHashParamFixture, + ::testing::Values( + AZStd::tuple("C:/test/foo", R"(c:\test/foo)"), + AZStd::tuple(R"(D:\test/bar/baz//foo)", "d:/test/bar/baz\\\\\\foo"), + AZStd::tuple(R"(foO/Bar)", "foo/bar") + )); + class PathSingleParamFixture : public ScopedAllocatorSetupFixture , public ::testing::WithParamInterface> diff --git a/Code/Tools/CrashHandler/CMakeLists.txt b/Code/Tools/CrashHandler/CMakeLists.txt index b436251619..259a7d2891 100644 --- a/Code/Tools/CrashHandler/CMakeLists.txt +++ b/Code/Tools/CrashHandler/CMakeLists.txt @@ -39,8 +39,19 @@ ly_add_target( string(REPLACE "." ";" version_list "${LY_VERSION_STRING}") list(GET version_list 0 EXE_VERSION_INFO_0) list(GET version_list 1 EXE_VERSION_INFO_1) -list(GET version_list 2 EXE_VERSION_INFO_2) -list(GET version_list 3 EXE_VERSION_INFO_3) + +list(LENGTH version_list version_component_count) +if(${version_component_count} GREATER_EQUAL 3) + list(GET version_list 2 EXE_VERSION_INFO_2) +else() + set(EXE_VERSION_INFO_2 0) +endif() + +if(${version_component_count} GREATER_EQUAL 4) + list(GET version_list 3 EXE_VERSION_INFO_3) +else() + set(EXE_VERSION_INFO_3 0) +endif() ly_add_source_properties( SOURCES Shared/CrashHandler.cpp diff --git a/Code/Tools/ProjectManager/Platform/Linux/PAL_linux_files.cmake b/Code/Tools/ProjectManager/Platform/Linux/PAL_linux_files.cmake index 54b4b649d9..11222602d5 100644 --- a/Code/Tools/ProjectManager/Platform/Linux/PAL_linux_files.cmake +++ b/Code/Tools/ProjectManager/Platform/Linux/PAL_linux_files.cmake @@ -9,4 +9,6 @@ set(FILES Python_linux.cpp ProjectBuilderWorker_linux.cpp + ProjectUtils_linux.cpp + ProjectManagerDefs_linux.cpp ) diff --git a/Code/Tools/ProjectManager/Platform/Linux/ProjectManagerDefs_linux.cpp b/Code/Tools/ProjectManager/Platform/Linux/ProjectManagerDefs_linux.cpp new file mode 100644 index 0000000000..5ccfbf8eff --- /dev/null +++ b/Code/Tools/ProjectManager/Platform/Linux/ProjectManagerDefs_linux.cpp @@ -0,0 +1,15 @@ +/* + * Copyright (c) Contributors to the Open 3D Engine Project. For complete copyright and license terms please see the LICENSE at the root of this distribution. + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + * + */ + +#include + + +namespace O3DE::ProjectManager +{ + const QString ProjectBuildPathPostfix = ProjectBuildDirectoryName + "/linux"; + +} // namespace O3DE::ProjectManager diff --git a/Code/Tools/ProjectManager/Platform/Linux/ProjectUtils_linux.cpp b/Code/Tools/ProjectManager/Platform/Linux/ProjectUtils_linux.cpp new file mode 100644 index 0000000000..64d18ec605 --- /dev/null +++ b/Code/Tools/ProjectManager/Platform/Linux/ProjectUtils_linux.cpp @@ -0,0 +1,21 @@ +/* + * Copyright (c) Contributors to the Open 3D Engine Project. For complete copyright and license terms please see the LICENSE at the root of this distribution. + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + * + */ + +#include + +namespace O3DE::ProjectManager +{ + namespace ProjectUtils + { + AZ::Outcome FindSupportedCompilerForPlatform() + { + // Compiler detection not supported on platform + return AZ::Success(); + } + + } // namespace ProjectUtils +} // namespace O3DE::ProjectManager diff --git a/Code/Tools/ProjectManager/Platform/Mac/PAL_mac_files.cmake b/Code/Tools/ProjectManager/Platform/Mac/PAL_mac_files.cmake index 586c66515a..54b35f0d3c 100644 --- a/Code/Tools/ProjectManager/Platform/Mac/PAL_mac_files.cmake +++ b/Code/Tools/ProjectManager/Platform/Mac/PAL_mac_files.cmake @@ -9,4 +9,6 @@ set(FILES Python_mac.cpp ProjectBuilderWorker_mac.cpp + ProjectUtils_mac.cpp + ProjectManagerDefs_mac.cpp ) diff --git a/Code/Tools/ProjectManager/Platform/Mac/ProjectManagerDefs_mac.cpp b/Code/Tools/ProjectManager/Platform/Mac/ProjectManagerDefs_mac.cpp new file mode 100644 index 0000000000..01a7f9e375 --- /dev/null +++ b/Code/Tools/ProjectManager/Platform/Mac/ProjectManagerDefs_mac.cpp @@ -0,0 +1,15 @@ +/* + * Copyright (c) Contributors to the Open 3D Engine Project. For complete copyright and license terms please see the LICENSE at the root of this distribution. + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + * + */ + +#include + + +namespace O3DE::ProjectManager +{ + const QString ProjectBuildPathPostfix = ProjectBuildDirectoryName + "/mac_xcode"; + +} // namespace O3DE::ProjectManager diff --git a/Code/Tools/ProjectManager/Platform/Mac/ProjectUtils_mac.cpp b/Code/Tools/ProjectManager/Platform/Mac/ProjectUtils_mac.cpp new file mode 100644 index 0000000000..64d18ec605 --- /dev/null +++ b/Code/Tools/ProjectManager/Platform/Mac/ProjectUtils_mac.cpp @@ -0,0 +1,21 @@ +/* + * Copyright (c) Contributors to the Open 3D Engine Project. For complete copyright and license terms please see the LICENSE at the root of this distribution. + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + * + */ + +#include + +namespace O3DE::ProjectManager +{ + namespace ProjectUtils + { + AZ::Outcome FindSupportedCompilerForPlatform() + { + // Compiler detection not supported on platform + return AZ::Success(); + } + + } // namespace ProjectUtils +} // namespace O3DE::ProjectManager diff --git a/Code/Tools/ProjectManager/Platform/Windows/PAL_windows_files.cmake b/Code/Tools/ProjectManager/Platform/Windows/PAL_windows_files.cmake index 17c571ea92..d95b0d2502 100644 --- a/Code/Tools/ProjectManager/Platform/Windows/PAL_windows_files.cmake +++ b/Code/Tools/ProjectManager/Platform/Windows/PAL_windows_files.cmake @@ -9,4 +9,6 @@ set(FILES Python_windows.cpp ProjectBuilderWorker_windows.cpp + ProjectUtils_windows.cpp + ProjectManagerDefs_windows.cpp ) diff --git a/Code/Tools/ProjectManager/Platform/Windows/ProjectBuilderWorker_windows.cpp b/Code/Tools/ProjectManager/Platform/Windows/ProjectBuilderWorker_windows.cpp index bbcd3ee7d3..a228f58e51 100644 --- a/Code/Tools/ProjectManager/Platform/Windows/ProjectBuilderWorker_windows.cpp +++ b/Code/Tools/ProjectManager/Platform/Windows/ProjectBuilderWorker_windows.cpp @@ -76,8 +76,17 @@ namespace O3DE::ProjectManager m_configProjectProcess->start( "cmake", - QStringList{ "-B", QDir(m_projectInfo.m_path).filePath(ProjectBuildPathPostfix), "-S", m_projectInfo.m_path, "-G", - "Visual Studio 16", "-DLY_3RDPARTY_PATH=" + engineInfo.m_thirdPartyPath }); + QStringList + { + "-B", + QDir(m_projectInfo.m_path).filePath(ProjectBuildPathPostfix), + "-S", + m_projectInfo.m_path, + "-G", + "Visual Studio 16", + "-DLY_3RDPARTY_PATH=" + engineInfo.m_thirdPartyPath, + "-DLY_UNITY_BUILD=1" + }); if (!m_configProjectProcess->waitForStarted()) { @@ -125,8 +134,16 @@ namespace O3DE::ProjectManager m_buildProjectProcess->start( "cmake", - QStringList{ "--build", QDir(m_projectInfo.m_path).filePath(ProjectBuildPathPostfix), "--target", - m_projectInfo.m_projectName + ".GameLauncher", "Editor", "--config", "profile" }); + QStringList + { + "--build", + QDir(m_projectInfo.m_path).filePath(ProjectBuildPathPostfix), + "--target", + m_projectInfo.m_projectName + ".GameLauncher", + "Editor", + "--config", + "profile" + }); if (!m_buildProjectProcess->waitForStarted()) { diff --git a/Code/Tools/ProjectManager/Platform/Windows/ProjectManagerDefs_windows.cpp b/Code/Tools/ProjectManager/Platform/Windows/ProjectManagerDefs_windows.cpp new file mode 100644 index 0000000000..6bd2194967 --- /dev/null +++ b/Code/Tools/ProjectManager/Platform/Windows/ProjectManagerDefs_windows.cpp @@ -0,0 +1,14 @@ +/* + * Copyright (c) Contributors to the Open 3D Engine Project. For complete copyright and license terms please see the LICENSE at the root of this distribution. + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + * + */ + +#include + +namespace O3DE::ProjectManager +{ + const QString ProjectBuildPathPostfix = ProjectBuildDirectoryName + "/windows_vs2019"; + +} // namespace O3DE::ProjectManager diff --git a/Code/Tools/ProjectManager/Platform/Windows/ProjectUtils_windows.cpp b/Code/Tools/ProjectManager/Platform/Windows/ProjectUtils_windows.cpp new file mode 100644 index 0000000000..c0d977a5f5 --- /dev/null +++ b/Code/Tools/ProjectManager/Platform/Windows/ProjectUtils_windows.cpp @@ -0,0 +1,60 @@ +/* + * Copyright (c) Contributors to the Open 3D Engine Project. For complete copyright and license terms please see the LICENSE at the root of this distribution. + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + * + */ + +#include + +#include +#include +#include +#include + +namespace O3DE::ProjectManager +{ + namespace ProjectUtils + { + AZ::Outcome FindSupportedCompilerForPlatform() + { + QProcessEnvironment environment = QProcessEnvironment::systemEnvironment(); + QString programFilesPath = environment.value("ProgramFiles(x86)"); + QString vsWherePath = QDir(programFilesPath).filePath("Microsoft Visual Studio/Installer/vswhere.exe"); + + QFileInfo vsWhereFile(vsWherePath); + if (vsWhereFile.exists() && vsWhereFile.isFile()) + { + QProcess vsWhereProcess; + vsWhereProcess.setProcessChannelMode(QProcess::MergedChannels); + + vsWhereProcess.start( + vsWherePath, + QStringList{ + "-version", + "16.0", + "-latest", + "-requires", + "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", + "-property", + "isComplete" + }); + + if (vsWhereProcess.waitForStarted() && vsWhereProcess.waitForFinished()) + { + QString vsWhereOutput(vsWhereProcess.readAllStandardOutput()); + if (vsWhereOutput.startsWith("1")) + { + return AZ::Success(); + } + } + } + + return AZ::Failure(QObject::tr("Visual Studio 2019 not found.\n\n" + "Visual Studio 2019 is required to build this project." + " Install any edition of Visual Studio 2019" + " before proceeding to the next step.")); + } + + } // namespace ProjectUtils +} // namespace O3DE::ProjectManager diff --git a/Code/Tools/ProjectManager/Source/CreateProjectCtrl.cpp b/Code/Tools/ProjectManager/Source/CreateProjectCtrl.cpp index 68d88d2f1c..20d564d8b0 100644 --- a/Code/Tools/ProjectManager/Source/CreateProjectCtrl.cpp +++ b/Code/Tools/ProjectManager/Source/CreateProjectCtrl.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -223,38 +224,42 @@ namespace O3DE::ProjectManager void CreateProjectCtrl::CreateProject() { - if (m_newProjectSettingsScreen->Validate()) + if (ProjectUtils::FindSupportedCompiler(this)) { - ProjectInfo projectInfo = m_newProjectSettingsScreen->GetProjectInfo(); - QString projectTemplatePath = m_newProjectSettingsScreen->GetProjectTemplatePath(); - - auto result = PythonBindingsInterface::Get()->CreateProject(projectTemplatePath, projectInfo); - if (result.IsSuccess()) + if (m_newProjectSettingsScreen->Validate()) { - // automatically register the project - PythonBindingsInterface::Get()->AddProject(projectInfo.m_path); + ProjectInfo projectInfo = m_newProjectSettingsScreen->GetProjectInfo(); + QString projectTemplatePath = m_newProjectSettingsScreen->GetProjectTemplatePath(); -#ifdef TEMPLATE_GEM_CONFIGURATION_ENABLED - if (!m_gemCatalogScreen->EnableDisableGemsForProject(projectInfo.m_path)) + auto result = PythonBindingsInterface::Get()->CreateProject(projectTemplatePath, projectInfo); + if (result.IsSuccess()) { - QMessageBox::critical(this, tr("Failed to configure gems"), tr("Failed to configure gems for template.")); - return; - } + // automatically register the project + PythonBindingsInterface::Get()->AddProject(projectInfo.m_path); + +#ifdef TEMPLATE_GEM_CONFIGURATION_ENABLED + if (!m_gemCatalogScreen->EnableDisableGemsForProject(projectInfo.m_path)) + { + QMessageBox::critical(this, tr("Failed to configure gems"), tr("Failed to configure gems for template.")); + return; + } #endif // TEMPLATE_GEM_CONFIGURATION_ENABLED - projectInfo.m_needsBuild = true; - emit NotifyBuildProject(projectInfo); - emit ChangeScreenRequest(ProjectManagerScreen::Projects); + projectInfo.m_needsBuild = true; + emit NotifyBuildProject(projectInfo); + emit ChangeScreenRequest(ProjectManagerScreen::Projects); + } + else + { + QMessageBox::critical(this, tr("Project creation failed"), tr("Failed to create project.")); + } } else { - QMessageBox::critical(this, tr("Project creation failed"), tr("Failed to create project.")); + QMessageBox::warning( + this, tr("Invalid project settings"), tr("Please correct the indicated project settings and try again.")); } } - else - { - QMessageBox::warning(this, tr("Invalid project settings"), tr("Please correct the indicated project settings and try again.")); - } } void CreateProjectCtrl::ReinitGemCatalogForSelectedTemplate() diff --git a/Code/Tools/ProjectManager/Source/ProjectBuilderController.cpp b/Code/Tools/ProjectManager/Source/ProjectBuilderController.cpp index b5565a4ccc..e782f0b57f 100644 --- a/Code/Tools/ProjectManager/Source/ProjectBuilderController.cpp +++ b/Code/Tools/ProjectManager/Source/ProjectBuilderController.cpp @@ -71,8 +71,9 @@ namespace O3DE::ProjectManager m_lastProgress = progress; if (m_projectButton) { - m_projectButton->SetButtonOverlayText(QString("%1 (%2%)\n\n").arg(tr("Building Project..."), QString::number(progress))); + m_projectButton->SetButtonOverlayText(QString("%1 (%2%)
%3
").arg(tr("Building Project..."), QString::number(progress), tr("Click to view logs."))); m_projectButton->SetProgressBarValue(progress); + m_projectButton->SetBuildLogsLink(m_worker->GetLogFilePath()); } } diff --git a/Code/Tools/ProjectManager/Source/ProjectBuilderWorker.cpp b/Code/Tools/ProjectManager/Source/ProjectBuilderWorker.cpp index c3f2f767f0..2fe1c6db15 100644 --- a/Code/Tools/ProjectManager/Source/ProjectBuilderWorker.cpp +++ b/Code/Tools/ProjectManager/Source/ProjectBuilderWorker.cpp @@ -15,8 +15,6 @@ namespace O3DE::ProjectManager { - const QString ProjectBuilderWorker::BuildCancelled = QObject::tr("Build Cancelled."); - ProjectBuilderWorker::ProjectBuilderWorker(const ProjectInfo& projectInfo) : QObject() , m_projectInfo(projectInfo) diff --git a/Code/Tools/ProjectManager/Source/ProjectBuilderWorker.h b/Code/Tools/ProjectManager/Source/ProjectBuilderWorker.h index 5b0dfe5c17..f42c87f47b 100644 --- a/Code/Tools/ProjectManager/Source/ProjectBuilderWorker.h +++ b/Code/Tools/ProjectManager/Source/ProjectBuilderWorker.h @@ -23,7 +23,7 @@ namespace O3DE::ProjectManager // QProcess::waitForFinished uses -1 to indicate that the process should not timeout static constexpr int MaxBuildTimeMSecs = -1; // Build was cancelled - static const QString BuildCancelled; + inline static const QString BuildCancelled = QObject::tr("Build Cancelled."); Q_OBJECT diff --git a/Code/Tools/ProjectManager/Source/ProjectButtonWidget.cpp b/Code/Tools/ProjectManager/Source/ProjectButtonWidget.cpp index c00490f0b9..82b4e8d84a 100644 --- a/Code/Tools/ProjectManager/Source/ProjectButtonWidget.cpp +++ b/Code/Tools/ProjectManager/Source/ProjectButtonWidget.cpp @@ -40,7 +40,9 @@ namespace O3DE::ProjectManager m_overlayLabel->setObjectName("labelButtonOverlay"); m_overlayLabel->setWordWrap(true); m_overlayLabel->setAlignment(Qt::AlignCenter); + m_overlayLabel->setTextInteractionFlags(Qt::LinksAccessibleByMouse); m_overlayLabel->setVisible(false); + connect(m_overlayLabel, &QLabel::linkActivated, this, &LabelButton::OnLinkActivated); vLayout->addWidget(m_overlayLabel); m_buildOverlayLayout = new QVBoxLayout(); @@ -232,7 +234,7 @@ namespace O3DE::ProjectManager AzQtComponents::ShowFileOnDesktop(m_projectInfo.m_path); }); menu->addSeparator(); - menu->addAction(tr("Duplicate"), this, [this]() { emit CopyProject(m_projectInfo.m_path); }); + menu->addAction(tr("Duplicate"), this, [this]() { emit CopyProject(m_projectInfo); }); menu->addSeparator(); menu->addAction(tr("Remove from O3DE"), this, [this]() { emit RemoveProject(m_projectInfo.m_path); }); menu->addAction(tr("Delete this Project"), this, [this]() { emit DeleteProject(m_projectInfo.m_path); }); @@ -267,6 +269,11 @@ namespace O3DE::ProjectManager SetProjectButtonAction(tr("Build Project"), [this]() { emit BuildProject(m_projectInfo); }); } + void ProjectButton::SetBuildLogsLink(const QUrl& logUrl) + { + m_projectImageLabel->SetLogUrl(logUrl); + } + void ProjectButton::ShowBuildFailed(bool show, const QUrl& logUrl) { if (!logUrl.isEmpty()) diff --git a/Code/Tools/ProjectManager/Source/ProjectButtonWidget.h b/Code/Tools/ProjectManager/Source/ProjectButtonWidget.h index fa6ec8a675..ff352e0afd 100644 --- a/Code/Tools/ProjectManager/Source/ProjectButtonWidget.h +++ b/Code/Tools/ProjectManager/Source/ProjectButtonWidget.h @@ -78,6 +78,7 @@ namespace O3DE::ProjectManager void SetProjectButtonAction(const QString& text, AZStd::function lambda); void SetProjectBuildButtonAction(); + void SetBuildLogsLink(const QUrl& logUrl); void ShowBuildFailed(bool show, const QUrl& logUrl); void SetLaunchButtonEnabled(bool enabled); @@ -88,7 +89,7 @@ namespace O3DE::ProjectManager signals: void OpenProject(const QString& projectName); void EditProject(const QString& projectName); - void CopyProject(const QString& projectName); + void CopyProject(const ProjectInfo& projectInfo); void RemoveProject(const QString& projectName); void DeleteProject(const QString& projectName); void BuildProject(const ProjectInfo& projectInfo); diff --git a/Code/Tools/ProjectManager/Source/ProjectManagerDefs.h b/Code/Tools/ProjectManager/Source/ProjectManagerDefs.h index 74505d5e25..8ab5128741 100644 --- a/Code/Tools/ProjectManager/Source/ProjectManagerDefs.h +++ b/Code/Tools/ProjectManager/Source/ProjectManagerDefs.h @@ -15,8 +15,10 @@ namespace O3DE::ProjectManager inline constexpr static int ProjectPreviewImageHeight = 280; inline constexpr static int ProjectTemplateImageWidth = 92; - static const QString ProjectBuildPathPostfix = "build/windows_vs2019"; + static const QString ProjectBuildDirectoryName = "build"; + extern const QString ProjectBuildPathPostfix; static const QString ProjectBuildPathCmakeFiles = "CMakeFiles"; static const QString ProjectBuildErrorLogName = "CMakeProjectBuildError.log"; + static const QString ProjectCacheDirectoryName = "Cache"; static const QString ProjectPreviewImagePath = "preview.png"; } // namespace O3DE::ProjectManager diff --git a/Code/Tools/ProjectManager/Source/ProjectUtils.cpp b/Code/Tools/ProjectManager/Source/ProjectUtils.cpp index 9742d875e4..52900f1b28 100644 --- a/Code/Tools/ProjectManager/Source/ProjectUtils.cpp +++ b/Code/Tools/ProjectManager/Source/ProjectUtils.cpp @@ -7,6 +7,7 @@ */ #include +#include #include #include @@ -18,6 +19,8 @@ #include #include #include +#include +#include namespace O3DE::ProjectManager { @@ -58,29 +61,63 @@ namespace O3DE::ProjectManager return false; } + static bool SkipFilePaths(const QString& curPath, QStringList& skippedPaths, QStringList& deeperSkippedPaths) + { + bool skip = false; + for (const QString& skippedPath : skippedPaths) + { + QString nativeSkippedPath = QDir::toNativeSeparators(skippedPath); + QString firstSectionSkippedPath = nativeSkippedPath.section(QDir::separator(), 0, 0); + if (curPath == firstSectionSkippedPath) + { + // We are at the end of the path to skip, so skip it + if (nativeSkippedPath == firstSectionSkippedPath) + { + skippedPaths.removeAll(skippedPath); + skip = true; + break; + } + // Append the next section of the skipped path + else + { + deeperSkippedPaths.append(nativeSkippedPath.section(QDir::separator(), 1)); + } + } + } + + return skip; + } + typedef AZStd::function StatusFunction; - static void RecursiveGetAllFiles(const QDir& directory, QStringList& outFileList, qint64& outTotalSizeInBytes, StatusFunction statusCallback) + static void RecursiveGetAllFiles(const QDir& directory, QStringList& skippedPaths, int& outFileCount, qint64& outTotalSizeInBytes, StatusFunction statusCallback) { const QStringList entries = directory.entryList(QDir::Dirs | QDir::Files | QDir::NoSymLinks | QDir::NoDotAndDotDot); for (const QString& entryPath : entries) { const QString filePath = QDir::toNativeSeparators(QString("%1/%2").arg(directory.path()).arg(entryPath)); + + QStringList deeperSkippedPaths; + if (SkipFilePaths(entryPath, skippedPaths, deeperSkippedPaths)) + { + continue; + } + QFileInfo fileInfo(filePath); if (fileInfo.isDir()) { QDir subDirectory(filePath); - RecursiveGetAllFiles(subDirectory, outFileList, outTotalSizeInBytes, statusCallback); + RecursiveGetAllFiles(subDirectory, deeperSkippedPaths, outFileCount, outTotalSizeInBytes, statusCallback); } else { - outFileList.push_back(filePath); + ++outFileCount; outTotalSizeInBytes += fileInfo.size(); const int updateStatusEvery = 64; - if (outFileList.size() % updateStatusEvery == 0) + if (outFileCount % updateStatusEvery == 0) { - statusCallback(outFileList.size(), outTotalSizeInBytes); + statusCallback(outFileCount, outTotalSizeInBytes); } } } @@ -89,7 +126,8 @@ namespace O3DE::ProjectManager static bool CopyDirectory(QProgressDialog* progressDialog, const QString& origPath, const QString& newPath, - QStringList& filesToCopy, + QStringList& skippedPaths, + int filesToCopyCount, int& outNumCopiedFiles, qint64 totalSizeToCopy, qint64& outCopiedFileSize, @@ -101,18 +139,24 @@ namespace O3DE::ProjectManager return false; } - for (QString directory : original.entryList(QDir::Dirs | QDir::NoDotAndDotDot)) + for (const QString& directory : original.entryList(QDir::Dirs | QDir::NoDotAndDotDot)) { if (progressDialog->wasCanceled()) { return false; } + QStringList deeperSkippedPaths; + if (SkipFilePaths(directory, skippedPaths, deeperSkippedPaths)) + { + continue; + } + QString newDirectoryPath = newPath + QDir::separator() + directory; original.mkpath(newDirectoryPath); - if (!CopyDirectory(progressDialog, origPath + QDir::separator() + directory, - newDirectoryPath, filesToCopy, outNumCopiedFiles, totalSizeToCopy, outCopiedFileSize, showIgnoreFileDialog)) + if (!CopyDirectory(progressDialog, origPath + QDir::separator() + directory, newDirectoryPath, deeperSkippedPaths, + filesToCopyCount, outNumCopiedFiles, totalSizeToCopy, outCopiedFileSize, showIgnoreFileDialog)) { return false; } @@ -120,18 +164,25 @@ namespace O3DE::ProjectManager QLocale locale; const float progressDialogRangeHalf = qFabs(progressDialog->maximum() - progressDialog->minimum()) * 0.5f; - for (QString file : original.entryList(QDir::Files)) + for (const QString& file : original.entryList(QDir::Files)) { if (progressDialog->wasCanceled()) { return false; } + // Unused by this function but neccesary to pass in to SkipFilePaths + QStringList deeperSkippedPaths; + if (SkipFilePaths(file, skippedPaths, deeperSkippedPaths)) + { + continue; + } + // Progress window update { // Weight in the number of already copied files as well as the copied bytes to get a better progress indication // for cases combining many small files and some really large files. - const float normalizedNumFiles = static_cast(outNumCopiedFiles) / filesToCopy.count(); + const float normalizedNumFiles = static_cast(outNumCopiedFiles) / filesToCopyCount; const float normalizedFileSize = static_cast(outCopiedFileSize) / totalSizeToCopy; const int progress = normalizedNumFiles * progressDialogRangeHalf + normalizedFileSize * progressDialogRangeHalf; progressDialog->setValue(progress); @@ -139,7 +190,7 @@ namespace O3DE::ProjectManager const QString copiedFileSizeString = locale.formattedDataSize(outCopiedFileSize); const QString totalFileSizeString = locale.formattedDataSize(totalSizeToCopy); progressDialog->setLabelText(QString("Coping file %1 of %2 (%3 of %4) ...").arg(QString::number(outNumCopiedFiles), - QString::number(filesToCopy.count()), + QString::number(filesToCopyCount), copiedFileSizeString, totalFileSizeString)); qApp->processEvents(QEventLoop::ExcludeUserInputEvents); @@ -193,6 +244,39 @@ namespace O3DE::ProjectManager return true; } + static bool ClearProjectBuildArtifactsAndCache(const QString& origPath, const QString& newPath, QWidget* parent) + { + QDir buildDirectory = QDir(newPath); + if ((!buildDirectory.cd(ProjectBuildDirectoryName) || !DeleteProjectFiles(buildDirectory.path(), true)) + && QDir(origPath).cd(ProjectBuildDirectoryName)) + { + QMessageBox::warning( + parent, + QObject::tr("Clear Build Artifacts"), + QObject::tr("Build artifacts failed to delete for moved project. Please manually delete build directory at \"%1\"") + .arg(buildDirectory.path()), + QMessageBox::Close); + + return false; + } + + QDir cacheDirectory = QDir(newPath); + if ((!cacheDirectory.cd(ProjectCacheDirectoryName) || !DeleteProjectFiles(cacheDirectory.path(), true)) + && QDir(origPath).cd(ProjectCacheDirectoryName)) + { + QMessageBox::warning( + parent, + QObject::tr("Clear Asset Cache"), + QObject::tr("Asset cache failed to delete for moved project. Please manually delete cache directory at \"%1\"") + .arg(cacheDirectory.path()), + QMessageBox::Close); + + return false; + } + + return false; + } + bool AddProjectDialog(QWidget* parent) { QString path = QDir::toNativeSeparators(QFileDialog::getExistingDirectory(parent, QObject::tr("Select Project Directory"))); @@ -214,7 +298,7 @@ namespace O3DE::ProjectManager return PythonBindingsInterface::Get()->RemoveProject(path); } - bool CopyProjectDialog(const QString& origPath, QWidget* parent) + bool CopyProjectDialog(const QString& origPath, ProjectInfo& newProjectInfo, QWidget* parent) { bool copyResult = false; @@ -224,6 +308,8 @@ namespace O3DE::ProjectManager QFileDialog::getExistingDirectory(parent, QObject::tr("Select New Project Directory"), parentOrigDir.path())); if (!newPath.isEmpty()) { + newProjectInfo.m_path = newPath; + if (!WarnDirectoryOverwrite(newPath, parent)) { return false; @@ -235,7 +321,7 @@ namespace O3DE::ProjectManager return copyResult; } - bool CopyProject(const QString& origPath, const QString& newPath, QWidget* parent) + bool CopyProject(const QString& origPath, const QString& newPath, QWidget* parent, bool skipRegister) { // Disallow copying from or into subdirectory if (IsDirectoryDescedent(origPath, newPath) || IsDirectoryDescedent(newPath, origPath)) @@ -243,8 +329,13 @@ namespace O3DE::ProjectManager return false; } - QStringList filesToCopy; + int filesToCopyCount = 0; qint64 totalSizeInBytes = 0; + QStringList skippedPaths + { + ProjectBuildDirectoryName, + ProjectCacheDirectoryName + }; QProgressDialog* progressDialog = new QProgressDialog(parent); progressDialog->setAutoClose(true); @@ -255,7 +346,8 @@ namespace O3DE::ProjectManager progressDialog->show(); QLocale locale; - RecursiveGetAllFiles(origPath, filesToCopy, totalSizeInBytes, [=](int fileCount, int sizeInBytes) + QStringList getFilesSkippedPaths(skippedPaths); + RecursiveGetAllFiles(origPath, getFilesSkippedPaths, filesToCopyCount, totalSizeInBytes, [=](int fileCount, int sizeInBytes) { // Create a human-readable version of the file size. const QString fileSizeString = locale.formattedDataSize(sizeInBytes); @@ -274,8 +366,10 @@ namespace O3DE::ProjectManager // Phase 1: Copy files bool showIgnoreFileDialog = true; - bool success = CopyDirectory(progressDialog, origPath, newPath, filesToCopy, numFilesCopied, totalSizeInBytes, copiedFileSize, showIgnoreFileDialog); - if (success) + QStringList copyFilesSkippedPaths(skippedPaths); + bool success = CopyDirectory(progressDialog, origPath, newPath, copyFilesSkippedPaths, filesToCopyCount, numFilesCopied, + totalSizeInBytes, copiedFileSize, showIgnoreFileDialog); + if (success && !skipRegister) { // Phase 2: Register project success = RegisterProject(newPath); @@ -298,7 +392,7 @@ namespace O3DE::ProjectManager QDir projectDirectory(path); if (projectDirectory.exists()) { - // Check if there is an actual project hereor just force it + // Check if there is an actual project here or just force it if (force || PythonBindingsInterface::Get()->GetProject(path).IsSuccess()) { return projectDirectory.removeRecursively(); @@ -308,12 +402,12 @@ namespace O3DE::ProjectManager return false; } - bool MoveProject(QString origPath, QString newPath, QWidget* parent, bool ignoreRegister) + bool MoveProject(QString origPath, QString newPath, QWidget* parent, bool skipRegister) { origPath = QDir::toNativeSeparators(origPath); newPath = QDir::toNativeSeparators(newPath); - if (!WarnDirectoryOverwrite(newPath, parent) || (!ignoreRegister && !UnregisterProject(origPath))) + if (!WarnDirectoryOverwrite(newPath, parent) || (!skipRegister && !UnregisterProject(origPath))) { return false; } @@ -333,8 +427,13 @@ namespace O3DE::ProjectManager DeleteProjectFiles(origPath, true); } + else + { + // If directoy rename succeeded then build and cache directories need to be deleted seperately + ClearProjectBuildArtifactsAndCache(origPath, newPath, parent); + } - if (!ignoreRegister && !RegisterProject(newPath)) + if (!skipRegister && !RegisterProject(newPath)) { return false; } @@ -375,46 +474,27 @@ namespace O3DE::ProjectManager return true; } - static bool IsVS2019Installed_internal() + bool FindSupportedCompiler(QWidget* parent) { - QProcessEnvironment environment = QProcessEnvironment::systemEnvironment(); - QString programFilesPath = environment.value("ProgramFiles(x86)"); - QString vsWherePath = programFilesPath + "\\Microsoft Visual Studio\\Installer\\vswhere.exe"; + auto findCompilerResult = FindSupportedCompilerForPlatform(); - QFileInfo vsWhereFile(vsWherePath); - if (vsWhereFile.exists() && vsWhereFile.isFile()) + if (!findCompilerResult.IsSuccess()) { - QProcess vsWhereProcess; - vsWhereProcess.setProcessChannelMode(QProcess::MergedChannels); - - vsWhereProcess.start( - vsWherePath, - QStringList{ "-version", "16.0", "-latest", "-requires", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", - "-property", "isComplete" }); - - if (!vsWhereProcess.waitForStarted()) - { - return false; - } - - while (vsWhereProcess.waitForReadyRead()) - { - } - - QString vsWhereOutput(vsWhereProcess.readAllStandardOutput()); - if (vsWhereOutput.startsWith("1")) - { - return true; - } + QMessageBox vsWarningMessage(parent); + vsWarningMessage.setIcon(QMessageBox::Warning); + vsWarningMessage.setWindowTitle(QObject::tr("Create Project")); + // Makes link clickable + vsWarningMessage.setTextFormat(Qt::RichText); + vsWarningMessage.setText(findCompilerResult.GetError()); + vsWarningMessage.setStandardButtons(QMessageBox::Close); + + QSpacerItem* horizontalSpacer = new QSpacerItem(600, 0, QSizePolicy::Minimum, QSizePolicy::Expanding); + QGridLayout* layout = reinterpret_cast(vsWarningMessage.layout()); + layout->addItem(horizontalSpacer, layout->rowCount(), 0, 1, layout->columnCount()); + vsWarningMessage.exec(); } - return false; - } - - bool IsVS2019Installed() - { - static bool vs2019Installed = IsVS2019Installed_internal(); - return vs2019Installed; + return findCompilerResult.IsSuccess(); } ProjectManagerScreen GetProjectManagerScreen(const QString& screen) diff --git a/Code/Tools/ProjectManager/Source/ProjectUtils.h b/Code/Tools/ProjectManager/Source/ProjectUtils.h index a79c62fd95..d06b8c2c8b 100644 --- a/Code/Tools/ProjectManager/Source/ProjectUtils.h +++ b/Code/Tools/ProjectManager/Source/ProjectUtils.h @@ -8,7 +8,10 @@ #pragma once #include +#include + #include +#include namespace O3DE::ProjectManager { @@ -17,14 +20,15 @@ namespace O3DE::ProjectManager bool AddProjectDialog(QWidget* parent = nullptr); bool RegisterProject(const QString& path); bool UnregisterProject(const QString& path); - bool CopyProjectDialog(const QString& origPath, QWidget* parent = nullptr); - bool CopyProject(const QString& origPath, const QString& newPath, QWidget* parent); + bool CopyProjectDialog(const QString& origPath, ProjectInfo& newProjectInfo, QWidget* parent = nullptr); + bool CopyProject(const QString& origPath, const QString& newPath, QWidget* parent, bool skipRegister = false); bool DeleteProjectFiles(const QString& path, bool force = false); - bool MoveProject(QString origPath, QString newPath, QWidget* parent = nullptr, bool ignoreRegister = false); + bool MoveProject(QString origPath, QString newPath, QWidget* parent, bool skipRegister = false); bool ReplaceFile(const QString& origFile, const QString& newFile, QWidget* parent = nullptr, bool interactive = true); - bool IsVS2019Installed(); + bool FindSupportedCompiler(QWidget* parent = nullptr); + AZ::Outcome FindSupportedCompilerForPlatform(); ProjectManagerScreen GetProjectManagerScreen(const QString& screen); } // namespace ProjectUtils diff --git a/Code/Tools/ProjectManager/Source/ProjectsScreen.cpp b/Code/Tools/ProjectManager/Source/ProjectsScreen.cpp index a6295716c9..e27eedd122 100644 --- a/Code/Tools/ProjectManager/Source/ProjectsScreen.cpp +++ b/Code/Tools/ProjectManager/Source/ProjectsScreen.cpp @@ -385,14 +385,17 @@ namespace O3DE::ProjectManager emit ChangeScreenRequest(ProjectManagerScreen::UpdateProject); } } - void ProjectsScreen::HandleCopyProject(const QString& projectPath) + void ProjectsScreen::HandleCopyProject(const ProjectInfo& projectInfo) { - if (!WarnIfInBuildQueue(projectPath)) + if (!WarnIfInBuildQueue(projectInfo.m_path)) { + ProjectInfo newProjectInfo(projectInfo); + // Open file dialog and choose location for copied project then register copy with O3DE - if (ProjectUtils::CopyProjectDialog(projectPath, this)) + if (ProjectUtils::CopyProjectDialog(projectInfo.m_path, newProjectInfo, this)) { ResetProjectsContent(); + emit NotifyBuildProject(newProjectInfo); emit ChangeScreenRequest(ProjectManagerScreen::Projects); } } @@ -517,7 +520,7 @@ namespace O3DE::ProjectManager bool ProjectsScreen::StartProjectBuild(const ProjectInfo& projectInfo) { - if (ProjectUtils::IsVS2019Installed()) + if (ProjectUtils::FindSupportedCompiler(this)) { QMessageBox::StandardButton buildProject = QMessageBox::information( this, diff --git a/Code/Tools/ProjectManager/Source/ProjectsScreen.h b/Code/Tools/ProjectManager/Source/ProjectsScreen.h index 62867e3a61..45605ab678 100644 --- a/Code/Tools/ProjectManager/Source/ProjectsScreen.h +++ b/Code/Tools/ProjectManager/Source/ProjectsScreen.h @@ -45,7 +45,7 @@ namespace O3DE::ProjectManager void HandleAddProjectButton(); void HandleOpenProject(const QString& projectPath); void HandleEditProject(const QString& projectPath); - void HandleCopyProject(const QString& projectPath); + void HandleCopyProject(const ProjectInfo& projectInfo); void HandleRemoveProject(const QString& projectPath); void HandleDeleteProject(const QString& projectPath); diff --git a/Code/Tools/ProjectManager/Source/UpdateProjectCtrl.cpp b/Code/Tools/ProjectManager/Source/UpdateProjectCtrl.cpp index dc6cb4467e..981a9352f7 100644 --- a/Code/Tools/ProjectManager/Source/UpdateProjectCtrl.cpp +++ b/Code/Tools/ProjectManager/Source/UpdateProjectCtrl.cpp @@ -220,11 +220,13 @@ namespace O3DE::ProjectManager // Move project first to avoid trying to update settings at the new location before it has been moved there if (newProjectSettings.m_path != m_projectInfo.m_path) { - if (!ProjectUtils::MoveProject(m_projectInfo.m_path, newProjectSettings.m_path)) + if (!ProjectUtils::MoveProject(m_projectInfo.m_path, newProjectSettings.m_path, this)) { QMessageBox::critical(this, tr("Project move failed"), tr("Failed to move project.")); return false; } + + emit NotifyBuildProject(newProjectSettings); } // Update project if settings changed diff --git a/Code/Tools/ProjectManager/tests/UtilsTests.cpp b/Code/Tools/ProjectManager/tests/UtilsTests.cpp index ee766638e5..bfe26ba760 100644 --- a/Code/Tools/ProjectManager/tests/UtilsTests.cpp +++ b/Code/Tools/ProjectManager/tests/UtilsTests.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -26,16 +27,31 @@ namespace O3DE::ProjectManager : public ::UnitTest::ScopedAllocatorSetupFixture { public: + static inline QString ReplaceFirstAWithB(const QString& originalString) + { + QString bString(originalString); + return bString.replace(bString.indexOf('A'), 1, 'B'); + } + ProjectManagerUtilsTests() { m_application = AZStd::make_unique(); m_application->Init(false); + m_projectAPath = "ProjectA"; + + // Replaces first 'A' with 'B' + m_projectBPath = ReplaceFirstAWithB(m_projectAPath); + m_projectABuildPath = QString("%1%2%3").arg(m_projectAPath, QDir::separator(), ProjectBuildDirectoryName); + m_projectBBuildPath = ReplaceFirstAWithB(m_projectABuildPath); + QDir dir; - dir.mkdir("ProjectA"); - dir.mkdir("ProjectB"); + dir.mkpath(m_projectABuildPath); + dir.mkdir(m_projectBPath); - QFile origFile("ProjectA/origFile.txt"); + m_projectAOrigFilePath = QString("%1%2%3").arg(m_projectAPath, QDir::separator(), "origFile.txt"); + m_projectBOrigFilePath = ReplaceFirstAWithB(m_projectAOrigFilePath); + QFile origFile(m_projectAOrigFilePath); if (origFile.open(QIODevice::ReadWrite)) { QTextStream stream(&origFile); @@ -43,63 +59,153 @@ namespace O3DE::ProjectManager origFile.close(); } - QFile replaceFile("ProjectA/replaceFile.txt"); + m_projectAReplaceFilePath = QString("%1%2%3").arg(m_projectAPath, QDir::separator(), "replaceFile.txt"); + m_projectBReplaceFilePath = ReplaceFirstAWithB(m_projectAReplaceFilePath); + QFile replaceFile(m_projectAReplaceFilePath); if (replaceFile.open(QIODevice::ReadWrite)) { QTextStream stream(&replaceFile); stream << "replace" << Qt::endl; replaceFile.close(); } + + m_projectABuildFilePath = QString("%1%2%3").arg(m_projectABuildPath, QDir::separator(), "build.obj"); + m_projectBBuildFilePath = ReplaceFirstAWithB(m_projectABuildFilePath); + QFile buildFile(m_projectABuildFilePath); + if (buildFile.open(QIODevice::ReadWrite)) + { + QTextStream stream(&buildFile); + stream << "x0FFFFFFFF" << Qt::endl; + buildFile.close(); + } } ~ProjectManagerUtilsTests() { - QDir dirA("ProjectA"); + QDir dirA(m_projectAPath); dirA.removeRecursively(); - QDir dirB("ProjectB"); + QDir dirB(m_projectBPath); dirB.removeRecursively(); m_application.reset(); } AZStd::unique_ptr m_application; + + QString m_projectAPath; + QString m_projectAOrigFilePath; + QString m_projectAReplaceFilePath; + QString m_projectABuildPath; + QString m_projectABuildFilePath; + QString m_projectBPath; + QString m_projectBOrigFilePath; + QString m_projectBReplaceFilePath; + QString m_projectBBuildPath; + QString m_projectBBuildFilePath; + }; #if AZ_TRAIT_DISABLE_FAILED_PROJECT_MANAGER_TESTS - TEST_F(ProjectManagerUtilsTests, DISABLED_MoveProject_Succeeds) + TEST_F(ProjectManagerUtilsTests, DISABLED_MoveProject_MovesExpectedFiles) +#else + TEST_F(ProjectManagerUtilsTests, MoveProject_MovesExpectedFiles) +#endif // !AZ_TRAIT_DISABLE_FAILED_PROJECT_MANAGER_TESTS + { + EXPECT_TRUE(MoveProject( + QDir::currentPath() + QDir::separator() + m_projectAPath, + QDir::currentPath() + QDir::separator() + m_projectBPath, + nullptr, true)); + + QFileInfo origFile(m_projectAOrigFilePath); + EXPECT_FALSE(origFile.exists()); + + QFileInfo replaceFile(m_projectAReplaceFilePath); + EXPECT_FALSE(replaceFile.exists()); + + QFileInfo origFileMoved(m_projectBOrigFilePath); + EXPECT_TRUE(origFileMoved.exists() && origFileMoved.isFile()); + + QFileInfo replaceFileMoved(m_projectBReplaceFilePath); + EXPECT_TRUE(replaceFileMoved.exists() && replaceFileMoved.isFile()); + } + +#if AZ_TRAIT_DISABLE_FAILED_PROJECT_MANAGER_TESTS + TEST_F(ProjectManagerUtilsTests, DISABLED_MoveProject_DoesntMoveBuild) #else - TEST_F(ProjectManagerUtilsTests, MoveProject_Succeeds) + TEST_F(ProjectManagerUtilsTests, MoveProject_DoesntMoveBuild) #endif // !AZ_TRAIT_DISABLE_FAILED_PROJECT_MANAGER_TESTS { EXPECT_TRUE(MoveProject( - QDir::currentPath() + QDir::separator() + "ProjectA", - QDir::currentPath() + QDir::separator() + "ProjectB", + QDir::currentPath() + QDir::separator() + m_projectAPath, + QDir::currentPath() + QDir::separator() + m_projectBPath, nullptr, true)); - QFileInfo origFile("ProjectA/origFile.txt"); - EXPECT_TRUE(!origFile.exists()); + QFileInfo origFile(m_projectAOrigFilePath); + EXPECT_FALSE(origFile.exists()); - QFileInfo replaceFile("ProjectA/replaceFile.txt"); - EXPECT_TRUE(!replaceFile.exists()); + QFileInfo origFileMoved(m_projectBOrigFilePath); + EXPECT_TRUE(origFileMoved.exists() && origFileMoved.isFile()); + + QDir buildDir(m_projectBBuildPath); + EXPECT_FALSE(buildDir.exists()); + } + +#if AZ_TRAIT_DISABLE_FAILED_PROJECT_MANAGER_TESTS + TEST_F(ProjectManagerUtilsTests, DISABLED_CopyProject_CopiesExpectedFiles) +#else + TEST_F(ProjectManagerUtilsTests, CopyProject_CopiesExpectedFiles) +#endif // !AZ_TRAIT_DISABLE_FAILED_PROJECT_MANAGER_TESTS + { + EXPECT_TRUE(CopyProject( + QDir::currentPath() + QDir::separator() + m_projectAPath, + QDir::currentPath() + QDir::separator() + m_projectBPath, + nullptr, true)); - QFileInfo origFileMoved("ProjectB/origFile.txt"); + QFileInfo origFile(m_projectAOrigFilePath); + EXPECT_TRUE(origFile.exists()); + + QFileInfo replaceFile(m_projectAReplaceFilePath); + EXPECT_TRUE(replaceFile.exists()); + + QFileInfo origFileMoved(m_projectBOrigFilePath); EXPECT_TRUE(origFileMoved.exists() && origFileMoved.isFile()); - QFileInfo replaceFileMoved("ProjectB/replaceFile.txt"); + QFileInfo replaceFileMoved(m_projectBReplaceFilePath); EXPECT_TRUE(replaceFileMoved.exists() && replaceFileMoved.isFile()); } +#if AZ_TRAIT_DISABLE_FAILED_PROJECT_MANAGER_TESTS + TEST_F(ProjectManagerUtilsTests, DISABLED_CopyProject_DoesntCopyBuild) +#else + TEST_F(ProjectManagerUtilsTests, CopyProject_DoesntCopyBuild) +#endif // !AZ_TRAIT_DISABLE_FAILED_PROJECT_MANAGER_TESTS + { + EXPECT_TRUE(CopyProject( + QDir::currentPath() + QDir::separator() + m_projectAPath, + QDir::currentPath() + QDir::separator() + m_projectBPath, + nullptr, true)); + + QFileInfo origFile(m_projectAOrigFilePath); + EXPECT_TRUE(origFile.exists()); + + QFileInfo origFileMoved(m_projectBOrigFilePath); + EXPECT_TRUE(origFileMoved.exists() && origFileMoved.isFile()); + + QDir buildDir(m_projectBBuildPath); + EXPECT_FALSE(buildDir.exists()); + } + #if AZ_TRAIT_DISABLE_FAILED_PROJECT_MANAGER_TESTS TEST_F(ProjectManagerUtilsTests, DISABLED_ReplaceFile_Succeeds) #else TEST_F(ProjectManagerUtilsTests, ReplaceFile_Succeeds) #endif // !AZ_TRAIT_DISABLE_FAILED_PROJECT_MANAGER_TESTS { - EXPECT_TRUE(ReplaceFile("ProjectA/origFile.txt", "ProjectA/replaceFile.txt", nullptr, false)); + EXPECT_TRUE(ReplaceFile(m_projectAOrigFilePath, m_projectAReplaceFilePath, nullptr, false)); - QFile origFile("ProjectA/origFile.txt"); - if (origFile.open(QIODevice::ReadOnly)) + QFile origFile(m_projectAOrigFilePath); + EXPECT_TRUE(origFile.open(QIODevice::ReadOnly)); { QTextStream stream(&origFile); QString line = stream.readLine(); @@ -107,10 +213,6 @@ namespace O3DE::ProjectManager origFile.close(); } - else - { - FAIL(); - } } } // namespace ProjectUtils } // namespace O3DE::ProjectManager diff --git a/Gems/AssetMemoryAnalyzer/gem.json b/Gems/AssetMemoryAnalyzer/gem.json index 8c63d51e1a..45610e417c 100644 --- a/Gems/AssetMemoryAnalyzer/gem.json +++ b/Gems/AssetMemoryAnalyzer/gem.json @@ -6,7 +6,7 @@ "type": "Code", "summary": "The Asset Memory Analyzer Gem provides tools to profile asset memory usage in Open 3D Engine through ImGUI (Immediate Mode Graphical User Interface).", "canonical_tags": ["Gem"], - "user_tags": ["Debug", "Utillity", "Tools"], + "user_tags": ["Debug", "Utility", "Tools"], "icon_path": "preview.png", "requirements": "" } diff --git a/Gems/Atom/CMakeLists.txt b/Gems/Atom/CMakeLists.txt index 3a5793d6d9..a71afa64e0 100644 --- a/Gems/Atom/CMakeLists.txt +++ b/Gems/Atom/CMakeLists.txt @@ -15,11 +15,3 @@ add_subdirectory(RPI) add_subdirectory(Tools) add_subdirectory(Utils) -# The "Atom" Gem will alias the real Atom_AtomBridge target variants -# allows the enabling and disabling the "Atom" Gem to build the pre-requisite dependencies -ly_create_alias(NAME Atom.Clients NAMESPACE Gem TARGETS Gem::Atom_AtomBridge.Clients) -ly_create_alias(NAME Atom.Servers NAMESPACE Gem TARGETS Gem::Atom_AtomBridge.Servers) -if(PAL_TRAIT_BUILD_HOST_TOOLS) - ly_create_alias(NAME Atom.Builders NAMESPACE Gem TARGETS Gem::Atom_AtomBridge.Builders) - ly_create_alias(NAME Atom.Tools NAMESPACE Gem TARGETS Gem::Atom_AtomBridge.Tools) -endif() diff --git a/Gems/AtomContent/CMakeLists.txt b/Gems/AtomContent/CMakeLists.txt index e93010eec1..ee06f07bc3 100644 --- a/Gems/AtomContent/CMakeLists.txt +++ b/Gems/AtomContent/CMakeLists.txt @@ -7,3 +7,7 @@ # add_subdirectory(ReferenceMaterials) add_subdirectory(Sponza) + +if(PAL_TRAIT_BUILD_HOST_TOOLS) + ly_create_alias(NAME AtomContent.Builders NAMESPACE Gem TARGETS Gem::AtomContent_ReferenceMaterials.Builders Gem::AtomContent_Sponza.Builders) +endif() diff --git a/Gems/AtomLyIntegration/AtomBridge/Code/CMakeLists.txt b/Gems/AtomLyIntegration/AtomBridge/Code/CMakeLists.txt index becaa71a36..e53a0c6281 100644 --- a/Gems/AtomLyIntegration/AtomBridge/Code/CMakeLists.txt +++ b/Gems/AtomLyIntegration/AtomBridge/Code/CMakeLists.txt @@ -116,3 +116,18 @@ if(PAL_TRAIT_BUILD_HOST_TOOLS) ly_create_alias(NAME Atom_AtomBridge.Builders NAMESPACE Gem TARGETS Gem::Atom_AtomBridge.Editor) ly_create_alias(NAME Atom_AtomBridge.Tools NAMESPACE Gem TARGETS Gem::Atom_AtomBridge.Editor) endif() + +# The "Atom" Gem will alias the real Atom_AtomBridge target variants +# allows the enabling and disabling the "Atom" Gem to build the pre-requisite dependencies +# The "AtomLyIntegration" Gem will also alias the real Atom_AtomBridge target variants +# The Atom Gem does the same at the moment. +ly_create_alias(NAME Atom.Clients NAMESPACE Gem TARGETS Gem::Atom_AtomBridge.Clients) +ly_create_alias(NAME Atom.Servers NAMESPACE Gem TARGETS Gem::Atom_AtomBridge.Servers) +ly_create_alias(NAME AtomLyIntegration.Clients NAMESPACE Gem TARGETS Gem::Atom_AtomBridge.Clients) +ly_create_alias(NAME AtomLyIntegration.Servers NAMESPACE Gem TARGETS Gem::Atom_AtomBridge.Servers) +if(PAL_TRAIT_BUILD_HOST_TOOLS) + ly_create_alias(NAME Atom.Builders NAMESPACE Gem TARGETS Gem::Atom_AtomBridge.Builders) + ly_create_alias(NAME Atom.Tools NAMESPACE Gem TARGETS Gem::Atom_AtomBridge.Tools) + ly_create_alias(NAME AtomLyIntegration.Builders NAMESPACE Gem TARGETS Gem::Atom_AtomBridge.Builders) + ly_create_alias(NAME AtomLyIntegration.Tools NAMESPACE Gem TARGETS Gem::Atom_AtomBridge.Tools) +endif() diff --git a/Gems/AtomLyIntegration/CMakeLists.txt b/Gems/AtomLyIntegration/CMakeLists.txt index 6b817b5ee5..4ab976d68f 100644 --- a/Gems/AtomLyIntegration/CMakeLists.txt +++ b/Gems/AtomLyIntegration/CMakeLists.txt @@ -16,11 +16,3 @@ add_subdirectory(AtomBridge) add_subdirectory(AtomViewportDisplayInfo) add_subdirectory(AtomViewportDisplayIcons) -# The "AtomLyIntegration" Gem will also alias the real Atom_AtomBridge target variants -# The Atom Gem does the same at the moment. -ly_create_alias(NAME AtomLyIntegration.Clients NAMESPACE Gem TARGETS Gem::Atom_AtomBridge.Clients) -ly_create_alias(NAME AtomLyIntegration.Servers NAMESPACE Gem TARGETS Gem::Atom_AtomBridge.Servers) -if(PAL_TRAIT_BUILD_HOST_TOOLS) - ly_create_alias(NAME AtomLyIntegration.Builders NAMESPACE Gem TARGETS Gem::Atom_AtomBridge.Builders) - ly_create_alias(NAME AtomLyIntegration.Tools NAMESPACE Gem TARGETS Gem::Atom_AtomBridge.Tools) -endif() diff --git a/Gems/AtomLyIntegration/EMotionFXAtom/Code/CMakeLists.txt b/Gems/AtomLyIntegration/EMotionFXAtom/Code/CMakeLists.txt index 2c172c1f86..5a08bab4fa 100644 --- a/Gems/AtomLyIntegration/EMotionFXAtom/Code/CMakeLists.txt +++ b/Gems/AtomLyIntegration/EMotionFXAtom/Code/CMakeLists.txt @@ -58,5 +58,7 @@ if(PAL_TRAIT_BUILD_HOST_TOOLS) PRIVATE AZ::AzCore Gem::EMotionFX_Atom.Static + RUNTIME_DEPENDENCIES + Gem::EMotionFX.Editor ) endif() diff --git a/Gems/AudioEngineWwise/gem.json b/Gems/AudioEngineWwise/gem.json index 8c6b5767b0..dc5f968bc7 100644 --- a/Gems/AudioEngineWwise/gem.json +++ b/Gems/AudioEngineWwise/gem.json @@ -6,7 +6,7 @@ "type": "Code", "summary": "The Wwise Audio Engine Gem provides support for Audiokinetic Wave Works Interactive Sound Engine (Wwise).", "canonical_tags": ["Gem"], - "user_tags": ["Audio", "Utiltity", "Tools"], + "user_tags": ["Audio", "Utility", "Tools"], "icon_path": "preview.png", "requirements": "Users will need to download WWise from the AudioKinetic web site: https://www.audiokinetic.com/download/" } diff --git a/Gems/AudioSystem/gem.json b/Gems/AudioSystem/gem.json index a1dbe9406f..a9058cb42e 100644 --- a/Gems/AudioSystem/gem.json +++ b/Gems/AudioSystem/gem.json @@ -6,7 +6,7 @@ "type": "Code", "summary": "The Audio System Gem provides the Audio Translation Layer (ATL) and Audio Controls Editor, which add support for audio in Open 3D Engine.", "canonical_tags": ["Gem"], - "user_tags": ["Audio", "Utiltity", "Tools"], + "user_tags": ["Audio", "Utility", "Tools"], "icon_path": "preview.png", "requirements": "" } diff --git a/Gems/ExpressionEvaluation/gem.json b/Gems/ExpressionEvaluation/gem.json index 9302af152a..92d5963891 100644 --- a/Gems/ExpressionEvaluation/gem.json +++ b/Gems/ExpressionEvaluation/gem.json @@ -6,7 +6,7 @@ "type": "Code", "summary": "The Expression Evaluation Gem provides a method for parsing and executing string expressions in Open 3D Engine.", "canonical_tags": ["Gem"], - "user_tags": ["Scripting", "Utiltity"], + "user_tags": ["Scripting", "Utility"], "icon_path": "preview.png", "requirements": "" } diff --git a/Gems/GameStateSamples/gem.json b/Gems/GameStateSamples/gem.json index ce0fcbd868..0241a8a1b7 100644 --- a/Gems/GameStateSamples/gem.json +++ b/Gems/GameStateSamples/gem.json @@ -6,7 +6,7 @@ "type": "Code", "summary": "The Game State Samples Gem provides a set of sample game states (built on top of the Game State Gem), including primary user selection, main menu, level loading, level running, and level paused.", "canonical_tags": ["Gem"], - "user_tags": ["Gameplay", "Samples", "Assets"], + "user_tags": ["Gameplay", "Sample", "Assets"], "icon_path": "preview.png", "requirements": "" } diff --git a/Gems/ScriptCanvas/gem.json b/Gems/ScriptCanvas/gem.json index 680ce6ef1f..413e58ef7e 100644 --- a/Gems/ScriptCanvas/gem.json +++ b/Gems/ScriptCanvas/gem.json @@ -6,7 +6,7 @@ "type": "Tool", "summary": "The Script Canvas Gem provides Open 3D Engine's visual scripting environment, Script Canvas.", "canonical_tags": ["Gem"], - "user_tags": ["Scripting", "Tools", "Utiltiy"], + "user_tags": ["Scripting", "Tools", "Utility"], "icon_path": "preview.png", "requirements": "" } diff --git a/Gems/SurfaceData/gem.json b/Gems/SurfaceData/gem.json index 1fb521ef58..24f7f1cb21 100644 --- a/Gems/SurfaceData/gem.json +++ b/Gems/SurfaceData/gem.json @@ -6,7 +6,7 @@ "type": "Code", "summary": "The Surface Data Gem provides functionality to emit signals or tags from surfaces such as meshes and terrain.", "canonical_tags": ["Gem"], - "user_tags": ["Environment", "Utiltiy", "Design"], + "user_tags": ["Environment", "Utility", "Design"], "icon_path": "preview.png", "requirements": "" } diff --git a/cmake/3rdPartyPackages.cmake b/cmake/3rdPartyPackages.cmake index ac37ffd141..2228e36653 100644 --- a/cmake/3rdPartyPackages.cmake +++ b/cmake/3rdPartyPackages.cmake @@ -530,6 +530,12 @@ function(ly_force_download_package package_name) execute_process(COMMAND ${CMAKE_COMMAND} -E tar xf ${temp_download_target} WORKING_DIRECTORY ${final_folder} COMMAND_ECHO STDOUT OUTPUT_VARIABLE unpack_result) + # For the runtime dependencies cases, we need the timestamps of the files coming from 3rdParty to be newer than the ones + # from the output so the new versions get copied over. The untar from the previous step preserves timestamps so they + # can produce binaries with older timestamps to the ones that are in the build output. + file(GLOB_RECURSE package_files LIST_DIRECTORIES false ${final_folder}/*) + file(TOUCH_NOCREATE ${package_files}) + if (NOT ${unpack_result} EQUAL 0) message(SEND_ERROR "ly_package: required package {package_name} could not be unpacked. Compile may fail! Enable LY_PACKAGE_DEBUG to debug.") return() diff --git a/cmake/FileUtil.cmake b/cmake/FileUtil.cmake index edb254e8b6..69a6ecc377 100644 --- a/cmake/FileUtil.cmake +++ b/cmake/FileUtil.cmake @@ -125,3 +125,42 @@ function(ly_file_read path content) set(${content} ${file_content} PARENT_SCOPE) set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${path}) endfunction() + + +#! ly_get_last_path_segment_concat_sha256 : Concatenates the last path segment of the absolute path +# with the first 8 characters of the absolute path SHA256 hash to make a unique relative path segment +function(ly_get_last_path_segment_concat_sha256 absolute_path output_path) + string(SHA256 target_source_hash ${absolute_path}) + string(SUBSTRING ${target_source_hash} 0 8 target_source_hash) + cmake_path(GET absolute_path FILENAME last_path_segment) + cmake_path(SET last_path_segment_sha256_path "${last_path_segment}-${target_source_hash}") + + set(${output_path} ${last_path_segment_sha256_path} PARENT_SCOPE) +endfunction() + +#! ly_get_engine_relative_source_dir: Attempts to form a path relative to the BASE_DIRECTORY. +# If that fails the last path segment of the absolute_target_source_dir concatenated with a SHA256 hash to form a target directory +# \arg:BASE_DIRECTORY - Directory to base relative path against. Defaults to LY_ROOT_FOLDER +function(ly_get_engine_relative_source_dir absolute_target_source_dir output_source_dir) + + set(options) + set(oneValueArgs BASE_DIRECTORY) + set(multiValueArgs) + cmake_parse_arguments(ly_get_engine_relative_source_dir "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + if(NOT ly_get_engine_relative_source_dir_BASE_DIRECTORY) + set(ly_get_engine_relative_source_dir_BASE_DIRECTORY ${LY_ROOT_FOLDER}) + endif() + + # Get a relative target source directory to the LY root folder if possible + # Otherwise use the final component name + cmake_path(IS_PREFIX LY_ROOT_FOLDER ${absolute_target_source_dir} is_target_source_dir_subdirectory_of_engine) + if(is_target_source_dir_subdirectory_of_engine) + cmake_path(RELATIVE_PATH absolute_target_source_dir BASE_DIRECTORY ${LY_ROOT_FOLDER} OUTPUT_VARIABLE relative_target_source_dir) + else() + ly_get_last_path_segment_concat_sha256(${absolute_target_source_dir} target_source_dir_last_path_segment) + unset(relative_target_source_dir) + cmake_path(APPEND relative_target_source_dir "External" ${target_source_dir_last_path_segment}) + endif() + + set(${output_source_dir} ${relative_target_source_dir} PARENT_SCOPE) +endfunction() diff --git a/cmake/Gems.cmake b/cmake/Gems.cmake index 67edda9ac6..bcb619fd8d 100644 --- a/cmake/Gems.cmake +++ b/cmake/Gems.cmake @@ -91,6 +91,13 @@ function(ly_create_alias) # Replace the CMake list separator with a space to replicate the space separated TARGETS arguments string(REPLACE ";" " " create_alias_args "${ly_create_alias_NAME},${ly_create_alias_NAMESPACE},${ly_create_alias_TARGETS}") set_property(DIRECTORY APPEND PROPERTY LY_CREATE_ALIAS_ARGUMENTS "${create_alias_args}") + + # Store the directory path in the GLOBAL property so that it can be accessed + # in the layout install logic. Skip if the directory has already been added + get_property(ly_all_target_directories GLOBAL PROPERTY LY_ALL_TARGET_DIRECTORIES) + if(NOT CMAKE_CURRENT_SOURCE_DIR IN_LIST ly_all_target_directories) + set_property(GLOBAL APPEND PROPERTY LY_ALL_TARGET_DIRECTORIES ${CMAKE_CURRENT_SOURCE_DIR}) + endif() endfunction() # ly_enable_gems diff --git a/cmake/Packaging.cmake b/cmake/Packaging.cmake index 7b7ce5b02b..c4d5be534c 100644 --- a/cmake/Packaging.cmake +++ b/cmake/Packaging.cmake @@ -126,37 +126,32 @@ install(FILES ${_cmake_package_dest} DESTINATION ./Tools/Redistributables/CMake ) -# temporary workaround for acquiring the 3rd party SPDX license manifest, the desired location is from -# another git repository that's private. once it's public, only how the URL is formed should change -set(LY_INSTALLER_3RD_PARTY_LICENSE_URL "" CACHE STRING "URL to the 3rd party SPDX license manifest file for inclusion in packaging.") - -if(${LY_VERSION_STRING} VERSION_GREATER "0.0.0.0" AND NOT LY_INSTALLER_3RD_PARTY_LICENSE_URL) - message(FATAL_ERROR "Missing required URL for the 3rd party SPDX license manifest file. " - "Please specifiy where to acquire the file via LY_INSTALLER_3RD_PARTY_LICENSE_URL") -endif() - -string(REPLACE "/" ";" _url_components ${LY_INSTALLER_3RD_PARTY_LICENSE_URL}) -list(POP_BACK _url_components _3rd_party_license_filename) - -set(_3rd_party_license_dest ${CPACK_BINARY_DIR}/${_3rd_party_license_filename}) - -# use the plain file downloader as we don't have the file hash available and using a dummy will -# delete the file once it fails hash verification -file(DOWNLOAD - ${LY_INSTALLER_3RD_PARTY_LICENSE_URL} - ${_3rd_party_license_dest} - STATUS _status - TLS_VERIFY ON -) -list(POP_FRONT _status _status_code) - -if (${_status_code} EQUAL 0 AND EXISTS ${_3rd_party_license_dest}) - install(FILES ${_3rd_party_license_dest} - DESTINATION . +# the version string and git tags are intended to be synchronized so it should be safe to use that instead +# of directly calling into git which could get messy in certain scenarios +if(${CPACK_PACKAGE_VERSION} VERSION_GREATER "0.0.0.0") + set(_3rd_party_license_filename SPDX-Licenses.txt) + + set(_3rd_party_license_url "https://raw.githubusercontent.com/o3de/3p-package-source/${CPACK_PACKAGE_VERSION}/${_3rd_party_license_filename}") + set(_3rd_party_license_dest ${CPACK_BINARY_DIR}/${_3rd_party_license_filename}) + + # use the plain file downloader as we don't have the file hash available and using a dummy will + # delete the file once it fails hash verification + file(DOWNLOAD + ${_3rd_party_license_url} + ${_3rd_party_license_dest} + STATUS _status + TLS_VERIFY ON ) -else() - file(REMOVE ${_3rd_party_license_dest}) - message(FATAL_ERROR "Failed to acquire the 3rd Party license manifest file. Error: ${_status}") + list(POP_FRONT _status _status_code) + + if (${_status_code} EQUAL 0 AND EXISTS ${_3rd_party_license_dest}) + install(FILES ${_3rd_party_license_dest} + DESTINATION . + ) + else() + file(REMOVE ${_3rd_party_license_dest}) + message(FATAL_ERROR "Failed to acquire the 3rd Party license manifest file at ${_3rd_party_license_url}. Error: ${_status}") + endif() endif() # checks for and removes trailing slash diff --git a/cmake/Platform/Common/Directory.Build.props b/cmake/Platform/Common/Directory.Build.props index 239b5578d0..1adf33ee89 100644 --- a/cmake/Platform/Common/Directory.Build.props +++ b/cmake/Platform/Common/Directory.Build.props @@ -10,6 +10,7 @@ SPDX-License-Identifier: Apache-2.0 OR MIT true true +@VCPKG_CONFIGURATION_MAPPING@ diff --git a/cmake/Platform/Common/Install_common.cmake b/cmake/Platform/Common/Install_common.cmake index b503b9366d..beb2802486 100644 --- a/cmake/Platform/Common/Install_common.cmake +++ b/cmake/Platform/Common/Install_common.cmake @@ -6,6 +6,8 @@ # # +include(cmake/FileUtil.cmake) + set(CMAKE_INSTALL_MESSAGE NEVER) # Simplify messages to reduce output noise ly_set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME Core) @@ -19,26 +21,6 @@ cmake_path(RELATIVE_PATH CMAKE_LIBRARY_OUTPUT_DIRECTORY BASE_DIRECTORY ${CMAKE_B set(install_output_folder "\${CMAKE_INSTALL_PREFIX}/${runtime_output_directory}/${PAL_PLATFORM_NAME}/$") -function(ly_get_engine_relative_source_dir absolute_target_source_dir output_source_dir) - # Get a relative target source directory to the LY root folder if possible - # Otherwise use the final component name - cmake_path(IS_PREFIX LY_ROOT_FOLDER ${absolute_target_source_dir} is_target_prefix_of_engine_root) - if(is_target_prefix_of_engine_root) - cmake_path(RELATIVE_PATH absolute_target_source_dir BASE_DIRECTORY ${LY_ROOT_FOLDER} OUTPUT_VARIABLE relative_target_source_dir) - else() - # In this case the target source directory is outside of the engine root of the target source directory and concatenate the first - # is used first 8 characters of the absolute path SHA256 hash to make a unique relative directory - # that can be used to install the generated CMakeLists.txt - # of a SHA256 hash - string(SHA256 target_source_hash ${absolute_target_source_dir}) - string(SUBSTRING ${target_source_hash} 0 8 target_source_hash) - cmake_path(GET absolute_target_source_dir FILENAME target_source_dirname) - cmake_path(SET relative_target_source_dir "${target_source_dirname}-${target_source_hash}") - endif() - - set(${output_source_dir} ${relative_target_source_dir} PARENT_SCOPE) -endfunction() - #! ly_setup_target: Setup the data needed to re-create the cmake target commands for a single target function(ly_setup_target OUTPUT_CONFIGURED_TARGET ALIAS_TARGET_NAME absolute_target_source_dir) # De-alias target name @@ -266,7 +248,6 @@ endfunction() #! ly_setup_subdirectory: setup all targets in the subdirectory function(ly_setup_subdirectory absolute_target_source_dir) - # Get the target source directory relative to the LY roo folder ly_get_engine_relative_source_dir(${absolute_target_source_dir} relative_target_source_dir) # The builtin BUILDSYSTEM_TARGETS property isn't being used here as that returns the de-alised @@ -560,56 +541,74 @@ function(ly_setup_others) ) # Exclude transient artifacts that shouldn't be copied to the install layout list(FILTER external_subdir_files EXCLUDE REGEX "/([Bb]uild|[Cc]ache|[Uu]ser)$") - list(APPEND filtered_asset_paths ${external_subdir_files}) + # Storing a "mapping" of gem candidate directories, to external_subdirectory files using + # a DIRECTORY property for the "value" and the GLOBAL property for the "key" + set_property(DIRECTORY ${gem_candidate_dir} APPEND PROPERTY directory_filtered_asset_paths "${external_subdir_files}") + set_property(GLOBAL APPEND PROPERTY global_gem_candidate_dirs_prop ${gem_candidate_dir}) endforeach() - # At this point the filtered_assets_paths contains the list of all directories and files - # that are non-excluded candidates that can be scanned for target directories and files - # to copy over to the install layout - foreach(filtered_asset_path IN LISTS filtered_asset_paths) - if(IS_DIRECTORY ${filtered_asset_path}) - file(GLOB_RECURSE - recurse_assets_paths - LIST_DIRECTORIES TRUE - "${filtered_asset_path}/*" - ) - set(gem_file_paths ${recurse_assets_paths}) - # Make sure to prepend the current path iteration to the gem_dirs_path to filter - set(gem_dir_paths ${filtered_asset_path} ${recurse_assets_paths}) - - # Gather directories to copy over - # Currently only the Assets, Registry and Config directories are copied over - list(FILTER gem_dir_paths INCLUDE REGEX "/(Assets|Registry|Config)$") - list(APPEND gems_assets_dir_path ${gem_dir_paths}) - else() - set(gem_file_paths ${filtered_asset_path}) - endif() + # Iterate over each gem candidate directories and read populate a directory property + # containing the files to copy over + get_property(gem_candidate_dirs GLOBAL PROPERTY global_gem_candidate_dirs_prop) + foreach(gem_candidate_dir IN LISTS gem_candidate_dirs) + get_property(filtered_asset_paths DIRECTORY ${gem_candidate_dir} PROPERTY directory_filtered_asset_paths) + ly_get_last_path_segment_concat_sha256(${gem_candidate_dir} last_gem_root_path_segment) + # Check if the gem is a subdirectory of the engine + cmake_path(IS_PREFIX LY_ROOT_FOLDER ${gem_candidate_dir} is_gem_subdirectory_of_engine) + + # At this point the filtered_assets_paths contains the list of all directories and files + # that are non-excluded candidates that can be scanned for target directories and files + # to copy over to the install layout + foreach(filtered_asset_path IN LISTS filtered_asset_paths) + if(IS_DIRECTORY ${filtered_asset_path}) + file(GLOB_RECURSE + recurse_assets_paths + LIST_DIRECTORIES TRUE + "${filtered_asset_path}/*" + ) + set(gem_file_paths ${recurse_assets_paths}) + # Make sure to prepend the current path iteration to the gem_dirs_path to filter + set(gem_dir_paths ${filtered_asset_path} ${recurse_assets_paths}) + + # Gather directories to copy over + # Currently only the Assets, Registry and Config directories are copied over + list(FILTER gem_dir_paths INCLUDE REGEX "/(Assets|Registry|Config|Editor/Scripts)$") + set_property(DIRECTORY ${gem_candidate_dir} APPEND PROPERTY gems_assets_paths ${gem_dir_paths}) + else() + set(gem_file_paths ${filtered_asset_path}) + endif() - # Gather files to copy over - # Currently only the gem.json file is copied over - list(FILTER gem_file_paths INCLUDE REGEX "/(gem.json)$") - list(APPEND gems_assets_file_path "${gem_file_paths}") - endforeach() + # Gather files to copy over + # Currently only the gem.json file is copied over + list(FILTER gem_file_paths INCLUDE REGEX "/(gem.json|preview.png)$") + set_property(DIRECTORY ${gem_candidate_dir} APPEND PROPERTY gems_assets_paths "${gem_file_paths}") + endforeach() - # gem directories to install - foreach(gem_absolute_dir_path ${gems_assets_dir_path}) - cmake_path(RELATIVE_PATH gem_absolute_dir_path BASE_DIRECTORY ${LY_ROOT_FOLDER} OUTPUT_VARIABLE gem_relative_dir_path) - if (EXISTS ${gem_absolute_dir_path}) - # The trailing slash is IMPORTANT here as that is needed to prevent - # the "Assets" folder from being copied underneath the /Assets folder - install(DIRECTORY "${gem_absolute_dir_path}/" - DESTINATION ${gem_relative_dir_path} - ) - endif() - endforeach() + # gem directories and files to install + get_property(gems_assets_paths DIRECTORY ${gem_candidate_dir} PROPERTY gems_assets_paths) + foreach(gem_absolute_path IN LISTS gems_assets_paths) + if(is_gem_subdirectory_of_engine) + cmake_path(RELATIVE_PATH gem_absolute_path BASE_DIRECTORY ${LY_ROOT_FOLDER} OUTPUT_VARIABLE gem_install_dest_dir) + else() + # The gem resides outside of the LY_ROOT_FOLDER, so the destination is made relative to the + # gem candidate directory and placed under the "External" directory" + # directory + cmake_path(RELATIVE_PATH gem_absolute_path BASE_DIRECTORY ${gem_candidate_dir} OUTPUT_VARIABLE gem_relative_path) + unset(gem_install_dest_dir) + cmake_path(APPEND gem_install_dest_dir "External" ${last_gem_root_path_segment} ${gem_relative_path}) + endif() + + cmake_path(GET gem_install_dest_dir PARENT_PATH gem_install_dest_dir) + if (NOT gem_install_dest_dir) + cmake_path(SET gem_install_dest_dir .) + endif() + if(IS_DIRECTORY ${gem_absolute_path}) + install(DIRECTORY "${gem_absolute_path}" DESTINATION ${gem_install_dest_dir}) + elseif (EXISTS ${gem_absolute_path}) + install(FILES ${gem_absolute_path} DESTINATION ${gem_install_dest_dir}) + endif() + endforeach() - # gem files to install - foreach(gem_absolute_file_path ${gems_assets_file_path}) - cmake_path(RELATIVE_PATH gem_absolute_file_path BASE_DIRECTORY ${LY_ROOT_FOLDER} OUTPUT_VARIABLE gem_relative_file_path) - cmake_path(GET gem_relative_file_path PARENT_PATH gem_relative_parent_dir) - install(FILES ${gem_absolute_file_path} - DESTINATION ${gem_relative_parent_dir} - ) endforeach() # Templates @@ -646,3 +645,46 @@ function(ly_setup_target_generator) ) endfunction() + +#! ly_add_install_paths: Adds the list of path to copy to the install layout relative to the same folder +# \arg:PATHS - Paths to copy over to the install layout. The DESTINATION sub argument is optional +# The INPUT sub-argument is required +# \arg:BASE_DIRECTORY(Optional) - Absolute path where a relative path from the each input path will be +# based off of. Defaults to LY_ROOT_FOLDER if not supplied +function(ly_add_install_paths) + set(options) + set(oneValueArgs BASE_DIRECTORY) + set(multiValueArgs PATHS) + cmake_parse_arguments(ly_add_install_paths "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + if(NOT ly_add_install_paths_PATHS) + message(FATAL_ERROR "ly_add_install_paths requires at least one input path to copy to the destination") + endif() + + # The default is the "." directory if not supplied + if(NOT ly_add_install_paths_BASE_DIRECTORY) + cmake_path(SET ly_add_install_paths_BASE_DIRECTORY ${LY_ROOT_FOLDER}) + endif() + + # Separate each path into an INPUT and DESTINATION parameter + set(options) + set(oneValueArgs INPUT DESTINATION) + set(multiValueArgs) + foreach(install_path IN LISTS ly_add_install_paths_PATHS) + string(REPLACE " " ";" install_path ${install_path}) + cmake_parse_arguments(install "${options}" "${oneValueArgs}" "${multiValueArgs}" ${install_path}) + if(NOT install_DESTINATION) + ly_get_engine_relative_source_dir(${install_INPUT} rel_to_root_input_path + BASE_DIRECTORY ${ly_add_install_paths_BASE_DIRECTORY}) + cmake_path(GET rel_to_root_input_path PARENT_PATH install_DESTINATION) + endif() + if(NOT install_DESTINATION) + cmake_path(SET install_DESTINATION .) + endif() + if(IS_DIRECTORY ${install_INPUT}) + install(DIRECTORY ${install_INPUT} DESTINATION ${install_DESTINATION}) + elseif(EXISTS ${install_INPUT}) + install(FILES ${install_INPUT} DESTINATION ${install_DESTINATION}) + endif() + endforeach() + +endfunction() diff --git a/cmake/Platform/Common/MSVC/Configurations_msvc.cmake b/cmake/Platform/Common/MSVC/Configurations_msvc.cmake index 6fcbd6a057..d61558bd49 100644 --- a/cmake/Platform/Common/MSVC/Configurations_msvc.cmake +++ b/cmake/Platform/Common/MSVC/Configurations_msvc.cmake @@ -9,39 +9,16 @@ include(cmake/Platform/Common/Configurations_common.cmake) include(cmake/Platform/Common/VisualStudio_common.cmake) -set(LY_MSVC_SUPPORTED_GENERATORS - "Visual Studio 15" - "Visual Studio 16" -) -set(FOUND_SUPPORTED_GENERATOR) -foreach(supported_generator ${LY_MSVC_SUPPORTED_GENERATORS}) - if(CMAKE_GENERATOR MATCHES ${supported_generator}) - set(FOUND_SUPPORTED_GENERATOR TRUE) - break() - endif() -endforeach() -# VS2017's checks since it defaults the toolchain and target architecture to x86 -if(CMAKE_GENERATOR MATCHES "Visual Studio 15") - if(CMAKE_VS_PLATFORM_NAME AND CMAKE_VS_PLATFORM_NAME STREQUAL "Win32") # VS2017 has Win32 as the default architecture - message(FATAL_ERROR "Win32 architecture not supported, specify \"-A x64\" when invoking cmake") - endif() - if(NOT CMAKE_VS_PLATFORM_TOOLSET_HOST_ARCHITECTURE STREQUAL "x64") # There is at least one library (EditorLib) that make the x86 linker to run out of memory - message(FATAL_ERROR "x86 toolset not supported, specify \"-T host=x64\" when invoking cmake") - endif() -else() - # For the other cases, verify that it wasn't invoked with an unsupported architecture. defaults to x86 architecture - if(SUPPORTED_VS_PLATFORM_NAME_OVERRIDE) - set(SUPPORTED_VS_PLATFORM_NAME ${SUPPORTED_VS_PLATFORM_NAME_OVERRIDE}) - else() - set(SUPPORTED_VS_PLATFORM_NAME x64) - endif() +if(NOT CMAKE_GENERATOR MATCHES "Visual Studio 1[6-7]") + message(FATAL_ERROR "Generator ${CMAKE_GENERATOR} not supported") +endif() - if(CMAKE_VS_PLATFORM_NAME AND NOT CMAKE_VS_PLATFORM_NAME STREQUAL "${SUPPORTED_VS_PLATFORM_NAME}") - message(FATAL_ERROR "${CMAKE_VS_PLATFORM_NAME} architecture not supported") - endif() - if(CMAKE_VS_PLATFORM_TOOLSET_HOST_ARCHITECTURE AND NOT CMAKE_VS_PLATFORM_TOOLSET_HOST_ARCHITECTURE STREQUAL "x64") - message(FATAL_ERROR "${CMAKE_VS_PLATFORM_TOOLSET_HOST_ARCHITECTURE} toolset not supported") - endif() +# Verify that it wasn't invoked with an unsupported target/host architecture. Currently only supports x64/x64 +if(CMAKE_VS_PLATFORM_NAME AND NOT CMAKE_VS_PLATFORM_NAME STREQUAL "x64") + message(FATAL_ERROR "${CMAKE_VS_PLATFORM_NAME} target architecture is not supported, it must be 'x64'") +endif() +if(CMAKE_VS_PLATFORM_TOOLSET_HOST_ARCHITECTURE AND NOT CMAKE_VS_PLATFORM_TOOLSET_HOST_ARCHITECTURE STREQUAL "x64") + message(FATAL_ERROR "${CMAKE_VS_PLATFORM_TOOLSET_HOST_ARCHITECTURE} host toolset is not supported, it must be 'x64'") endif() ly_append_configurations_options( diff --git a/cmake/Platform/Common/VisualStudio_common.cmake b/cmake/Platform/Common/VisualStudio_common.cmake index fcb93908f0..9124758b18 100644 --- a/cmake/Platform/Common/VisualStudio_common.cmake +++ b/cmake/Platform/Common/VisualStudio_common.cmake @@ -6,6 +6,13 @@ # # -if(CMAKE_GENERATOR MATCHES "Visual Studio 16") - configure_file("${CMAKE_CURRENT_LIST_DIR}/Directory.Build.props" "${CMAKE_BINARY_DIR}/Directory.Build.props" COPYONLY) -endif() \ No newline at end of file +foreach(conf IN LISTS CMAKE_CONFIGURATION_TYPES) + if(conf STREQUAL debug) + string(APPEND VCPKG_CONFIGURATION_MAPPING " Debug\n") + else() + string(APPEND VCPKG_CONFIGURATION_MAPPING " Release\n") + endif() +endforeach() + +configure_file("${CMAKE_CURRENT_LIST_DIR}/Directory.Build.props" "${CMAKE_BINARY_DIR}/Directory.Build.props" @ONLY) + diff --git a/cmake/Platform/Common/runtime_dependencies_common.cmake.in b/cmake/Platform/Common/runtime_dependencies_common.cmake.in index d05007c3b1..bb280f895b 100644 --- a/cmake/Platform/Common/runtime_dependencies_common.cmake.in +++ b/cmake/Platform/Common/runtime_dependencies_common.cmake.in @@ -16,6 +16,7 @@ function(ly_copy source_file target_directory) if("${source_file}" IS_NEWER_THAN "${target_directory}/${target_filename}") message(STATUS "Copying \"${source_file}\" to \"${target_directory}\"...") file(COPY "${source_file}" DESTINATION "${target_directory}" FILE_PERMISSIONS @LY_COPY_PERMISSIONS@ FOLLOW_SYMLINK_CHAIN) + file(TOUCH_NOCREATE ${target_directory}/${target_filename}) endif() endif() endfunction() diff --git a/cmake/Platform/Mac/runtime_dependencies_mac.cmake.in b/cmake/Platform/Mac/runtime_dependencies_mac.cmake.in index be597a3591..d65578f82a 100644 --- a/cmake/Platform/Mac/runtime_dependencies_mac.cmake.in +++ b/cmake/Platform/Mac/runtime_dependencies_mac.cmake.in @@ -126,7 +126,7 @@ function(ly_copy source_file target_directory) file(LOCK ${target_directory}/${target_filename}.lock GUARD FUNCTION TIMEOUT 300) endif() file(COPY "${source_file}" DESTINATION "${target_directory}" FILE_PERMISSIONS @LY_COPY_PERMISSIONS@ FOLLOW_SYMLINK_CHAIN) - file(TOUCH ${target_directory}/${target_filename}) + file(TOUCH_NOCREATE ${target_directory}/${target_filename}) set(anything_new TRUE PARENT_SCOPE) endif() endif() diff --git a/scripts/build/bootstrap/incremental_build_util.py b/scripts/build/bootstrap/incremental_build_util.py index 633c504299..10543d5951 100644 --- a/scripts/build/bootstrap/incremental_build_util.py +++ b/scripts/build/bootstrap/incremental_build_util.py @@ -5,18 +5,18 @@ # import argparse -import ast import boto3 import datetime import urllib.request, urllib.error, urllib.parse import os import psutil import time -import requests import subprocess import sys import tempfile -import traceback +from contextlib import contextmanager +import threading +import _thread DEFAULT_REGION = 'us-west-2' DEFAULT_DISK_SIZE = 300 @@ -43,14 +43,18 @@ if os.name == 'nt': kernel32 = ctypes.WinDLL('kernel32', use_last_error=True) kernel32.GetDiskFreeSpaceExW.argtypes = (ctypes.c_wchar_p,) + (PULARGE_INTEGER,) * 3 + class UsageTuple(collections.namedtuple('UsageTuple', 'total, used, free')): def __str__(self): # Add thousands separator to numbers displayed return self.__class__.__name__ + '(total={:n}, used={:n}, free={:n})'.format(*self) + def is_dir_symlink(path): FILE_ATTRIBUTE_REPARSE_POINT = 0x0400 - return os.path.isdir(path) and (ctypes.windll.kernel32.GetFileAttributesW(str(path)) & FILE_ATTRIBUTE_REPARSE_POINT) + return os.path.isdir(path) and ( + ctypes.windll.kernel32.GetFileAttributesW(str(path)) & FILE_ATTRIBUTE_REPARSE_POINT) + def get_free_space_mb(path): if sys.version_info < (3,): # Python 2? @@ -78,16 +82,39 @@ if os.name == 'nt': used = total.value - free.value - return free.value / 1024 / 1024#for now + return free.value / 1024 / 1024 # for now else: def get_free_space_mb(dirname): st = os.statvfs(dirname) return st.f_bavail * st.f_frsize / 1024 / 1024 + def error(message): print(message) exit(1) + +@contextmanager +def timeout(duration, timeout_message): + timer = threading.Timer(duration, lambda: _thread.interrupt_main()) + timer.start() + try: + yield + except KeyboardInterrupt: + print(timeout_message) + raise TimeoutError + finally: + # If the action ends in specified time, timer is canceled + timer.cancel() + + +def print_drives(): + if os.name == 'nt': + drives_before = win32api.GetLogicalDriveStrings() + drives_before = drives_before.split('\000')[:-1] + print(drives_before) + + def parse_args(): parser = argparse.ArgumentParser() parser.add_argument('-a', '--action', dest="action", help="Action (mount|unmount|delete)") @@ -98,8 +125,10 @@ def parse_args(): parser.add_argument('-b', '--branch', dest="branch", help="Branch") parser.add_argument('-plat', '--platform', dest="platform", help="Platform") parser.add_argument('-c', '--build_type', dest="build_type", help="Build type") - parser.add_argument('-ds', '--disk_size', dest="disk_size", help="Disk size in Gigabytes (defaults to {})".format(DEFAULT_DISK_SIZE), default=DEFAULT_DISK_SIZE) - parser.add_argument('-dt', '--disk_type', dest="disk_type", help="Disk type (defaults to {})".format(DEFAULT_DISK_TYPE), default=DEFAULT_DISK_TYPE) + parser.add_argument('-ds', '--disk_size', dest="disk_size", + help=f"Disk size in Gigabytes (defaults to {DEFAULT_DISK_SIZE})", default=DEFAULT_DISK_SIZE) + parser.add_argument('-dt', '--disk_type', dest="disk_type", help=f"Disk type (defaults to {DEFAULT_DISK_TYPE})", + default=DEFAULT_DISK_TYPE) args = parser.parse_args() # Input validation @@ -119,19 +148,30 @@ def parse_args(): error('No platform specified') if args.build_type is None: error('No build_type specified') - + return args + def get_mount_name(repository_name, project, pipeline, branch, platform, build_type): - mount_name = "{}_{}_{}_{}_{}_{}".format(repository_name, project, pipeline, branch, platform, build_type) - mount_name = mount_name.replace('/','_').replace('\\','_') + mount_name = f"{repository_name}_{project}_{pipeline}_{branch}_{platform}_{build_type}" + mount_name = mount_name.replace('/', '_').replace('\\', '_') return mount_name + def get_pipeline_and_branch(pipeline, branch): - pipeline_and_branch = "{}_{}".format(pipeline, branch) - pipeline_and_branch = pipeline_and_branch.replace('/','_').replace('\\','_') + pipeline_and_branch = f"{pipeline}_{branch}" + pipeline_and_branch = pipeline_and_branch.replace('/', '_').replace('\\', '_') return pipeline_and_branch + +def get_region_name(): + session = boto3.session.Session() + region = session.region_name + if region is None: + region = DEFAULT_REGION + return region + + def get_ec2_client(region): client = boto3.client('ec2', region_name=region) return client @@ -142,38 +182,39 @@ def get_ec2_instance_id(): instance_id = urllib.request.urlopen('http://169.254.169.254/latest/meta-data/instance-id').read() return instance_id.decode("utf-8") except Exception as e: - print(e.message) + print(e) error('No EC2 metadata! Check if you are running this script on an EC2 instance.') def get_availability_zone(): try: - availability_zone = urllib.request.urlopen('http://169.254.169.254/latest/meta-data/placement/availability-zone').read() + availability_zone = urllib.request.urlopen( + 'http://169.254.169.254/latest/meta-data/placement/availability-zone').read() return availability_zone.decode("utf-8") except Exception as e: - print(e.message) + print(e) error('No EC2 metadata! Check if you are running this script on an EC2 instance.') def kill_processes(workspace='/dev/'): - ''' + """ Kills all processes that have open file paths associated with the workspace. Uses PSUtil for cross-platform compatibility - ''' + """ print('Checking for any stuck processes...') for proc in psutil.process_iter(): try: if workspace in str(proc.open_files()): - print("{} has open files in {}. Terminating".format(proc.name(), proc.open_files())) + print(f"{proc.name()} has open files in {proc.open_files()}. Terminating") proc.kill() - time.sleep(1) # Just to make sure a parent process has time to close + time.sleep(1) # Just to make sure a parent process has time to close except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess): continue def delete_volume(ec2_client, volume_id): response = ec2_client.delete_volume(VolumeId=volume_id) - print('Volume {} deleted'.format(volume_id)) + print(f'Volume {volume_id} deleted') def find_snapshot_id(ec2_client, snapshot_hint, repository_name, project, pipeline, platform, build_type, disk_size): mount_name = get_mount_name(repository_name, project, pipeline, snapshot_hint, platform, build_type) @@ -183,7 +224,7 @@ def find_snapshot_id(ec2_client, snapshot_hint, repository_name, project, pipeli snapshot_id = None if 'Snapshots' in response and len(response['Snapshots']) > 0: - snapshot_start_time_max = None # find the latest snapshot + snapshot_start_time_max = None # find the latest snapshot for snapshot in response['Snapshots']: if snapshot['State'] == 'completed' and snapshot['VolumeSize'] == disk_size: snapshot_start_time = snapshot['StartTime'] @@ -195,26 +236,30 @@ def find_snapshot_id(ec2_client, snapshot_hint, repository_name, project, pipeli def create_volume(ec2_client, availability_zone, snapshot_hint, repository_name, project, pipeline, branch, platform, build_type, disk_size, disk_type): # The actual EBS default calculation for IOps is a floating point number, the closest approxmiation is 4x of the disk size for simplicity mount_name = get_mount_name(repository_name, project, pipeline, branch, platform, build_type) - pipeline_and_branch = get_pipeline_and_branch(pipeline, branch) + pipeline_and_branch = get_pipeline_and_branch(pipeline, branch) parameters = dict( - AvailabilityZone = availability_zone, + AvailabilityZone=availability_zone, VolumeType=disk_type, Encrypted=True, TagSpecifications= [{ 'ResourceType': 'volume', 'Tags': [ - { 'Key': 'Name', 'Value': mount_name }, - { 'Key': 'RepositoryName', 'Value': repository_name}, - { 'Key': 'Project', 'Value': project }, - { 'Key': 'Pipeline', 'Value': pipeline }, - { 'Key': 'BranchName', 'Value': branch }, - { 'Key': 'Platform', 'Value': platform }, - { 'Key': 'BuildType', 'Value': build_type }, - { 'Key': 'PipelineAndBranch', 'Value': pipeline_and_branch }, # used so the snapshoting easily identifies which volumes to snapshot + {'Key': 'Name', 'Value': mount_name}, + {'Key': 'RepositoryName', 'Value': repository_name}, + {'Key': 'Project', 'Value': project}, + {'Key': 'Pipeline', 'Value': pipeline}, + {'Key': 'BranchName', 'Value': branch}, + {'Key': 'Platform', 'Value': platform}, + {'Key': 'BuildType', 'Value': build_type}, + # Used so the snapshoting easily identifies which volumes to snapshot + {'Key': 'PipelineAndBranch', 'Value': pipeline_and_branch}, + ] }] ) - if 'io1' in disk_type.lower(): + # The actual EBS default calculation for IOps is a floating point number, + # the closest approxmiation is 4x of the disk size for simplicity + if 'io1' in disk_type.lower(): parameters['Iops'] = (4 * disk_size) snapshot_id = find_snapshot_id(ec2_client, snapshot_hint, repository_name, project, pipeline, platform, build_type, disk_size) @@ -233,16 +278,17 @@ def create_volume(ec2_client, availability_zone, snapshot_hint, repository_name, time.sleep(1) response = ec2_client.describe_volumes(VolumeIds=[volume_id, ]) - while (response['Volumes'][0]['State'] != 'available'): - time.sleep(1) - response = ec2_client.describe_volumes(VolumeIds=[volume_id, ]) + with timeout(DEFAULT_TIMEOUT, 'ERROR: Timeout reached trying to create EBS.'): + while response['Volumes'][0]['State'] != 'available': + time.sleep(1) + response = ec2_client.describe_volumes(VolumeIds=[volume_id, ]) - print(("Volume {} created\n\tSnapshot: {}\n\tRepository {}\n\tProject {}\n\tPipeline {}\n\tBranch {}\n\tPlatform: {}\n\tBuild type: {}" - .format(volume_id, snapshot_id, repository_name, project, pipeline, branch, platform, build_type))) + print(f"Volume {volume_id} created\n\tSnapshot: {snapshot_id}\n\tRepository {repository_name}\n\t" + f"Project {project}\n\tPipeline {pipeline}\n\tBranch {branch}\n\tPlatform: {platform}\n\tBuild type: {build_type}") return volume_id, created -def mount_volume(created): +def mount_volume_to_device(created): print('Mounting volume...') if os.name == 'nt': f = tempfile.NamedTemporaryFile(delete=False) @@ -250,7 +296,7 @@ def mount_volume(created): select disk 1 online disk attribute disk clear readonly - """.encode('utf-8')) # assume disk # for now + """.encode('utf-8')) # assume disk # for now if created: print('Creating filesystem on new volume') @@ -262,18 +308,12 @@ def mount_volume(created): """.encode('utf-8')) f.close() - + subprocess.call(['diskpart', '/s', f.name]) time.sleep(5) - drives_after = win32api.GetLogicalDriveStrings() - drives_after = drives_after.split('\000')[:-1] - - print(drives_after) - - #drive_letter = next(item for item in drives_after if item not in drives_before) - drive_letter = MOUNT_PATH + print_drives() os.unlink(f.name) @@ -286,8 +326,8 @@ def mount_volume(created): subprocess.call(['mount', '/dev/xvdf', MOUNT_PATH]) -def attach_volume(volume, volume_id, instance_id, timeout=DEFAULT_TIMEOUT): - print('Attaching volume {} to instance {}'.format(volume_id, instance_id)) +def attach_volume_to_ec2_instance(volume, volume_id, instance_id, timeout_duration=DEFAULT_TIMEOUT): + print(f'Attaching volume {volume_id} to instance {instance_id}') volume.attach_to_instance(Device='xvdf', InstanceId=instance_id, VolumeId=volume_id) @@ -295,13 +335,10 @@ def attach_volume(volume, volume_id, instance_id, timeout=DEFAULT_TIMEOUT): time.sleep(2) # reload the volume just in case volume.load() - timeout_init = time.clock() - while (len(volume.attachments) and volume.attachments[0]['State'] != 'attached'): - time.sleep(1) - volume.load() - if (time.clock() - timeout_init) > timeout: - print('ERROR: Timeout reached trying to mount EBS') - exit(1) + with timeout(timeout_duration, 'ERROR: Timeout reached trying to mount EBS.'): + while len(volume.attachments) and volume.attachments[0]['State'] != 'attached': + time.sleep(1) + volume.load() volume.create_tags( Tags=[ { @@ -310,11 +347,11 @@ def attach_volume(volume, volume_id, instance_id, timeout=DEFAULT_TIMEOUT): }, ] ) - print('Volume {} has been attached to instance {}'.format(volume_id, instance_id)) + print(f'Volume {volume_id} has been attached to instance {instance_id}') -def unmount_volume(): - print('Umounting volume...') +def unmount_volume_from_device(): + print('Unmounting EBS volume from device...') if os.name == 'nt': kill_processes(MOUNT_PATH + 'workspace') f = tempfile.NamedTemporaryFile(delete=False) @@ -330,44 +367,28 @@ def unmount_volume(): subprocess.call(['umount', '-f', MOUNT_PATH]) -def detach_volume(volume, ec2_instance_id, force, timeout=DEFAULT_TIMEOUT): - print('Detaching volume {} from instance {}'.format(volume.volume_id, ec2_instance_id)) +def detach_volume_from_ec2_instance(volume, ec2_instance_id, force, timeout_duration=DEFAULT_TIMEOUT): + print(f'Detaching volume {volume.volume_id} from instance {ec2_instance_id}') volume.detach_from_instance(Device='xvdf', Force=force, InstanceId=ec2_instance_id, VolumeId=volume.volume_id) - timeout_init = time.clock() - while len(volume.attachments) and volume.attachments[0]['State'] != 'detached': - time.sleep(1) - volume.load() - if (time.clock() - timeout_init) > timeout: - print('ERROR: Timeout reached trying to unmount EBS.') - volume.detach_from_instance(Device='xvdf',Force=True,InstanceId=ec2_instance_id,VolumeId=volume.volume_id) - exit(1) - - print('Volume {} has been detached from instance {}'.format(volume.volume_id, ec2_instance_id)) + try: + with timeout(timeout_duration, 'ERROR: Timeout reached trying to unmount EBS.'): + while len(volume.attachments) and volume.attachments[0]['State'] != 'detached': + time.sleep(1) + volume.load() + except TimeoutError: + print('Force detaching EBS.') + volume.detach_from_instance(Device='xvdf', Force=True, InstanceId=ec2_instance_id, VolumeId=volume.volume_id) + + print(f'Volume {volume.volume_id} has been detached from instance {ec2_instance_id}') volume.load() if len(volume.attachments): print('Volume still has attachments') for attachment in volume.attachments: - print('Volume {} {} to instance {}'.format(attachment['VolumeId'], attachment['State'], attachment['InstanceId'])) - - -def attach_ebs_and_create_partition_with_retry(volume, volume_id, ec2_instance_id, created): - attach_volume(volume, volume_id, ec2_instance_id) - mount_volume(created) - attempt = 1 - while attempt <= MAX_EBS_MOUNTING_ATTEMPT: - if os.name == 'nt': - drives_after = win32api.GetLogicalDriveStrings() - drives_after = drives_after.split('\000')[:-1] - if MOUNT_PATH not in drives_after: - print('Disk partitioning failed, retrying...') - unmount_volume() - detach_volume(volume, ec2_instance_id, False) - attach_volume(volume, volume_id, ec2_instance_id) - mount_volume(created) - attempt += 1 + print(f"Volume {attachment['VolumeId']} {attachment['State']} to instance {attachment['InstanceId']}") + def mount_ebs(snapshot_hint, repository_name, project, pipeline, branch, platform, build_type, disk_size, disk_type): session = boto3.session.Session() @@ -382,70 +403,67 @@ def mount_ebs(snapshot_hint, repository_name, project, pipeline, branch, platfor for volume in ec2_instance.volumes.all(): for attachment in volume.attachments: - print('attachment device: {}'.format(attachment['Device'])) + print(f"attachment device: {attachment['Device']}") if 'xvdf' in attachment['Device'] and attachment['State'] != 'detached': - print('A device is already attached to xvdf. This likely means a previous build failed to detach its ' \ + print('A device is already attached to xvdf. This likely means a previous build failed to detach its ' 'build volume. This volume is considered orphaned and will be detached from this instance.') - unmount_volume() - detach_volume(volume, ec2_instance_id, False) # Force unmounts should not be used, as that will cause the EBS block device driver to fail the remount + unmount_volume_from_device() + detach_volume_from_ec2_instance(volume, ec2_instance_id, + False) # Force unmounts should not be used, as that will cause the EBS block device driver to fail the remount mount_name = get_mount_name(repository_name, project, pipeline, branch, platform, build_type) response = ec2_client.describe_volumes(Filters=[{ 'Name': 'tag:Name', 'Values': [mount_name] - }]) + }]) created = False if 'Volumes' in response and not len(response['Volumes']): - print('Volume for {} doesn\'t exist creating it...'.format(mount_name)) + print(f'Volume for {mount_name} doesn\'t exist creating it...') # volume doesn't exist, create it volume_id, created = create_volume(ec2_client, ec2_availability_zone, snapshot_hint, repository_name, project, pipeline, branch, platform, build_type, disk_size, disk_type) else: volume = response['Volumes'][0] volume_id = volume['VolumeId'] - print('Current volume {} is a {} GB {}'.format(volume_id, volume['Size'], volume['VolumeType'])) - if (volume['Size'] != disk_size or volume['VolumeType'] != disk_type): - print('Override disk attributes does not match the existing volume, deleting {} and replacing the volume'.format(volume_id)) + print(f"Current volume {volume_id} is a {volume['Size']} GB {volume['VolumeType']}") + if volume['Size'] != disk_size or volume['VolumeType'] != disk_type: + print( + f'Override disk attributes does not match the existing volume, deleting {volume_id} and replacing the volume') delete_volume(ec2_client, volume_id) volume_id, created = create_volume(ec2_client, ec2_availability_zone, snapshot_hint, repository_name, project, pipeline, branch, platform, build_type, disk_size, disk_type) if len(volume['Attachments']): # this is bad we shouldn't be attached, we should have detached at the end of a build attachment = volume['Attachments'][0] - print(('Volume already has attachment {}, detaching...'.format(attachment))) - detach_volume(ec2_resource.Volume(volume_id), attachment['InstanceId'], True) + print(f'Volume already has attachment {attachment}, detaching...') + detach_volume_from_ec2_instance(ec2_resource.Volume(volume_id), attachment['InstanceId'], True) volume = ec2_resource.Volume(volume_id) - if os.name == 'nt': - drives_before = win32api.GetLogicalDriveStrings() - drives_before = drives_before.split('\000')[:-1] - - print(drives_before) - - attach_ebs_and_create_partition_with_retry(volume, volume_id, ec2_instance_id, created) + print_drives() + attach_volume_to_ec2_instance(volume, volume_id, ec2_instance_id) + mount_volume_to_device(created) + print_drives() free_space_mb = get_free_space_mb(MOUNT_PATH) - print('Free disk space {}MB'.format(free_space_mb)) - + print(f'Free disk space {free_space_mb}MB') + if free_space_mb < LOW_EBS_DISK_SPACE_LIMIT: - print('Volume is running below EBS free disk space treshhold {}MB. Recreating volume and running clean build.'.format(LOW_EBS_DISK_SPACE_LIMIT)) - unmount_volume() - detach_volume(volume, ec2_instance_id, False) + print(f'Volume is running below EBS free disk space treshhold {LOW_EBS_DISK_SPACE_LIMIT}MB. Recreating volume and running clean build.') + unmount_volume_from_device() + detach_volume_from_ec2_instance(volume, ec2_instance_id, False) delete_volume(ec2_client, volume_id) new_disk_size = int(volume.size * 1.25) if new_disk_size > MAX_EBS_DISK_SIZE: - print('Error: EBS disk size reached to the allowed maximum disk size {}MB, please contact ly-infra@ and ly-build@ to investigate.'.format(MAX_EBS_DISK_SIZE)) + print(f'Error: EBS disk size reached to the allowed maximum disk size {MAX_EBS_DISK_SIZE}MB, please contact ly-infra@ and ly-build@ to investigate.') exit(1) print('Recreating the EBS with disk size {}'.format(new_disk_size)) volume_id, created = create_volume(ec2_client, ec2_availability_zone, snapshot_hint, repository_name, project, pipeline, branch, platform, build_type, new_disk_size, disk_type) volume = ec2_resource.Volume(volume_id) - attach_ebs_and_create_partition_with_retry(volume, volume_id, ec2_instance_id, created) + attach_volume_to_ec2_instance(volume, volume_id, ec2_instance_id) + mount_volume_to_device(created) + def unmount_ebs(): - session = boto3.session.Session() - region = session.region_name - if region is None: - region = DEFAULT_REGION - ec2_client = get_ec2_client(region) + region = get_region_name() ec2_instance_id = get_ec2_instance_id() ec2_resource = boto3.resource('ec2', region_name=region) ec2_instance = ec2_resource.Instance(ec2_instance_id) @@ -457,7 +475,7 @@ def unmount_ebs(): for attached_volume in ec2_instance.volumes.all(): for attachment in attached_volume.attachments: - print('attachment device: {}'.format(attachment['Device'])) + print(f"attachment device: {attachment['Device']}") if attachment['Device'] == 'xvdf': volume = attached_volume @@ -465,24 +483,18 @@ def unmount_ebs(): # volume is not mounted print('Volume is not mounted') else: - unmount_volume() - detach_volume(volume, ec2_instance_id, False) + unmount_volume_from_device() + detach_volume_from_ec2_instance(volume, ec2_instance_id, False) + def delete_ebs(repository_name, project, pipeline, branch, platform, build_type): unmount_ebs() - - session = boto3.session.Session() - region = session.region_name - if region is None: - region = DEFAULT_REGION + region = get_region_name() ec2_client = get_ec2_client(region) - ec2_instance_id = get_ec2_instance_id() - ec2_resource = boto3.resource('ec2', region_name=region) - ec2_instance = ec2_resource.Instance(ec2_instance_id) mount_name = get_mount_name(repository_name, project, pipeline, branch, platform, build_type) response = ec2_client.describe_volumes(Filters=[ - { 'Name': 'tag:Name', 'Values': [mount_name] } + {'Name': 'tag:Name', 'Values': [mount_name]} ]) if 'Volumes' in response and len(response['Volumes']): @@ -499,6 +511,7 @@ def main(action, snapshot_hint, repository_name, project, pipeline, branch, plat elif action == 'delete': delete_ebs(repository_name, project, pipeline, branch, platform, build_type) + if __name__ == "__main__": args = parse_args() ret = main(args.action, args.snapshot_hint, args.repository_name, args.project, args.pipeline, args.branch, args.platform, args.build_type, args.disk_size, args.disk_type)