O3DE.exe Project-Centric "Open Editor" fix (#5852)

* The O3DE.exe Open Editor button now attempts to open the Editor in the
build directory of the project being opened.

If their is no Editor within the build directory of the Project, it uses
the Editor.exe in the current O3DE.exe executable directory if it exists

Signed-off-by: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com>

* Engine .gitignore now ignores the build directory if placed in the
AutomatedTesting project

Previously it was just ignoring a `[Bb]uild` directory if it was
directly within the engine root.
This change matches the behavior of the project templates.

Signed-off-by: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com>

* Renamed the ProjectUtils GetEditorDirectory function to
GetEditorExecutablePath

Added a platform specific implementation for retrieving the path to the
Editor executable in the GetEditorExectuablePath function.

It first attempts to locate the Editor via checking the project build
directory for an Editor executable before falling back to checking the
binary directory of the currently running O3DE executable.

Signed-off-by: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com>

* Correct the MacOS GetEditorExecutablePath to return the Editor path

Signed-off-by: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com>

.gitignore

@@ -3,7 +3,7 @@
 .vscode/
 __pycache__
 AssetProcessorTemp/**
-[Bb]uild/**
+[Bb]uild/
 [Oo]ut/**
 CMakeUserPresets.json
 [Cc]ache/

Code/Tools/ProjectManager/Platform/Linux/ProjectUtils_linux.cpp

@@ -1,6 +1,6 @@
 /*
  * Copyright (c) Contributors to the Open 3D Engine Project. For complete copyright and license terms please see the LICENSE at the root of this distribution.
- * 
+ *
  * SPDX-License-Identifier: Apache-2.0 OR MIT
  *
  */
@@ -10,6 +10,8 @@
 #include <QProcessEnvironment>
 #include <QDir>
 
+#include <AzCore/Settings/SettingsRegistryImpl.h>
+#include <AzCore/Settings/SettingsRegistryMergeUtils.h>
 #include <AzCore/Utils/Utils.h>
 
 namespace O3DE::ProjectManager
@@ -68,7 +70,7 @@ namespace O3DE::ProjectManager
 
             QProcess process;
 
-            // if the project build path is relative, it should be relative to the project path 
+            // if the project build path is relative, it should be relative to the project path
             process.setWorkingDirectory(projectPath);
 
             process.setProgram("cmake-gui");
@@ -80,7 +82,7 @@ namespace O3DE::ProjectManager
 
             return AZ::Success();
         }
-        
+
         AZ::Outcome<QString, QString> RunGetPythonScript(const QString& engineRoot)
         {
             return ExecuteCommandResultModalDialog(
@@ -89,9 +91,53 @@ namespace O3DE::ProjectManager
                 QObject::tr("Running get_python script..."));
         }
 
-        AZ::IO::FixedMaxPath GetEditorDirectory()
+        AZ::IO::FixedMaxPath GetEditorExecutablePath(const AZ::IO::PathView& projectPath)
         {
-            return AZ::Utils::GetExecutableDirectory();
+            AZ::IO::FixedMaxPath editorPath;
+            AZ::IO::FixedMaxPath fixedProjectPath{ projectPath };
+            // First attempt to launch the Editor.exe within the project build directory if it exists
+            AZ::IO::FixedMaxPath buildPathSetregPath = fixedProjectPath
+                / AZ::SettingsRegistryInterface::DevUserRegistryFolder
+                / "Platform" / AZ_TRAIT_OS_PLATFORM_CODENAME / "build_path.setreg";
+            if (AZ::IO::SystemFile::Exists(buildPathSetregPath.c_str()))
+            {
+                AZ::SettingsRegistryImpl settingsRegistry;
+                // Merge the build_path.setreg into the local SettingsRegistry instance
+                if (AZ::IO::FixedMaxPath projectBuildPath;
+                    settingsRegistry.MergeSettingsFile(buildPathSetregPath.Native(),
+                        AZ::SettingsRegistryInterface::Format::JsonMergePatch)
+                    && settingsRegistry.Get(projectBuildPath.Native(), AZ::SettingsRegistryMergeUtils::ProjectBuildPath))
+                {
+                    // local Settings Registry will be used to merge the build_path.setreg for the supplied projectPath
+                    AZ::IO::FixedMaxPath buildConfigurationPath = (fixedProjectPath / projectBuildPath).LexicallyNormal();
+
+                    // First try <project-build-path>/bin/$<CONFIG> and if that path doesn't exist
+                    // try <project-build-path>/bin/$<PLATFORM>/$<CONFIG>
+                    buildConfigurationPath /= "bin";
+                    if (editorPath = (buildConfigurationPath / AZ_BUILD_CONFIGURATION_TYPE / "Editor").
+                        ReplaceExtension(AZ_TRAIT_OS_EXECUTABLE_EXTENSION);
+                        AZ::IO::SystemFile::Exists(editorPath.c_str()))
+                    {
+                        return editorPath;
+                    }
+                    else if (editorPath = (buildConfigurationPath / AZ_TRAIT_OS_PLATFORM_CODENAME
+                    / AZ_BUILD_CONFIGURATION_TYPE / "Editor").
+                        ReplaceExtension(AZ_TRAIT_OS_EXECUTABLE_EXTENSION);
+                        AZ::IO::SystemFile::Exists(editorPath.c_str()))
+                    {
+                        return editorPath;
+                    }
+                }
+            }
+
+            // Fall back to checking if an Editor exists in O3DE executable directory
+            editorPath = AZ::IO::FixedMaxPath(AZ::Utils::GetExecutableDirectory()) / "Editor";
+            editorPath.ReplaceExtension(AZ_TRAIT_OS_EXECUTABLE_EXTENSION);
+            if (AZ::IO::SystemFile::Exists(editorPath.c_str()))
+            {
+                return editorPath;
+            }
+            return {};
         }
 
         AZ::Outcome<QString, QString> CreateDesktopShortcut([[maybe_unused]] const QString& filename, [[maybe_unused]] const QString& targetPath, [[maybe_unused]] const QStringList& arguments)

Code/Tools/ProjectManager/Platform/Mac/ProjectUtils_mac.cpp

@@ -1,6 +1,6 @@
 /*
  * Copyright (c) Contributors to the Open 3D Engine Project. For complete copyright and license terms please see the LICENSE at the root of this distribution.
- * 
+ *
  * SPDX-License-Identifier: Apache-2.0 OR MIT
  *
  */
@@ -11,8 +11,9 @@
 #include <QStandardPaths>
 #include <QDir>
 
-#include <AzCore/Utils/Utils.h>
+#include <AzCore/Settings/SettingsRegistryImpl.h>
 #include <AzCore/Settings/SettingsRegistryMergeUtils.h>
+#include <AzCore/Utils/Utils.h>
 
 namespace O3DE::ProjectManager
 {
@@ -21,7 +22,7 @@ namespace O3DE::ProjectManager
         AZ::Outcome<void, QString> SetupCommandLineProcessEnvironment()
         {
             // For CMake on Mac, if its installed through home-brew, then it will be installed
-            // under /usr/local/bin, which may not be in the system PATH environment. 
+            // under /usr/local/bin, which may not be in the system PATH environment.
             // Add that path for the command line process so that it will be able to locate
             // a home-brew installed version of CMake
             QString pathEnv = qEnvironmentVariable("PATH");
@@ -73,11 +74,11 @@ namespace O3DE::ProjectManager
 
 
             return AZ::Success(xcodeBuilderVersionNumber);
-        }       
+        }
 
         AZ::Outcome<void, QString> OpenCMakeGUI(const QString& projectPath)
         {
-            const QString cmakeHelp = QObject::tr("Please verify you've installed CMake.app from " 
+            const QString cmakeHelp = QObject::tr("Please verify you've installed CMake.app from "
                             "<a href=\"https://cmake.org\">cmake.org</a> or, if using HomeBrew, "
                             "have installed it with <pre>brew install --cask cmake</pre>");
             QString cmakeAppPath = QStandardPaths::locate(QStandardPaths::ApplicationsLocation, "CMake.app", QStandardPaths::LocateDirectory);
@@ -95,7 +96,7 @@ namespace O3DE::ProjectManager
 
             QProcess process;
 
-            // if the project build path is relative, it should be relative to the project path 
+            // if the project build path is relative, it should be relative to the project path
             process.setWorkingDirectory(projectPath);
             process.setProgram("open");
             process.setArguments({"-a", "CMake", "--args", "-S", projectPath, "-B", projectBuildPath});
@@ -106,7 +107,7 @@ namespace O3DE::ProjectManager
 
             return AZ::Success();
         }
-        
+
         AZ::Outcome<QString, QString> RunGetPythonScript(const QString& engineRoot)
         {
             return ExecuteCommandResultModalDialog(
@@ -115,30 +116,74 @@ namespace O3DE::ProjectManager
                 QObject::tr("Running get_python script..."));
         }
 
-        AZ::IO::FixedMaxPath GetEditorDirectory()
+        AZ::IO::FixedMaxPath GetEditorExecutablePath(const AZ::IO::PathView& projectPath)
         {
-            AZ::IO::FixedMaxPath executableDirectory = AZ::Utils::GetExecutableDirectory();
-            AZ::IO::FixedMaxPath editorPath{ executableDirectory };
-            editorPath /= "../../../Editor.app/Contents/MacOS";
-            editorPath = editorPath.LexicallyNormal();
-            if (!AZ::IO::SystemFile::IsDirectory(editorPath.c_str()))
+            AZ::IO::FixedMaxPath editorPath;
+            AZ::IO::FixedMaxPath fixedProjectPath{ projectPath };
+
+            // First attempt to launch the Editor.exe within the project build directory if it exists
+            AZ::IO::FixedMaxPath buildPathSetregPath = fixedProjectPath
+            / AZ::SettingsRegistryInterface::DevUserRegistryFolder
+                / "Platform" / AZ_TRAIT_OS_PLATFORM_CODENAME / "build_path.setreg";
+            if (AZ::IO::SystemFile::Exists(buildPathSetregPath.c_str()))
+            {
+                AZ::SettingsRegistryImpl localRegistry;
+                // Merge the build_path.setreg into the local SettingsRegistry instance
+                if (AZ::IO::FixedMaxPath projectBuildPath;
+                    localRegistry.MergeSettingsFile(buildPathSetregPath.Native(),
+                        AZ::SettingsRegistryInterface::Format::JsonMergePatch)
+                    && localRegistry.Get(projectBuildPath.Native(), AZ::SettingsRegistryMergeUtils::ProjectBuildPath))
+                {
+                    //  local Settings Registry will be used to merge the build_path.setreg for the supplied projectPath
+                    AZ::IO::FixedMaxPath buildConfigurationPath = (fixedProjectPath / projectBuildPath).LexicallyNormal();
+
+                    // First try "<project-build-path>/bin/$<CONFIG>/Editor.app/Contents/MacOS"
+                    // Followed by "<project-build-path>/bin/$<PLATFORM>/$<CONFIG>/Editor.app/Contents/MacOS"
+                    // Directory existence is checked in this case
+                    buildConfigurationPath /= "bin";
+                    if (editorPath = (buildConfigurationPath
+                        / AZ_BUILD_CONFIGURATION_TYPE / "Editor.app/Contents/MacOS/Editor");
+                        AZ::IO::SystemFile::Exists(editorPath.c_str()))
+                    {
+                        return editorPath;
+                    }
+                    else if (editorPath = (buildConfigurationPath / AZ_TRAIT_OS_PLATFORM_CODENAME
+                        / AZ_BUILD_CONFIGURATION_TYPE / "Editor.app/Contents/MacOS/Editor");
+                        AZ::IO::SystemFile::Exists(editorPath.c_str()))
+                    {
+                        return editorPath;
+                    }
+                }
+            }
+
+            // Fall back to locating the Editor.app bundle which should exists
+            // outside of the current O3DE.app bundle
+            editorPath = (AZ::IO::FixedMaxPath(AZ::Utils::GetExecutableDirectory()) /
+                "../../../Editor.app/Contents/MacOS/Editor").LexicallyNormal();
+
+            if (!AZ::IO::SystemFile::Exists(editorPath.c_str()))
             {
+                // Attempt to search the O3DE.app global settings registry for an InstalledBinaryFolder
+                // key which indicates the relative path to an SDK binary directory on MacOS
                 if (auto settingsRegistry = AZ::SettingsRegistry::Get(); settingsRegistry != nullptr)
                 {
                     if (AZ::IO::FixedMaxPath installedBinariesPath;
-                        settingsRegistry->Get(installedBinariesPath.Native(), AZ::SettingsRegistryMergeUtils::FilePathKey_InstalledBinaryFolder))
+                        settingsRegistry->Get(installedBinariesPath.Native(),
+                        AZ::SettingsRegistryMergeUtils::FilePathKey_InstalledBinaryFolder))
                     {
                         if (AZ::IO::FixedMaxPath engineRootFolder;
-                            settingsRegistry->Get(engineRootFolder.Native(), AZ::SettingsRegistryMergeUtils::FilePathKey_EngineRootFolder))
+                            settingsRegistry->Get(engineRootFolder.Native(),
+                            AZ::SettingsRegistryMergeUtils::FilePathKey_EngineRootFolder))
                         {
-                            editorPath = engineRootFolder / installedBinariesPath / "Editor.app/Contents/MacOS";
+                            editorPath = engineRootFolder / installedBinariesPath / "Editor.app/Contents/MacOS/Editor";
                         }
                     }
                 }
 
-                if (!AZ::IO::SystemFile::IsDirectory(editorPath.c_str()))
+                if (!AZ::IO::SystemFile::Exists(editorPath.c_str()))
                 {
                     AZ_Error("ProjectManager", false, "Unable to find the Editor app bundle!");
+                    return {};
                 }
             }
 

Code/Tools/ProjectManager/Platform/Windows/ProjectUtils_windows.cpp

@@ -1,6 +1,6 @@
 /*
  * Copyright (c) Contributors to the Open 3D Engine Project. For complete copyright and license terms please see the LICENSE at the root of this distribution.
- * 
+ *
  * SPDX-License-Identifier: Apache-2.0 OR MIT
  *
  */
@@ -15,6 +15,8 @@
 #include <QProcessEnvironment>
 #include <QStandardPaths>
 
+#include <AzCore/Settings/SettingsRegistryImpl.h>
+#include <AzCore/Settings/SettingsRegistryMergeUtils.h>
 #include <AzCore/Utils/Utils.h>
 
 namespace O3DE::ProjectManager
@@ -27,7 +29,7 @@ namespace O3DE::ProjectManager
             auto engineInfoResult = PythonBindingsInterface::Get()->GetEngineInfo();
             if (!engineInfoResult.IsSuccess())
             {
-                return AZ::Failure(QObject::tr("Failed to get engine info"));    
+                return AZ::Failure(QObject::tr("Failed to get engine info"));
             }
             auto engineInfo = engineInfoResult.GetValue();
 
@@ -52,7 +54,7 @@ namespace O3DE::ProjectManager
 
         AZ::Outcome<QString, QString> FindSupportedCompilerForPlatform()
         {
-            // Validate that cmake is installed 
+            // Validate that cmake is installed
             auto cmakeProcessEnvResult = SetupCommandLineProcessEnvironment();
             if (!cmakeProcessEnvResult.IsSuccess())
             {
@@ -109,7 +111,7 @@ namespace O3DE::ProjectManager
                 " or update to a newer version before proceeding to the next step."
                 " While installing configure Visual Studio with these <a href='https://o3de.org/docs/welcome-guide/setup/requirements/#visual-studio-configuration'>workloads</a>."));
         }
-        
+
         AZ::Outcome<void, QString> OpenCMakeGUI(const QString& projectPath)
         {
             AZ::Outcome processEnvResult = SetupCommandLineProcessEnvironment();
@@ -127,7 +129,7 @@ namespace O3DE::ProjectManager
 
             QProcess process;
 
-            // if the project build path is relative, it should be relative to the project path 
+            // if the project build path is relative, it should be relative to the project path
             process.setWorkingDirectory(projectPath);
 
             process.setProgram("cmake-gui");
@@ -149,9 +151,53 @@ namespace O3DE::ProjectManager
                 QObject::tr("Running get_python script..."));
         }
 
-        AZ::IO::FixedMaxPath GetEditorDirectory()
+        AZ::IO::FixedMaxPath GetEditorExecutablePath(const AZ::IO::PathView& projectPath)
         {
-            return AZ::Utils::GetExecutableDirectory();
+            AZ::IO::FixedMaxPath editorPath;
+            AZ::IO::FixedMaxPath fixedProjectPath{ projectPath };
+            // First attempt to launch the Editor.exe within the project build directory if it exists
+            AZ::IO::FixedMaxPath buildPathSetregPath = fixedProjectPath
+                / AZ::SettingsRegistryInterface::DevUserRegistryFolder
+                / "Platform" / AZ_TRAIT_OS_PLATFORM_CODENAME / "build_path.setreg";
+            if (AZ::IO::SystemFile::Exists(buildPathSetregPath.c_str()))
+            {
+                AZ::SettingsRegistryImpl settingsRegistry;
+                // Merge the build_path.setreg into the local SettingsRegistry instance
+                if (AZ::IO::FixedMaxPath projectBuildPath;
+                    settingsRegistry.MergeSettingsFile(buildPathSetregPath.Native(),
+                        AZ::SettingsRegistryInterface::Format::JsonMergePatch)
+                    && settingsRegistry.Get(projectBuildPath.Native(), AZ::SettingsRegistryMergeUtils::ProjectBuildPath))
+                {
+                    // local Settings Registry will be used to merge the build_path.setreg for the supplied projectPath
+                    AZ::IO::FixedMaxPath buildConfigurationPath = (fixedProjectPath / projectBuildPath).LexicallyNormal();
+
+                    // First try <project-build-path>/bin/$<CONFIG> and if that path doesn't exist
+                    // try <project-build-path>/bin/$<PLATFORM>/$<CONFIG>
+                    buildConfigurationPath /= "bin";
+                    if (editorPath = (buildConfigurationPath / AZ_BUILD_CONFIGURATION_TYPE / "Editor").
+                        ReplaceExtension(AZ_TRAIT_OS_EXECUTABLE_EXTENSION);
+                        AZ::IO::SystemFile::Exists(editorPath.c_str()))
+                    {
+                        return editorPath;
+                    }
+                    else if (editorPath = (buildConfigurationPath / AZ_TRAIT_OS_PLATFORM_CODENAME
+                        / AZ_BUILD_CONFIGURATION_TYPE / "Editor").
+                        ReplaceExtension(AZ_TRAIT_OS_EXECUTABLE_EXTENSION);
+                        AZ::IO::SystemFile::Exists(editorPath.c_str()))
+                    {
+                        return editorPath;
+                    }
+                }
+            }
+
+            // Fall back to checking if an Editor exists in O3DE executable directory
+            editorPath = AZ::IO::FixedMaxPath(AZ::Utils::GetExecutableDirectory()) / "Editor";
+            editorPath.ReplaceExtension(AZ_TRAIT_OS_EXECUTABLE_EXTENSION);
+            if (AZ::IO::SystemFile::Exists(editorPath.c_str()))
+            {
+                return editorPath;
+            }
+            return {};
         }
 
         AZ::Outcome<QString, QString> CreateDesktopShortcut(const QString& filename, const QString& targetPath, const QStringList& arguments)

Code/Tools/ProjectManager/Source/ProjectButtonWidget.cpp

@@ -215,9 +215,7 @@ namespace O3DE::ProjectManager
 #if AZ_TRAIT_PROJECT_MANAGER_CREATE_DESKTOP_SHORTCUT
             menu->addAction(tr("Create Editor desktop shortcut..."), this, [this]()
             {
-                AZ::IO::FixedMaxPath executableDirectory = ProjectUtils::GetEditorDirectory();
-                AZStd::string executableFilename = "Editor";
-                AZ::IO::FixedMaxPath editorExecutablePath = executableDirectory / (executableFilename + AZ_TRAIT_OS_EXECUTABLE_EXTENSION);
+                AZ::IO::FixedMaxPath editorExecutablePath = ProjectUtils::GetEditorExecutablePath(m_projectInfo.m_path.toUtf8().constData());
 
                 const QString shortcutName = QString("%1 Editor").arg(m_projectInfo.m_displayName); 
                 const QString arg = QString("--regset=\"/Amazon/AzCore/Bootstrap/project_path=%1\"").arg(m_projectInfo.m_path);

Code/Tools/ProjectManager/Source/ProjectUtils.h

@@ -76,7 +76,27 @@ namespace O3DE::ProjectManager
          */
         AZ::Outcome<QString, QString> CreateDesktopShortcut(const QString& filename, const QString& targetPath, const QStringList& arguments);
         
-        AZ::IO::FixedMaxPath GetEditorDirectory();
+        /**
+         * Lookup the location of an Editor executable executable that can be used with the
+         * supplied project path
+         * First the method attempts to locate a build directory with the project path
+         * via querying the <project-path>/user/Registry/Platform/<platform>/build_path.setreg
+         * Once that is done a path is formed to locate the Editor executable within the that build
+         * directory.
+         * Two paths will checked for the existence of an Editor
+         * - "<project-build-directory>/bin/$<CONFIG>/Editor"
+         * - "<project-build-directory>/bin/<platform>/$<CONFIG>/Editor"
+         * Where <platform> is the current platform the O3DE executable is running on and $<CONFIG> is the
+         * current build configuration the O3DE executable
+         *
+         * If neiether of the above paths contain an Editor application, then a path to the Editor
+         * is formed by combinding the O3DE executable directory with the filename of Editor
+         * - "<executable-directory>/Editor"
+         *
+         * @param projectPath Path to the root of the project
+         * @return path of the Editor Executable if found or an empty path if not
+         */
+        AZ::IO::FixedMaxPath GetEditorExecutablePath(const AZ::IO::PathView& projectPath);
 
     } // namespace ProjectUtils
 } // namespace O3DE::ProjectManager

Code/Tools/ProjectManager/Source/ProjectsScreen.cpp

@@ -23,7 +23,6 @@
 #include <AzFramework/Process/ProcessCommon.h>
 #include <AzFramework/Process/ProcessWatcher.h>
 #include <AzCore/Utils/Utils.h>
-#include <AzCore/Settings/SettingsRegistry.h>
 
 #include <QVBoxLayout>
 #include <QHBoxLayout>
@@ -408,18 +407,26 @@ namespace O3DE::ProjectManager
             emit ChangeScreenRequest(ProjectManagerScreen::Projects);
         }
     }
+
     void ProjectsScreen::HandleOpenProject(const QString& projectPath)
     {
         if (!projectPath.isEmpty())
         {
             if (!WarnIfInBuildQueue(projectPath))
             {
-                AZ::IO::FixedMaxPath executableDirectory = ProjectUtils::GetEditorDirectory();
-                AZStd::string executableFilename = "Editor";
-                AZ::IO::FixedMaxPath editorExecutablePath = executableDirectory / (executableFilename + AZ_TRAIT_OS_EXECUTABLE_EXTENSION);
+                AZ::IO::FixedMaxPath fixedProjectPath = projectPath.toUtf8().constData();
+                AZ::IO::FixedMaxPath editorExecutablePath = ProjectUtils::GetEditorExecutablePath(fixedProjectPath);
+                if (editorExecutablePath.empty())
+                {
+                    AZ_Error("ProjectManager", false, "Failed to locate editor");
+                    QMessageBox::critical(
+                        this, tr("Error"), tr("Failed to locate the Editor, please verify that it is built."));
+                    return;
+                }
+
                 auto cmdPath = AZ::IO::FixedMaxPathString::format(
                     "%s --regset=\"/Amazon/AzCore/Bootstrap/project_path=%s\"", editorExecutablePath.c_str(),
-                    projectPath.toStdString().c_str());
+                    fixedProjectPath.c_str());
 
                 AzFramework::ProcessLauncher::ProcessLaunchInfo processLaunchInfo;
                 processLaunchInfo.m_commandlineParameters = cmdPath;