From 6d592d78cdc3bd7e2e42127a87de7eacea322ba8 Mon Sep 17 00:00:00 2001 From: AMZN-stankowi <4838196+AMZN-stankowi@users.noreply.github.com> Date: Tue, 2 Nov 2021 09:03:38 -0700 Subject: [PATCH] Fixes for release builds with DCO fix (#5164) * This set of changes is work toward allowing release builds to work with asset bundler generated bundles and legacy, non-prefab levels. This requires some other in-flight changes before this work is complete. Updated engine seed list + fixed automated test ComponentApplicationLifecycle has the ability to automatically register events if asked to register a handler and the event doesn't exist. This is only intended for cases where you need to register a handler early in startup before the settings registry file is loaded. Added two new lifecycle events: One after the system entity has been activated, and one after the system interface has been created. If you load an archive before the system entity has been activated, archive.cpp caches information about those archives until that time, so it can finish registration. This is because the serialization system and BundlingSystemComponent both need to be available to do this registration, but the bundles have to be loaded before those are initialized so that the settings registry file can be loaded. Fixed an error were mounted pak files were searching for levels.pak and not level.pak, and not finding them. I'm pretty sure this logic doesn't do anything functional either way, but I've been testing legacy levels with this change and they work now. Moved wildcard pak loading to where engine.pak is loaded. This is because the settings registry file that defines the IO stack to spin up must be available early in application startup, and this file must be within a mounted pak file. If you're using asset bundler generated bundles, they need to be loaded at this time so that file can be loaded. Atom's BootstrapSystemComponent.cpp no longer initializes on AssetCatalogLoaded, and instead initializes on the ApplicationLifecycle event SystemInterfaceCreated. This is because the base assetcatalog.xml file is really just a development time concept, this file should not be used in packaged release builds, because those builds will make use of delta catalogs in each bundle loaded. The asset catalog contains the list of all assets that were in the cache at development time, and this contains content that developers don't want to ship, and they may want to specifically hide from their customers, so data miners don't find secrets about upcoming game content. Recovering from a branch that had incorrect DCO Signed-off-by: stankowi <4838196+AMZN-stankowi@users.noreply.github.com> * Fixed an incorrect ebus disconnect and removed an include that's no longer needed Signed-off-by: stankowi <4838196+AMZN-stankowi@users.noreply.github.com> * Fixed a copy and paste typo from trying to recover the previous pull request Signed-off-by: stankowi <4838196+AMZN-stankowi@users.noreply.github.com> * Updated product IDs for the settings registry builder to no longer collide with the JSON builder. Now they are based on a hash of the configuration. Updated the engine default seed list to include the new asset ID info for the renamed bootstrap file Signed-off-by: stankowi <4838196+AMZN-stankowi@users.noreply.github.com> * Updated the path to the application lifecycle events, because runtime settings aren't included in the merged bootstrap file. Addressed some feedback on printing out a string view on an error Signed-off-by: stankowi <4838196+AMZN-stankowi@users.noreply.github.com> * Removed a test that uses old assets that aren't relevant. We may not need this test anymore, but if we do we've backlogged a task to create a new test to cover this behavior without using old assets. Signed-off-by: stankowi <4838196+AMZN-stankowi@users.noreply.github.com> * Renamed SystemInterfaceCreated event to LegacySystemInterfaceCreated Removed SystemEntityActivated event. Now that I have the rest of the fixes in this pull request, this new event wasn't needed, the already existing SystemComponentsActivated event does what I need. Changed list to vector Signed-off-by: stankowi <4838196+AMZN-stankowi@users.noreply.github.com> --- Assets/Engine/Engine_Dependencies.xml | 23 +- Assets/Engine/SeedAssetList.seed | 877 ++++++------------ .../asset_processor_batch_dependency_tests.py | 76 -- Code/Editor/CryEdit.cpp | 6 + .../ComponentApplicationLifecycle.cpp | 12 +- .../Component/ComponentApplicationLifecycle.h | 6 +- .../AzFramework/Application/Application.cpp | 1 - .../AzFramework/Archive/Archive.cpp | 76 +- .../AzFramework/AzFramework/Archive/Archive.h | 37 + .../Application/GameApplication.cpp | 7 + Code/LauncherUnified/Launcher.cpp | 3 + Code/Legacy/CrySystem/System.h | 2 +- Code/Legacy/CrySystem/SystemInit.cpp | 25 +- .../SettingsRegistryBuilder.cpp | 10 +- .../Code/Source/BootstrapSystemComponent.cpp | 25 +- .../Code/Source/BootstrapSystemComponent.h | 15 +- .../PreviewRendererSystemComponent.cpp | 23 +- .../PreviewRendererSystemComponent.h | 5 - Registry/application_lifecycle_events.setreg | 35 +- 19 files changed, 470 insertions(+), 794 deletions(-) diff --git a/Assets/Engine/Engine_Dependencies.xml b/Assets/Engine/Engine_Dependencies.xml index 0d6541e18e..b969b3bc97 100644 --- a/Assets/Engine/Engine_Dependencies.xml +++ b/Assets/Engine/Engine_Dependencies.xml @@ -1,16 +1,9 @@ - - - - - - - - - - - - - - - \ No newline at end of file + + + + + + + + diff --git a/Assets/Engine/SeedAssetList.seed b/Assets/Engine/SeedAssetList.seed index aafbffbe8f..18ccd19dc3 100644 --- a/Assets/Engine/SeedAssetList.seed +++ b/Assets/Engine/SeedAssetList.seed @@ -1,621 +1,260 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - diff --git a/AutomatedTesting/Gem/PythonTests/assetpipeline/asset_processor_tests/asset_processor_batch_dependency_tests.py b/AutomatedTesting/Gem/PythonTests/assetpipeline/asset_processor_tests/asset_processor_batch_dependency_tests.py index 264f690534..303a012cda 100755 --- a/AutomatedTesting/Gem/PythonTests/assetpipeline/asset_processor_tests/asset_processor_batch_dependency_tests.py +++ b/AutomatedTesting/Gem/PythonTests/assetpipeline/asset_processor_tests/asset_processor_batch_dependency_tests.py @@ -47,82 +47,6 @@ class TestsAssetProcessorBatch_DependenycyTests(object): """ AssetProcessorBatch Dependency tests """ - - @pytest.mark.test_case_id("C16877166") - @pytest.mark.BAT - @pytest.mark.assetpipeline - # fmt:off - def test_WindowsMacPlatforms_RunAPBatch_NotMissingDependency(self, ap_setup_fixture, asset_processor, - workspace): - # fmt:on - """ - Engine Schema - This test case has a conditional scenario depending on the existence of surfacetypes.xml in a project. - Some projects have this file and others do not. Run the conditional scenario depending on the existence - of the file in the project - libs/materialeffects/surfacetypes.xml is listed as an entry engine_dependencies.xml - libs/materialeffects/surfacetypes.xml is not listed as a missing dependency - in the 'assetprocessorbatch' console output - - Test Steps: - 1. Assets are pre-processed - 2. Verify that engine_dependencies.xml exists - 3. Verify engine_dependencies.xml has surfacetypes.xml present - 4. Run Missing Dependency scanner against the engine_dependenciese.xml - 5. Verify that Surfacetypes.xml is NOT in the missing depdencies output - 6. Add the schema file which allows our xml parser to understand dependencies for our engine_dependencies file - 7. Process assets - 8. Run Missing Dependency scanner against the engine_dependenciese.xml - 9. Verify that surfacetypes.xml is in the missing dependencies out - """ - - env = ap_setup_fixture - BATCH_LOG_PATH = env["ap_batch_log_file"] - asset_processor.create_temp_asset_root() - asset_processor.add_relative_source_asset(os.path.join("Assets", "Engine", "Engine_Dependencies.xml")) - asset_processor.add_scan_folder(os.path.join("Assets", "Engine")) - asset_processor.add_relative_source_asset(os.path.join("Assets", "Engine", "Libs", "MaterialEffects", "surfacetypes.xml")) - - # Precondition: Assets are all processed - asset_processor.batch_process() - - DEPENDENCIES_PATH = os.path.join(asset_processor.temp_project_cache(), "engine_dependencies.xml") - assert os.path.exists(DEPENDENCIES_PATH), "The engine_dependencies.xml does not exist." - surfacetypes_in_dependencies = False - surfacetypes_missing_logline = False - - # Read engine_dependencies.xml to see if surfacetypes.xml is present - with open(DEPENDENCIES_PATH, "r") as dependencies_file: - for line in dependencies_file.readlines(): - if "surfacetypes.xml" in line: - surfacetypes_in_dependencies = True - logger.info("Surfacetypes.xml was listed in the engine_dependencies.xml file.") - break - - if not surfacetypes_in_dependencies: - logger.info("Surfacetypes.xml was not listed in the engine_dependencies.xml file.") - - _, output = asset_processor.batch_process(capture_output=True, - extra_params="--dsp=%engine_dependencies.xml") - log = APOutputParser(output) - for _ in log.get_lines(run=-1, contains=["surfacetypes.xml", "Missing"]): - surfacetypes_missing_logline = True - - assert surfacetypes_missing_logline, "Surfacetypes.xml not seen in the batch log as missing." - - # Add the schema file which allows our xml parser to understand dependencies for our engine_dependencies file - asset_processor.add_relative_source_asset(os.path.join("Assets", "Engine", "Schema", "enginedependency.xmlschema")) - asset_processor.batch_process() - - _, output = asset_processor.batch_process(capture_output=True, - extra_params="--dsp=%engine_dependencies.xml") - log = APOutputParser(output) - surfacetypes_missing_logline = False - for _ in log.get_lines(run=-1, contains=["surfacetypes.xml", "Missing"]): - surfacetypes_missing_logline = True - - assert not surfacetypes_missing_logline, "Surfacetypes.xml not seen in the batch log as missing." - schemas = [ ("C16877167", ".ent"), ("C16877168", "Environment.xml"), diff --git a/Code/Editor/CryEdit.cpp b/Code/Editor/CryEdit.cpp index 0277abe60d..c5c3acd2f6 100644 --- a/Code/Editor/CryEdit.cpp +++ b/Code/Editor/CryEdit.cpp @@ -45,6 +45,7 @@ AZ_POP_DISABLE_WARNING // AzCore #include +#include #include #include #include @@ -1686,6 +1687,11 @@ bool CCryEditApp::InitInstance() return false; } + if (AZ::SettingsRegistryInterface* settingsRegistry = AZ::SettingsRegistry::Get()) + { + AZ::ComponentApplicationLifecycle::SignalEvent(*settingsRegistry, "LegacySystemInterfaceCreated", R"({})"); + } + // Process some queued events come from system init // Such as asset catalog loaded notification. // There are some systems need to load configurations from assets for post initialization but before loading level diff --git a/Code/Framework/AzCore/AzCore/Component/ComponentApplicationLifecycle.cpp b/Code/Framework/AzCore/AzCore/Component/ComponentApplicationLifecycle.cpp index 533d55ed27..8e15871bc0 100644 --- a/Code/Framework/AzCore/AzCore/Component/ComponentApplicationLifecycle.cpp +++ b/Code/Framework/AzCore/AzCore/Component/ComponentApplicationLifecycle.cpp @@ -56,15 +56,21 @@ namespace AZ::ComponentApplicationLifecycle } bool RegisterHandler(AZ::SettingsRegistryInterface& settingsRegistry, AZ::SettingsRegistryInterface::NotifyEventHandler& handler, - AZ::SettingsRegistryInterface::NotifyCallback callback, AZStd::string_view eventName) + AZ::SettingsRegistryInterface::NotifyCallback callback, AZStd::string_view eventName, bool autoRegisterEvent) { using FixedValueString = AZ::SettingsRegistryInterface::FixedValueString; using Type = AZ::SettingsRegistryInterface::Type; using NotifyEventHandler = AZ::SettingsRegistryInterface::NotifyEventHandler; - if (!ValidateEvent(settingsRegistry, eventName)) + // Some systems may attempt to register a handler before the settings registry has been loaded + // If so, this flag lets them automatically register an event if it hasn't yet been registered. + // RegisterEvent calls validate event. + if ((!autoRegisterEvent && !ValidateEvent(settingsRegistry, eventName)) || + (autoRegisterEvent && !RegisterEvent(settingsRegistry, eventName))) { - AZ_Warning("ComponentApplicationLifecycle", false, R"(Cannot register event %.*s. Name does is not a field of object "%.*s".)" + AZ_Warning( + "ComponentApplicationLifecycle", false, + R"(Cannot register event %.*s. Name is not a field of object "%.*s".)" R"( Please make sure the entry exists in the '/Registry/application_lifecycle_events.setreg")" " or in *.setreg within the project", AZ_STRING_ARG(eventName), AZ_STRING_ARG(ApplicationLifecycleEventRegistrationKey)); return false; diff --git a/Code/Framework/AzCore/AzCore/Component/ComponentApplicationLifecycle.h b/Code/Framework/AzCore/AzCore/Component/ComponentApplicationLifecycle.h index 5fc6c4b2c3..6f07c0da3f 100644 --- a/Code/Framework/AzCore/AzCore/Component/ComponentApplicationLifecycle.h +++ b/Code/Framework/AzCore/AzCore/Component/ComponentApplicationLifecycle.h @@ -14,7 +14,7 @@ namespace AZ::ComponentApplicationLifecycle { //! Root Key where lifecycle events should be registered under - inline constexpr AZStd::string_view ApplicationLifecycleEventRegistrationKey = "/O3DE/Runtime/Application/LifecycleEvents"; + inline constexpr AZStd::string_view ApplicationLifecycleEventRegistrationKey = "/O3DE/Application/LifecycleEvents"; //! Validates that the event @eventName is stored in the array at ApplicationLifecycleEventRegistrationKey @@ -48,7 +48,9 @@ namespace AZ::ComponentApplicationLifecycle //! if the specified @eventName passes validation //! @param callback will be moved into the handler if the specified @eventName is valid //! @param eventName name of key underneath the ApplicationLifecycleEventRegistrationKey to register + //! @param autoRegisterEvent automatically register this event if it hasn't been registered yet. This is useful + //! when registering a handler before the settings registry has been loaded. //! @return true if the handler was registered with the SettingsRegistry NotifyEvent bool RegisterHandler(AZ::SettingsRegistryInterface& settingsRegistry, AZ::SettingsRegistryInterface::NotifyEventHandler& handler, - AZ::SettingsRegistryInterface::NotifyCallback callback, AZStd::string_view eventName); + AZ::SettingsRegistryInterface::NotifyCallback callback, AZStd::string_view eventName, bool autoRegisterEvent = false); } diff --git a/Code/Framework/AzFramework/AzFramework/Application/Application.cpp b/Code/Framework/AzFramework/AzFramework/Application/Application.cpp index 8fb986802e..323e834413 100644 --- a/Code/Framework/AzFramework/AzFramework/Application/Application.cpp +++ b/Code/Framework/AzFramework/AzFramework/Application/Application.cpp @@ -204,7 +204,6 @@ namespace AzFramework systemEntity->Activate(); AZ_Assert(systemEntity->GetState() == AZ::Entity::State::Active, "System Entity failed to activate."); - if (m_isStarted = (systemEntity->GetState() == AZ::Entity::State::Active); m_isStarted) { if (m_startupParameters.m_loadAssetCatalog) diff --git a/Code/Framework/AzFramework/AzFramework/Archive/Archive.cpp b/Code/Framework/AzFramework/AzFramework/Archive/Archive.cpp index cd30b753b5..c1c5775958 100644 --- a/Code/Framework/AzFramework/AzFramework/Archive/Archive.cpp +++ b/Code/Framework/AzFramework/AzFramework/Archive/Archive.cpp @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -363,6 +364,23 @@ namespace AZ::IO , m_mainThreadId{ AZStd::this_thread::get_id() } { CompressionBus::Handler::BusConnect(); + + // If the settings registry is not available at this point, + // then something catastrophic has happened in the application startup. + // That should have been caught and messaged out earlier in startup. + if (auto settingsRegistry = AZ::SettingsRegistry::Get(); settingsRegistry != nullptr) + { + // Automatically register the event if it's not registered, because + // this system is initialized before the settings registry has loaded the event list. + AZ::ComponentApplicationLifecycle::RegisterHandler( + *settingsRegistry, m_componentApplicationLifecycleHandler, + [this](AZStd::string_view /*path*/, AZ::SettingsRegistryInterface::Type /*type*/) + { + OnSystemEntityActivated(); + }, + "SystemComponentsActivated", + /*autoRegisterEvent*/ true); + } } ////////////////////////////////////////////////////////////////////////// @@ -1175,13 +1193,20 @@ namespace AZ::IO } } - auto bundleManifest = GetBundleManifest(desc.pZip); AZStd::shared_ptr bundleCatalog; + auto bundleManifest = GetBundleManifest(desc.pZip); if (bundleManifest) { bundleCatalog = GetBundleCatalog(desc.pZip, bundleManifest->GetCatalogName()); } + // If this archive is loaded before the serialize context is available, then the manifest and catalog will need to be loaded later. + if (!bundleManifest || !bundleCatalog) + { + m_archivesWithCatalogsToLoad.push_back( + ArchivesWithCatalogsToLoad(szFullPath, szBindRoot, flags, nextBundle, desc.m_strFileName)); + } + bool usePrefabSystemForLevels = false; AzFramework::ApplicationRequests::Bus::BroadcastResult( usePrefabSystemForLevels, &AzFramework::ApplicationRequests::IsPrefabSystemForLevelsEnabled); @@ -1219,12 +1244,17 @@ namespace AZ::IO m_levelOpenEvent.Signal(levelDirs); } - AZ::IO::ArchiveNotificationBus::Broadcast([](AZ::IO::ArchiveNotifications* archiveNotifications, const char* bundleName, - AZStd::shared_ptr bundleManifest, const AZ::IO::FixedMaxPath& nextBundle, AZStd::shared_ptr bundleCatalog) + if (bundleManifest && bundleCatalog) { - archiveNotifications->BundleOpened(bundleName, bundleManifest, nextBundle.c_str(), bundleCatalog); - }, desc.m_strFileName.c_str(), bundleManifest, nextBundle, bundleCatalog); - + AZ::IO::ArchiveNotificationBus::Broadcast( + [](AZ::IO::ArchiveNotifications* archiveNotifications, const char* bundleName, + AZStd::shared_ptr bundleManifest, const AZ::IO::FixedMaxPath& nextBundle, + AZStd::shared_ptr bundleCatalog) + { + archiveNotifications->BundleOpened(bundleName, bundleManifest, nextBundle.c_str(), bundleCatalog); + }, + desc.m_strFileName.c_str(), bundleManifest, nextBundle, bundleCatalog); + } return true; } @@ -2138,7 +2168,7 @@ namespace AZ::IO } currentDirPattern = currentDir + AZ_FILESYSTEM_SEPARATOR_WILDCARD; - currentFilePattern = currentDir + AZ_CORRECT_FILESYSTEM_SEPARATOR_STRING + "levels.pak"; + currentFilePattern = currentDir + AZ_CORRECT_FILESYSTEM_SEPARATOR_STRING + "level.pak"; ZipDir::FileEntry* fileEntry = findFile.FindExact(currentFilePattern.c_str()); if (fileEntry) @@ -2175,4 +2205,36 @@ namespace AZ::IO return catalogInfo; } + + void Archive::OnSystemEntityActivated() + { + for (const auto& archiveInfo : m_archivesWithCatalogsToLoad) + { + AZStd::intrusive_ptr archive = + OpenArchive(archiveInfo.m_fullPath, archiveInfo.m_bindRoot, archiveInfo.m_flags, nullptr); + if (!archive) + { + continue; + } + + ZipDir::CachePtr pZip = static_cast(archive.get())->GetCache(); + + AZStd::shared_ptr bundleCatalog; + auto bundleManifest = GetBundleManifest(pZip); + if (bundleManifest) + { + bundleCatalog = GetBundleCatalog(pZip, bundleManifest->GetCatalogName()); + } + + AZ::IO::ArchiveNotificationBus::Broadcast( + [](AZ::IO::ArchiveNotifications* archiveNotifications, const char* bundleName, + AZStd::shared_ptr bundleManifest, const AZ::IO::FixedMaxPath& nextBundle, + AZStd::shared_ptr bundleCatalog) + { + archiveNotifications->BundleOpened(bundleName, bundleManifest, nextBundle.c_str(), bundleCatalog); + }, + archiveInfo.m_strFileName.c_str(), bundleManifest, archiveInfo.m_nextBundle, bundleCatalog); + } + m_archivesWithCatalogsToLoad.clear(); + } } diff --git a/Code/Framework/AzFramework/AzFramework/Archive/Archive.h b/Code/Framework/AzFramework/AzFramework/Archive/Archive.h index f08d90a66e..279702b433 100644 --- a/Code/Framework/AzFramework/AzFramework/Archive/Archive.h +++ b/Code/Framework/AzFramework/AzFramework/Archive/Archive.h @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -271,6 +272,11 @@ namespace AZ::IO ZipDir::CachePtr* pZip = {}) const; private: + // Archives can't be fully mounted until the system entity has been activated, + // because mounting them requires the BundlingSystemComponent and the serialization system + // to both be available. + void OnSystemEntityActivated(); + bool OpenPackCommon(AZStd::string_view szBindRoot, AZStd::string_view pName, AZStd::intrusive_ptr pData = nullptr, bool addLevels = true); bool OpenPacksCommon(AZStd::string_view szDir, AZStd::string_view pWildcardIn, AZStd::vector* pFullPaths = nullptr, bool addLevels = true); @@ -313,6 +319,8 @@ namespace AZ::IO mutable AZStd::shared_mutex m_csZips; ZipArray m_arrZips; + AZ::SettingsRegistryInterface::NotifyEventHandler m_componentApplicationLifecycleHandler; + ////////////////////////////////////////////////////////////////////////// // Opened files collector. ////////////////////////////////////////////////////////////////////////// @@ -339,5 +347,34 @@ namespace AZ::IO // [LYN-2376] Remove once legacy slice support is removed LevelPackOpenEvent m_levelOpenEvent; LevelPackCloseEvent m_levelCloseEvent; + + // If pak files are loaded before the serialization and bundling system + // are ready to go, their asset catalogs can't be loaded. + // In this case, cache information about those archives, + // and attempt to load the catalogs later, when the required systems are enabled. + struct ArchivesWithCatalogsToLoad + { + ArchivesWithCatalogsToLoad( + AZStd::string_view fullPath, + AZStd::string_view bindRoot, + int flags, + AZ::IO::PathView nextBundle, + AZ::IO::Path strFileName) + : m_fullPath(fullPath) + , m_bindRoot(bindRoot) + , m_flags(flags) + , m_nextBundle(nextBundle) + , m_strFileName(strFileName) + { + } + + AZ::IO::Path m_strFileName; + AZStd::string m_fullPath; + AZStd::string m_bindRoot; + AZ::IO::PathView m_nextBundle; + int m_flags; + }; + + AZStd::vector m_archivesWithCatalogsToLoad; }; } diff --git a/Code/Framework/AzGameFramework/AzGameFramework/Application/GameApplication.cpp b/Code/Framework/AzGameFramework/AzGameFramework/Application/GameApplication.cpp index 6957844452..f0417d206e 100644 --- a/Code/Framework/AzGameFramework/AzGameFramework/Application/GameApplication.cpp +++ b/Code/Framework/AzGameFramework/AzGameFramework/Application/GameApplication.cpp @@ -45,6 +45,13 @@ namespace AzGameFramework enginePakPath = AZ::IO::FixedMaxPath(AZ::Utils::GetExecutableDirectory()) / "engine.pak"; m_archive->OpenPack("@products@", enginePakPath.Native()); } + + // By default, load all archives in the products folder. + // If you want to adjust this for your project, make sure that the archive containing + // the bootstrap for the settings registry is still loaded here, and any archives containing + // assets used early in startup, like default shaders, are loaded here. + constexpr AZStd::string_view paksFolder = "@products@/*.pak"; // (@products@ assumed) + m_archive->OpenPacks(paksFolder); } GameApplication::~GameApplication() diff --git a/Code/LauncherUnified/Launcher.cpp b/Code/LauncherUnified/Launcher.cpp index 0ef9cdfc3b..0f832ff1a5 100644 --- a/Code/LauncherUnified/Launcher.cpp +++ b/Code/LauncherUnified/Launcher.cpp @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -664,6 +665,8 @@ namespace O3DELauncher systemInitParams.pSystem = CreateSystemInterface(systemInitParams); #endif // !defined(AZ_MONOLITHIC_BUILD) + AZ::ComponentApplicationLifecycle::SignalEvent(*settingsRegistry, "LegacySystemInterfaceCreated", R"({})"); + ReturnCode status = ReturnCode::Success; if (systemInitParams.pSystem) diff --git a/Code/Legacy/CrySystem/System.h b/Code/Legacy/CrySystem/System.h index 015b09a69b..a3a0f12278 100644 --- a/Code/Legacy/CrySystem/System.h +++ b/Code/Legacy/CrySystem/System.h @@ -590,7 +590,7 @@ public: bool InitVTuneProfiler(); - void OpenBasicPaks(); + void OpenPlatformPaks(); void OpenLanguagePak(const char* sLanguage); void OpenLanguageAudioPak(const char* sLanguage); void GetLocalizedPath(const char* sLanguage, AZStd::string& sLocalizedPath); diff --git a/Code/Legacy/CrySystem/SystemInit.cpp b/Code/Legacy/CrySystem/SystemInit.cpp index ee1c498a9a..ac0683cf1b 100644 --- a/Code/Legacy/CrySystem/SystemInit.cpp +++ b/Code/Legacy/CrySystem/SystemInit.cpp @@ -649,7 +649,7 @@ bool CSystem::InitFileSystem_LoadEngineFolders(const SSystemInitParams&) auto projectName = AZ::Utils::GetProjectName(); AZ_Printf(AZ_TRACE_SYSTEM_WINDOW, "Project Name: %s\n", projectName.empty() ? "None specified" : projectName.c_str()); - OpenBasicPaks(); + OpenPlatformPaks(); // Load game-specific folder. LoadConfiguration("game.cfg"); @@ -786,29 +786,19 @@ void CSystem::InitLocalization() OpenLanguageAudioPak(language.c_str()); } -void CSystem::OpenBasicPaks() +void CSystem::OpenPlatformPaks() { - static bool bBasicPaksLoaded = false; - if (bBasicPaksLoaded) + static bool bPlatformPaksLoaded = false; + if (bPlatformPaksLoaded) { return; } - bBasicPaksLoaded = true; - - // open pak files - constexpr AZStd::string_view paksFolder = "@products@/*.pak"; // (@products@ assumed) - m_env.pCryPak->OpenPacks(paksFolder); - - InlineInitializationProcessing("CSystem::OpenBasicPaks OpenPacks( paksFolder.c_str() )"); + bPlatformPaksLoaded = true; ////////////////////////////////////////////////////////////////////////// // Open engine packs ////////////////////////////////////////////////////////////////////////// - const char* const assetsDir = "@products@"; - - // After game paks to have same search order as with files on disk - m_env.pCryPak->OpenPack(assetsDir, "engine.pak"); #if defined(AZ_RESTRICTED_PLATFORM) #define AZ_RESTRICTED_SECTION SYSTEMINIT_CPP_SECTION_15 @@ -816,6 +806,7 @@ void CSystem::OpenBasicPaks() #endif #ifdef AZ_PLATFORM_ANDROID + const char* const assetsDir = "@products@"; // Load Android Obb files if available const char* obbStorage = AZ::Android::Utils::GetObbStoragePath(); AZStd::string mainObbPath = AZStd::move(AZStd::string::format("%s/%s", obbStorage, AZ::Android::Utils::GetObbFileName(true))); @@ -824,7 +815,7 @@ void CSystem::OpenBasicPaks() m_env.pCryPak->OpenPack(assetsDir, patchObbPath.c_str()); #endif //AZ_PLATFORM_ANDROID - InlineInitializationProcessing("CSystem::OpenBasicPaks OpenPacks( Engine... )"); + InlineInitializationProcessing("CSystem::OpenPlatformPaks OpenPacks( Engine... )"); } ////////////////////////////////////////////////////////////////////////// @@ -1328,7 +1319,7 @@ AZ_POP_DISABLE_WARNING ////////////////////////////////////////////////////////////////////////// // Open basic pak files after intro movie playback started ////////////////////////////////////////////////////////////////////////// - OpenBasicPaks(); + OpenPlatformPaks(); ////////////////////////////////////////////////////////////////////////// // AUDIO diff --git a/Code/Tools/AssetProcessor/native/InternalBuilders/SettingsRegistryBuilder.cpp b/Code/Tools/AssetProcessor/native/InternalBuilders/SettingsRegistryBuilder.cpp index 6f15fa5ea2..7823a0582f 100644 --- a/Code/Tools/AssetProcessor/native/InternalBuilders/SettingsRegistryBuilder.cpp +++ b/Code/Tools/AssetProcessor/native/InternalBuilders/SettingsRegistryBuilder.cpp @@ -159,6 +159,7 @@ namespace AssetProcessor builderDesc.m_busId = m_builderId; builderDesc.m_createJobFunction = AZStd::bind(&SettingsRegistryBuilder::CreateJobs, this, AZStd::placeholders::_1, AZStd::placeholders::_2); builderDesc.m_processJobFunction = AZStd::bind(&SettingsRegistryBuilder::ProcessJob, this, AZStd::placeholders::_1, AZStd::placeholders::_2); + builderDesc.m_version = 1; AssetBuilderSDK::AssetBuilderBus::Broadcast(&AssetBuilderSDK::AssetBuilderBusTraits::RegisterBuilderInformation, builderDesc); @@ -301,7 +302,6 @@ namespace AssetProcessor if (!platformCodes.empty()) { AZStd::string_view platform = platformCodes.front(); - constexpr AZ::u32 productSubID = 0; for (size_t i = 0; i < AZStd::size(specializations); ++i) { const AZ::SettingsRegistryInterface::Specializations& specialization = specializations[i]; @@ -413,7 +413,8 @@ namespace AssetProcessor return; } - outputPath += specialization.GetSpecialization(0); // Append configuration + AZStd::string_view specializationString(specialization.GetSpecialization(0)); + outputPath += specializationString; // Append configuration outputPath += ".setreg"; AZ::IO::SystemFile file; @@ -430,7 +431,10 @@ namespace AssetProcessor } file.Close(); - response.m_outputProducts.emplace_back(outputPath, m_assetType, productSubID + aznumeric_cast(i)); + AZ::u32 hashedSpecialization = static_cast(AZStd::hash{}(specializationString)); + AZ_Assert(hashedSpecialization != 0, "Product ID generation failed for specialization %.*s. This can result in a product ID collision with other builders for this asset.", + AZ_STRING_ARG(specializationString)); + response.m_outputProducts.emplace_back(outputPath, m_assetType, hashedSpecialization); response.m_outputProducts.back().m_dependenciesHandled = true; outputPath.erase(extensionOffset); diff --git a/Gems/Atom/Bootstrap/Code/Source/BootstrapSystemComponent.cpp b/Gems/Atom/Bootstrap/Code/Source/BootstrapSystemComponent.cpp index ed3241b312..84fc58718c 100644 --- a/Gems/Atom/Bootstrap/Code/Source/BootstrapSystemComponent.cpp +++ b/Gems/Atom/Bootstrap/Code/Source/BootstrapSystemComponent.cpp @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -132,7 +133,6 @@ namespace AZ m_createDefaultScene = false; } - AzFramework::AssetCatalogEventBus::Handler::BusConnect(); TickBus::Handler::BusConnect(); // Listen for window system requests (e.g. requests for default window handle) @@ -143,6 +143,20 @@ namespace AZ Render::Bootstrap::DefaultWindowBus::Handler::BusConnect(); Render::Bootstrap::RequestBus::Handler::BusConnect(); + + // If the settings registry isn't available, something earlier in startup will report that failure. + if (auto settingsRegistry = AZ::SettingsRegistry::Get(); settingsRegistry != nullptr) + { + // Automatically register the event if it's not registered, because + // this system is initialized before the settings registry has loaded the event list. + AZ::ComponentApplicationLifecycle::RegisterHandler( + *settingsRegistry, m_componentApplicationLifecycleHandler, + [this](AZStd::string_view /*path*/, AZ::SettingsRegistryInterface::Type /*type*/) + { + Initialize(); + }, + "LegacySystemInterfaceCreated"); + } } void BootstrapSystemComponent::Deactivate() @@ -153,7 +167,6 @@ namespace AZ AzFramework::WindowSystemRequestBus::Handler::BusDisconnect(); AzFramework::WindowSystemNotificationBus::Handler::BusDisconnect(); TickBus::Handler::BusDisconnect(); - AzFramework::AssetCatalogEventBus::Handler::BusDisconnect(); m_brdfTexture = nullptr; RemoveRenderPipeline(); @@ -164,14 +177,14 @@ namespace AZ m_windowHandle = nullptr; } - void BootstrapSystemComponent::OnCatalogLoaded(const char* /*catalogFile*/) + void BootstrapSystemComponent::Initialize() { - if (m_isAssetCatalogLoaded) + if (m_isInitialized) { return; } - m_isAssetCatalogLoaded = true; + m_isInitialized = true; if (!RPI::RPISystemInterface::Get()->IsInitialized()) { @@ -216,7 +229,7 @@ namespace AZ { m_windowHandle = windowHandle; - if (m_isAssetCatalogLoaded) + if (m_isInitialized) { CreateWindowContext(); if (m_createDefaultScene) diff --git a/Gems/Atom/Bootstrap/Code/Source/BootstrapSystemComponent.h b/Gems/Atom/Bootstrap/Code/Source/BootstrapSystemComponent.h index 566d19b1a4..74679f5753 100644 --- a/Gems/Atom/Bootstrap/Code/Source/BootstrapSystemComponent.h +++ b/Gems/Atom/Bootstrap/Code/Source/BootstrapSystemComponent.h @@ -8,10 +8,10 @@ #pragma once #include -#include #include +#include +#include -#include #include #include #include @@ -29,7 +29,6 @@ #include #include - namespace AZ { namespace Render @@ -40,7 +39,6 @@ namespace AZ : public Component , public TickBus::Handler , public AzFramework::WindowNotificationBus::Handler - , public AzFramework::AssetCatalogEventBus::Handler , public AzFramework::WindowSystemNotificationBus::Handler , public AzFramework::WindowSystemRequestBus::Handler , public Render::Bootstrap::DefaultWindowBus::Handler @@ -82,13 +80,12 @@ namespace AZ void OnTick(float deltaTime, AZ::ScriptTimePoint time) override; int GetTickOrder() override; - // AzFramework::AssetCatalogEventBus::Handler overrides ... - void OnCatalogLoaded(const char* catalogFile) override; - // AzFramework::WindowSystemNotificationBus::Handler overrides ... void OnWindowCreated(AzFramework::NativeWindowHandle windowHandle) override; private: + void Initialize(); + void CreateDefaultRenderPipeline(); void CreateDefaultScene(); void DestroyDefaultScene(); @@ -105,7 +102,7 @@ namespace AZ RPI::ScenePtr m_defaultScene = nullptr; AZStd::shared_ptr m_defaultFrameworkScene = nullptr; - bool m_isAssetCatalogLoaded = false; + bool m_isInitialized = false; // The id of the render pipeline created by this component RPI::RenderPipelineId m_renderPipelineId; @@ -119,6 +116,8 @@ namespace AZ // Maps AZ scenes to RPI scene weak pointers to allow looking up a ScenePtr instead of a raw Scene* AZStd::unordered_map> m_azSceneToAtomSceneMap; + + AZ::SettingsRegistryInterface::NotifyEventHandler m_componentApplicationLifecycleHandler; }; } // namespace Bootstrap } // namespace Render diff --git a/Gems/Atom/Tools/AtomToolsFramework/Code/Source/PreviewRenderer/PreviewRendererSystemComponent.cpp b/Gems/Atom/Tools/AtomToolsFramework/Code/Source/PreviewRenderer/PreviewRendererSystemComponent.cpp index fc5b31297a..d46e7b27be 100644 --- a/Gems/Atom/Tools/AtomToolsFramework/Code/Source/PreviewRenderer/PreviewRendererSystemComponent.cpp +++ b/Gems/Atom/Tools/AtomToolsFramework/Code/Source/PreviewRenderer/PreviewRendererSystemComponent.cpp @@ -47,30 +47,27 @@ namespace AtomToolsFramework void PreviewRendererSystemComponent::Activate() { - AzFramework::AssetCatalogEventBus::Handler::BusConnect(); AzFramework::ApplicationLifecycleEvents::Bus::Handler::BusConnect(); PreviewRendererSystemRequestBus::Handler::BusConnect(); + + AZ::TickBus::QueueFunction( + [this]() + { + if (!m_previewRenderer) + { + m_previewRenderer.reset(aznew AtomToolsFramework::PreviewRenderer( + "PreviewRendererSystemComponent Preview Scene", "PreviewRendererSystemComponent Preview Pipeline")); + } + }); } void PreviewRendererSystemComponent::Deactivate() { PreviewRendererSystemRequestBus::Handler::BusDisconnect(); AzFramework::ApplicationLifecycleEvents::Bus::Handler::BusDisconnect(); - AzFramework::AssetCatalogEventBus::Handler::BusDisconnect(); m_previewRenderer.reset(); } - void PreviewRendererSystemComponent::OnCatalogLoaded([[maybe_unused]] const char* catalogFile) - { - AZ::TickBus::QueueFunction([this](){ - if (!m_previewRenderer) - { - m_previewRenderer.reset(aznew AtomToolsFramework::PreviewRenderer( - "PreviewRendererSystemComponent Preview Scene", "PreviewRendererSystemComponent Preview Pipeline")); - } - }); - } - void PreviewRendererSystemComponent::OnApplicationAboutToStop() { m_previewRenderer.reset(); diff --git a/Gems/Atom/Tools/AtomToolsFramework/Code/Source/PreviewRenderer/PreviewRendererSystemComponent.h b/Gems/Atom/Tools/AtomToolsFramework/Code/Source/PreviewRenderer/PreviewRendererSystemComponent.h index 8110d84794..0b145bbdf1 100644 --- a/Gems/Atom/Tools/AtomToolsFramework/Code/Source/PreviewRenderer/PreviewRendererSystemComponent.h +++ b/Gems/Atom/Tools/AtomToolsFramework/Code/Source/PreviewRenderer/PreviewRendererSystemComponent.h @@ -9,7 +9,6 @@ #pragma once #include -#include #include #include #include @@ -19,7 +18,6 @@ namespace AtomToolsFramework //! System component that manages a global PreviewRenderer. class PreviewRendererSystemComponent final : public AZ::Component - , public AzFramework::AssetCatalogEventBus::Handler , public AzFramework::ApplicationLifecycleEvents::Bus::Handler , public PreviewRendererSystemRequestBus::Handler { @@ -38,9 +36,6 @@ namespace AtomToolsFramework void Deactivate() override; private: - // AzFramework::AssetCatalogEventBus::Handler overrides ... - void OnCatalogLoaded(const char* catalogFile) override; - // AzFramework::ApplicationLifecycleEvents overrides... void OnApplicationAboutToStop() override; diff --git a/Registry/application_lifecycle_events.setreg b/Registry/application_lifecycle_events.setreg index 58baf267b0..0d9cd0f170 100644 --- a/Registry/application_lifecycle_events.setreg +++ b/Registry/application_lifecycle_events.setreg @@ -7,24 +7,23 @@ // related to the event { "O3DE" : { - "Runtime": { - "Application": { - "LifecycleEvents": { - "SystemComponentsActivated": {}, - "SystemComponentsDeactivated": {}, - "ReflectionManagerAvailable": {}, - "ReflectionManagerUnavailable": {}, - "SystemAllocatorCreated": {}, - "SystemAllocatorPendingDestruction": {}, - "SettingsRegistryAvailable": {}, - "SettingsRegistryUnavailable": {}, - "ConsoleAvailable": {}, - "ConsoleUnavailable": {}, - "GemsLoaded": {}, - "GemsUnloaded": {}, - "FileIOAvailable": {}, - "FileIOUnavailable": {} - } + "Application": { + "LifecycleEvents": { + "SystemComponentsActivated": {}, + "SystemComponentsDeactivated": {}, + "ReflectionManagerAvailable": {}, + "ReflectionManagerUnavailable": {}, + "SystemAllocatorCreated": {}, + "SystemAllocatorPendingDestruction": {}, + "SettingsRegistryAvailable": {}, + "SettingsRegistryUnavailable": {}, + "ConsoleAvailable": {}, + "ConsoleUnavailable": {}, + "GemsLoaded": {}, + "GemsUnloaded": {}, + "FileIOAvailable": {}, + "FileIOUnavailable": {}, + "LegacySystemInterfaceCreated": {} } } }