From 37b53c06800f5eb4dcb4a86de53abb2bea51645b Mon Sep 17 00:00:00 2001 From: AMZN-koppersr <82230785+AMZN-koppersr@users.noreply.github.com> Date: Tue, 25 May 2021 09:13:11 -0700 Subject: [PATCH 01/19] Spawnables can no longer be moved. Spawnables had support for moving, but as the base class AZ::Data::AssetData doesn't support moving this was causing subtle issues. Moving spawnables wasn't used so it was removed. --- .../AzFramework/Spawnable/Spawnable.cpp | 16 ---------------- .../AzFramework/Spawnable/Spawnable.h | 4 ++-- .../Prefab/Spawnable/SpawnableUtils.cpp | 11 ----------- .../Prefab/Spawnable/SpawnableUtils.h | 1 - 4 files changed, 2 insertions(+), 30 deletions(-) diff --git a/Code/Framework/AzFramework/AzFramework/Spawnable/Spawnable.cpp b/Code/Framework/AzFramework/AzFramework/Spawnable/Spawnable.cpp index 7ab2d48814..46b3dfe87c 100644 --- a/Code/Framework/AzFramework/AzFramework/Spawnable/Spawnable.cpp +++ b/Code/Framework/AzFramework/AzFramework/Spawnable/Spawnable.cpp @@ -21,22 +21,6 @@ namespace AzFramework { } - Spawnable::Spawnable(Spawnable&& other) - : m_entities(AZStd::move(other.m_entities)) - { - } - - - Spawnable& Spawnable::operator=(Spawnable&& other) - { - if (this != &other) - { - m_entities = AZStd::move(other.m_entities); - } - - return *this; - } - const Spawnable::EntityList& Spawnable::GetEntities() const { return m_entities; diff --git a/Code/Framework/AzFramework/AzFramework/Spawnable/Spawnable.h b/Code/Framework/AzFramework/AzFramework/Spawnable/Spawnable.h index 79cea647e4..677f0326cf 100644 --- a/Code/Framework/AzFramework/AzFramework/Spawnable/Spawnable.h +++ b/Code/Framework/AzFramework/AzFramework/Spawnable/Spawnable.h @@ -41,11 +41,11 @@ namespace AzFramework Spawnable() = default; explicit Spawnable(const AZ::Data::AssetId& id, AssetStatus status = AssetStatus::NotLoaded); Spawnable(const Spawnable& rhs) = delete; - Spawnable(Spawnable&& other); + Spawnable(Spawnable&& other) = delete; ~Spawnable() override = default; Spawnable& operator=(const Spawnable& rhs) = delete; - Spawnable& operator=(Spawnable&& other); + Spawnable& operator=(Spawnable&& other) = delete; const EntityList& GetEntities() const; EntityList& GetEntities(); diff --git a/Code/Framework/AzToolsFramework/AzToolsFramework/Prefab/Spawnable/SpawnableUtils.cpp b/Code/Framework/AzToolsFramework/AzToolsFramework/Prefab/Spawnable/SpawnableUtils.cpp index 716c3098d9..3c91f99b05 100644 --- a/Code/Framework/AzToolsFramework/AzToolsFramework/Prefab/Spawnable/SpawnableUtils.cpp +++ b/Code/Framework/AzToolsFramework/AzToolsFramework/Prefab/Spawnable/SpawnableUtils.cpp @@ -24,17 +24,6 @@ namespace AzToolsFramework::Prefab::SpawnableUtils { - - AzFramework::Spawnable CreateSpawnable(const PrefabDom& prefabDom) - { - AzFramework::Spawnable spawnable; - AZStd::vector> referencedAssets; - [[maybe_unused]] bool result = CreateSpawnable(spawnable, prefabDom, referencedAssets); - AZ_Assert(result, - "Failed to Load Prefab Instance from given Prefab DOM while Spawnable creation."); - return spawnable; - } - bool CreateSpawnable(AzFramework::Spawnable& spawnable, const PrefabDom& prefabDom) { AZStd::vector> referencedAssets; diff --git a/Code/Framework/AzToolsFramework/AzToolsFramework/Prefab/Spawnable/SpawnableUtils.h b/Code/Framework/AzToolsFramework/AzToolsFramework/Prefab/Spawnable/SpawnableUtils.h index 3b5ea488cb..cdf07346d0 100644 --- a/Code/Framework/AzToolsFramework/AzToolsFramework/Prefab/Spawnable/SpawnableUtils.h +++ b/Code/Framework/AzToolsFramework/AzToolsFramework/Prefab/Spawnable/SpawnableUtils.h @@ -17,7 +17,6 @@ namespace AzToolsFramework::Prefab::SpawnableUtils { - AzFramework::Spawnable CreateSpawnable(const PrefabDom& prefabDom); bool CreateSpawnable(AzFramework::Spawnable& spawnable, const PrefabDom& prefabDom); bool CreateSpawnable(AzFramework::Spawnable& spawnable, const PrefabDom& prefabDom, AZStd::vector>& referencedAssets); From 76827ff95eab5bfbba5e2b763fb46ef4156db9aa Mon Sep 17 00:00:00 2001 From: AMZN-koppersr <82230785+AMZN-koppersr@users.noreply.github.com> Date: Wed, 26 May 2021 09:23:53 -0700 Subject: [PATCH 02/19] Added support for a priority lane for entity spawning It's now possible to have high and normal priority calls on the spawnable entities manager. This allows for events like (de)spawning and retrieving information to be executed before already queued requests, though requests cannot be reordered on the same ticket. High priority calls are executed twice per frame, while normal priority calls are called only once. --- .../Spawnable/SpawnableEntitiesContainer.cpp | 18 ++- .../Spawnable/SpawnableEntitiesInterface.h | 58 ++++++-- .../Spawnable/SpawnableEntitiesManager.cpp | 127 +++++++++--------- .../Spawnable/SpawnableEntitiesManager.h | 64 ++++++--- .../Spawnable/SpawnableSystemComponent.cpp | 18 ++- .../Spawnable/SpawnableSystemComponent.h | 8 ++ .../SpawnableEntitiesManagerTests.cpp | 84 ++++++++++-- 7 files changed, 266 insertions(+), 111 deletions(-) diff --git a/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesContainer.cpp b/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesContainer.cpp index 673701cac4..e9a78dccde 100644 --- a/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesContainer.cpp +++ b/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesContainer.cpp @@ -38,19 +38,20 @@ namespace AzFramework void SpawnableEntitiesContainer::SpawnAllEntities() { AZ_Assert(m_threadData, "Calling SpawnAllEntities on a Spawnable container that's not set."); - SpawnableEntitiesInterface::Get()->SpawnAllEntities(m_threadData->m_spawnedEntitiesTicket); + SpawnableEntitiesInterface::Get()->SpawnAllEntities(m_threadData->m_spawnedEntitiesTicket, SpawnablePriorty_Default); } void SpawnableEntitiesContainer::SpawnEntities(AZStd::vector entityIndices) { AZ_Assert(m_threadData, "Calling SpawnEntities on a Spawnable container that's not set."); - SpawnableEntitiesInterface::Get()->SpawnEntities(m_threadData->m_spawnedEntitiesTicket, AZStd::move(entityIndices)); + SpawnableEntitiesInterface::Get()->SpawnEntities( + m_threadData->m_spawnedEntitiesTicket, SpawnablePriorty_Default, AZStd::move(entityIndices)); } void SpawnableEntitiesContainer::DespawnAllEntities() { AZ_Assert(m_threadData, "Calling DespawnEntities on a Spawnable container that's not set."); - SpawnableEntitiesInterface::Get()->DespawnAllEntities(m_threadData->m_spawnedEntitiesTicket); + SpawnableEntitiesInterface::Get()->DespawnAllEntities(m_threadData->m_spawnedEntitiesTicket, SpawnablePriorty_Default); } void SpawnableEntitiesContainer::Reset(AZ::Data::Asset spawnable) @@ -66,7 +67,9 @@ namespace AzFramework m_monitor.Disconnect(); m_monitor.m_threadData.reset(); - SpawnableEntitiesInterface::Get()->Barrier(m_threadData->m_spawnedEntitiesTicket, + SpawnableEntitiesInterface::Get()->Barrier( + m_threadData->m_spawnedEntitiesTicket, + SpawnablePriorty_Default, [threadData = m_threadData](EntitySpawnTicket&) mutable { threadData.reset(); @@ -83,7 +86,9 @@ namespace AzFramework void SpawnableEntitiesContainer::Alert(AlertCallback callback) { AZ_Assert(m_threadData, "Calling DespawnEntities on a Spawnable container that's not set."); - SpawnableEntitiesInterface::Get()->Barrier(m_threadData->m_spawnedEntitiesTicket, + SpawnableEntitiesInterface::Get()->Barrier( + m_threadData->m_spawnedEntitiesTicket, + SpawnablePriorty_Default, [generation = m_threadData->m_generation, callback = AZStd::move(callback)](EntitySpawnTicket&) { callback(generation); @@ -110,6 +115,7 @@ namespace AzFramework AZ_Assert(m_threadData, "SpawnableEntitiesContainer is monitoring a spawnable, but doesn't have the associated data."); AZ_TracePrintf("Spawnables", "Reloading spawnable '%s'.\n", replacementAsset.GetHint().c_str()); - SpawnableEntitiesInterface::Get()->ReloadSpawnable(m_threadData->m_spawnedEntitiesTicket, AZStd::move(replacementAsset)); + SpawnableEntitiesInterface::Get()->ReloadSpawnable( + m_threadData->m_spawnedEntitiesTicket, SpawnablePriorty_Default, AZStd::move(replacementAsset)); } } // namespace AzFramework diff --git a/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesInterface.h b/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesInterface.h index 69bca8e111..97d06e1f37 100644 --- a/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesInterface.h +++ b/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesInterface.h @@ -14,6 +14,7 @@ #include #include +#include #include #include @@ -24,6 +25,14 @@ namespace AZ namespace AzFramework { + AZ_TYPE_SAFE_INTEGRAL(SpawnablePriority, uint8_t); + + inline static constexpr SpawnablePriority SpawnablePriorty_Highest { 0 }; + inline static constexpr SpawnablePriority SpawnablePriorty_High { 32 }; + inline static constexpr SpawnablePriority SpawnablePriorty_Default { 128 }; + inline static constexpr SpawnablePriority SpawnablePriorty_Low { 192 }; + inline static constexpr SpawnablePriority SpawnablePriorty_Lowest { 255 }; + class SpawnableEntityContainerView { public: @@ -124,10 +133,10 @@ namespace AzFramework SpawnableIndexEntityIterator m_end; }; - //! Requests to the SpawnableEntitiesInterface require a ticket with a valid spawnable that be used as a template. A ticket can - //! be reused for multiple calls on the same spawnable and is safe to use by multiple threads at the same time. Entities created + //! Requests to the SpawnableEntitiesInterface require a ticket with a valid spawnable that is used as a template. A ticket can + //! be reused for multiple calls on the same spawnable and is safe to be used by multiple threads at the same time. Entities created //! from the spawnable may be tracked by the ticket and so using the same ticket is needed to despawn the exact entities created - //! by a call so spawn entities. The life cycle of the spawned entities is tied to the ticket and all entities spawned using a + //! by a call to spawn entities. The life cycle of the spawned entities is tied to the ticket and all entities spawned using a //! ticket will be despawned when it's deleted. class EntitySpawnTicket { @@ -159,10 +168,19 @@ namespace AzFramework using BarrierCallback = AZStd::function; //! Interface definition to (de)spawn entities from a spawnable into the game world. + //! //! While the callbacks of the individual calls are being processed they will block processing any other request. Callbacks can be //! issued from threads other than the one that issued the call, including the main thread. + //! //! Calls on the same ticket are guaranteed to be executed in the order they are issued. Note that when issuing requests from //! multiple threads on the same ticket the order in which the requests are assigned to the ticket is not guaranteed. + //! + //! Most calls have a priority where values closer to 0 mean higher priority than values closer to 255. The implementation of this + //! interface may choose to use priority lanes which doesn't guarantee that higher priority requests happen before lower priority + //! requests if they don't pass the priority lane threshold. Priority lanes and their thresholds are implementation specific and may + //! differ between platforms. Note that if a call happened on a ticket with lower priority followed by a one with a higher priority + //! the first lower priority call will still needs to complete before the second higher priority call can be executed and the priority + //! of the first call will not be updated. class SpawnableEntitiesDefinition { public: @@ -173,40 +191,48 @@ namespace AzFramework virtual ~SpawnableEntitiesDefinition() = default; //! Spawn instances of all entities in the spawnable. - //! @param spawnable The Spawnable asset that will be used to create entity instances from. //! @param ticket Stores the results of the call. Use this ticket to spawn additional entities or to despawn them. + //! @param priority The priority at which this call will be executed. //! @param completionCallback Optional callback that's called when spawning entities has completed. This can be called from //! a different thread than the one that made the function call. The returned list of entities contains all the newly //! created entities. - virtual void SpawnAllEntities(EntitySpawnTicket& ticket, EntityPreInsertionCallback preInsertionCallback = {}, + virtual void SpawnAllEntities( + EntitySpawnTicket& ticket, SpawnablePriority priority, EntityPreInsertionCallback preInsertionCallback = {}, EntitySpawnCallback completionCallback = {}) = 0; //! Spawn instances of some entities in the spawnable. //! @param ticket Stores the results of the call. Use this ticket to spawn additional entities or to despawn them. + //! @param priority The priority at which this call will be executed. //! @param entityIndices The indices into the template entities stored in the spawnable that will be used to spawn entities from. //! @param completionCallback Optional callback that's called when spawning entities has completed. This can be called from //! a different thread than the one that made this function call. The returned list of entities contains all the newly //! created entities. - virtual void SpawnEntities(EntitySpawnTicket& ticket, AZStd::vector entityIndices, + virtual void SpawnEntities( + EntitySpawnTicket& ticket, SpawnablePriority priority, AZStd::vector entityIndices, EntityPreInsertionCallback preInsertionCallback = {}, EntitySpawnCallback completionCallback = {}) = 0; //! Removes all entities in the provided list from the environment. //! @param ticket The ticket previously used to spawn entities with. + //! @param priority The priority at which this call will be executed. //! @param completionCallback Optional callback that's called when despawning entities has completed. This can be called from //! a different thread than the one that made this function call. - virtual void DespawnAllEntities(EntitySpawnTicket& ticket, EntityDespawnCallback completionCallback = {}) = 0; + virtual void DespawnAllEntities( + EntitySpawnTicket& ticket, SpawnablePriority priority, EntityDespawnCallback completionCallback = {}) = 0; //! Removes all entities in the provided list from the environment and reconstructs the entities from the provided spawnable. - //! @param ticket Stores the results of the call. Use this ticket to spawn additional entities or to despawn them. + //! @param ticket Holds the information on the entities to reload. + //! @param priority The priority at which this call will be executed. //! @param spawnable The spawnable that will replace the existing spawnable. Both need to have the same asset id. //! @param completionCallback Optional callback that's called when the entities have been reloaded. This can be called from //! a different thread than the one that made this function call. The returned list of entities contains all the replacement //! entities. - virtual void ReloadSpawnable(EntitySpawnTicket& ticket, AZ::Data::Asset spawnable, + virtual void ReloadSpawnable( + EntitySpawnTicket& ticket, SpawnablePriority priority, AZ::Data::Asset spawnable, ReloadSpawnableCallback completionCallback = {}) = 0; //! List all entities that are spawned using this ticket. //! @param ticket Only the entities associated with this ticket will be listed. + //! @param priority The priority at which this call will be executed. //! @param listCallback Required callback that will be called to list the entities on. - virtual void ListEntities(EntitySpawnTicket& ticket, ListEntitiesCallback listCallback) = 0; + virtual void ListEntities(EntitySpawnTicket& ticket, SpawnablePriority priority, ListEntitiesCallback listCallback) = 0; //! List all entities that are spawned using this ticket with their spawnable index. //! Spawnables contain a flat list of entities, which are used as templates to spawn entities from. For every spawned entity //! the index of the entity in the spawnable that was used as a template is stored. This version of ListEntities will return @@ -214,17 +240,23 @@ namespace AzFramework //! the same index may appear multiple times as there are no restriction on how many instance of a specific entity can be //! created. //! @param ticket Only the entities associated with this ticket will be listed. + //! @param priority The priority at which this call will be executed. //! @param listCallback Required callback that will be called to list the entities and indices on. - virtual void ListIndicesAndEntities(EntitySpawnTicket& ticket, ListIndicesEntitiesCallback listCallback) = 0; + virtual void ListIndicesAndEntities( + EntitySpawnTicket& ticket, SpawnablePriority priority, ListIndicesEntitiesCallback listCallback) = 0; //! Claim all entities that are spawned using this ticket. Ownership of the entities is transferred from the ticket to the //! caller through the callback. After this call the ticket will have no entities associated with it. The caller of //! this function will need to manage the entities after this call. //! @param ticket Only the entities associated with this ticket will be released. + //! @param priority The priority at which this call will be executed. //! @param listCallback Required callback that will be called to transfer the entities through. - virtual void ClaimEntities(EntitySpawnTicket& ticket, ClaimEntitiesCallback listCallback) = 0; + virtual void ClaimEntities(EntitySpawnTicket& ticket, SpawnablePriority priority, ClaimEntitiesCallback listCallback) = 0; //! Blocks until all operations made on the provided ticket before the barrier call have completed. - virtual void Barrier(EntitySpawnTicket& ticket, BarrierCallback completionCallback) = 0; + //! @param ticket The ticket to monitor. + //! @param priority The priority at which this call will be executed. + //! @param completionCallback Required callback that will be called as soon as the barrier has been reached. + virtual void Barrier(EntitySpawnTicket& ticket, SpawnablePriority priority, BarrierCallback completionCallback) = 0; //! Register a handler for OnSpawned events. virtual void AddOnSpawnedHandler(AZ::Event>::Handler& handler) = 0; diff --git a/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesManager.cpp b/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesManager.cpp index 7e20f7b265..959d2ab64f 100644 --- a/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesManager.cpp +++ b/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesManager.cpp @@ -22,7 +22,19 @@ namespace AzFramework { - void SpawnableEntitiesManager::SpawnAllEntities(EntitySpawnTicket& ticket, EntityPreInsertionCallback preInsertionCallback, + template + void SpawnableEntitiesManager::QueueRequest(EntitySpawnTicket& ticket, SpawnablePriority priority, T&& request) + { + Queue& queue = priority <= HighPriorityThreshold ? m_highPriorityQueue : m_regularPriorityQueue; + { + AZStd::scoped_lock queueLock(queue.m_pendingRequestMutex); + request.m_ticketId = GetTicketPayload(ticket).m_nextTicketId++; + queue.m_pendingRequest.push(AZStd::move(request)); + } + } + + void SpawnableEntitiesManager::SpawnAllEntities( + EntitySpawnTicket& ticket, SpawnablePriority priority, EntityPreInsertionCallback preInsertionCallback, EntitySpawnCallback completionCallback) { AZ_Assert(ticket.IsValid(), "Ticket provided to SpawnAllEntities hasn't been initialized."); @@ -31,15 +43,11 @@ namespace AzFramework queueEntry.m_ticket = &ticket; queueEntry.m_completionCallback = AZStd::move(completionCallback); queueEntry.m_preInsertionCallback = AZStd::move(preInsertionCallback); - { - AZStd::scoped_lock queueLock(m_pendingRequestQueueMutex); - queueEntry.m_ticketId = GetTicketPayload(ticket).m_nextTicketId++; - m_pendingRequestQueue.push(AZStd::move(queueEntry)); - } + QueueRequest(ticket, priority, AZStd::move(queueEntry)); } void SpawnableEntitiesManager::SpawnEntities( - EntitySpawnTicket& ticket, AZStd::vector entityIndices, + EntitySpawnTicket& ticket, SpawnablePriority priority, AZStd::vector entityIndices, EntityPreInsertionCallback preInsertionCallback, EntitySpawnCallback completionCallback) { AZ_Assert(ticket.IsValid(), "Ticket provided to SpawnEntities hasn't been initialized."); @@ -49,28 +57,22 @@ namespace AzFramework queueEntry.m_entityIndices = AZStd::move(entityIndices); queueEntry.m_completionCallback = AZStd::move(completionCallback); queueEntry.m_preInsertionCallback = AZStd::move(preInsertionCallback); - { - AZStd::scoped_lock queueLock(m_pendingRequestQueueMutex); - queueEntry.m_ticketId = GetTicketPayload(ticket).m_nextTicketId++; - m_pendingRequestQueue.push(AZStd::move(queueEntry)); - } + QueueRequest(ticket, priority, AZStd::move(queueEntry)); } - void SpawnableEntitiesManager::DespawnAllEntities(EntitySpawnTicket& ticket, EntityDespawnCallback completionCallback) + void SpawnableEntitiesManager::DespawnAllEntities( + EntitySpawnTicket& ticket, SpawnablePriority priority, EntityDespawnCallback completionCallback) { AZ_Assert(ticket.IsValid(), "Ticket provided to DespawnAllEntities hasn't been initialized."); DespawnAllEntitiesCommand queueEntry; queueEntry.m_ticket = &ticket; queueEntry.m_completionCallback = AZStd::move(completionCallback); - { - AZStd::scoped_lock queueLock(m_pendingRequestQueueMutex); - queueEntry.m_ticketId = GetTicketPayload(ticket).m_nextTicketId++; - m_pendingRequestQueue.push(AZStd::move(queueEntry)); - } + QueueRequest(ticket, priority, AZStd::move(queueEntry)); } - void SpawnableEntitiesManager::ReloadSpawnable(EntitySpawnTicket& ticket, AZ::Data::Asset spawnable, + void SpawnableEntitiesManager::ReloadSpawnable( + EntitySpawnTicket& ticket, SpawnablePriority priority, AZ::Data::Asset spawnable, ReloadSpawnableCallback completionCallback) { AZ_Assert(ticket.IsValid(), "Ticket provided to ReloadSpawnable hasn't been initialized."); @@ -79,14 +81,10 @@ namespace AzFramework queueEntry.m_ticket = &ticket; queueEntry.m_spawnable = AZStd::move(spawnable); queueEntry.m_completionCallback = AZStd::move(completionCallback); - { - AZStd::scoped_lock queueLock(m_pendingRequestQueueMutex); - queueEntry.m_ticketId = GetTicketPayload(ticket).m_nextTicketId++; - m_pendingRequestQueue.push(AZStd::move(queueEntry)); - } + QueueRequest(ticket, priority, AZStd::move(queueEntry)); } - void SpawnableEntitiesManager::ListEntities(EntitySpawnTicket& ticket, ListEntitiesCallback listCallback) + void SpawnableEntitiesManager::ListEntities(EntitySpawnTicket& ticket, SpawnablePriority priority, ListEntitiesCallback listCallback) { AZ_Assert(listCallback, "ListEntities called on spawnable entities without a valid callback to use."); AZ_Assert(ticket.IsValid(), "Ticket provided to ListEntities hasn't been initialized."); @@ -94,14 +92,11 @@ namespace AzFramework ListEntitiesCommand queueEntry; queueEntry.m_ticket = &ticket; queueEntry.m_listCallback = AZStd::move(listCallback); - { - AZStd::scoped_lock queueLock(m_pendingRequestQueueMutex); - queueEntry.m_ticketId = GetTicketPayload(ticket).m_nextTicketId++; - m_pendingRequestQueue.push(AZStd::move(queueEntry)); - } + QueueRequest(ticket, priority, AZStd::move(queueEntry)); } - void SpawnableEntitiesManager::ListIndicesAndEntities(EntitySpawnTicket& ticket, ListIndicesEntitiesCallback listCallback) + void SpawnableEntitiesManager::ListIndicesAndEntities( + EntitySpawnTicket& ticket, SpawnablePriority priority, ListIndicesEntitiesCallback listCallback) { AZ_Assert(listCallback, "ListEntities called on spawnable entities without a valid callback to use."); AZ_Assert(ticket.IsValid(), "Ticket provided to ListEntities hasn't been initialized."); @@ -109,14 +104,10 @@ namespace AzFramework ListIndicesEntitiesCommand queueEntry; queueEntry.m_ticket = &ticket; queueEntry.m_listCallback = AZStd::move(listCallback); - { - AZStd::scoped_lock queueLock(m_pendingRequestQueueMutex); - queueEntry.m_ticketId = GetTicketPayload(ticket).m_nextTicketId++; - m_pendingRequestQueue.push(AZStd::move(queueEntry)); - } + QueueRequest(ticket, priority, AZStd::move(queueEntry)); } - void SpawnableEntitiesManager::ClaimEntities(EntitySpawnTicket& ticket, ClaimEntitiesCallback listCallback) + void SpawnableEntitiesManager::ClaimEntities(EntitySpawnTicket& ticket, SpawnablePriority priority, ClaimEntitiesCallback listCallback) { AZ_Assert(listCallback, "ClaimEntities called on spawnable entities without a valid callback to use."); AZ_Assert(ticket.IsValid(), "Ticket provided to ClaimEntities hasn't been initialized."); @@ -124,14 +115,10 @@ namespace AzFramework ClaimEntitiesCommand queueEntry; queueEntry.m_ticket = &ticket; queueEntry.m_listCallback = AZStd::move(listCallback); - { - AZStd::scoped_lock queueLock(m_pendingRequestQueueMutex); - queueEntry.m_ticketId = GetTicketPayload(ticket).m_nextTicketId++; - m_pendingRequestQueue.push(AZStd::move(queueEntry)); - } + QueueRequest(ticket, priority, AZStd::move(queueEntry)); } - void SpawnableEntitiesManager::Barrier(EntitySpawnTicket& ticket, BarrierCallback completionCallback) + void SpawnableEntitiesManager::Barrier(EntitySpawnTicket& ticket, SpawnablePriority priority, BarrierCallback completionCallback) { AZ_Assert(completionCallback, "Barrier on spawnable entities called without a valid callback to use."); AZ_Assert(ticket.IsValid(), "Ticket provided to Barrier hasn't been initialized."); @@ -139,11 +126,7 @@ namespace AzFramework BarrierCommand queueEntry; queueEntry.m_ticket = &ticket; queueEntry.m_completionCallback = AZStd::move(completionCallback); - { - AZStd::scoped_lock queueLock(m_pendingRequestQueueMutex); - queueEntry.m_ticketId = GetTicketPayload(ticket).m_nextTicketId++; - m_pendingRequestQueue.push(AZStd::move(queueEntry)); - } + QueueRequest(ticket, priority, AZStd::move(queueEntry)); } void SpawnableEntitiesManager::AddOnSpawnedHandler(AZ::Event>::Handler& handler) @@ -156,34 +139,54 @@ namespace AzFramework handler.Connect(m_onDespawnedEvent); } - auto SpawnableEntitiesManager::ProcessQueue() -> CommandQueueStatus + auto SpawnableEntitiesManager::ProcessQueue(CommandQueuePriority priority) -> CommandQueueStatus + { + CommandQueueStatus result = CommandQueueStatus::NoCommandsLeft; + if ((priority & CommandQueuePriority::High) == CommandQueuePriority::High) + { + if (ProcessQueue(m_highPriorityQueue) == CommandQueueStatus::HasCommandsLeft) + { + result = CommandQueueStatus::HasCommandsLeft; + } + } + if ((priority & CommandQueuePriority::Regular) == CommandQueuePriority::Regular) + { + if (ProcessQueue(m_regularPriorityQueue) == CommandQueueStatus::HasCommandsLeft) + { + result = CommandQueueStatus::HasCommandsLeft; + } + } + return result; + } + + auto SpawnableEntitiesManager::ProcessQueue(Queue& queue) -> CommandQueueStatus { AZStd::queue pendingRequestQueue; { - AZStd::scoped_lock queueLock(m_pendingRequestQueueMutex); - m_pendingRequestQueue.swap(pendingRequestQueue); + AZStd::scoped_lock queueLock(queue.m_pendingRequestMutex); + queue.m_pendingRequest.swap(pendingRequestQueue); } - if (!pendingRequestQueue.empty() || !m_delayedQueue.empty()) + if (!pendingRequestQueue.empty() || !queue.m_delayed.empty()) { AZ::SerializeContext* serializeContext = nullptr; AZ::ComponentApplicationBus::BroadcastResult(serializeContext, &AZ::ComponentApplicationBus::Events::GetSerializeContext); AZ_Assert(serializeContext, "Failed to retrieve serialization context."); // Only process the requests that are currently in this queue, not the ones that could be re-added if they still can't complete. - size_t delayedSize = m_delayedQueue.size(); + size_t delayedSize = queue.m_delayed.size(); for (size_t i = 0; i < delayedSize; ++i) { - Requests& request = m_delayedQueue.front(); + Requests& request = queue.m_delayed.front(); bool result = AZStd::visit([this, serializeContext](auto&& args) -> bool { return ProcessRequest(args, *serializeContext); }, request); if (!result) { - m_delayedQueue.emplace_back(AZStd::move(request)); + queue.m_delayed.emplace_back(AZStd::move(request)); } - m_delayedQueue.pop_front(); + queue.m_delayed.pop_front(); } do @@ -197,7 +200,7 @@ namespace AzFramework }, request); if (!result) { - m_delayedQueue.emplace_back(AZStd::move(request)); + queue.m_delayed.emplace_back(AZStd::move(request)); } pendingRequestQueue.pop(); } @@ -205,13 +208,13 @@ namespace AzFramework // Spawning entities can result in more entities being queued to spawn. Repeat spawning until the queue is // empty to avoid a chain of entity spawning getting dragged out over multiple frames. { - AZStd::scoped_lock queueLock(m_pendingRequestQueueMutex); - m_pendingRequestQueue.swap(pendingRequestQueue); + AZStd::scoped_lock queueLock(queue.m_pendingRequestMutex); + queue.m_pendingRequest.swap(pendingRequestQueue); } } while (!pendingRequestQueue.empty()); } - return m_delayedQueue.empty() ? CommandQueueStatus::NoCommandLeft : CommandQueueStatus::HasCommandsLeft; + return queue.m_delayed.empty() ? CommandQueueStatus::NoCommandsLeft : CommandQueueStatus::HasCommandsLeft; } void* SpawnableEntitiesManager::CreateTicket(AZ::Data::Asset&& spawnable) @@ -226,9 +229,9 @@ namespace AzFramework DestroyTicketCommand queueEntry; queueEntry.m_ticket = reinterpret_cast(ticket); { - AZStd::scoped_lock queueLock(m_pendingRequestQueueMutex); + AZStd::scoped_lock queueLock(m_regularPriorityQueue.m_pendingRequestMutex); queueEntry.m_ticketId = reinterpret_cast(ticket)->m_nextTicketId++; - m_pendingRequestQueue.push(AZStd::move(queueEntry)); + m_regularPriorityQueue.m_pendingRequest.push(AZStd::move(queueEntry)); } } diff --git a/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesManager.h b/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesManager.h index 3481ab180a..137376ac7b 100644 --- a/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesManager.h +++ b/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesManager.h @@ -29,8 +29,6 @@ namespace AZ namespace AzFramework { - using EntityIdMap = AZStd::unordered_map; - class SpawnableEntitiesManager : public SpawnableEntitiesInterface::Registrar { @@ -38,31 +36,48 @@ namespace AzFramework AZ_RTTI(AzFramework::SpawnableEntitiesManager, "{6E14333F-128C-464C-94CA-A63B05A5E51C}"); AZ_CLASS_ALLOCATOR(SpawnableEntitiesManager, AZ::SystemAllocator, 0); + using EntityIdMap = AZStd::unordered_map; + enum class CommandQueueStatus : bool { HasCommandsLeft, - NoCommandLeft + NoCommandsLeft }; + enum class CommandQueuePriority + { + High = 1 << 0, + Regular = 1 << 1 + }; + + static constexpr SpawnablePriority HighPriorityThreshold = SpawnablePriority { 64 }; + ~SpawnableEntitiesManager() override = default; // // The following functions are thread safe // - void SpawnAllEntities(EntitySpawnTicket& ticket, EntityPreInsertionCallback preInsertionCallback = {}, EntitySpawnCallback completionCallback = {}) override; - void SpawnEntities(EntitySpawnTicket& ticket, AZStd::vector entityIndices, EntityPreInsertionCallback preInsertionCallback = {}, + void SpawnAllEntities( + EntitySpawnTicket& ticket, SpawnablePriority priority, EntityPreInsertionCallback preInsertionCallback = {}, EntitySpawnCallback completionCallback = {}) override; - void DespawnAllEntities(EntitySpawnTicket& ticket, EntityDespawnCallback completionCallback = {}) override; + void SpawnEntities( + EntitySpawnTicket& ticket, SpawnablePriority priority, AZStd::vector entityIndices, + EntityPreInsertionCallback preInsertionCallback = {}, + EntitySpawnCallback completionCallback = {}) override; + void DespawnAllEntities( + EntitySpawnTicket& ticket, SpawnablePriority priority, EntityDespawnCallback completionCallback = {}) override; - void ReloadSpawnable(EntitySpawnTicket& ticket, AZ::Data::Asset spawnable, + void ReloadSpawnable( + EntitySpawnTicket& ticket, SpawnablePriority priority, AZ::Data::Asset spawnable, ReloadSpawnableCallback completionCallback = {}) override; - void ListEntities(EntitySpawnTicket& ticket, ListEntitiesCallback listCallback) override; - void ListIndicesAndEntities(EntitySpawnTicket& ticket, ListIndicesEntitiesCallback listCallback) override; - void ClaimEntities(EntitySpawnTicket& ticket, ClaimEntitiesCallback listCallback) override; + void ListEntities(EntitySpawnTicket& ticket, SpawnablePriority priority, ListEntitiesCallback listCallback) override; + void ListIndicesAndEntities( + EntitySpawnTicket& ticket, SpawnablePriority priority, ListIndicesEntitiesCallback listCallback) override; + void ClaimEntities(EntitySpawnTicket& ticket, SpawnablePriority priority, ClaimEntitiesCallback listCallback) override; - void Barrier(EntitySpawnTicket& spawnInfo, BarrierCallback completionCallback) override; + void Barrier(EntitySpawnTicket& spawnInfo, SpawnablePriority priority, BarrierCallback completionCallback) override; void AddOnSpawnedHandler(AZ::Event>::Handler& handler) override; void AddOnDespawnedHandler(AZ::Event>::Handler& handler) override; @@ -71,13 +86,9 @@ namespace AzFramework // The following function is thread safe but intended to be run from the main thread. // - CommandQueueStatus ProcessQueue(); + CommandQueueStatus ProcessQueue(CommandQueuePriority priority); protected: - void* CreateTicket(AZ::Data::Asset&& spawnable) override; - void DestroyTicket(void* ticket) override; - - private: struct Ticket { AZ_CLASS_ALLOCATOR(Ticket, AZ::ThreadPoolAllocator, 0); @@ -153,6 +164,20 @@ namespace AzFramework SpawnAllEntitiesCommand, SpawnEntitiesCommand, DespawnAllEntitiesCommand, ReloadSpawnableCommand, ListEntitiesCommand, ListIndicesEntitiesCommand, ClaimEntitiesCommand, BarrierCommand, DestroyTicketCommand>; + struct Queue + { + AZStd::deque m_delayed; //!< Requests that were processed before, but couldn't be completed. + AZStd::queue m_pendingRequest; //!< Requests waiting to be processed for the first time. + AZStd::mutex m_pendingRequestMutex; + }; + + template + void QueueRequest(EntitySpawnTicket& ticket, SpawnablePriority priority, T&& request); + void* CreateTicket(AZ::Data::Asset&& spawnable) override; + void DestroyTicket(void* ticket) override; + + CommandQueueStatus ProcessQueue(Queue& queue); + AZ::Entity* SpawnSingleEntity(const AZ::Entity& entityTemplate, AZ::SerializeContext& serializeContext); @@ -174,11 +199,12 @@ namespace AzFramework [[nodiscard]] static bool IsEqualTicket(const EntitySpawnTicket* lhs, const Ticket* rhs); [[nodiscard]] static bool IsEqualTicket(const Ticket* lhs, const Ticket* rhs); - AZStd::deque m_delayedQueue; //!< Requests that were processed before, but couldn't be completed. - AZStd::queue m_pendingRequestQueue; - AZStd::mutex m_pendingRequestQueueMutex; + Queue m_highPriorityQueue; + Queue m_regularPriorityQueue; AZ::Event> m_onSpawnedEvent; AZ::Event> m_onDespawnedEvent; }; + + AZ_DEFINE_ENUM_BITWISE_OPERATORS(AzFramework::SpawnableEntitiesManager::CommandQueuePriority); } // namespace AzFramework diff --git a/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableSystemComponent.cpp b/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableSystemComponent.cpp index 262f006f15..32cc61914e 100644 --- a/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableSystemComponent.cpp +++ b/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableSystemComponent.cpp @@ -48,10 +48,23 @@ namespace AzFramework void SpawnableSystemComponent::OnTick(float /*deltaTime*/, AZ::ScriptTimePoint /*time*/) { - m_entitiesManager.ProcessQueue(); + m_entitiesManager.ProcessQueue( + SpawnableEntitiesManager::CommandQueuePriority::High | SpawnableEntitiesManager::CommandQueuePriority::Regular); RootSpawnableNotificationBus::ExecuteQueuedEvents(); } + int SpawnableSystemComponent::GetTickOrder() + { + return AZ::ComponentTickBus::TICK_GAME; + } + + void SpawnableSystemComponent::OnSystemTick() + { + // Handle only high priority spawning events such as those created from network. These need to happen even if the server + // doesn't have focus to avoid + m_entitiesManager.ProcessQueue(SpawnableEntitiesManager::CommandQueuePriority::High); + } + void SpawnableSystemComponent::OnCatalogLoaded([[maybe_unused]] const char* catalogFile) { if (!m_catalogAvailable) @@ -168,7 +181,8 @@ namespace AzFramework SpawnableEntitiesManager::CommandQueueStatus queueStatus; do { - queueStatus = m_entitiesManager.ProcessQueue(); + queueStatus = m_entitiesManager.ProcessQueue( + SpawnableEntitiesManager::CommandQueuePriority::High | SpawnableEntitiesManager::CommandQueuePriority::Regular); } while (queueStatus == SpawnableEntitiesManager::CommandQueueStatus::HasCommandsLeft); } diff --git a/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableSystemComponent.h b/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableSystemComponent.h index b29fbea5e4..1549b86385 100644 --- a/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableSystemComponent.h +++ b/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableSystemComponent.h @@ -28,6 +28,7 @@ namespace AzFramework class SpawnableSystemComponent : public AZ::Component , public AZ::TickBus::Handler + , public AZ::SystemTickBus::Handler , public AssetCatalogEventBus::Handler , public RootSpawnableInterface::Registrar , public RootSpawnableNotificationBus::Handler @@ -58,6 +59,13 @@ namespace AzFramework // void OnTick(float deltaTime, AZ::ScriptTimePoint time) override; + int GetTickOrder() override; + + // + // SystemTickBus + // + + void OnSystemTick() override; // // AssetCatalogEventBus diff --git a/Code/Framework/Tests/Spawnable/SpawnableEntitiesManagerTests.cpp b/Code/Framework/Tests/Spawnable/SpawnableEntitiesManagerTests.cpp index 1fa50fff52..191b14c31d 100644 --- a/Code/Framework/Tests/Spawnable/SpawnableEntitiesManagerTests.cpp +++ b/Code/Framework/Tests/Spawnable/SpawnableEntitiesManagerTests.cpp @@ -55,7 +55,11 @@ namespace UnitTest delete m_ticket; m_ticket = nullptr; // One more tick on the spawnable entities manager in order to delete the ticket fully. - m_manager->ProcessQueue(); + while (m_manager->ProcessQueue( + AzFramework::SpawnableEntitiesManager::CommandQueuePriority::High | + AzFramework::SpawnableEntitiesManager::CommandQueuePriority::Regular) != + AzFramework::SpawnableEntitiesManager::CommandQueueStatus::NoCommandsLeft) + ; delete m_spawnableAsset; m_spawnableAsset = nullptr; @@ -96,8 +100,8 @@ namespace UnitTest { spawnedEntitiesCount += entities.size(); }; - m_manager->SpawnAllEntities(*m_ticket, {}, AZStd::move(callback)); - m_manager->ProcessQueue(); + m_manager->SpawnAllEntities(*m_ticket, AzFramework::SpawnablePriorty_Default, {}, AZStd::move(callback)); + m_manager->ProcessQueue(AzFramework::SpawnableEntitiesManager::CommandQueuePriority::Regular); EXPECT_EQ(NumEntities, spawnedEntitiesCount); } @@ -119,9 +123,9 @@ namespace UnitTest spawnedEntitiesCount += entities.size(); }; - m_manager->SpawnAllEntities(*m_ticket); - m_manager->ListEntities(*m_ticket, AZStd::move(callback)); - m_manager->ProcessQueue(); + m_manager->SpawnAllEntities(*m_ticket, AzFramework::SpawnablePriorty_Default); + m_manager->ListEntities(*m_ticket, AzFramework::SpawnablePriorty_Default, AZStd::move(callback)); + m_manager->ProcessQueue(AzFramework::SpawnableEntitiesManager::CommandQueuePriority::Regular); EXPECT_TRUE(allValidEntityIds); EXPECT_EQ(NumEntities, spawnedEntitiesCount); @@ -148,11 +152,73 @@ namespace UnitTest } }; - m_manager->SpawnAllEntities(*m_ticket); - m_manager->ListIndicesAndEntities(*m_ticket, AZStd::move(callback)); - m_manager->ProcessQueue(); + m_manager->SpawnAllEntities(*m_ticket, AzFramework::SpawnablePriorty_Default); + m_manager->ListIndicesAndEntities(*m_ticket, AzFramework::SpawnablePriorty_Default, AZStd::move(callback)); + m_manager->ProcessQueue(AzFramework::SpawnableEntitiesManager::CommandQueuePriority::Regular); EXPECT_TRUE(allValidEntityIds); EXPECT_EQ(NumEntities, spawnedEntitiesCount); } + + TEST_F(SpawnableEntitiesManagerTest, Priority_HighBeforeDefault_HigherPriorityCallHappensBeforeDefaultPriorityEvenWhenQueuedLater) + { + static constexpr size_t NumEntities = 4; + FillSpawnable(NumEntities); + + AzFramework::EntitySpawnTicket highPriorityTicket(*m_spawnableAsset); + + size_t callCounter = 1; + size_t highPriorityCallId = 0; + size_t defaultPriorityCallId = 0; + auto highCallback = [&callCounter, &highPriorityCallId] + (AzFramework::EntitySpawnTicket&, AzFramework::SpawnableConstEntityContainerView) + { + highPriorityCallId = callCounter++; + }; + auto defaultCallback = [&callCounter, &defaultPriorityCallId] + (AzFramework::EntitySpawnTicket&, AzFramework::SpawnableConstEntityContainerView) + { + defaultPriorityCallId = callCounter++; + }; + + m_manager->SpawnAllEntities(*m_ticket, AzFramework::SpawnablePriorty_Default, {}, AZStd::move(defaultCallback)); + m_manager->SpawnAllEntities(highPriorityTicket, AzFramework::SpawnablePriorty_High, {}, AZStd::move(highCallback)); + m_manager->ProcessQueue( + AzFramework::SpawnableEntitiesManager::CommandQueuePriority::High | + AzFramework::SpawnableEntitiesManager::CommandQueuePriority::Regular); + + EXPECT_LT(highPriorityCallId, defaultPriorityCallId); + } + + TEST_F(SpawnableEntitiesManagerTest, Priority_SameTicket_DefaultPriorityCallHappensBeforeHighPriority) + { + static constexpr size_t NumEntities = 4; + FillSpawnable(NumEntities); + + size_t callCounter = 1; + size_t highPriorityCallId = 0; + size_t defaultPriorityCallId = 0; + auto highCallback = + [&callCounter, &highPriorityCallId](AzFramework::EntitySpawnTicket&, AzFramework::SpawnableConstEntityContainerView) + { + highPriorityCallId = callCounter++; + }; + auto defaultCallback = + [&callCounter, &defaultPriorityCallId](AzFramework::EntitySpawnTicket&, AzFramework::SpawnableConstEntityContainerView) + { + defaultPriorityCallId = callCounter++; + }; + + m_manager->SpawnAllEntities(*m_ticket, AzFramework::SpawnablePriorty_Default, {}, AZStd::move(defaultCallback)); + m_manager->SpawnAllEntities(*m_ticket, AzFramework::SpawnablePriorty_High, {}, AZStd::move(highCallback)); + m_manager->ProcessQueue( + AzFramework::SpawnableEntitiesManager::CommandQueuePriority::High | + AzFramework::SpawnableEntitiesManager::CommandQueuePriority::Regular); + // Run a second time as the high priority task will be pending at this point. + m_manager->ProcessQueue( + AzFramework::SpawnableEntitiesManager::CommandQueuePriority::High | + AzFramework::SpawnableEntitiesManager::CommandQueuePriority::Regular); + + EXPECT_LT(defaultPriorityCallId, highPriorityCallId); + } } // namespace UnitTest From e0948a26bc1cd169de8ae5a2f82bc8d69794aa1e Mon Sep 17 00:00:00 2001 From: AMZN-koppersr <82230785+AMZN-koppersr@users.noreply.github.com> Date: Wed, 26 May 2021 14:26:16 -0700 Subject: [PATCH 03/19] Fixed early ticket delete crash This commit fixes a crash that could happen when a spawnable ticket was deleted before all requests in the queue had completed. Because of this crash the requests now only hold on to the payload of the ticket but not the ticket itself. As a side effect, callbacks can no longer provide the ticket itself so instead a unique id for the ticket is returned. --- .../Spawnable/SpawnableEntitiesContainer.cpp | 4 +- .../Spawnable/SpawnableEntitiesInterface.cpp | 13 +- .../Spawnable/SpawnableEntitiesInterface.h | 22 +-- .../Spawnable/SpawnableEntitiesManager.cpp | 101 ++++++------ .../Spawnable/SpawnableEntitiesManager.h | 48 +++--- .../SpawnableEntitiesManagerTests.cpp | 150 ++++++++++++++++-- 6 files changed, 242 insertions(+), 96 deletions(-) diff --git a/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesContainer.cpp b/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesContainer.cpp index e9a78dccde..9b06eb1f20 100644 --- a/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesContainer.cpp +++ b/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesContainer.cpp @@ -70,7 +70,7 @@ namespace AzFramework SpawnableEntitiesInterface::Get()->Barrier( m_threadData->m_spawnedEntitiesTicket, SpawnablePriorty_Default, - [threadData = m_threadData](EntitySpawnTicket&) mutable + [threadData = m_threadData](EntitySpawnTicket::Id) mutable { threadData.reset(); }); @@ -89,7 +89,7 @@ namespace AzFramework SpawnableEntitiesInterface::Get()->Barrier( m_threadData->m_spawnedEntitiesTicket, SpawnablePriorty_Default, - [generation = m_threadData->m_generation, callback = AZStd::move(callback)](EntitySpawnTicket&) + [generation = m_threadData->m_generation, callback = AZStd::move(callback)](EntitySpawnTicket::Id) { callback(generation); }); diff --git a/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesInterface.cpp b/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesInterface.cpp index 97169ebeb3..26a10933b5 100644 --- a/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesInterface.cpp +++ b/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesInterface.cpp @@ -239,7 +239,9 @@ namespace AzFramework { auto manager = SpawnableEntitiesInterface::Get(); AZ_Assert(manager, "Attempting to create an entity spawn ticket while the SpawnableEntitiesInterface has no implementation."); - m_payload = manager->CreateTicket(AZStd::move(spawnable)); + AZStd::pair result = manager->CreateTicket(AZStd::move(spawnable)); + m_id = result.first; + m_payload = result.second; } EntitySpawnTicket::~EntitySpawnTicket() @@ -250,6 +252,7 @@ namespace AzFramework AZ_Assert(manager, "Attempting to destroy an entity spawn ticket while the SpawnableEntitiesInterface has no implementation."); manager->DestroyTicket(m_payload); m_payload = nullptr; + m_id = 0; } } @@ -263,12 +266,20 @@ namespace AzFramework AZ_Assert(manager, "Attempting to destroy an entity spawn ticket while the SpawnableEntitiesInterface has no implementation."); manager->DestroyTicket(m_payload); } + m_id = rhs.m_id; + rhs.m_id = 0; + m_payload = rhs.m_payload; rhs.m_payload = nullptr; } return *this; } + uint64_t EntitySpawnTicket::GetId() const + { + return m_id; + } + bool EntitySpawnTicket::IsValid() const { return m_payload != nullptr; diff --git a/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesInterface.h b/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesInterface.h index 97d06e1f37..b40136def7 100644 --- a/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesInterface.h +++ b/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesInterface.h @@ -143,6 +143,8 @@ namespace AzFramework public: friend class SpawnableEntitiesDefinition; + using Id = uint64_t; + EntitySpawnTicket() = default; EntitySpawnTicket(const EntitySpawnTicket&) = delete; EntitySpawnTicket(EntitySpawnTicket&& rhs); @@ -152,20 +154,22 @@ namespace AzFramework EntitySpawnTicket& operator=(const EntitySpawnTicket&) = delete; EntitySpawnTicket& operator=(EntitySpawnTicket&& rhs); + uint64_t GetId() const; bool IsValid() const; private: void* m_payload{ nullptr }; + Id m_id { 0 }; //!< An id that uniquely identifies a ticket. }; - using EntitySpawnCallback = AZStd::function; - using EntityPreInsertionCallback = AZStd::function; - using EntityDespawnCallback = AZStd::function; - using ReloadSpawnableCallback = AZStd::function; - using ListEntitiesCallback = AZStd::function; - using ListIndicesEntitiesCallback = AZStd::function; - using ClaimEntitiesCallback = AZStd::function; - using BarrierCallback = AZStd::function; + using EntitySpawnCallback = AZStd::function; + using EntityPreInsertionCallback = AZStd::function; + using EntityDespawnCallback = AZStd::function; + using ReloadSpawnableCallback = AZStd::function; + using ListEntitiesCallback = AZStd::function; + using ListIndicesEntitiesCallback = AZStd::function; + using ClaimEntitiesCallback = AZStd::function; + using BarrierCallback = AZStd::function; //! Interface definition to (de)spawn entities from a spawnable into the game world. //! @@ -265,7 +269,7 @@ namespace AzFramework virtual void AddOnDespawnedHandler(AZ::Event>::Handler& handler) = 0; protected: - [[nodiscard]] virtual void* CreateTicket(AZ::Data::Asset&& spawnable) = 0; + [[nodiscard]] virtual AZStd::pair CreateTicket(AZ::Data::Asset&& spawnable) = 0; virtual void DestroyTicket(void* ticket) = 0; template diff --git a/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesManager.cpp b/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesManager.cpp index 959d2ab64f..caf1112e9b 100644 --- a/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesManager.cpp +++ b/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesManager.cpp @@ -25,10 +25,11 @@ namespace AzFramework template void SpawnableEntitiesManager::QueueRequest(EntitySpawnTicket& ticket, SpawnablePriority priority, T&& request) { + request.m_ticket = &GetTicketPayload(ticket); Queue& queue = priority <= HighPriorityThreshold ? m_highPriorityQueue : m_regularPriorityQueue; { AZStd::scoped_lock queueLock(queue.m_pendingRequestMutex); - request.m_ticketId = GetTicketPayload(ticket).m_nextTicketId++; + request.m_requestId = GetTicketPayload(ticket).m_nextRequestId++; queue.m_pendingRequest.push(AZStd::move(request)); } } @@ -40,7 +41,7 @@ namespace AzFramework AZ_Assert(ticket.IsValid(), "Ticket provided to SpawnAllEntities hasn't been initialized."); SpawnAllEntitiesCommand queueEntry; - queueEntry.m_ticket = &ticket; + queueEntry.m_ticketId = ticket.GetId(); queueEntry.m_completionCallback = AZStd::move(completionCallback); queueEntry.m_preInsertionCallback = AZStd::move(preInsertionCallback); QueueRequest(ticket, priority, AZStd::move(queueEntry)); @@ -53,7 +54,7 @@ namespace AzFramework AZ_Assert(ticket.IsValid(), "Ticket provided to SpawnEntities hasn't been initialized."); SpawnEntitiesCommand queueEntry; - queueEntry.m_ticket = &ticket; + queueEntry.m_ticketId = ticket.GetId(); queueEntry.m_entityIndices = AZStd::move(entityIndices); queueEntry.m_completionCallback = AZStd::move(completionCallback); queueEntry.m_preInsertionCallback = AZStd::move(preInsertionCallback); @@ -66,7 +67,7 @@ namespace AzFramework AZ_Assert(ticket.IsValid(), "Ticket provided to DespawnAllEntities hasn't been initialized."); DespawnAllEntitiesCommand queueEntry; - queueEntry.m_ticket = &ticket; + queueEntry.m_ticketId = ticket.GetId(); queueEntry.m_completionCallback = AZStd::move(completionCallback); QueueRequest(ticket, priority, AZStd::move(queueEntry)); } @@ -78,7 +79,7 @@ namespace AzFramework AZ_Assert(ticket.IsValid(), "Ticket provided to ReloadSpawnable hasn't been initialized."); ReloadSpawnableCommand queueEntry; - queueEntry.m_ticket = &ticket; + queueEntry.m_ticketId = ticket.GetId(); queueEntry.m_spawnable = AZStd::move(spawnable); queueEntry.m_completionCallback = AZStd::move(completionCallback); QueueRequest(ticket, priority, AZStd::move(queueEntry)); @@ -90,7 +91,7 @@ namespace AzFramework AZ_Assert(ticket.IsValid(), "Ticket provided to ListEntities hasn't been initialized."); ListEntitiesCommand queueEntry; - queueEntry.m_ticket = &ticket; + queueEntry.m_ticketId = ticket.GetId(); queueEntry.m_listCallback = AZStd::move(listCallback); QueueRequest(ticket, priority, AZStd::move(queueEntry)); } @@ -102,7 +103,7 @@ namespace AzFramework AZ_Assert(ticket.IsValid(), "Ticket provided to ListEntities hasn't been initialized."); ListIndicesEntitiesCommand queueEntry; - queueEntry.m_ticket = &ticket; + queueEntry.m_ticketId = ticket.GetId(); queueEntry.m_listCallback = AZStd::move(listCallback); QueueRequest(ticket, priority, AZStd::move(queueEntry)); } @@ -113,7 +114,7 @@ namespace AzFramework AZ_Assert(ticket.IsValid(), "Ticket provided to ClaimEntities hasn't been initialized."); ClaimEntitiesCommand queueEntry; - queueEntry.m_ticket = &ticket; + queueEntry.m_ticketId = ticket.GetId(); queueEntry.m_listCallback = AZStd::move(listCallback); QueueRequest(ticket, priority, AZStd::move(queueEntry)); } @@ -124,7 +125,7 @@ namespace AzFramework AZ_Assert(ticket.IsValid(), "Ticket provided to Barrier hasn't been initialized."); BarrierCommand queueEntry; - queueEntry.m_ticket = &ticket; + queueEntry.m_ticketId = ticket.GetId(); queueEntry.m_completionCallback = AZStd::move(completionCallback); QueueRequest(ticket, priority, AZStd::move(queueEntry)); } @@ -217,11 +218,13 @@ namespace AzFramework return queue.m_delayed.empty() ? CommandQueueStatus::NoCommandsLeft : CommandQueueStatus::HasCommandsLeft; } - void* SpawnableEntitiesManager::CreateTicket(AZ::Data::Asset&& spawnable) + AZStd::pair SpawnableEntitiesManager::CreateTicket(AZ::Data::Asset&& spawnable) { + static AZStd::atomic_uint64_t idCounter { 1 }; + auto result = aznew Ticket(); result->m_spawnable = AZStd::move(spawnable); - return result; + return AZStd::make_pair(idCounter++, result); } void SpawnableEntitiesManager::DestroyTicket(void* ticket) @@ -230,7 +233,7 @@ namespace AzFramework queueEntry.m_ticket = reinterpret_cast(ticket); { AZStd::scoped_lock queueLock(m_regularPriorityQueue.m_pendingRequestMutex); - queueEntry.m_ticketId = reinterpret_cast(ticket)->m_nextTicketId++; + queueEntry.m_requestId = reinterpret_cast(ticket)->m_nextRequestId++; m_regularPriorityQueue.m_pendingRequest.push(AZStd::move(queueEntry)); } } @@ -254,8 +257,8 @@ namespace AzFramework bool SpawnableEntitiesManager::ProcessRequest(SpawnAllEntitiesCommand& request, AZ::SerializeContext& serializeContext) { - Ticket& ticket = GetTicketPayload(*request.m_ticket); - if (ticket.m_spawnable.IsReady() && request.m_ticketId == ticket.m_currentTicketId) + Ticket& ticket = *request.m_ticket; + if (ticket.m_spawnable.IsReady() && request.m_requestId == ticket.m_currentRequestId) { AZStd::vector& spawnedEntities = ticket.m_spawnedEntities; AZStd::vector& spawnedEntityIndices = ticket.m_spawnedEntityIndices; @@ -303,7 +306,7 @@ namespace AzFramework // Let other systems know about newly spawned entities for any pre-processing before adding to the scene/game context. if (request.m_preInsertionCallback) { - request.m_preInsertionCallback(*request.m_ticket, SpawnableEntityContainerView( + request.m_preInsertionCallback(request.m_ticketId, SpawnableEntityContainerView( ticket.m_spawnedEntities.begin() + spawnedEntitiesInitialCount, ticket.m_spawnedEntities.end())); } @@ -317,13 +320,13 @@ namespace AzFramework // Let other systems know about newly spawned entities for any post-processing after adding to the scene/game context. if (request.m_completionCallback) { - request.m_completionCallback(*request.m_ticket, SpawnableConstEntityContainerView( + request.m_completionCallback(request.m_ticketId, SpawnableConstEntityContainerView( ticket.m_spawnedEntities.begin() + spawnedEntitiesInitialCount, ticket.m_spawnedEntities.end())); } m_onSpawnedEvent.Signal(ticket.m_spawnable); - ticket.m_currentTicketId++; + ticket.m_currentRequestId++; return true; } else @@ -334,8 +337,8 @@ namespace AzFramework bool SpawnableEntitiesManager::ProcessRequest(SpawnEntitiesCommand& request, AZ::SerializeContext& serializeContext) { - Ticket& ticket = GetTicketPayload(*request.m_ticket); - if (ticket.m_spawnable.IsReady() && request.m_ticketId == ticket.m_currentTicketId) + Ticket& ticket = *request.m_ticket; + if (ticket.m_spawnable.IsReady() && request.m_requestId == ticket.m_currentRequestId) { AZStd::vector& spawnedEntities = ticket.m_spawnedEntities; AZStd::vector& spawnedEntityIndices = ticket.m_spawnedEntityIndices; @@ -370,9 +373,7 @@ namespace AzFramework // Let other systems know about newly spawned entities for any pre-processing before adding to the scene/game context. if (request.m_preInsertionCallback) { - request.m_preInsertionCallback( - *request.m_ticket, - SpawnableEntityContainerView( + request.m_preInsertionCallback(request.m_ticketId, SpawnableEntityContainerView( ticket.m_spawnedEntities.begin() + spawnedEntitiesInitialCount, ticket.m_spawnedEntities.end())); } @@ -385,13 +386,13 @@ namespace AzFramework if (request.m_completionCallback) { - request.m_completionCallback(*request.m_ticket, SpawnableConstEntityContainerView( + request.m_completionCallback(request.m_ticketId, SpawnableConstEntityContainerView( ticket.m_spawnedEntities.begin() + spawnedEntitiesInitialCount, ticket.m_spawnedEntities.end())); } m_onSpawnedEvent.Signal(ticket.m_spawnable); - ticket.m_currentTicketId++; + ticket.m_currentRequestId++; return true; } else @@ -403,8 +404,8 @@ namespace AzFramework bool SpawnableEntitiesManager::ProcessRequest(DespawnAllEntitiesCommand& request, [[maybe_unused]] AZ::SerializeContext& serializeContext) { - Ticket& ticket = GetTicketPayload(*request.m_ticket); - if (request.m_ticketId == ticket.m_currentTicketId) + Ticket& ticket = *request.m_ticket; + if (request.m_requestId == ticket.m_currentRequestId) { for (AZ::Entity* entity : ticket.m_spawnedEntities) { @@ -420,12 +421,12 @@ namespace AzFramework if (request.m_completionCallback) { - request.m_completionCallback(*request.m_ticket); + request.m_completionCallback(request.m_ticketId); } m_onDespawnedEvent.Signal(ticket.m_spawnable); - ticket.m_currentTicketId++; + ticket.m_currentRequestId++; return true; } else @@ -436,11 +437,11 @@ namespace AzFramework bool SpawnableEntitiesManager::ProcessRequest(ReloadSpawnableCommand& request, AZ::SerializeContext& serializeContext) { - Ticket& ticket = GetTicketPayload(*request.m_ticket); + Ticket& ticket = *request.m_ticket; AZ_Assert(ticket.m_spawnable.GetId() == request.m_spawnable.GetId(), "Spawnable is being reloaded, but the provided spawnable has a different asset id. " "This will likely result in unexpected entities being created."); - if (ticket.m_spawnable.IsReady() && request.m_ticketId == ticket.m_currentTicketId) + if (ticket.m_spawnable.IsReady() && request.m_requestId == ticket.m_currentRequestId) { // Delete the original entities. for (AZ::Entity* entity : ticket.m_spawnedEntities) @@ -496,11 +497,11 @@ namespace AzFramework if (request.m_completionCallback) { - request.m_completionCallback(*request.m_ticket, SpawnableConstEntityContainerView( + request.m_completionCallback(request.m_ticketId, SpawnableConstEntityContainerView( ticket.m_spawnedEntities.begin(), ticket.m_spawnedEntities.end())); } - ticket.m_currentTicketId++; + ticket.m_currentRequestId++; m_onSpawnedEvent.Signal(ticket.m_spawnable); @@ -514,12 +515,12 @@ namespace AzFramework bool SpawnableEntitiesManager::ProcessRequest(ListEntitiesCommand& request, [[maybe_unused]] AZ::SerializeContext& serializeContext) { - Ticket& ticket = GetTicketPayload(*request.m_ticket); - if (request.m_ticketId == ticket.m_currentTicketId) + Ticket& ticket = *request.m_ticket; + if (request.m_requestId == ticket.m_currentRequestId) { - request.m_listCallback(*request.m_ticket, SpawnableConstEntityContainerView( + request.m_listCallback(request.m_ticketId, SpawnableConstEntityContainerView( ticket.m_spawnedEntities.begin(), ticket.m_spawnedEntities.end())); - ticket.m_currentTicketId++; + ticket.m_currentRequestId++; return true; } else @@ -530,17 +531,15 @@ namespace AzFramework bool SpawnableEntitiesManager::ProcessRequest(ListIndicesEntitiesCommand& request, [[maybe_unused]] AZ::SerializeContext& serializeContext) { - Ticket& ticket = GetTicketPayload(*request.m_ticket); - if (request.m_ticketId == ticket.m_currentTicketId) + Ticket& ticket = *request.m_ticket; + if (request.m_requestId == ticket.m_currentRequestId) { AZ_Assert( ticket.m_spawnedEntities.size() == ticket.m_spawnedEntityIndices.size(), "Entities and indices on spawnable ticket have gone out of sync."); - request.m_listCallback( - *request.m_ticket, - SpawnableConstIndexEntityContainerView( + request.m_listCallback(request.m_ticketId, SpawnableConstIndexEntityContainerView( ticket.m_spawnedEntities.begin(), ticket.m_spawnedEntityIndices.begin(), ticket.m_spawnedEntities.size())); - ticket.m_currentTicketId++; + ticket.m_currentRequestId++; return true; } else @@ -551,16 +550,16 @@ namespace AzFramework bool SpawnableEntitiesManager::ProcessRequest(ClaimEntitiesCommand& request, [[maybe_unused]] AZ::SerializeContext& serializeContext) { - Ticket& ticket = GetTicketPayload(*request.m_ticket); - if (request.m_ticketId == ticket.m_currentTicketId) + Ticket& ticket = *request.m_ticket; + if (request.m_requestId == ticket.m_currentRequestId) { - request.m_listCallback(*request.m_ticket, SpawnableEntityContainerView( + request.m_listCallback(request.m_ticketId, SpawnableEntityContainerView( ticket.m_spawnedEntities.begin(), ticket.m_spawnedEntities.end())); ticket.m_spawnedEntities.clear(); ticket.m_spawnedEntityIndices.clear(); - ticket.m_currentTicketId++; + ticket.m_currentRequestId++; return true; } else @@ -571,15 +570,15 @@ namespace AzFramework bool SpawnableEntitiesManager::ProcessRequest(BarrierCommand& request, [[maybe_unused]] AZ::SerializeContext& serializeContext) { - Ticket& ticket = GetTicketPayload(*request.m_ticket); - if (request.m_ticketId == ticket.m_currentTicketId) + Ticket& ticket = *request.m_ticket; + if (request.m_requestId == ticket.m_currentRequestId) { if (request.m_completionCallback) { - request.m_completionCallback(*request.m_ticket); + request.m_completionCallback(request.m_ticketId); } - ticket.m_currentTicketId++; + ticket.m_currentRequestId++; return true; } else @@ -590,7 +589,7 @@ namespace AzFramework bool SpawnableEntitiesManager::ProcessRequest(DestroyTicketCommand& request, [[maybe_unused]] AZ::SerializeContext& serializeContext) { - if (request.m_ticketId == request.m_ticket->m_currentTicketId) + if (request.m_requestId == request.m_ticket->m_currentRequestId) { for (AZ::Entity* entity : request.m_ticket->m_spawnedEntities) { diff --git a/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesManager.h b/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesManager.h index 137376ac7b..b98be60145 100644 --- a/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesManager.h +++ b/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesManager.h @@ -97,8 +97,8 @@ namespace AzFramework AZStd::vector m_spawnedEntities; AZStd::vector m_spawnedEntityIndices; AZ::Data::Asset m_spawnable; - uint32_t m_nextTicketId{ 0 }; //!< Next id for this ticket. - uint32_t m_currentTicketId{ 0 }; //!< The id for the command that should be executed. + uint32_t m_nextRequestId{ 0 }; //!< Next id for this ticket. + uint32_t m_currentRequestId { 0 }; //!< The id for the command that should be executed. bool m_loadAll{ true }; }; @@ -106,58 +106,66 @@ namespace AzFramework { EntitySpawnCallback m_completionCallback; EntityPreInsertionCallback m_preInsertionCallback; - EntitySpawnTicket* m_ticket; - uint32_t m_ticketId; + Ticket* m_ticket; + EntitySpawnTicket::Id m_ticketId; + uint32_t m_requestId; }; struct SpawnEntitiesCommand { AZStd::vector m_entityIndices; EntitySpawnCallback m_completionCallback; EntityPreInsertionCallback m_preInsertionCallback; - EntitySpawnTicket* m_ticket; - uint32_t m_ticketId; + Ticket* m_ticket; + EntitySpawnTicket::Id m_ticketId; + uint32_t m_requestId; }; struct DespawnAllEntitiesCommand { EntityDespawnCallback m_completionCallback; - EntitySpawnTicket* m_ticket; - uint32_t m_ticketId; + Ticket* m_ticket; + EntitySpawnTicket::Id m_ticketId; + uint32_t m_requestId; }; struct ReloadSpawnableCommand { AZ::Data::Asset m_spawnable; ReloadSpawnableCallback m_completionCallback; - EntitySpawnTicket* m_ticket; - uint32_t m_ticketId; + Ticket* m_ticket; + EntitySpawnTicket::Id m_ticketId; + uint32_t m_requestId; }; struct ListEntitiesCommand { ListEntitiesCallback m_listCallback; - EntitySpawnTicket* m_ticket; - uint32_t m_ticketId; + Ticket* m_ticket; + EntitySpawnTicket::Id m_ticketId; + uint32_t m_requestId; }; struct ListIndicesEntitiesCommand { ListIndicesEntitiesCallback m_listCallback; - EntitySpawnTicket* m_ticket; - uint32_t m_ticketId; + Ticket* m_ticket; + EntitySpawnTicket::Id m_ticketId; + uint32_t m_requestId; }; struct ClaimEntitiesCommand { ClaimEntitiesCallback m_listCallback; - EntitySpawnTicket* m_ticket; - uint32_t m_ticketId; + Ticket* m_ticket; + EntitySpawnTicket::Id m_ticketId; + uint32_t m_requestId; }; struct BarrierCommand { BarrierCallback m_completionCallback; - EntitySpawnTicket* m_ticket; - uint32_t m_ticketId; + Ticket* m_ticket; + EntitySpawnTicket::Id m_ticketId; + uint32_t m_requestId; }; struct DestroyTicketCommand { Ticket* m_ticket; - uint32_t m_ticketId; + uint32_t m_requestId; }; using Requests = AZStd::variant< @@ -173,7 +181,7 @@ namespace AzFramework template void QueueRequest(EntitySpawnTicket& ticket, SpawnablePriority priority, T&& request); - void* CreateTicket(AZ::Data::Asset&& spawnable) override; + AZStd::pair CreateTicket(AZ::Data::Asset&& spawnable) override; void DestroyTicket(void* ticket) override; CommandQueueStatus ProcessQueue(Queue& queue); diff --git a/Code/Framework/Tests/Spawnable/SpawnableEntitiesManagerTests.cpp b/Code/Framework/Tests/Spawnable/SpawnableEntitiesManagerTests.cpp index 191b14c31d..320e2ec435 100644 --- a/Code/Framework/Tests/Spawnable/SpawnableEntitiesManagerTests.cpp +++ b/Code/Framework/Tests/Spawnable/SpawnableEntitiesManagerTests.cpp @@ -89,6 +89,10 @@ namespace UnitTest TestApplication* m_application { nullptr }; }; + // + // SpawnAllEntitities + // + TEST_F(SpawnableEntitiesManagerTest, SpawnAllEntities_Call_AllEntitiesSpawned) { static constexpr size_t NumEntities = 4; @@ -96,7 +100,7 @@ namespace UnitTest size_t spawnedEntitiesCount = 0; auto callback = - [&spawnedEntitiesCount](AzFramework::EntitySpawnTicket&, AzFramework::SpawnableConstEntityContainerView entities) + [&spawnedEntitiesCount](AzFramework::EntitySpawnTicket::Id, AzFramework::SpawnableConstEntityContainerView entities) { spawnedEntitiesCount += entities.size(); }; @@ -106,6 +110,62 @@ namespace UnitTest EXPECT_EQ(NumEntities, spawnedEntitiesCount); } + TEST_F(SpawnableEntitiesManagerTest, SpawnAllEntities_DeleteTicketBeforeCall_NoCrash) + { + { + AzFramework::EntitySpawnTicket ticket(*m_spawnableAsset); + m_manager->SpawnAllEntities(ticket, AzFramework::SpawnablePriorty_Default); + } + m_manager->ProcessQueue(AzFramework::SpawnableEntitiesManager::CommandQueuePriority::Regular); + } + + + // + // SpawnEntities + // + + TEST_F(SpawnableEntitiesManagerTest, SpawnEntities_DeleteTicketBeforeCall_NoCrash) + { + { + AzFramework::EntitySpawnTicket ticket(*m_spawnableAsset); + m_manager->SpawnEntities(ticket, AzFramework::SpawnablePriorty_Default, {}); + } + m_manager->ProcessQueue(AzFramework::SpawnableEntitiesManager::CommandQueuePriority::Regular); + } + + + // + // DespawnAllEntities + // + + TEST_F(SpawnableEntitiesManagerTest, DespawnAllEntities_DeleteTicketBeforeCall_NoCrash) + { + { + AzFramework::EntitySpawnTicket ticket(*m_spawnableAsset); + m_manager->DespawnAllEntities(ticket, AzFramework::SpawnablePriorty_Default); + } + m_manager->ProcessQueue(AzFramework::SpawnableEntitiesManager::CommandQueuePriority::Regular); + } + + + // + // ReloadSpawnable + // + + TEST_F(SpawnableEntitiesManagerTest, ReloadSpawnable_DeleteTicketBeforeCall_NoCrash) + { + { + AzFramework::EntitySpawnTicket ticket(*m_spawnableAsset); + m_manager->ReloadSpawnable(ticket, AzFramework::SpawnablePriorty_Default, *m_spawnableAsset); + } + m_manager->ProcessQueue(AzFramework::SpawnableEntitiesManager::CommandQueuePriority::Regular); + } + + + // + // ListEntitities + // + TEST_F(SpawnableEntitiesManagerTest, ListEntities_Call_AllEntitiesAreReported) { static constexpr size_t NumEntities = 4; @@ -114,7 +174,7 @@ namespace UnitTest bool allValidEntityIds = true; size_t spawnedEntitiesCount = 0; auto callback = [&allValidEntityIds, &spawnedEntitiesCount] - (AzFramework::EntitySpawnTicket&, AzFramework::SpawnableConstEntityContainerView entities) + (AzFramework::EntitySpawnTicket::Id, AzFramework::SpawnableConstEntityContainerView entities) { for (auto&& entity : entities) { @@ -131,6 +191,22 @@ namespace UnitTest EXPECT_EQ(NumEntities, spawnedEntitiesCount); } + TEST_F(SpawnableEntitiesManagerTest, ListEntities_DeleteTicketBeforeCall_NoCrash) + { + auto callback = [](AzFramework::EntitySpawnTicket::Id, AzFramework::SpawnableConstEntityContainerView) {}; + + { + AzFramework::EntitySpawnTicket ticket(*m_spawnableAsset); + m_manager->ListEntities(ticket, AzFramework::SpawnablePriorty_Default, AZStd::move(callback)); + } + m_manager->ProcessQueue(AzFramework::SpawnableEntitiesManager::CommandQueuePriority::Regular); + } + + + // + // ListIndicesAndEntities + // + TEST_F(SpawnableEntitiesManagerTest, ListIndicesAndEntities_Call_AllEntitiesAreReportedAndIncrementByOne) { static constexpr size_t NumEntities = 4; @@ -139,7 +215,7 @@ namespace UnitTest bool allValidEntityIds = true; size_t spawnedEntitiesCount = 0; auto callback = [&allValidEntityIds, &spawnedEntitiesCount] - (AzFramework::EntitySpawnTicket&, AzFramework::SpawnableConstIndexEntityContainerView entities) + (AzFramework::EntitySpawnTicket::Id, AzFramework::SpawnableConstIndexEntityContainerView entities) { for (auto&& indexEntityPair : entities) { @@ -160,6 +236,54 @@ namespace UnitTest EXPECT_EQ(NumEntities, spawnedEntitiesCount); } + TEST_F(SpawnableEntitiesManagerTest, ListIndicesAndEntities_DeleteTicketBeforeCall_NoCrash) + { + auto callback = [](AzFramework::EntitySpawnTicket::Id, AzFramework::SpawnableConstIndexEntityContainerView) {}; + + { + AzFramework::EntitySpawnTicket ticket(*m_spawnableAsset); + m_manager->ListIndicesAndEntities(ticket, AzFramework::SpawnablePriorty_Default, AZStd::move(callback)); + } + m_manager->ProcessQueue(AzFramework::SpawnableEntitiesManager::CommandQueuePriority::Regular); + } + + + // + // ClaimEntities + // + + TEST_F(SpawnableEntitiesManagerTest, ClaimEntities_DeleteTicketBeforeCall_NoCrash) + { + auto callback = [](AzFramework::EntitySpawnTicket::Id, AzFramework::SpawnableEntityContainerView) {}; + + { + AzFramework::EntitySpawnTicket ticket(*m_spawnableAsset); + m_manager->ClaimEntities(ticket, AzFramework::SpawnablePriorty_Default, AZStd::move(callback)); + } + m_manager->ProcessQueue(AzFramework::SpawnableEntitiesManager::CommandQueuePriority::Regular); + } + + + // + // Barrier + // + + TEST_F(SpawnableEntitiesManagerTest, Barrier_DeleteTicketBeforeCall_NoCrash) + { + auto callback = [](AzFramework::EntitySpawnTicket::Id) {}; + + { + AzFramework::EntitySpawnTicket ticket(*m_spawnableAsset); + m_manager->Barrier(ticket, AzFramework::SpawnablePriorty_Default, AZStd::move(callback)); + } + m_manager->ProcessQueue(AzFramework::SpawnableEntitiesManager::CommandQueuePriority::Regular); + } + + + // + // Misc. - Priority tests + // + TEST_F(SpawnableEntitiesManagerTest, Priority_HighBeforeDefault_HigherPriorityCallHappensBeforeDefaultPriorityEvenWhenQueuedLater) { static constexpr size_t NumEntities = 4; @@ -171,12 +295,12 @@ namespace UnitTest size_t highPriorityCallId = 0; size_t defaultPriorityCallId = 0; auto highCallback = [&callCounter, &highPriorityCallId] - (AzFramework::EntitySpawnTicket&, AzFramework::SpawnableConstEntityContainerView) + (AzFramework::EntitySpawnTicket::Id, AzFramework::SpawnableConstEntityContainerView) { highPriorityCallId = callCounter++; }; auto defaultCallback = [&callCounter, &defaultPriorityCallId] - (AzFramework::EntitySpawnTicket&, AzFramework::SpawnableConstEntityContainerView) + (AzFramework::EntitySpawnTicket::Id, AzFramework::SpawnableConstEntityContainerView) { defaultPriorityCallId = callCounter++; }; @@ -199,15 +323,15 @@ namespace UnitTest size_t highPriorityCallId = 0; size_t defaultPriorityCallId = 0; auto highCallback = - [&callCounter, &highPriorityCallId](AzFramework::EntitySpawnTicket&, AzFramework::SpawnableConstEntityContainerView) - { - highPriorityCallId = callCounter++; - }; + [&callCounter, &highPriorityCallId](AzFramework::EntitySpawnTicket::Id, AzFramework::SpawnableConstEntityContainerView) + { + highPriorityCallId = callCounter++; + }; auto defaultCallback = - [&callCounter, &defaultPriorityCallId](AzFramework::EntitySpawnTicket&, AzFramework::SpawnableConstEntityContainerView) - { - defaultPriorityCallId = callCounter++; - }; + [&callCounter, &defaultPriorityCallId](AzFramework::EntitySpawnTicket::Id, AzFramework::SpawnableConstEntityContainerView) + { + defaultPriorityCallId = callCounter++; + }; m_manager->SpawnAllEntities(*m_ticket, AzFramework::SpawnablePriorty_Default, {}, AZStd::move(defaultCallback)); m_manager->SpawnAllEntities(*m_ticket, AzFramework::SpawnablePriorty_High, {}, AZStd::move(highCallback)); From 1248dc5fb43889530de28bc38feca79fd75ca414 Mon Sep 17 00:00:00 2001 From: AMZN-koppersr <82230785+AMZN-koppersr@users.noreply.github.com> Date: Wed, 26 May 2021 16:39:24 -0700 Subject: [PATCH 04/19] Spawning priority threshold through SetReg The priority threshold to consider a task high priority can now be configured through the Settings Registry under key "/O3DE/AzFramework/Spawnables/HighPriorityThreshold". --- .../Spawnable/SpawnableEntitiesManager.cpp | 34 +++++++------------ .../Spawnable/SpawnableEntitiesManager.h | 10 ++---- Registry/prefab.setreg | 14 ++++++++ 3 files changed, 30 insertions(+), 28 deletions(-) create mode 100644 Registry/prefab.setreg diff --git a/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesManager.cpp b/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesManager.cpp index caf1112e9b..2c80aa8cbd 100644 --- a/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesManager.cpp +++ b/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesManager.cpp @@ -10,9 +10,11 @@ * */ +#include #include #include #include +#include #include #include #include @@ -26,7 +28,7 @@ namespace AzFramework void SpawnableEntitiesManager::QueueRequest(EntitySpawnTicket& ticket, SpawnablePriority priority, T&& request) { request.m_ticket = &GetTicketPayload(ticket); - Queue& queue = priority <= HighPriorityThreshold ? m_highPriorityQueue : m_regularPriorityQueue; + Queue& queue = priority <= m_highPriorityThreshold ? m_highPriorityQueue : m_regularPriorityQueue; { AZStd::scoped_lock queueLock(queue.m_pendingRequestMutex); request.m_requestId = GetTicketPayload(ticket).m_nextRequestId++; @@ -34,6 +36,16 @@ namespace AzFramework } } + SpawnableEntitiesManager::SpawnableEntitiesManager() + { + if (auto settingsRegistry = AZ::SettingsRegistry::Get(); settingsRegistry != nullptr) + { + AZ::u64 value = 64; + settingsRegistry->Get(value, "/O3DE/AzFramework/Spawnables/HighPriorityThreshold"); + m_highPriorityThreshold = aznumeric_cast(AZStd::clamp(value, 0llu, 255llu)); + } + } + void SpawnableEntitiesManager::SpawnAllEntities( EntitySpawnTicket& ticket, SpawnablePriority priority, EntityPreInsertionCallback preInsertionCallback, EntitySpawnCallback completionCallback) @@ -608,24 +620,4 @@ namespace AzFramework return false; } } - - bool SpawnableEntitiesManager::IsEqualTicket(const EntitySpawnTicket* lhs, const EntitySpawnTicket* rhs) - { - return GetTicketPayload(lhs) == GetTicketPayload(rhs); - } - - bool SpawnableEntitiesManager::IsEqualTicket(const Ticket* lhs, const EntitySpawnTicket* rhs) - { - return lhs == GetTicketPayload(rhs); - } - - bool SpawnableEntitiesManager::IsEqualTicket(const EntitySpawnTicket* lhs, const Ticket* rhs) - { - return GetTicketPayload(lhs) == rhs; - } - - bool SpawnableEntitiesManager::IsEqualTicket(const Ticket* lhs, const Ticket* rhs) - { - return lhs = rhs; - } } // namespace AzFramework diff --git a/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesManager.h b/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesManager.h index b98be60145..373a4db9cb 100644 --- a/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesManager.h +++ b/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesManager.h @@ -50,8 +50,7 @@ namespace AzFramework Regular = 1 << 1 }; - static constexpr SpawnablePriority HighPriorityThreshold = SpawnablePriority { 64 }; - + SpawnableEntitiesManager(); ~SpawnableEntitiesManager() override = default; // @@ -202,16 +201,13 @@ namespace AzFramework bool ProcessRequest(BarrierCommand& request, AZ::SerializeContext& serializeContext); bool ProcessRequest(DestroyTicketCommand& request, AZ::SerializeContext& serializeContext); - [[nodiscard]] static bool IsEqualTicket(const EntitySpawnTicket* lhs, const EntitySpawnTicket* rhs); - [[nodiscard]] static bool IsEqualTicket(const Ticket* lhs, const EntitySpawnTicket* rhs); - [[nodiscard]] static bool IsEqualTicket(const EntitySpawnTicket* lhs, const Ticket* rhs); - [[nodiscard]] static bool IsEqualTicket(const Ticket* lhs, const Ticket* rhs); - Queue m_highPriorityQueue; Queue m_regularPriorityQueue; AZ::Event> m_onSpawnedEvent; AZ::Event> m_onDespawnedEvent; + + SpawnablePriority m_highPriorityThreshold { 64 }; }; AZ_DEFINE_ENUM_BITWISE_OPERATORS(AzFramework::SpawnableEntitiesManager::CommandQueuePriority); diff --git a/Registry/prefab.setreg b/Registry/prefab.setreg new file mode 100644 index 0000000000..903dc0c4f8 --- /dev/null +++ b/Registry/prefab.setreg @@ -0,0 +1,14 @@ +{ + "O3DE": + { + "AzFramework": + { + "Spawnables": + { + // Any requests with a priorty value equal or smaller than this will be considered a high priority request. + // The range for this value is between 0 and 255. + "HighPriorityThreshold" : 64 + } + } + } +} \ No newline at end of file From c946d579282340877d8e11cba758a82b38e37cb8 Mon Sep 17 00:00:00 2001 From: AMZN-koppersr <82230785+AMZN-koppersr@users.noreply.github.com> Date: Thu, 27 May 2021 11:14:21 -0700 Subject: [PATCH 05/19] Addressed PR feedback. --- .../Spawnable/SpawnableEntitiesContainer.cpp | 12 +++---- .../Spawnable/SpawnableEntitiesInterface.cpp | 2 +- .../Spawnable/SpawnableEntitiesInterface.h | 16 ++++----- .../Spawnable/SpawnableEntitiesManager.h | 4 +++ .../Spawnable/SpawnableSystemComponent.cpp | 4 +-- .../SpawnableEntitiesManagerTests.cpp | 34 +++++++++---------- 6 files changed, 38 insertions(+), 34 deletions(-) diff --git a/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesContainer.cpp b/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesContainer.cpp index 9b06eb1f20..808de74e71 100644 --- a/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesContainer.cpp +++ b/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesContainer.cpp @@ -38,20 +38,20 @@ namespace AzFramework void SpawnableEntitiesContainer::SpawnAllEntities() { AZ_Assert(m_threadData, "Calling SpawnAllEntities on a Spawnable container that's not set."); - SpawnableEntitiesInterface::Get()->SpawnAllEntities(m_threadData->m_spawnedEntitiesTicket, SpawnablePriorty_Default); + SpawnableEntitiesInterface::Get()->SpawnAllEntities(m_threadData->m_spawnedEntitiesTicket, SpawnablePriority_Default); } void SpawnableEntitiesContainer::SpawnEntities(AZStd::vector entityIndices) { AZ_Assert(m_threadData, "Calling SpawnEntities on a Spawnable container that's not set."); SpawnableEntitiesInterface::Get()->SpawnEntities( - m_threadData->m_spawnedEntitiesTicket, SpawnablePriorty_Default, AZStd::move(entityIndices)); + m_threadData->m_spawnedEntitiesTicket, SpawnablePriority_Default, AZStd::move(entityIndices)); } void SpawnableEntitiesContainer::DespawnAllEntities() { AZ_Assert(m_threadData, "Calling DespawnEntities on a Spawnable container that's not set."); - SpawnableEntitiesInterface::Get()->DespawnAllEntities(m_threadData->m_spawnedEntitiesTicket, SpawnablePriorty_Default); + SpawnableEntitiesInterface::Get()->DespawnAllEntities(m_threadData->m_spawnedEntitiesTicket, SpawnablePriority_Default); } void SpawnableEntitiesContainer::Reset(AZ::Data::Asset spawnable) @@ -69,7 +69,7 @@ namespace AzFramework SpawnableEntitiesInterface::Get()->Barrier( m_threadData->m_spawnedEntitiesTicket, - SpawnablePriorty_Default, + SpawnablePriority_Default, [threadData = m_threadData](EntitySpawnTicket::Id) mutable { threadData.reset(); @@ -88,7 +88,7 @@ namespace AzFramework AZ_Assert(m_threadData, "Calling DespawnEntities on a Spawnable container that's not set."); SpawnableEntitiesInterface::Get()->Barrier( m_threadData->m_spawnedEntitiesTicket, - SpawnablePriorty_Default, + SpawnablePriority_Default, [generation = m_threadData->m_generation, callback = AZStd::move(callback)](EntitySpawnTicket::Id) { callback(generation); @@ -116,6 +116,6 @@ namespace AzFramework AZ_TracePrintf("Spawnables", "Reloading spawnable '%s'.\n", replacementAsset.GetHint().c_str()); SpawnableEntitiesInterface::Get()->ReloadSpawnable( - m_threadData->m_spawnedEntitiesTicket, SpawnablePriorty_Default, AZStd::move(replacementAsset)); + m_threadData->m_spawnedEntitiesTicket, SpawnablePriority_Default, AZStd::move(replacementAsset)); } } // namespace AzFramework diff --git a/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesInterface.cpp b/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesInterface.cpp index 26a10933b5..ad1bf032c8 100644 --- a/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesInterface.cpp +++ b/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesInterface.cpp @@ -275,7 +275,7 @@ namespace AzFramework return *this; } - uint64_t EntitySpawnTicket::GetId() const + auto EntitySpawnTicket::GetId() const -> Id { return m_id; } diff --git a/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesInterface.h b/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesInterface.h index b40136def7..27f45064b6 100644 --- a/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesInterface.h +++ b/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesInterface.h @@ -27,11 +27,11 @@ namespace AzFramework { AZ_TYPE_SAFE_INTEGRAL(SpawnablePriority, uint8_t); - inline static constexpr SpawnablePriority SpawnablePriorty_Highest { 0 }; - inline static constexpr SpawnablePriority SpawnablePriorty_High { 32 }; - inline static constexpr SpawnablePriority SpawnablePriorty_Default { 128 }; - inline static constexpr SpawnablePriority SpawnablePriorty_Low { 192 }; - inline static constexpr SpawnablePriority SpawnablePriorty_Lowest { 255 }; + inline static constexpr SpawnablePriority SpawnablePriority_Highest { 0 }; + inline static constexpr SpawnablePriority SpawnablePriority_High { 32 }; + inline static constexpr SpawnablePriority SpawnablePriority_Default { 128 }; + inline static constexpr SpawnablePriority SpawnablePriority_Low { 192 }; + inline static constexpr SpawnablePriority SpawnablePriority_Lowest { 255 }; class SpawnableEntityContainerView { @@ -154,7 +154,7 @@ namespace AzFramework EntitySpawnTicket& operator=(const EntitySpawnTicket&) = delete; EntitySpawnTicket& operator=(EntitySpawnTicket&& rhs); - uint64_t GetId() const; + Id GetId() const; bool IsValid() const; private: @@ -179,11 +179,11 @@ namespace AzFramework //! Calls on the same ticket are guaranteed to be executed in the order they are issued. Note that when issuing requests from //! multiple threads on the same ticket the order in which the requests are assigned to the ticket is not guaranteed. //! - //! Most calls have a priority where values closer to 0 mean higher priority than values closer to 255. The implementation of this + //! Most calls have a priority with values that range from 0 (highest priority) to 255 (lowest priority). The implementation of this //! interface may choose to use priority lanes which doesn't guarantee that higher priority requests happen before lower priority //! requests if they don't pass the priority lane threshold. Priority lanes and their thresholds are implementation specific and may //! differ between platforms. Note that if a call happened on a ticket with lower priority followed by a one with a higher priority - //! the first lower priority call will still needs to complete before the second higher priority call can be executed and the priority + //! the first lower priority call will still need to complete before the second higher priority call can be executed and the priority //! of the first call will not be updated. class SpawnableEntitiesDefinition { diff --git a/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesManager.h b/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesManager.h index 373a4db9cb..afffdab8b5 100644 --- a/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesManager.h +++ b/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesManager.h @@ -207,6 +207,10 @@ namespace AzFramework AZ::Event> m_onSpawnedEvent; AZ::Event> m_onDespawnedEvent; + //! The threshold used to determine if a request goes in the regular (if bigger than the value) or high priority queue (if smaller + //! or equal to this value). The starting value of 64 is chosen as it's between default values SpawnablePriority_High and + //! SpawnablePriority_Default which gives users a bit of room to fine tune the priorities as this value can be configured + //! through the Settings Registry under the key "/O3DE/AzFramework/Spawnables/HighPriorityThreshold". SpawnablePriority m_highPriorityThreshold { 64 }; }; diff --git a/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableSystemComponent.cpp b/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableSystemComponent.cpp index 32cc61914e..300ff1441e 100644 --- a/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableSystemComponent.cpp +++ b/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableSystemComponent.cpp @@ -60,8 +60,8 @@ namespace AzFramework void SpawnableSystemComponent::OnSystemTick() { - // Handle only high priority spawning events such as those created from network. These need to happen even if the server - // doesn't have focus to avoid + // Handle only high priority spawning events such as those created from network. These need to happen even if the client + // doesn't have focus to avoid time-out issues for instance. m_entitiesManager.ProcessQueue(SpawnableEntitiesManager::CommandQueuePriority::High); } diff --git a/Code/Framework/Tests/Spawnable/SpawnableEntitiesManagerTests.cpp b/Code/Framework/Tests/Spawnable/SpawnableEntitiesManagerTests.cpp index 320e2ec435..484b7f46d7 100644 --- a/Code/Framework/Tests/Spawnable/SpawnableEntitiesManagerTests.cpp +++ b/Code/Framework/Tests/Spawnable/SpawnableEntitiesManagerTests.cpp @@ -104,7 +104,7 @@ namespace UnitTest { spawnedEntitiesCount += entities.size(); }; - m_manager->SpawnAllEntities(*m_ticket, AzFramework::SpawnablePriorty_Default, {}, AZStd::move(callback)); + m_manager->SpawnAllEntities(*m_ticket, AzFramework::SpawnablePriority_Default, {}, AZStd::move(callback)); m_manager->ProcessQueue(AzFramework::SpawnableEntitiesManager::CommandQueuePriority::Regular); EXPECT_EQ(NumEntities, spawnedEntitiesCount); @@ -114,7 +114,7 @@ namespace UnitTest { { AzFramework::EntitySpawnTicket ticket(*m_spawnableAsset); - m_manager->SpawnAllEntities(ticket, AzFramework::SpawnablePriorty_Default); + m_manager->SpawnAllEntities(ticket, AzFramework::SpawnablePriority_Default); } m_manager->ProcessQueue(AzFramework::SpawnableEntitiesManager::CommandQueuePriority::Regular); } @@ -128,7 +128,7 @@ namespace UnitTest { { AzFramework::EntitySpawnTicket ticket(*m_spawnableAsset); - m_manager->SpawnEntities(ticket, AzFramework::SpawnablePriorty_Default, {}); + m_manager->SpawnEntities(ticket, AzFramework::SpawnablePriority_Default, {}); } m_manager->ProcessQueue(AzFramework::SpawnableEntitiesManager::CommandQueuePriority::Regular); } @@ -142,7 +142,7 @@ namespace UnitTest { { AzFramework::EntitySpawnTicket ticket(*m_spawnableAsset); - m_manager->DespawnAllEntities(ticket, AzFramework::SpawnablePriorty_Default); + m_manager->DespawnAllEntities(ticket, AzFramework::SpawnablePriority_Default); } m_manager->ProcessQueue(AzFramework::SpawnableEntitiesManager::CommandQueuePriority::Regular); } @@ -156,7 +156,7 @@ namespace UnitTest { { AzFramework::EntitySpawnTicket ticket(*m_spawnableAsset); - m_manager->ReloadSpawnable(ticket, AzFramework::SpawnablePriorty_Default, *m_spawnableAsset); + m_manager->ReloadSpawnable(ticket, AzFramework::SpawnablePriority_Default, *m_spawnableAsset); } m_manager->ProcessQueue(AzFramework::SpawnableEntitiesManager::CommandQueuePriority::Regular); } @@ -183,8 +183,8 @@ namespace UnitTest spawnedEntitiesCount += entities.size(); }; - m_manager->SpawnAllEntities(*m_ticket, AzFramework::SpawnablePriorty_Default); - m_manager->ListEntities(*m_ticket, AzFramework::SpawnablePriorty_Default, AZStd::move(callback)); + m_manager->SpawnAllEntities(*m_ticket, AzFramework::SpawnablePriority_Default); + m_manager->ListEntities(*m_ticket, AzFramework::SpawnablePriority_Default, AZStd::move(callback)); m_manager->ProcessQueue(AzFramework::SpawnableEntitiesManager::CommandQueuePriority::Regular); EXPECT_TRUE(allValidEntityIds); @@ -197,7 +197,7 @@ namespace UnitTest { AzFramework::EntitySpawnTicket ticket(*m_spawnableAsset); - m_manager->ListEntities(ticket, AzFramework::SpawnablePriorty_Default, AZStd::move(callback)); + m_manager->ListEntities(ticket, AzFramework::SpawnablePriority_Default, AZStd::move(callback)); } m_manager->ProcessQueue(AzFramework::SpawnableEntitiesManager::CommandQueuePriority::Regular); } @@ -228,8 +228,8 @@ namespace UnitTest } }; - m_manager->SpawnAllEntities(*m_ticket, AzFramework::SpawnablePriorty_Default); - m_manager->ListIndicesAndEntities(*m_ticket, AzFramework::SpawnablePriorty_Default, AZStd::move(callback)); + m_manager->SpawnAllEntities(*m_ticket, AzFramework::SpawnablePriority_Default); + m_manager->ListIndicesAndEntities(*m_ticket, AzFramework::SpawnablePriority_Default, AZStd::move(callback)); m_manager->ProcessQueue(AzFramework::SpawnableEntitiesManager::CommandQueuePriority::Regular); EXPECT_TRUE(allValidEntityIds); @@ -242,7 +242,7 @@ namespace UnitTest { AzFramework::EntitySpawnTicket ticket(*m_spawnableAsset); - m_manager->ListIndicesAndEntities(ticket, AzFramework::SpawnablePriorty_Default, AZStd::move(callback)); + m_manager->ListIndicesAndEntities(ticket, AzFramework::SpawnablePriority_Default, AZStd::move(callback)); } m_manager->ProcessQueue(AzFramework::SpawnableEntitiesManager::CommandQueuePriority::Regular); } @@ -258,7 +258,7 @@ namespace UnitTest { AzFramework::EntitySpawnTicket ticket(*m_spawnableAsset); - m_manager->ClaimEntities(ticket, AzFramework::SpawnablePriorty_Default, AZStd::move(callback)); + m_manager->ClaimEntities(ticket, AzFramework::SpawnablePriority_Default, AZStd::move(callback)); } m_manager->ProcessQueue(AzFramework::SpawnableEntitiesManager::CommandQueuePriority::Regular); } @@ -274,7 +274,7 @@ namespace UnitTest { AzFramework::EntitySpawnTicket ticket(*m_spawnableAsset); - m_manager->Barrier(ticket, AzFramework::SpawnablePriorty_Default, AZStd::move(callback)); + m_manager->Barrier(ticket, AzFramework::SpawnablePriority_Default, AZStd::move(callback)); } m_manager->ProcessQueue(AzFramework::SpawnableEntitiesManager::CommandQueuePriority::Regular); } @@ -305,8 +305,8 @@ namespace UnitTest defaultPriorityCallId = callCounter++; }; - m_manager->SpawnAllEntities(*m_ticket, AzFramework::SpawnablePriorty_Default, {}, AZStd::move(defaultCallback)); - m_manager->SpawnAllEntities(highPriorityTicket, AzFramework::SpawnablePriorty_High, {}, AZStd::move(highCallback)); + m_manager->SpawnAllEntities(*m_ticket, AzFramework::SpawnablePriority_Default, {}, AZStd::move(defaultCallback)); + m_manager->SpawnAllEntities(highPriorityTicket, AzFramework::SpawnablePriority_High, {}, AZStd::move(highCallback)); m_manager->ProcessQueue( AzFramework::SpawnableEntitiesManager::CommandQueuePriority::High | AzFramework::SpawnableEntitiesManager::CommandQueuePriority::Regular); @@ -333,8 +333,8 @@ namespace UnitTest defaultPriorityCallId = callCounter++; }; - m_manager->SpawnAllEntities(*m_ticket, AzFramework::SpawnablePriorty_Default, {}, AZStd::move(defaultCallback)); - m_manager->SpawnAllEntities(*m_ticket, AzFramework::SpawnablePriorty_High, {}, AZStd::move(highCallback)); + m_manager->SpawnAllEntities(*m_ticket, AzFramework::SpawnablePriority_Default, {}, AZStd::move(defaultCallback)); + m_manager->SpawnAllEntities(*m_ticket, AzFramework::SpawnablePriority_High, {}, AZStd::move(highCallback)); m_manager->ProcessQueue( AzFramework::SpawnableEntitiesManager::CommandQueuePriority::High | AzFramework::SpawnableEntitiesManager::CommandQueuePriority::Regular); From 2550c3e1ff1f9cfb2241a1c3e488ef6091d77168 Mon Sep 17 00:00:00 2001 From: AMZN-koppersr <82230785+AMZN-koppersr@users.noreply.github.com> Date: Thu, 27 May 2021 16:13:32 -0700 Subject: [PATCH 06/19] Spawnable Entity Manager threshold default to "m_highPriorityThreshold" --- .../AzFramework/Spawnable/SpawnableEntitiesManager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesManager.cpp b/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesManager.cpp index 2c80aa8cbd..7b767d2a72 100644 --- a/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesManager.cpp +++ b/Code/Framework/AzFramework/AzFramework/Spawnable/SpawnableEntitiesManager.cpp @@ -40,7 +40,7 @@ namespace AzFramework { if (auto settingsRegistry = AZ::SettingsRegistry::Get(); settingsRegistry != nullptr) { - AZ::u64 value = 64; + AZ::u64 value = aznumeric_caster(m_highPriorityThreshold); settingsRegistry->Get(value, "/O3DE/AzFramework/Spawnables/HighPriorityThreshold"); m_highPriorityThreshold = aznumeric_cast(AZStd::clamp(value, 0llu, 255llu)); } From d4ce2849c7995b24311a973d731dc85fb33f82ad Mon Sep 17 00:00:00 2001 From: AMZN-koppersr <82230785+AMZN-koppersr@users.noreply.github.com> Date: Thu, 27 May 2021 16:46:33 -0700 Subject: [PATCH 07/19] Post-merge fixup. --- .../Code/Source/Pipeline/NetBindMarkerComponent.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Gems/Multiplayer/Code/Source/Pipeline/NetBindMarkerComponent.cpp b/Gems/Multiplayer/Code/Source/Pipeline/NetBindMarkerComponent.cpp index 1696e851a5..c93c09cfbc 100644 --- a/Gems/Multiplayer/Code/Source/Pipeline/NetBindMarkerComponent.cpp +++ b/Gems/Multiplayer/Code/Source/Pipeline/NetBindMarkerComponent.cpp @@ -59,7 +59,7 @@ namespace Multiplayer AZ::Transform worldTm = GetEntity()->FindComponent()->GetWorldTM(); auto preInsertionCallback = [worldTm = AZStd::move(worldTm), netEntityIndex = m_netEntityIndex, spawnableAssetId = m_networkSpawnableAsset.GetId()] - (AzFramework::EntitySpawnTicket&, AzFramework::SpawnableEntityContainerView entities) + (AzFramework::EntitySpawnTicket::Id, AzFramework::SpawnableEntityContainerView entities) { if (entities.size() == 1) { @@ -81,7 +81,8 @@ namespace Multiplayer }; m_netSpawnTicket = AzFramework::EntitySpawnTicket(m_networkSpawnableAsset); - AzFramework::SpawnableEntitiesInterface::Get()->SpawnEntities(m_netSpawnTicket, {m_netEntityIndex}, preInsertionCallback); + AzFramework::SpawnableEntitiesInterface::Get()->SpawnEntities( + m_netSpawnTicket, AzFramework::SpawnablePriority_Default, { m_netEntityIndex }, preInsertionCallback); } } @@ -89,7 +90,7 @@ namespace Multiplayer { if(m_netSpawnTicket.IsValid()) { - AzFramework::SpawnableEntitiesInterface::Get()->DespawnAllEntities(m_netSpawnTicket); + AzFramework::SpawnableEntitiesInterface::Get()->DespawnAllEntities(m_netSpawnTicket, AzFramework::SpawnablePriority_Default); } } From c45697fd510efd95fb41f861799763b9998aae17 Mon Sep 17 00:00:00 2001 From: AMZN-koppersr <82230785+AMZN-koppersr@users.noreply.github.com> Date: Thu, 27 May 2021 16:57:15 -0700 Subject: [PATCH 08/19] Fixed unit test compile error with spawnables --- .../Tests/Prefab/Benchmark/SpawnableCreateBenchmarks.cpp | 3 ++- .../AzToolsFramework/Tests/Prefab/SpawnableCreateTests.cpp | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Code/Framework/AzToolsFramework/Tests/Prefab/Benchmark/SpawnableCreateBenchmarks.cpp b/Code/Framework/AzToolsFramework/Tests/Prefab/Benchmark/SpawnableCreateBenchmarks.cpp index 821daaba82..8e4c2eaad7 100644 --- a/Code/Framework/AzToolsFramework/Tests/Prefab/Benchmark/SpawnableCreateBenchmarks.cpp +++ b/Code/Framework/AzToolsFramework/Tests/Prefab/Benchmark/SpawnableCreateBenchmarks.cpp @@ -34,7 +34,8 @@ namespace Benchmark { state.PauseTiming(); - auto spawnable = ::AzToolsFramework::Prefab::SpawnableUtils::CreateSpawnable(prefabDom); + AzFramework::Spawnable spawnable; + AzToolsFramework::Prefab::SpawnableUtils::CreateSpawnable(spawnable, prefabDom); state.ResumeTiming(); } diff --git a/Code/Framework/AzToolsFramework/Tests/Prefab/SpawnableCreateTests.cpp b/Code/Framework/AzToolsFramework/Tests/Prefab/SpawnableCreateTests.cpp index 2e1cd14b2a..af178ec7a4 100644 --- a/Code/Framework/AzToolsFramework/Tests/Prefab/SpawnableCreateTests.cpp +++ b/Code/Framework/AzToolsFramework/Tests/Prefab/SpawnableCreateTests.cpp @@ -40,7 +40,8 @@ namespace UnitTest //Create Spawnable auto& prefabDom = m_prefabSystemComponent->FindTemplateDom(instance->GetTemplateId()); - auto spawnable = ::AzToolsFramework::Prefab::SpawnableUtils::CreateSpawnable(prefabDom); + AzFramework::Spawnable spawnable; + AzToolsFramework::Prefab::SpawnableUtils::CreateSpawnable(spawnable, prefabDom); EXPECT_EQ(spawnable.GetEntities().size() - 1, normalEntityCount); // 1 for container entity const auto& spawnableEntities = spawnable.GetEntities(); @@ -84,7 +85,8 @@ namespace UnitTest //Create Spawnable auto& prefabDom = m_prefabSystemComponent->FindTemplateDom(thirdInstance->GetTemplateId()); - auto spawnable = ::AzToolsFramework::Prefab::SpawnableUtils::CreateSpawnable(prefabDom); + AzFramework::Spawnable spawnable; + AzToolsFramework::Prefab::SpawnableUtils::CreateSpawnable(spawnable, prefabDom); EXPECT_EQ(spawnable.GetEntities().size() - 1, normalEntityCount); // 1 for container entity const auto& spawnableEntities = spawnable.GetEntities(); From 147f0084a8e4c0f8a88535e00e7bef300888ecdd Mon Sep 17 00:00:00 2001 From: Chris Santora Date: Fri, 28 May 2021 15:47:25 -0700 Subject: [PATCH 09/19] Removed subsurface scattering and transmission features from StandardPBR.materialtype. ATOM-4120 Stabilize Standard PBR Regarding Subsurface and Translucency --- .../Materials/Types/EnhancedPBR.materialtype | 2 +- ...te.lua => EnhancedPBR_SubsurfaceState.lua} | 0 .../Assets/Materials/Types/Skin.materialtype | 2 +- .../Materials/Types/StandardPBR.materialtype | 207 ------------------ .../Materials/Types/StandardPBR_Common.azsli | 19 -- .../Types/StandardPBR_ForwardPass.azsl | 32 +-- .../Features/PBR/Lighting/LightingData.azsli | 8 +- .../PBR/Lighting/StandardLighting.azsli | 1 - .../PBR/Surfaces/StandardSurface.azsli | 2 +- .../atom_feature_common_asset_files.cmake | 2 +- 10 files changed, 15 insertions(+), 260 deletions(-) rename Gems/Atom/Feature/Common/Assets/Materials/Types/{StandardPBR_SubsurfaceState.lua => EnhancedPBR_SubsurfaceState.lua} (100%) diff --git a/Gems/Atom/Feature/Common/Assets/Materials/Types/EnhancedPBR.materialtype b/Gems/Atom/Feature/Common/Assets/Materials/Types/EnhancedPBR.materialtype index 79c4ca3cc3..ff0c4c59da 100644 --- a/Gems/Atom/Feature/Common/Assets/Materials/Types/EnhancedPBR.materialtype +++ b/Gems/Atom/Feature/Common/Assets/Materials/Types/EnhancedPBR.materialtype @@ -1636,7 +1636,7 @@ { "type": "Lua", "args": { - "file": "StandardPBR_SubsurfaceState.lua" + "file": "EnhancedPBR_SubsurfaceState.lua" } }, { diff --git a/Gems/Atom/Feature/Common/Assets/Materials/Types/StandardPBR_SubsurfaceState.lua b/Gems/Atom/Feature/Common/Assets/Materials/Types/EnhancedPBR_SubsurfaceState.lua similarity index 100% rename from Gems/Atom/Feature/Common/Assets/Materials/Types/StandardPBR_SubsurfaceState.lua rename to Gems/Atom/Feature/Common/Assets/Materials/Types/EnhancedPBR_SubsurfaceState.lua diff --git a/Gems/Atom/Feature/Common/Assets/Materials/Types/Skin.materialtype b/Gems/Atom/Feature/Common/Assets/Materials/Types/Skin.materialtype index 9ead0376bb..dfe2fad60f 100644 --- a/Gems/Atom/Feature/Common/Assets/Materials/Types/Skin.materialtype +++ b/Gems/Atom/Feature/Common/Assets/Materials/Types/Skin.materialtype @@ -1080,7 +1080,7 @@ { "type": "Lua", "args": { - "file": "StandardPBR_SubsurfaceState.lua" + "file": "EnhancedPBR_SubsurfaceState.lua" } }, { diff --git a/Gems/Atom/Feature/Common/Assets/Materials/Types/StandardPBR.materialtype b/Gems/Atom/Feature/Common/Assets/Materials/Types/StandardPBR.materialtype index 658aaeeee9..2d94f66edf 100644 --- a/Gems/Atom/Feature/Common/Assets/Materials/Types/StandardPBR.materialtype +++ b/Gems/Atom/Feature/Common/Assets/Materials/Types/StandardPBR.materialtype @@ -38,11 +38,6 @@ "displayName": "Emissive", "description": "Properties to add light emission, independent of other lights in the scene." }, - { - "id": "subsurfaceScattering", - "displayName": "Subsurface Scattering", - "description": "Properties for configuring subsurface scattering effects." - }, { "id": "clearCoat", "displayName": "Clear Coat", @@ -981,183 +976,6 @@ } } ], - "subsurfaceScattering": [ - { - "id": "enableSubsurfaceScattering", - "displayName": "Subsurface Scattering", - "description": "Enable subsurface scattering feature, this will disable metallic and parallax mapping property due to incompatibility", - "type": "Bool", - "defaultValue": false, - "connection": { - "type": "ShaderOption", - "id": "o_enableSubsurfaceScattering" - } - }, - { - "id": "subsurfaceScatterFactor", - "displayName": " Factor", - "description": "Strength factor for scaling percentage of subsurface scattering effect applied", - "type": "float", - "defaultValue": 1.0, - "min": 0.0, - "max": 1.0, - "connection": { - "type": "ShaderInput", - "id": "m_subsurfaceScatteringFactor" - } - }, - { - "id": "influenceMap", - "displayName": " Influence Map", - "description": "Texture for controlling the strength of subsurface scattering", - "type": "Image", - "connection": { - "type": "ShaderInput", - "id": "m_subsurfaceScatteringInfluenceMap" - } - }, - { - "id": "useInfluenceMap", - "displayName": " Use Influence Map", - "description": "Whether to use the influence map.", - "type": "Bool", - "defaultValue": true - }, - { - "id": "influenceMapUv", - "displayName": " UV", - "description": "Influence map UV set", - "type": "Enum", - "enumIsUv": true, - "defaultValue": "Tiled", - "connection": { - "type": "ShaderInput", - "id": "m_subsurfaceScatteringInfluenceMapUvIndex" - } - }, - { - "id": "scatterColor", - "displayName": " Scatter color", - "description": "Color of volume light traveled through", - "type": "Color", - "defaultValue": [ 1.0, 0.27, 0.13 ] - }, - { - "id": "scatterDistance", - "displayName": " Scatter distance", - "description": "How far light traveled inside the volume", - "type": "float", - "defaultValue": 8, - "min": 0.0, - "softMax": 20.0 - }, - { - "id": "quality", - "displayName": " Quality", - "description": "How much percent of sample will be used for each pixel, more samples improve quality and reduce artifacts, especially when the scatter distance is relatively large, but slow down computation time, 1.0 = full set 200 samples per pixel", - "type": "float", - "defaultValue": 0.4, - "min": 0.2, - "max": 1.0, - "connection": { - "type": "ShaderInput", - "id": "m_subsurfaceScatteringQuality" - } - }, - { - "id": "transmissionMode", - "displayName": "Transmission", - "description": "Algorithm used for calculating transmission", - "type": "Enum", - "enumValues": [ "None", "ThickObject", "ThinObject" ], - "defaultValue": "None", - "connection": { - "type": "ShaderOption", - "id": "o_transmission_mode" - } - }, - { - "id": "thickness", - "displayName": " Thickness", - "description": "Normalized global thickness, the maxima between this value (multiplied by thickness map if enabled) and thickness from shadow map (if applicable) will be used as final thickness of pixel", - "type": "float", - "defaultValue": 0.5, - "min": 0.0, - "max": 1.0 - }, - { - "id": "thicknessMap", - "displayName": " Thickness Map", - "description": "Texture for controlling per pixel thickness", - "type": "Image", - "connection": { - "type": "ShaderInput", - "id": "m_transmissionThicknessMap" - } - }, - { - "id": "useThicknessMap", - "displayName": " Use Thickness Map", - "description": "Whether to use the thickness map", - "type": "Bool", - "defaultValue": true - }, - { - "id": "thicknessMapUv", - "displayName": " UV", - "description": "Thickness map UV set", - "type": "Enum", - "enumIsUv": true, - "defaultValue": "Tiled", - "connection": { - "type": "ShaderInput", - "id": "m_transmissionThicknessMapUvIndex" - } - }, - { - "id": "transmissionTint", - "displayName": " Transmission Tint", - "description": "Color of the volume light traveling through", - "type": "Color", - "defaultValue": [ 1.0, 0.8, 0.6 ] - }, - { - "id": "transmissionPower", - "displayName": " Power", - "description": "How much transmitted light scatter radially ", - "type": "float", - "defaultValue": 6.0, - "min": 0.0, - "softMax": 20.0 - }, - { - "id": "transmissionDistortion", - "displayName": " Distortion", - "description": "How much light direction distorted towards surface normal", - "type": "float", - "defaultValue": 0.1, - "min": 0.0, - "max": 1.0 - }, - { - "id": "transmissionAttenuation", - "displayName": " Attenuation", - "description": "How fast transmitted light fade with thickness", - "type": "float", - "defaultValue": 4.0, - "min": 0.0, - "softMax": 20.0 - }, - { - "id": "transmissionScale", - "displayName": " Scale", - "description": "Strength of transmission", - "type": "float", - "defaultValue": 3.0, - "min": 0.0, - "softMax": 20.0 - } - ], "irradiance": [ // Note: this property group is used in the DiffuseGlobalIllumination pass and not by the main forward shader { @@ -1261,25 +1079,6 @@ "nitMinMax": [0.001, 100000.0] } }, - { - // Preprocess & build parameter set for subsurface scattering and translucency - "type": "HandleSubsurfaceScatteringParameters", - "args": { - "mode": "subsurfaceScattering.transmissionMode", - "scale": "subsurfaceScattering.transmissionScale", - "power": "subsurfaceScattering.transmissionPower", - "distortion": "subsurfaceScattering.transmissionDistortion", - "attenuation": "subsurfaceScattering.transmissionAttenuation", - "tintColor": "subsurfaceScattering.transmissionTint", - "thickness": "subsurfaceScattering.thickness", - "enabled": "subsurfaceScattering.enableSubsurfaceScattering", - "scatterDistanceColor": "subsurfaceScattering.scatterColor", - "scatterDistanceIntensity": "subsurfaceScattering.scatterDistance", - "scatterDistanceShaderInput": "m_scatterDistance", - "parametersShaderInput": "m_transmissionParams", - "tintThickenssShaderInput": "m_transmissionTintThickness" - } - }, { "type": "UseTexture", "args": { @@ -1364,12 +1163,6 @@ "file": "StandardPBR_Roughness.lua" } }, - { - "type": "Lua", - "args": { - "file": "StandardPBR_SubsurfaceState.lua" - } - }, { "type": "Lua", "args": { diff --git a/Gems/Atom/Feature/Common/Assets/Materials/Types/StandardPBR_Common.azsli b/Gems/Atom/Feature/Common/Assets/Materials/Types/StandardPBR_Common.azsli index 87562c3d20..f339642a4d 100644 --- a/Gems/Atom/Feature/Common/Assets/Materials/Types/StandardPBR_Common.azsli +++ b/Gems/Atom/Feature/Common/Assets/Materials/Types/StandardPBR_Common.azsli @@ -73,25 +73,6 @@ ShaderResourceGroup MaterialSrg : SRG_PerMaterial MagFilter = Linear; MipFilter = Linear; }; - - // Parameters for subsurface scattering - float m_subsurfaceScatteringFactor; - float m_subsurfaceScatteringQuality; - float3 m_scatterDistance; - Texture2D m_subsurfaceScatteringInfluenceMap; - uint m_subsurfaceScatteringInfluenceMapUvIndex; - - // Parameters for transmission - - // Elements of m_transmissionParams: - // Thick object mode: (attenuation coefficient, power, distortion, scale) - // Thin object mode: (float3 scatter distance, scale) - float4 m_transmissionParams; - - // (float3 TintColor, thickness) - float4 m_transmissionTintThickness; - Texture2D m_transmissionThicknessMap; - uint m_transmissionThicknessMapUvIndex; } // Callback function for ParallaxMapping.azsli diff --git a/Gems/Atom/Feature/Common/Assets/Materials/Types/StandardPBR_ForwardPass.azsl b/Gems/Atom/Feature/Common/Assets/Materials/Types/StandardPBR_ForwardPass.azsl index 286b9b23df..b221ee0a33 100644 --- a/Gems/Atom/Feature/Common/Assets/Materials/Types/StandardPBR_ForwardPass.azsl +++ b/Gems/Atom/Feature/Common/Assets/Materials/Types/StandardPBR_ForwardPass.azsl @@ -47,13 +47,6 @@ COMMON_OPTIONS_EMISSIVE() // Alpha #include "MaterialInputs/AlphaInput.azsli" -// Subsurface -#include "MaterialInputs/SubsurfaceInput.azsli" - -// Transmission -#include "MaterialInputs/TransmissionInput.azsli" - - // ---------- Vertex Shader ---------- struct VSInput @@ -113,7 +106,7 @@ PbrLightingOutput ForwardPassPS_Common(VSOutput IN, bool isFrontFace, out float float3 tangents[UvSetCount] = { IN.m_tangent.xyz, IN.m_tangent.xyz }; float3 bitangents[UvSetCount] = { IN.m_bitangent.xyz, IN.m_bitangent.xyz }; - if ((o_parallax_feature_enabled && !o_enableSubsurfaceScattering) || o_normal_useTexture || (o_clearCoat_enabled && o_clearCoat_normal_useTexture)) + if (ShouldHandleParallax() || o_normal_useTexture || (o_clearCoat_enabled && o_clearCoat_normal_useTexture)) { PrepareGeneratedTangent(IN.m_normal, IN.m_worldPosition, isFrontFace, IN.m_uv, UvSetCount, tangents, bitangents); } @@ -124,7 +117,6 @@ PbrLightingOutput ForwardPassPS_Common(VSOutput IN, bool isFrontFace, out float bool displacementIsClipped = false; - // Parallax mapping's non uniform uv transformations break screen space subsurface scattering, disable it when subsurface scatteirng is enabled if(ShouldHandleParallax()) { @@ -174,12 +166,8 @@ PbrLightingOutput ForwardPassPS_Common(VSOutput IN, bool isFrontFace, out float // ------- Metallic ------- - float metallic = 0; - if(!o_enableSubsurfaceScattering) // If subsurface scattering is enabled skip texture lookup for metallic, as this quantity won't be used anyway - { - float2 metallicUv = IN.m_uv[MaterialSrg::m_metallicMapUvIndex]; - metallic = GetMetallicInput(MaterialSrg::m_metallicMap, MaterialSrg::m_sampler, metallicUv, MaterialSrg::m_metallicFactor, o_metallic_useTexture); - } + float2 metallicUv = IN.m_uv[MaterialSrg::m_metallicMapUvIndex]; + float metallic = GetMetallicInput(MaterialSrg::m_metallicMap, MaterialSrg::m_sampler, metallicUv, MaterialSrg::m_metallicFactor, o_metallic_useTexture); // ------- Specular ------- @@ -195,11 +183,6 @@ PbrLightingOutput ForwardPassPS_Common(VSOutput IN, bool isFrontFace, out float MaterialSrg::m_roughnessLowerBound, MaterialSrg::m_roughnessUpperBound, o_roughness_useTexture); surface.CalculateRoughnessA(); - // ------- Subsurface ------- - - float surfaceScatteringFactor = 0.0f; - surface.transmission.InitializeToZero(); - // ------- Lighting Data ------- LightingData lightingData; @@ -271,7 +254,7 @@ PbrLightingOutput ForwardPassPS_Common(VSOutput IN, bool isFrontFace, out float ApplyIBL(surface, lightingData); // Finalize Lighting - lightingData.FinalizeLighting(surface.transmission.tint); + lightingData.FinalizeLighting(); if (o_opacity_mode == OpacityMode::Blended || o_opacity_mode == OpacityMode::TintedTransparent) @@ -312,13 +295,6 @@ PbrLightingOutput ForwardPassPS_Common(VSOutput IN, bool isFrontFace, out float lightingOutput.m_diffuseColor.rgb += lightingOutput.m_specularColor.rgb; // add specular lightingOutput.m_specularColor.rgb = baseColor * (1.0 - lightingOutput.m_diffuseColor.w); } - else - { - // Pack factor and quality, drawback: because of precision limit of float16 cannot represent exact 1, maximum representable value is 0.9961 - uint factorAndQuality = dot(round(float2(saturate(surfaceScatteringFactor), MaterialSrg::m_subsurfaceScatteringQuality) * 255), float2(256, 1)); - lightingOutput.m_diffuseColor.w = factorAndQuality * (o_enableSubsurfaceScattering ? 1.0 : -1.0); - lightingOutput.m_scatterDistance = MaterialSrg::m_scatterDistance; - } return lightingOutput; } diff --git a/Gems/Atom/Feature/Common/Assets/ShaderLib/Atom/Features/PBR/Lighting/LightingData.azsli b/Gems/Atom/Feature/Common/Assets/ShaderLib/Atom/Features/PBR/Lighting/LightingData.azsli index 37cae0acb1..e37aa55c09 100644 --- a/Gems/Atom/Feature/Common/Assets/ShaderLib/Atom/Features/PBR/Lighting/LightingData.azsli +++ b/Gems/Atom/Feature/Common/Assets/ShaderLib/Atom/Features/PBR/Lighting/LightingData.azsli @@ -54,6 +54,7 @@ class LightingData void Init(float3 positionWS, float3 normal, float roughnessLinear); void CalculateMultiscatterCompensation(float3 specularF0, bool enabled); + void FinalizeLighting(); void FinalizeLighting(float3 transmissionTint); }; @@ -80,10 +81,15 @@ void LightingData::CalculateMultiscatterCompensation(float3 specularF0, bool ena multiScatterCompensation = GetMultiScatterCompensation(specularF0, brdf, enabled); } -void LightingData::FinalizeLighting(float3 transmissionTint) +void LightingData::FinalizeLighting() { specularLighting *= specularOcclusion; specularLighting += emissiveLighting; +} + +void LightingData::FinalizeLighting(float3 transmissionTint) +{ + FinalizeLighting(); // Transmitted light if(o_transmission_mode != TransmissionMode::None) diff --git a/Gems/Atom/Feature/Common/Assets/ShaderLib/Atom/Features/PBR/Lighting/StandardLighting.azsli b/Gems/Atom/Feature/Common/Assets/ShaderLib/Atom/Features/PBR/Lighting/StandardLighting.azsli index 45aabeede4..f690163a58 100644 --- a/Gems/Atom/Feature/Common/Assets/ShaderLib/Atom/Features/PBR/Lighting/StandardLighting.azsli +++ b/Gems/Atom/Feature/Common/Assets/ShaderLib/Atom/Features/PBR/Lighting/StandardLighting.azsli @@ -75,7 +75,6 @@ struct PbrLightingOutput float4 m_albedo; float4 m_specularF0; float4 m_normal; - float3 m_scatterDistance; }; diff --git a/Gems/Atom/Feature/Common/Assets/ShaderLib/Atom/Features/PBR/Surfaces/StandardSurface.azsli b/Gems/Atom/Feature/Common/Assets/ShaderLib/Atom/Features/PBR/Surfaces/StandardSurface.azsli index 1a74a68e96..bc4e41d2c2 100644 --- a/Gems/Atom/Feature/Common/Assets/ShaderLib/Atom/Features/PBR/Surfaces/StandardSurface.azsli +++ b/Gems/Atom/Feature/Common/Assets/ShaderLib/Atom/Features/PBR/Surfaces/StandardSurface.azsli @@ -20,7 +20,7 @@ class Surface { ClearCoatSurfaceData clearCoat; - TransmissionSurfaceData transmission; + TransmissionSurfaceData transmission; // This is not actually used for Standard PBR, but must be present for common lighting code to compile // ------- BasePbrSurfaceData ------- diff --git a/Gems/Atom/Feature/Common/Assets/atom_feature_common_asset_files.cmake b/Gems/Atom/Feature/Common/Assets/atom_feature_common_asset_files.cmake index 841cdf67c8..359c0b9b20 100644 --- a/Gems/Atom/Feature/Common/Assets/atom_feature_common_asset_files.cmake +++ b/Gems/Atom/Feature/Common/Assets/atom_feature_common_asset_files.cmake @@ -23,6 +23,7 @@ set(FILES Materials/Types/EnhancedPBR_ForwardPass_EDS.shader Materials/Types/EnhancedPBR_Shadowmap_WithPS.azsl Materials/Types/EnhancedPBR_Shadowmap_WithPS.shader + Materials/Types/EnhancedPBR_SubsurfaceState.lua Materials/Types/Skin.azsl Materials/Types/Skin.materialtype Materials/Types/Skin.shader @@ -61,7 +62,6 @@ set(FILES Materials/Types/StandardPBR_ShaderEnable.lua Materials/Types/StandardPBR_Shadowmap_WithPS.azsl Materials/Types/StandardPBR_Shadowmap_WithPS.shader - Materials/Types/StandardPBR_SubsurfaceState.lua Materials/Types/MaterialInputs/AlphaInput.azsli Materials/Types/MaterialInputs/BaseColorInput.azsli Materials/Types/MaterialInputs/ClearCoatInput.azsli From 7ee55cce3a858ca5dae0fd1acfb130f9a468cd75 Mon Sep 17 00:00:00 2001 From: AMZN-koppersr <82230785+AMZN-koppersr@users.noreply.github.com> Date: Tue, 1 Jun 2021 11:19:03 -0700 Subject: [PATCH 10/19] Post merge fixes for spawning entities --- .../ScriptCanvas/Libraries/Spawning/SpawnNodeable.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Gems/ScriptCanvas/Code/Include/ScriptCanvas/Libraries/Spawning/SpawnNodeable.cpp b/Gems/ScriptCanvas/Code/Include/ScriptCanvas/Libraries/Spawning/SpawnNodeable.cpp index 21d1fc5264..ad53236108 100644 --- a/Gems/ScriptCanvas/Code/Include/ScriptCanvas/Libraries/Spawning/SpawnNodeable.cpp +++ b/Gems/ScriptCanvas/Code/Include/ScriptCanvas/Libraries/Spawning/SpawnNodeable.cpp @@ -105,7 +105,7 @@ namespace ScriptCanvas::Nodeables::Spawning return; } - auto preSpawnCB = [this, translation, rotation, scale]([[maybe_unused]] AzFramework::EntitySpawnTicket& ticket, + auto preSpawnCB = [this, translation, rotation, scale]([[maybe_unused]] AzFramework::EntitySpawnTicket::Id ticketId, AzFramework::SpawnableEntityContainerView view) { AZ::Entity* rootEntity = *view.begin(); @@ -122,7 +122,7 @@ namespace ScriptCanvas::Nodeables::Spawning } }; - auto spawnCompleteCB = [this]([[maybe_unused]] AzFramework::EntitySpawnTicket& ticket, + auto spawnCompleteCB = [this]([[maybe_unused]] AzFramework::EntitySpawnTicket::Id ticketId, AzFramework::SpawnableConstEntityContainerView view) { AZStd::lock_guard lock(m_idBatchMutex); @@ -134,6 +134,7 @@ namespace ScriptCanvas::Nodeables::Spawning m_spawnBatchSizes.push_back(view.size()); }; - AzFramework::SpawnableEntitiesInterface::Get()->SpawnAllEntities(m_spawnTicket, preSpawnCB, spawnCompleteCB); + AzFramework::SpawnableEntitiesInterface::Get()->SpawnAllEntities( + m_spawnTicket, AzFramework::SpawnablePriority_Default, preSpawnCB, spawnCompleteCB); } } From 4070a9ec303573915b2b98af0dea3e0665946970 Mon Sep 17 00:00:00 2001 From: Junbo Liang <68558268+junbo75@users.noreply.github.com> Date: Tue, 1 Jun 2021 13:57:55 -0700 Subject: [PATCH 11/19] [LYN-2243] Create end-to-end automation tests for the metrics gem (#23) [LYN-2243] Create end-to-end automation tests for the metrics gem --- .../AWS/Windows/aws_metrics/__init__.py | 10 + .../aws_metrics_automation_test.py | 237 ++++++++++++++++ .../Windows/aws_metrics/aws_metrics_utils.py | 252 ++++++++++++++++++ .../aws_metrics/aws_metrics_waiters.py | 142 ++++++++++ .../Gem/PythonTests/AWS/Windows/cdk/cdk.py | 57 +++- .../PythonTests/AWS/common/aws_credentials.py | 134 ++++++++++ .../Gem/PythonTests/AWS/common/aws_utils.py | 172 ++++++------ .../PythonTests/AWS/common/custom_waiter.py | 91 +++++++ .../Registry/awscoreconfiguration.setreg | 2 +- .../cdk/aws_metrics/batch_processing.py | 5 +- .../cdk/aws_metrics/data_ingestion.py | 4 +- 11 files changed, 1017 insertions(+), 89 deletions(-) create mode 100644 AutomatedTesting/Gem/PythonTests/AWS/Windows/aws_metrics/__init__.py create mode 100644 AutomatedTesting/Gem/PythonTests/AWS/Windows/aws_metrics/aws_metrics_automation_test.py create mode 100644 AutomatedTesting/Gem/PythonTests/AWS/Windows/aws_metrics/aws_metrics_utils.py create mode 100644 AutomatedTesting/Gem/PythonTests/AWS/Windows/aws_metrics/aws_metrics_waiters.py create mode 100644 AutomatedTesting/Gem/PythonTests/AWS/common/aws_credentials.py create mode 100644 AutomatedTesting/Gem/PythonTests/AWS/common/custom_waiter.py diff --git a/AutomatedTesting/Gem/PythonTests/AWS/Windows/aws_metrics/__init__.py b/AutomatedTesting/Gem/PythonTests/AWS/Windows/aws_metrics/__init__.py new file mode 100644 index 0000000000..cdee4b5a56 --- /dev/null +++ b/AutomatedTesting/Gem/PythonTests/AWS/Windows/aws_metrics/__init__.py @@ -0,0 +1,10 @@ +""" +All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +its licensors. + +For complete copyright and license terms please see the LICENSE at the root of this +distribution (the "License"). All use of this software is governed by the License, +or, if provided, by the license below or the license accompanying this file. Do not +remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +""" diff --git a/AutomatedTesting/Gem/PythonTests/AWS/Windows/aws_metrics/aws_metrics_automation_test.py b/AutomatedTesting/Gem/PythonTests/AWS/Windows/aws_metrics/aws_metrics_automation_test.py new file mode 100644 index 0000000000..04be31759d --- /dev/null +++ b/AutomatedTesting/Gem/PythonTests/AWS/Windows/aws_metrics/aws_metrics_automation_test.py @@ -0,0 +1,237 @@ +""" +All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +its licensors. + +For complete copyright and license terms please see the LICENSE at the root of this +distribution (the "License"). All use of this software is governed by the License, +or, if provided, by the license below or the license accompanying this file. Do not +remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +""" + +import logging +import os +import pytest +import time +import typing + +from datetime import datetime +import ly_test_tools.log.log_monitor + +from assetpipeline.ap_fixtures.asset_processor_fixture import asset_processor as asset_processor +from AWS.common.aws_utils import aws_utils +from AWS.common.aws_credentials import aws_credentials +from AWS.Windows.resource_mappings.resource_mappings import resource_mappings +from AWS.Windows.cdk.cdk import cdk +from .aws_metrics_utils import aws_metrics_utils + +AWS_METRICS_FEATURE_NAME = 'AWSMetrics' +GAME_LOG_NAME = 'Game.log' + +logger = logging.getLogger(__name__) + + +def setup(launcher: ly_test_tools.launchers.Launcher, + cdk: cdk, + asset_processor: asset_processor, + resource_mappings: resource_mappings, + context_variable: str = '') -> typing.Tuple[ly_test_tools.log.log_monitor.LogMonitor, str, str]: + """ + Set up the CDK application and start the log monitor. + :param launcher: Client launcher for running the test level. + :param cdk: CDK application for deploying the AWS resources. + :param asset_processor: asset_processor fixture. + :param resource_mappings: resource_mappings fixture. + :param context_variable: context_variable for enable optional CDK feature. + :return log monitor object, metrics file path and the metrics stack name. + """ + logger.info(f'Cdk stack names:\n{cdk.list()}') + stacks = cdk.deploy(context_variable=context_variable) + resource_mappings.populate_output_keys(stacks) + + asset_processor.start() + asset_processor.wait_for_idle() + + metrics_file_path = os.path.join(launcher.workspace.paths.project(), 'user', + AWS_METRICS_FEATURE_NAME, 'metrics.json') + remove_file(metrics_file_path) + + file_to_monitor = os.path.join(launcher.workspace.paths.project_log(), GAME_LOG_NAME) + remove_file(file_to_monitor) + + # Initialize the log monitor. + log_monitor = ly_test_tools.log.log_monitor.LogMonitor(launcher=launcher, log_file_path=file_to_monitor) + + return log_monitor, metrics_file_path, stacks[0] + + +def monitor_metrics_submission(log_monitor: ly_test_tools.log.log_monitor.LogMonitor) -> None: + """ + Monitor the messages and notifications for submitting metrics. + :param log_monitor: Log monitor to check the log messages. + """ + expected_lines = [ + '(Script) - Submitted metrics without buffer.', + '(Script) - Submitted metrics with buffer.', + '(Script) - Metrics is sent successfully.' + ] + + unexpected_lines = [ + '(Script) - Failed to submit metrics without buffer.', + '(Script) - Failed to submit metrics with buffer.', + '(Script) - Failed to send metrics.' + ] + + result = log_monitor.monitor_log_for_lines( + expected_lines=expected_lines, + unexpected_lines=unexpected_lines, + halt_on_unexpected=True) + + # Assert the log monitor detected expected lines and did not detect any unexpected lines. + assert result, ( + f'Log monitoring failed. Used expected_lines values: {expected_lines} & ' + f'unexpected_lines values: {unexpected_lines}') + + +def remove_file(file_path: str) -> None: + """ + Remove a local file and its directory. + :param file_path: Path to the local file. + """ + if os.path.exists(file_path): + os.remove(file_path) + + file_dir = os.path.dirname(file_path) + if os.path.exists(file_dir) and len(os.listdir(file_dir)) == 0: + os.rmdir(file_dir) + + +@pytest.mark.SUITE_periodic +@pytest.mark.usefixtures('automatic_process_killer') +@pytest.mark.parametrize('project', ['AutomatedTesting']) +@pytest.mark.parametrize('level', ['AWS/Metrics']) +@pytest.mark.parametrize('feature_name', [AWS_METRICS_FEATURE_NAME]) +@pytest.mark.parametrize('resource_mappings_filename', ['aws_resource_mappings.json']) +@pytest.mark.parametrize('profile_name', ['AWSAutomationTest']) +@pytest.mark.parametrize('region_name', ['us-west-2']) +@pytest.mark.parametrize('assume_role_arn', ['arn:aws:iam::645075835648:role/o3de-automation-tests']) +@pytest.mark.parametrize('session_name', ['o3de-Automation-session']) +class TestAWSMetrics_Windows(object): + def test_AWSMetrics_RealTimeAnalytics_MetricsSentToCloudWatch(self, + level: str, + launcher: ly_test_tools.launchers.Launcher, + asset_processor: pytest.fixture, + workspace: pytest.fixture, + aws_utils: aws_utils, + aws_credentials: aws_credentials, + resource_mappings: resource_mappings, + cdk: cdk, + aws_metrics_utils: aws_metrics_utils, + ): + """ + Tests that the submitted metrics are sent to CloudWatch for real-time analytics. + """ + log_monitor, metrics_file_path, stack_name = setup(launcher, cdk, asset_processor, resource_mappings) + + # Start the Kinesis Data Analytics application for real-time analytics. + analytics_application_name = f'{stack_name}-AnalyticsApplication' + aws_metrics_utils.start_kinesis_data_analytics_application(analytics_application_name) + + launcher.args = ['+LoadLevel', level] + launcher.args.extend(['-rhi=null']) + + with launcher.start(launch_ap=False): + start_time = datetime.utcnow() + monitor_metrics_submission(log_monitor) + # Verify that operational health metrics are delivered to CloudWatch. + aws_metrics_utils.verify_cloud_watch_delivery( + 'AWS/Lambda', + 'Invocations', + [{'Name': 'FunctionName', + 'Value': f'{stack_name}-AnalyticsProcessingLambda'}], + start_time) + logger.info('Operational health metrics sent to CloudWatch.') + + aws_metrics_utils.verify_cloud_watch_delivery( + AWS_METRICS_FEATURE_NAME, + 'TotalLogins', + [], + start_time) + logger.info('Real-time metrics sent to CloudWatch.') + + # Stop the Kinesis Data Analytics application. + aws_metrics_utils.stop_kinesis_data_analytics_application(analytics_application_name) + + def test_AWSMetrics_UnauthorizedUser_RequestRejected(self, + level: str, + launcher: ly_test_tools.launchers.Launcher, + cdk: cdk, + aws_credentials: aws_credentials, + asset_processor: pytest.fixture, + resource_mappings: resource_mappings, + workspace: pytest.fixture): + """ + Tests that unauthorized users cannot send metrics events to the AWS backed backend. + """ + log_monitor, metrics_file_path, stack_name = setup(launcher, cdk, asset_processor, resource_mappings) + # Set invalid AWS credentials. + launcher.args = ['+LoadLevel', level, '+cl_awsAccessKey', 'AKIAIOSFODNN7EXAMPLE', + '+cl_awsSecretKey', 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'] + launcher.args.extend(['-rhi=null']) + + with launcher.start(launch_ap=False): + result = log_monitor.monitor_log_for_lines( + expected_lines=['(Script) - Failed to send metrics.'], + unexpected_lines=['(Script) - Metrics is sent successfully.'], + halt_on_unexpected=True) + assert result, 'Metrics events are sent successfully by unauthorized user' + logger.info('Unauthorized user is rejected to send metrics.') + + def test_AWSMetrics_BatchAnalytics_MetricsDeliveredToS3(self, + level: str, + launcher: ly_test_tools.launchers.Launcher, + cdk: cdk, + aws_credentials: aws_credentials, + asset_processor: pytest.fixture, + resource_mappings: resource_mappings, + aws_utils: aws_utils, + aws_metrics_utils: aws_metrics_utils, + workspace: pytest.fixture): + """ + Tests that the submitted metrics are sent to the data lake for batch analytics. + """ + log_monitor, metrics_file_path, stack_name = setup(launcher, cdk, asset_processor, resource_mappings, + context_variable='batch_processing=true') + + analytics_bucket_name = aws_metrics_utils.get_analytics_bucket_name(stack_name) + + launcher.args = ['+LoadLevel', level] + launcher.args.extend(['-rhi=null']) + + with launcher.start(launch_ap=False): + start_time = datetime.utcnow() + monitor_metrics_submission(log_monitor) + # Verify that operational health metrics are delivered to CloudWatch. + aws_metrics_utils.verify_cloud_watch_delivery( + 'AWS/Lambda', + 'Invocations', + [{'Name': 'FunctionName', + 'Value': f'{stack_name}-EventsProcessingLambda'}], + start_time) + logger.info('Operational health metrics sent to CloudWatch.') + + aws_metrics_utils.verify_s3_delivery(analytics_bucket_name) + logger.info('Metrics sent to S3.') + + # Run the glue crawler to populate the AWS Glue Data Catalog with tables. + aws_metrics_utils.run_glue_crawler(f'{stack_name}-EventsCrawler') + # Run named queries on the table to verify the batch analytics. + aws_metrics_utils.run_named_queries(f'{stack_name}-AthenaWorkGroup') + logger.info('Query metrics from S3 successfully.') + + # Kinesis Data Firehose buffers incoming data before it delivers it to Amazon S3. Sleep for the + # default interval (60s) to make sure that all the metrics are sent to the bucket before cleanup. + time.sleep(60) + # Empty the S3 bucket. S3 buckets can only be deleted successfully when it doesn't contain any object. + aws_metrics_utils.empty_s3_bucket(analytics_bucket_name) + diff --git a/AutomatedTesting/Gem/PythonTests/AWS/Windows/aws_metrics/aws_metrics_utils.py b/AutomatedTesting/Gem/PythonTests/AWS/Windows/aws_metrics/aws_metrics_utils.py new file mode 100644 index 0000000000..686feda3d9 --- /dev/null +++ b/AutomatedTesting/Gem/PythonTests/AWS/Windows/aws_metrics/aws_metrics_utils.py @@ -0,0 +1,252 @@ +""" +All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +its licensors. + +For complete copyright and license terms please see the LICENSE at the root of this +distribution (the "License"). All use of this software is governed by the License, +or, if provided, by the license below or the license accompanying this file. Do not +remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +""" + +import logging +import pathlib +import pytest +import typing + +from datetime import datetime +from botocore.exceptions import WaiterError + +from AWS.common.aws_utils import AwsUtils +from .aws_metrics_waiters import KinesisAnalyticsApplicationUpdatedWaiter, \ + CloudWatchMetricsDeliveredWaiter, DataLakeMetricsDeliveredWaiter, GlueCrawlerReadyWaiter + +logging.getLogger('boto').setLevel(logging.CRITICAL) + +# Expected directory and file extension for the S3 objects. +EXPECTED_S3_DIRECTORY = 'firehose_events/' +EXPECTED_S3_OBJECT_EXTENSION = '.parquet' + + +class AWSMetricsUtils: + """ + Provide utils functions for the AWSMetrics gem to interact with the deployed resources. + """ + + def __init__(self, aws_utils: AwsUtils): + self._aws_util = aws_utils + + def start_kinesis_data_analytics_application(self, application_name: str) -> None: + """ + Start the Kenisis Data Analytics application for real-time analytics. + :param application_name: Name of the Kenisis Data Analytics application. + """ + input_id = self.get_kinesis_analytics_application_input_id(application_name) + assert input_id, 'invalid Kinesis Data Analytics application input.' + + client = self._aws_util.client('kinesisanalytics') + try: + client.start_application( + ApplicationName=application_name, + InputConfigurations=[ + { + 'Id': input_id, + 'InputStartingPositionConfiguration': { + 'InputStartingPosition': 'NOW' + } + }, + ] + ) + except client.exceptions.ResourceInUseException: + # The application has been started. + return + + try: + KinesisAnalyticsApplicationUpdatedWaiter(client, 'RUNNING').wait(application_name=application_name) + except WaiterError as e: + assert False, f'Failed to start the Kinesis Data Analytics application: {str(e)}.' + + def get_kinesis_analytics_application_input_id(self, application_name: str) -> str: + """ + Get the input ID for the Kenisis Data Analytics application. + :param application_name: Name of the Kenisis Data Analytics application. + :return: Input ID for the Kenisis Data Analytics application. + """ + client = self._aws_util.client('kinesisanalytics') + response = client.describe_application( + ApplicationName=application_name + ) + if not response: + return '' + input_descriptions = response.get('ApplicationDetail', {}).get('InputDescriptions', []) + if len(input_descriptions) != 1: + return '' + + return input_descriptions[0].get('InputId', '') + + def stop_kinesis_data_analytics_application(self, application_name: str) -> None: + """ + Stop the Kenisis Data Analytics application. + :param application_name: Name of the Kenisis Data Analytics application. + """ + client = self._aws_util.client('kinesisanalytics') + client.stop_application( + ApplicationName=application_name + ) + + try: + KinesisAnalyticsApplicationUpdatedWaiter(client, 'READY').wait(application_name=application_name) + except WaiterError as e: + assert False, f'Failed to stop the Kinesis Data Analytics application: {str(e)}.' + + def verify_cloud_watch_delivery(self, namespace: str, metrics_name: str, + dimensions: typing.List[dict], start_time: datetime) -> None: + """ + Verify that the expected metrics is delivered to CloudWatch. + :param namespace: Namespace of the metrics. + :param metrics_name: Name of the metrics. + :param dimensions: Dimensions of the metrics. + :param start_time: Start time for generating the metrics. + """ + client = self._aws_util.client('cloudwatch') + + try: + CloudWatchMetricsDeliveredWaiter(client).wait( + namespace=namespace, + metrics_name=metrics_name, + dimensions=dimensions, + start_time=start_time + ) + except WaiterError as e: + assert False, f'Failed to deliver metrics to CloudWatch: {str(e)}.' + + def verify_s3_delivery(self, analytics_bucket_name: str) -> None: + """ + Verify that metrics are delivered to S3 for batch analytics successfully. + :param analytics_bucket_name: Name of the deployed S3 bucket. + """ + client = self._aws_util.client('s3') + bucket_name = analytics_bucket_name + + try: + DataLakeMetricsDeliveredWaiter(client).wait(bucket_name=bucket_name, prefix=EXPECTED_S3_DIRECTORY) + except WaiterError as e: + assert False, f'Failed to find the S3 directory for storing metrics data: {str(e)}.' + + # Check whether the data is converted to the expected data format. + response = client.list_objects_v2( + Bucket=bucket_name, + Prefix=EXPECTED_S3_DIRECTORY + ) + assert response.get('KeyCount', 0) != 0, f'Failed to deliver metrics to the S3 bucket {bucket_name}.' + + s3_objects = response.get('Contents', []) + for s3_object in s3_objects: + key = s3_object.get('Key', '') + assert pathlib.Path(key).suffix == EXPECTED_S3_OBJECT_EXTENSION, \ + f'Invalid data format is found in the S3 bucket {bucket_name}' + + def run_glue_crawler(self, crawler_name: str) -> None: + """ + Run the Glue crawler and wait for it to finish. + :param crawler_name: Name of the Glue crawler + """ + client = self._aws_util.client('glue') + try: + client.start_crawler( + Name=crawler_name + ) + except client.exceptions.CrawlerRunningException: + # The crawler has already been started. + return + + try: + GlueCrawlerReadyWaiter(client).wait(crawler_name=crawler_name) + except WaiterError as e: + assert False, f'Failed to run the Glue crawler: {str(e)}.' + + def run_named_queries(self, work_group: str) -> None: + """ + Run the named queries under the specific Athena work group. + :param work_group: Name of the Athena work group. + """ + client = self._aws_util.client('athena') + # List all the named queries. + response = client.list_named_queries( + WorkGroup=work_group + ) + named_query_ids = response.get('NamedQueryIds', []) + + # Run each of the queries. + for named_query_id in named_query_ids: + get_named_query_response = client.get_named_query( + NamedQueryId=named_query_id + ) + named_query = get_named_query_response.get('NamedQuery', {}) + + start_query_execution_response = client.start_query_execution( + QueryString=named_query.get('QueryString', ''), + QueryExecutionContext={ + 'Database': named_query.get('Database', '') + }, + WorkGroup=work_group + ) + + # Wait for the query to finish. + state = 'RUNNING' + while state == 'QUEUED' or state == 'RUNNING': + get_query_execution_response = client.get_query_execution( + QueryExecutionId=start_query_execution_response.get('QueryExecutionId', '') + ) + + state = get_query_execution_response.get('QueryExecution', {}).get('Status', {}).get('State', '') + + assert state == 'SUCCEEDED', f'Failed to run the named query {named_query.get("Name", {})}' + + def empty_s3_bucket(self, bucket_name: str) -> None: + """ + Empty the S3 bucket following: + https://boto3.amazonaws.com/v1/documentation/api/latest/guide/migrations3.html + + :param bucket_name: Name of the S3 bucket. + """ + + s3 = self._aws_util.resource('s3') + bucket = s3.Bucket(bucket_name) + + for key in bucket.objects.all(): + key.delete() + + def get_analytics_bucket_name(self, stack_name: str) -> str: + """ + Get the name of the deployed S3 bucket. + :param stack_name: Name of the CloudFormation stack. + :return: Name of the deployed S3 bucket. + """ + + client = self._aws_util.client('cloudformation') + + response = client.describe_stack_resources( + StackName=stack_name + ) + resources = response.get('StackResources', []) + + for resource in resources: + if resource.get('ResourceType') == 'AWS::S3::Bucket': + return resource.get('PhysicalResourceId', '') + + return '' + + +@pytest.fixture(scope='function') +def aws_metrics_utils( + request: pytest.fixture, + aws_utils: pytest.fixture): + """ + Fixture for the AWS metrics util functions. + :param request: _pytest.fixtures.SubRequest class that handles getting + a pytest fixture from a pytest function/fixture. + :param aws_utils: aws_utils fixture. + """ + aws_utils_obj = AWSMetricsUtils(aws_utils) + return aws_utils_obj diff --git a/AutomatedTesting/Gem/PythonTests/AWS/Windows/aws_metrics/aws_metrics_waiters.py b/AutomatedTesting/Gem/PythonTests/AWS/Windows/aws_metrics/aws_metrics_waiters.py new file mode 100644 index 0000000000..7ce5551fd4 --- /dev/null +++ b/AutomatedTesting/Gem/PythonTests/AWS/Windows/aws_metrics/aws_metrics_waiters.py @@ -0,0 +1,142 @@ +""" +All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +its licensors. + +For complete copyright and license terms please see the LICENSE at the root of this +distribution (the "License"). All use of this software is governed by the License, +or, if provided, by the license below or the license accompanying this file. Do not +remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +""" + +import botocore.client +import logging + +from datetime import timedelta +from AWS.common.custom_waiter import CustomWaiter, WaitState + +logging.getLogger('boto').setLevel(logging.CRITICAL) + + +class KinesisAnalyticsApplicationUpdatedWaiter(CustomWaiter): + """ + Subclass of the base custom waiter class. + Wait for the Kinesis analytics application being updated to a specific status. + """ + def __init__(self, client: botocore.client, status: str): + """ + Initialize the waiter. + + :param client: Boto3 client to use. + :param status: Expected status. + """ + super().__init__( + 'KinesisAnalyticsApplicationUpdated', + 'DescribeApplication', + 'ApplicationDetail.ApplicationStatus', + {status: WaitState.SUCCESS}, + client) + + def wait(self, application_name: str): + """ + Wait for the expected status. + + :param application_name: Name of the Kinesis analytics application. + """ + self._wait(ApplicationName=application_name) + + +class GlueCrawlerReadyWaiter(CustomWaiter): + """ + Subclass of the base custom waiter class. + Wait for the Glue crawler to finish its processing. + """ + def __init__(self, client: botocore.client): + """ + Initialize the waiter. + + :param client: Boto3 client to use. + """ + super().__init__( + 'GlueCrawlerReady', + 'GetCrawler', + 'Crawler.State', + {'READY': WaitState.SUCCESS}, + client) + + def wait(self, crawler_name): + """ + Wait for the expected status. + + :param crawler_name: Name of the Glue crawler. + """ + self._wait(Name=crawler_name) + + +class DataLakeMetricsDeliveredWaiter(CustomWaiter): + """ + Subclass of the base custom waiter class. + Wait for the expected directory being created in the S3 bucket. + """ + def __init__(self, client: botocore.client): + """ + Initialize the waiter. + + :param client: Boto3 client to use. + """ + super().__init__( + 'DataLakeMetricsDelivered', + 'ListObjectsV2', + 'KeyCount > `0`', + {True: WaitState.SUCCESS}, + client) + + def wait(self, bucket_name, prefix): + """ + Wait for the expected directory being created. + + :param bucket_name: Name of the S3 bucket. + :param prefix: Name of the expected directory prefix. + """ + self._wait(Bucket=bucket_name, Prefix=prefix) + + +class CloudWatchMetricsDeliveredWaiter(CustomWaiter): + """ + Subclass of the base custom waiter class. + Wait for the expected metrics being delivered to CloudWatch. + """ + def __init__(self, client: botocore.client): + """ + Initialize the waiter. + + :param client: Boto3 client to use. + """ + super().__init__( + 'CloudWatchMetricsDelivered', + 'GetMetricStatistics', + 'length(Datapoints) > `0`', + {True: WaitState.SUCCESS}, + client) + + def wait(self, namespace, metrics_name, dimensions, start_time): + """ + Wait for the expected metrics being delivered. + + :param namespace: Namespace of the metrics. + :param metrics_name: Name of the metrics. + :param dimensions: Dimensions of the metrics. + :param start_time: Start time for generating the metrics. + """ + self._wait( + Namespace=namespace, + MetricName=metrics_name, + Dimensions=dimensions, + StartTime=start_time, + EndTime=start_time + timedelta(0, self.timeout), + Period=60, + Statistics=[ + 'SampleCount' + ], + Unit='Count' + ) diff --git a/AutomatedTesting/Gem/PythonTests/AWS/Windows/cdk/cdk.py b/AutomatedTesting/Gem/PythonTests/AWS/Windows/cdk/cdk.py index 455b3f94cb..5dbde29b9d 100644 --- a/AutomatedTesting/Gem/PythonTests/AWS/Windows/cdk/cdk.py +++ b/AutomatedTesting/Gem/PythonTests/AWS/Windows/cdk/cdk.py @@ -16,12 +16,15 @@ import boto3 import ly_test_tools.environment.process_utils as process_utils from typing import List +BOOTSTRAP_STACK_NAME = 'CDKToolkit' +BOOTSTRAP_STAGING_BUCKET_LOGIC_ID = 'StagingBucket' class Cdk: """ Cdk class that provides methods to run cdk application commands. Expects system to have NodeJS, AWS CLI and CDK installed globally and have their paths setup as env variables. """ + def __init__(self, cdk_path: str, project: str, account_id: str, workspace: pytest.fixture, session: boto3.session.Session): """ @@ -49,12 +52,24 @@ class Cdk: env=self._cdk_env, shell=True) + def bootstrap(self) -> None: + """ + Deploy the bootstrap stack. + """ + bootstrap_cmd = ['cdk', 'bootstrap', + f'aws://{self._cdk_env["O3DE_AWS_DEPLOY_ACCOUNT"]}/{self._cdk_env["O3DE_AWS_DEPLOY_REGION"]}'] + + process_utils.check_call( + bootstrap_cmd, + cwd=self._cdk_path, + env=self._cdk_env, + shell=True) + def list(self) -> List[str]: """ lists cdk stack names :return List of cdk stack names """ - if not self._cdk_path: return [] @@ -123,6 +138,38 @@ class Cdk: self._stacks = [] self._cdk_path = '' + @staticmethod + def remove_bootstrap_stack(aws_utils: pytest.fixture) -> None: + """ + Remove the CDK bootstrap stack. + :param aws_utils: aws_utils fixture. + """ + # Check if the bootstrap stack exists. + response = aws_utils.client('cloudformation').describe_stacks( + StackName=BOOTSTRAP_STACK_NAME + ) + stacks = response.get('Stacks', []) + if not stacks: + return + + # Clear the bootstrap staging bucket before deleting the bootstrap stack. + response = aws_utils.client('cloudformation').describe_stack_resource( + StackName=BOOTSTRAP_STACK_NAME, + LogicalResourceId=BOOTSTRAP_STAGING_BUCKET_LOGIC_ID + ) + + staging_bucket_name = response.get('StackResourceDetail', {}).get('PhysicalResourceId', '') + if staging_bucket_name: + s3 = aws_utils.resource('s3') + bucket = s3.Bucket(staging_bucket_name) + for key in bucket.objects.all(): + key.delete() + + # Delete the bootstrap stack. + aws_utils.client('cloudformation').delete_stack( + StackName=BOOTSTRAP_STACK_NAME + ) + @pytest.fixture(scope='function') def cdk( @@ -131,6 +178,7 @@ def cdk( feature_name: str, workspace: pytest.fixture, aws_utils: pytest.fixture, + bootstrap_required: bool = True, destroy_stacks_on_teardown: bool = True) -> Cdk: """ Fixture for setting up a Cdk @@ -140,6 +188,8 @@ def cdk( :param feature_name: Feature gem name to expect cdk folder in. :param workspace: ly_test_tools workspace fixture. :param aws_utils: aws_utils fixture. + :param bootstrap_required: Whether the bootstrap stack needs to be created to + provision resources the AWS CDK needs to perform the deployment. :param destroy_stacks_on_teardown: option to control calling destroy ot the end of test. :return Cdk class object. """ @@ -147,9 +197,14 @@ def cdk( cdk_path = f'{workspace.paths.engine_root()}/Gems/{feature_name}/cdk' cdk_obj = Cdk(cdk_path, project, aws_utils.assume_account_id(), workspace, aws_utils.assume_session()) + if bootstrap_required: + cdk_obj.bootstrap() + def teardown(): if destroy_stacks_on_teardown: cdk_obj.destroy() + cdk_obj.remove_bootstrap_stack(aws_utils) + request.addfinalizer(teardown) return cdk_obj diff --git a/AutomatedTesting/Gem/PythonTests/AWS/common/aws_credentials.py b/AutomatedTesting/Gem/PythonTests/AWS/common/aws_credentials.py new file mode 100644 index 0000000000..fbce772d40 --- /dev/null +++ b/AutomatedTesting/Gem/PythonTests/AWS/common/aws_credentials.py @@ -0,0 +1,134 @@ +""" +All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +its licensors. +For complete copyright and license terms please see the LICENSE at the root of this +distribution (the "License"). All use of this software is governed by the License, +or, if provided, by the license below or the license accompanying this file. Do not +remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +""" + +import boto3 +import configparser +import logging +import os +import pytest +import typing + +logger = logging.getLogger(__name__) +logging.getLogger('boto').setLevel(logging.CRITICAL) + + +class AwsCredentials: + def __init__(self, profile_name: str): + self._profile_name = profile_name + + self._credentials_path = os.environ.get('AWS_SHARED_CREDENTIALS_FILE') + if not self._credentials_path: + # Home directory location varies based on the operating system, but is referred to using the environment + # variables %UserProfile% in Windows and $HOME or ~ (tilde) in Unix-based systems. + self._credentials_path = os.path.join(os.environ.get('UserProfile', os.path.expanduser('~')), + '.aws', 'credentials') + self._credentials_file_exists = os.path.exists(self._credentials_path) + + self._credentials = configparser.ConfigParser() + self._credentials.read(self._credentials_path) + + def get_aws_credentials(self) -> typing.Tuple[str, str, str]: + """ + Get aws credentials stored in the specific named profile. + + :return AWS credentials. + """ + access_key_id = self._get_aws_credential_attribute_value('aws_access_key_id') + secret_access_key = self._get_aws_credential_attribute_value('aws_secret_access_key') + session_token = self._get_aws_credential_attribute_value('aws_session_token') + + return access_key_id, secret_access_key, session_token + + def set_aws_credentials_by_session(self, session: boto3.Session) -> None: + """ + Set AWS credentials stored in the specific named profile using an assumed role session. + + :param session: assumed role session. + """ + credentials = session.get_credentials().get_frozen_credentials() + self.set_aws_credentials(credentials.access_key, credentials.secret_key, credentials.token) + + def set_aws_credentials(self, aws_access_key_id: str, aws_secret_access_key: str, + aws_session_token: str) -> None: + """ + Set AWS credentials stored in the specific named profile. + + :param aws_access_key_id: AWS access key id. + :param aws_secret_access_key: AWS secrete access key. + :param aws_session_token: AWS assumed role session. + """ + self._set_aws_credential_attribute_value('aws_access_key_id', aws_access_key_id) + self._set_aws_credential_attribute_value('aws_secret_access_key', aws_secret_access_key) + self._set_aws_credential_attribute_value('aws_session_token', aws_session_token) + + if (len(self._credentials.sections()) == 0) and (not self._credentials_file_exists): + os.remove(self._credentials_path) + return + + with open(self._credentials_path, 'w+') as credential_file: + self._credentials.write(credential_file) + + def _get_aws_credential_attribute_value(self, attribute_name: str) -> str: + """ + Get the value of an AWS credential attribute stored in the specific named profile. + + :param attribute_name: Name of the AWS credential attribute. + :return Value of the AWS credential attribute. + """ + try: + value = self._credentials.get(self._profile_name, attribute_name) + except configparser.NoSectionError: + # Named profile or key doesn't exist + value = None + except configparser.NoOptionError: + # Named profile doesn't have the specified attribute + value = None + + return value + + def _set_aws_credential_attribute_value(self, attribute_name: str, attribute_value: str) -> None: + """ + Set the value of an AWS credential attribute stored in the specific named profile. + + :param attribute_name: Name of the AWS credential attribute. + :param attribute_value: Value of the AWS credential attribute. + """ + if self._profile_name not in self._credentials: + self._credentials[self._profile_name] = {} + + if attribute_value is None: + self._credentials.remove_option(self._profile_name, attribute_name) + # Remove the named profile if it doesn't have any AWS credential attribute. + if len(self._credentials[self._profile_name]) == 0: + self._credentials.remove_section(self._profile_name) + else: + self._credentials[self._profile_name][attribute_name] = attribute_value + + +@pytest.fixture(scope='function') +def aws_credentials(request: pytest.fixture, aws_utils: pytest.fixture, profile_name: str): + """ + Fixture for setting up temporary AWS credentials from assume role. + + :param request: _pytest.fixtures.SubRequest class that handles getting + a pytest fixture from a pytest function/fixture. + :param aws_utils: aws_utils fixture. + :param profile_name: Named AWS profile to store temporary credentials. + """ + aws_credentials_obj = AwsCredentials(profile_name) + original_access_key, original_secret_access_key, original_token = aws_credentials_obj.get_aws_credentials() + aws_credentials_obj.set_aws_credentials_by_session(aws_utils.assume_session()) + + def teardown(): + # Reset to the named profile using the original AWS credentials + aws_credentials_obj.set_aws_credentials(original_access_key, original_secret_access_key, original_token) + request.addfinalizer(teardown) + + return aws_credentials_obj diff --git a/AutomatedTesting/Gem/PythonTests/AWS/common/aws_utils.py b/AutomatedTesting/Gem/PythonTests/AWS/common/aws_utils.py index 7a15ba0abe..ff33f58d1d 100644 --- a/AutomatedTesting/Gem/PythonTests/AWS/common/aws_utils.py +++ b/AutomatedTesting/Gem/PythonTests/AWS/common/aws_utils.py @@ -1,82 +1,90 @@ -""" -All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or -its licensors. -For complete copyright and license terms please see the LICENSE at the root of this -distribution (the "License"). All use of this software is governed by the License, -or, if provided, by the license below or the license accompanying this file. Do not -remove or modify any license notices. This file is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -""" -import boto3 -import pytest -import logging - -logger = logging.getLogger(__name__) - - -class AwsUtils: - - def __init__(self, arn: str, session_name: str, region_name: str): - local_session = boto3.Session(profile_name='default') - local_sts_client = local_session.client('sts') - self._local_account_id = local_sts_client.get_caller_identity()["Account"] - logger.info(f'Local Account Id: {self._local_account_id}') - - response = local_sts_client.assume_role(RoleArn=arn, RoleSessionName=session_name) - - self._assume_session = boto3.Session(aws_access_key_id=response['Credentials']['AccessKeyId'], - aws_secret_access_key=response['Credentials']['SecretAccessKey'], - aws_session_token=response['Credentials']['SessionToken'], - region_name=region_name) - - assume_sts_client = self._assume_session.client('sts') - assume_account_id = assume_sts_client.get_caller_identity()["Account"] - logger.info(f'Assume Account Id: {assume_account_id}') - self._assume_account_id = assume_account_id - - def client(self, service: str): - """ - Get the client for a specific AWS service from configured session - :return: Client for the AWS service. - """ - return self._assume_session.client(service) - - def assume_session(self): - return self._assume_session - - def local_account_id(self): - return self._local_account_id - - def assume_account_id(self): - return self._assume_account_id - - def destroy(self) -> None: - """ - clears stored session - """ - self._assume_session = None - - -@pytest.fixture(scope='function') -def aws_utils( - request: pytest.fixture, - assume_role_arn: str, - session_name: str, - region_name: str): - """ - Fixture for setting up a Cdk - :param request: _pytest.fixtures.SubRequest class that handles getting - a pytest fixture from a pytest function/fixture. - :param assume_role_arn: Role used to fetch temporary aws credentials, configure service clients with obtained credentials. - :param session_name: Session name to set. - :param region_name: AWS account region to set for session. - :return AWSUtils class object. - """ - aws_utils_obj = AwsUtils(assume_role_arn, session_name, region_name) - - def teardown(): - aws_utils_obj.destroy() - - request.addfinalizer(teardown) - - return aws_utils_obj +""" +All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +its licensors. +For complete copyright and license terms please see the LICENSE at the root of this +distribution (the "License"). All use of this software is governed by the License, +or, if provided, by the license below or the license accompanying this file. Do not +remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +""" +import boto3 +import pytest +import logging + +logger = logging.getLogger(__name__) +logging.getLogger('boto').setLevel(logging.CRITICAL) + + +class AwsUtils: + + def __init__(self, arn: str, session_name: str, region_name: str): + local_session = boto3.Session(profile_name='default') + local_sts_client = local_session.client('sts') + self._local_account_id = local_sts_client.get_caller_identity()["Account"] + logger.info(f'Local Account Id: {self._local_account_id}') + + response = local_sts_client.assume_role(RoleArn=arn, RoleSessionName=session_name) + + self._assume_session = boto3.Session(aws_access_key_id=response['Credentials']['AccessKeyId'], + aws_secret_access_key=response['Credentials']['SecretAccessKey'], + aws_session_token=response['Credentials']['SessionToken'], + region_name=region_name) + + assume_sts_client = self._assume_session.client('sts') + assume_account_id = assume_sts_client.get_caller_identity()["Account"] + logger.info(f'Assume Account Id: {assume_account_id}') + self._assume_account_id = assume_account_id + + def client(self, service: str): + """ + Get the client for a specific AWS service from configured session + :return: Client for the AWS service. + """ + return self._assume_session.client(service) + + def resource(self, service: str): + """ + Get the resource for a specific AWS service from configured session + :return: Client for the AWS service. + """ + return self._assume_session.resource(service) + + def assume_session(self): + return self._assume_session + + def local_account_id(self): + return self._local_account_id + + def assume_account_id(self): + return self._assume_account_id + + def destroy(self) -> None: + """ + clears stored session + """ + self._assume_session = None + + +@pytest.fixture(scope='function') +def aws_utils( + request: pytest.fixture, + assume_role_arn: str, + session_name: str, + region_name: str): + """ + Fixture for AWS util functions + :param request: _pytest.fixtures.SubRequest class that handles getting + a pytest fixture from a pytest function/fixture. + :param assume_role_arn: Role used to fetch temporary aws credentials, configure service clients with obtained credentials. + :param session_name: Session name to set. + :param region_name: AWS account region to set for session. + :return AWSUtils class object. + """ + aws_utils_obj = AwsUtils(assume_role_arn, session_name, region_name) + + def teardown(): + aws_utils_obj.destroy() + + request.addfinalizer(teardown) + + return aws_utils_obj diff --git a/AutomatedTesting/Gem/PythonTests/AWS/common/custom_waiter.py b/AutomatedTesting/Gem/PythonTests/AWS/common/custom_waiter.py new file mode 100644 index 0000000000..7c0a65e8a3 --- /dev/null +++ b/AutomatedTesting/Gem/PythonTests/AWS/common/custom_waiter.py @@ -0,0 +1,91 @@ +""" +All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +its licensors. + +For complete copyright and license terms please see the LICENSE at the root of this +distribution (the "License"). All use of this software is governed by the License, +or, if provided, by the license below or the license accompanying this file. Do not +remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +""" + +from enum import Enum +import botocore.client +import botocore.waiter +import logging + +logging.getLogger('boto').setLevel(logging.CRITICAL) + + +class WaitState(Enum): + SUCCESS = 'success' + FAILURE = 'failure' + + +class CustomWaiter: + """ + Base class for a custom waiter. + + Modified from: + https://docs.aws.amazon.com/code-samples/latest/catalog/python-demo_tools-custom_waiter.py.html + """ + def __init__( + self, name: str, operation: str, argument: str, + acceptors: dict, client: botocore.client, delay: int = 30, max_tries: int = 10, + matcher='path'): + """ + Subclasses should pass specific operations, arguments, and acceptors to + their superclass. + + :param name: The name of the waiter. This can be any descriptive string. + :param operation: The operation to wait for. This must match the casing of + the underlying operation model, which is typically in + CamelCase. + :param argument: The dict keys used to access the result of the operation, in + dot notation. For example, 'Job.Status' will access + result['Job']['Status']. + :param acceptors: The list of acceptors that indicate the wait is over. These + can indicate either success or failure. The acceptor values + are compared to the result of the operation after the + argument keys are applied. + :param client: The Boto3 client. + :param delay: The number of seconds to wait between each call to the operation. Default to 30 seconds. + :param max_tries: The maximum number of tries before exiting. Default to 10. + :param matcher: The kind of matcher to use. Default to 'path'. + """ + self.name = name + self.operation = operation + self.argument = argument + self.client = client + self.waiter_model = botocore.waiter.WaiterModel({ + 'version': 2, + 'waiters': { + name: { + "delay": delay, + "operation": operation, + "maxAttempts": max_tries, + "acceptors": [{ + "state": state.value, + "matcher": matcher, + "argument": argument, + "expected": expected + } for expected, state in acceptors.items()] + }}}) + self.waiter = botocore.waiter.create_waiter_with_client( + self.name, self.waiter_model, self.client) + + self._timeout = delay * max_tries + + def _wait(self, **kwargs): + """ + Starts the botocore wait loop. + + :param kwargs: Keyword arguments that are passed to the operation being polled. + """ + self.waiter.wait(**kwargs) + + @property + def timeout(self): + return self._timeout + + diff --git a/AutomatedTesting/Registry/awscoreconfiguration.setreg b/AutomatedTesting/Registry/awscoreconfiguration.setreg index ca110eb103..b7c60b0fb9 100644 --- a/AutomatedTesting/Registry/awscoreconfiguration.setreg +++ b/AutomatedTesting/Registry/awscoreconfiguration.setreg @@ -3,7 +3,7 @@ { "AWSCore": { - "ProfileName": "default", + "ProfileName": "AWSAutomationTest", "ResourceMappingConfigFileName": "aws_resource_mappings.json" } } diff --git a/Gems/AWSMetrics/cdk/aws_metrics/batch_processing.py b/Gems/AWSMetrics/cdk/aws_metrics/batch_processing.py index fe85615d8d..22f6ad6903 100755 --- a/Gems/AWSMetrics/cdk/aws_metrics/batch_processing.py +++ b/Gems/AWSMetrics/cdk/aws_metrics/batch_processing.py @@ -111,8 +111,7 @@ class BatchProcessing: self._events_firehose_delivery_stream = kinesisfirehose.CfnDeliveryStream( self._stack, - id='EventsFirehoseDeliveryStream', - delivery_stream_name=f'{self._stack.stack_name}-EventsFirehoseDeliveryStream', + id=f'{self._stack.stack_name}-EventsFirehoseDeliveryStream', delivery_stream_type='KinesisStreamAsSource', kinesis_stream_source_configuration=kinesisfirehose.CfnDeliveryStream.KinesisStreamSourceConfigurationProperty( kinesis_stream_arn=self._input_stream_arn, @@ -327,7 +326,7 @@ class BatchProcessing: @property def delivery_stream_name(self) -> kinesisfirehose.CfnDeliveryStream.delivery_stream_name: - return self._events_firehose_delivery_stream.delivery_stream_name + return self._events_firehose_delivery_stream.ref @property def delivery_stream_role_arn(self) -> iam.Role.role_arn: diff --git a/Gems/AWSMetrics/cdk/aws_metrics/data_ingestion.py b/Gems/AWSMetrics/cdk/aws_metrics/data_ingestion.py index a3ab99fccf..fbd332a577 100755 --- a/Gems/AWSMetrics/cdk/aws_metrics/data_ingestion.py +++ b/Gems/AWSMetrics/cdk/aws_metrics/data_ingestion.py @@ -73,14 +73,14 @@ class DataIngestion: api_id_output = core.CfnOutput( self._stack, - id='RestApiId', + id='RESTApiId', description='Service API Id for the analytics pipeline', export_name=f"{application_name}:RestApiId", value=self._rest_api.rest_api_id) stage_output = core.CfnOutput( self._stack, - id='DeploymentStage', + id='RESTApiStage', description='Stage for the REST API deployment', export_name=f"{application_name}:DeploymentStage", value=self._rest_api.deployment_stage.stage_name) From 9e3b7b45d95e6a8fe4af6340569f1d7f356674c4 Mon Sep 17 00:00:00 2001 From: hershey5045 <43485729+hershey5045@users.noreply.github.com> Date: Tue, 1 Jun 2021 14:21:02 -0700 Subject: [PATCH 12/19] Fix window handle retrieval in frame capture system. (#1073) Capturing screenshots via hydra should work now. --- .../Common/Code/Source/FrameCaptureSystemComponent.cpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Gems/Atom/Feature/Common/Code/Source/FrameCaptureSystemComponent.cpp b/Gems/Atom/Feature/Common/Code/Source/FrameCaptureSystemComponent.cpp index f8c1258fc0..0d9e46a418 100644 --- a/Gems/Atom/Feature/Common/Code/Source/FrameCaptureSystemComponent.cpp +++ b/Gems/Atom/Feature/Common/Code/Source/FrameCaptureSystemComponent.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -289,11 +290,7 @@ namespace AZ bool FrameCaptureSystemComponent::CaptureScreenshot(const AZStd::string& filePath) { - AzFramework::NativeWindowHandle windowHandle = nullptr; - AzFramework::WindowSystemRequestBus::BroadcastResult( - windowHandle, - &AzFramework::WindowSystemRequestBus::Events::GetDefaultWindowHandle); - + AzFramework::NativeWindowHandle windowHandle = AZ::RPI::ViewportContextRequests::Get()->GetDefaultViewportContext()->GetWindowHandle(); if (windowHandle) { return CaptureScreenshotForWindow(filePath, windowHandle); From afe20906db9c9f02003eed21a460627b51523ea5 Mon Sep 17 00:00:00 2001 From: Alex Peterson <26804013+AMZN-alexpete@users.noreply.github.com> Date: Tue, 1 Jun 2021 15:30:14 -0700 Subject: [PATCH 13/19] Add Project Manager File menu options to Editor --- .../ProjectManager/ProjectManager.cpp | 6 ++-- .../ProjectManager/ProjectManager.h | 15 ++++++-- .../Editor/Core/LevelEditorMenuHandler.cpp | 11 +++--- Code/Sandbox/Editor/CryEdit.cpp | 34 +++++++++++++++++++ Code/Sandbox/Editor/CryEdit.h | 4 +++ Code/Sandbox/Editor/LyViewPaneNames.h | 2 +- Code/Sandbox/Editor/MainWindow.cpp | 3 ++ Code/Sandbox/Editor/Resource.h | 3 ++ .../Source/ProjectManagerWindow.cpp | 18 ++++++++-- .../Source/ProjectManagerWindow.h | 4 ++- .../ProjectManager/Source/ProjectUtils.cpp | 11 ++++++ .../ProjectManager/Source/ProjectUtils.h | 2 ++ Code/Tools/ProjectManager/Source/ScreenDefs.h | 23 ++++++++++++- Code/Tools/ProjectManager/Source/main.cpp | 28 ++++++++++++++- 14 files changed, 147 insertions(+), 17 deletions(-) diff --git a/Code/Framework/AzFramework/AzFramework/ProjectManager/ProjectManager.cpp b/Code/Framework/AzFramework/AzFramework/ProjectManager/ProjectManager.cpp index 2742b90f4c..22598595fc 100644 --- a/Code/Framework/AzFramework/AzFramework/ProjectManager/ProjectManager.cpp +++ b/Code/Framework/AzFramework/AzFramework/ProjectManager/ProjectManager.cpp @@ -78,7 +78,7 @@ namespace AzFramework::ProjectManager projectJsonPath.c_str()); } - if (LaunchProjectManager(engineRootPath)) + if (LaunchProjectManager()) { AZ_TracePrintf("ProjectManager", "Project Manager launched successfully, requesting exit."); return ProjectPathCheckResult::ProjectManagerLaunched; @@ -87,7 +87,7 @@ namespace AzFramework::ProjectManager return ProjectPathCheckResult::ProjectManagerLaunchFailed; } - bool LaunchProjectManager([[maybe_unused]] const AZ::IO::FixedMaxPath& engineRootPath) + bool LaunchProjectManager(const AZStd::string& commandLineArgs) { bool launchSuccess = false; #if (AZ_TRAIT_AZFRAMEWORK_USE_PROJECT_MANAGER) @@ -109,7 +109,7 @@ namespace AzFramework::ProjectManager } AzFramework::ProcessLauncher::ProcessLaunchInfo processLaunchInfo; - processLaunchInfo.m_commandlineParameters = executablePath.String(); + processLaunchInfo.m_commandlineParameters = executablePath.String() + commandLineArgs; launchSuccess = AzFramework::ProcessLauncher::LaunchUnwatchedProcess(processLaunchInfo); } if (ownsSystemAllocator) diff --git a/Code/Framework/AzFramework/AzFramework/ProjectManager/ProjectManager.h b/Code/Framework/AzFramework/AzFramework/ProjectManager/ProjectManager.h index cc79bd4184..d0ef7172b0 100644 --- a/Code/Framework/AzFramework/AzFramework/ProjectManager/ProjectManager.h +++ b/Code/Framework/AzFramework/AzFramework/ProjectManager/ProjectManager.h @@ -12,6 +12,7 @@ #pragma once #include +#include namespace AzFramework::ProjectManager { @@ -21,8 +22,16 @@ namespace AzFramework::ProjectManager ProjectManagerLaunched = 0, ProjectPathFound = 1 }; - // Check for a project name, if not found, attempts to launch project manager and returns false + + //! Check for a project name, if not found, attempts to launch project manager and returns false + //! @param argc the number of arguments in argv + //! @param argv arguments provided to this executable + //! @return a ProjectPathCheckResult ProjectPathCheckResult CheckProjectPathProvided(const int argc, char* argv[]); - // Attempt to Launch the project manager. Requires locating the engine root, project manager script, and python. - bool LaunchProjectManager(const AZ::IO::FixedMaxPath& engineRootPath); + + //! Attempt to Launch the project manager, assuming the o3de executable exists in same folder as + //! current executable. Requires the o3de cli and python. + //! @param commandLineArgs additional command line arguments to provide to the project manager + //! @return true on success, false if failed to find or launch the executable + bool LaunchProjectManager(const AZStd::string& commandLineArgs = ""); } // AzFramework::ProjectManager diff --git a/Code/Sandbox/Editor/Core/LevelEditorMenuHandler.cpp b/Code/Sandbox/Editor/Core/LevelEditorMenuHandler.cpp index 0ce3fa55f5..8f6e927a84 100644 --- a/Code/Sandbox/Editor/Core/LevelEditorMenuHandler.cpp +++ b/Code/Sandbox/Editor/Core/LevelEditorMenuHandler.cpp @@ -421,17 +421,18 @@ QMenu* LevelEditorMenuHandler::CreateFileMenu() fileMenu.AddSeparator(); // Project Settings - auto projectSettingMenu = fileMenu.AddMenu(tr("Project Settings")); + fileMenu.AddAction(ID_FILE_PROJECT_MANAGER_SETTINGS); - // Project Settings Tool + // Platform Settings - Project Settings Tool // Shortcut must be set while adding the action otherwise it doesn't work - projectSettingMenu.Get()->addAction( + fileMenu.Get()->addAction( tr(LyViewPane::ProjectSettingsTool), []() { QtViewPaneManager::instance()->OpenPane(LyViewPane::ProjectSettingsTool); }, tr("Ctrl+Shift+P")); - projectSettingMenu.AddSeparator(); - + fileMenu.AddSeparator(); + fileMenu.AddAction(ID_FILE_PROJECT_MANAGER_NEW); + fileMenu.AddAction(ID_FILE_PROJECT_MANAGER_OPEN); fileMenu.AddSeparator(); // NEWMENUS: NEEDS IMPLEMENTATION diff --git a/Code/Sandbox/Editor/CryEdit.cpp b/Code/Sandbox/Editor/CryEdit.cpp index a0ff5d7eff..c60f32560b 100644 --- a/Code/Sandbox/Editor/CryEdit.cpp +++ b/Code/Sandbox/Editor/CryEdit.cpp @@ -58,6 +58,7 @@ AZ_POP_DISABLE_WARNING #include #include #include +#include // AzToolsFramework #include @@ -477,6 +478,11 @@ void CCryEditApp::RegisterActionHandlers() ON_COMMAND(ID_FILE_SAVE_LEVEL, OnFileSave) ON_COMMAND(ID_FILE_EXPORTOCCLUSIONMESH, OnFileExportOcclusionMesh) + + // Project Manager + ON_COMMAND(ID_FILE_PROJECT_MANAGER_SETTINGS, OnOpenProjectManagerSettings) + ON_COMMAND(ID_FILE_PROJECT_MANAGER_NEW, OnOpenProjectManagerNew) + ON_COMMAND(ID_FILE_PROJECT_MANAGER_OPEN, OnOpenProjectManager) } CCryEditApp* CCryEditApp::s_currentInstance = nullptr; @@ -2854,6 +2860,34 @@ void CCryEditApp::OnPreferences() */ } +void CCryEditApp::OnOpenProjectManagerSettings() +{ + OpenProjectManager("UpdateProject"); +} + +void CCryEditApp::OnOpenProjectManagerNew() +{ + OpenProjectManager("CreateProject"); +} + +void CCryEditApp::OnOpenProjectManager() +{ + OpenProjectManager("Projects"); +} + +void CCryEditApp::OpenProjectManager(const AZStd::string& screen) +{ + // provide the current project path for in case we want to update the project + AZ::IO::FixedMaxPathString projectPath = AZ::Utils::GetProjectPath(); + const AZStd::string commandLineOptions = AZStd::string::format(" --screen %s --project_path %s", screen.c_str(), projectPath.c_str()); + bool launchSuccess = AzFramework::ProjectManager::LaunchProjectManager(commandLineOptions); + if (!launchSuccess) + { + QMessageBox::critical(AzToolsFramework::GetActiveWindow(), QObject::tr("Failed to launch O3DE Project Manager"), QObject::tr("Failed to find or start the O3dE Project Manager")); + } +} + + ////////////////////////////////////////////////////////////////////////// void CCryEditApp::OnUndo() { diff --git a/Code/Sandbox/Editor/CryEdit.h b/Code/Sandbox/Editor/CryEdit.h index d4c1304b6a..dc4f015faf 100644 --- a/Code/Sandbox/Editor/CryEdit.h +++ b/Code/Sandbox/Editor/CryEdit.h @@ -229,6 +229,9 @@ public: void OnFileResaveSlices(); void OnFileEditEditorini(); void OnPreferences(); + void OnOpenProjectManagerSettings(); + void OnOpenProjectManagerNew(); + void OnOpenProjectManager(); void OnRedo(); void OnUpdateRedo(QAction* action); void OnUpdateUndo(QAction* action); @@ -366,6 +369,7 @@ private: AZ_POP_DISABLE_DLL_EXPORT_MEMBER_WARNING friend struct PythonTestOutputHandler; + void OpenProjectManager(const AZStd::string& screen); void OnWireframe(); void OnUpdateWireframe(QAction* action); void OnViewConfigureLayout(); diff --git a/Code/Sandbox/Editor/LyViewPaneNames.h b/Code/Sandbox/Editor/LyViewPaneNames.h index e95191ce06..b94cda3c52 100644 --- a/Code/Sandbox/Editor/LyViewPaneNames.h +++ b/Code/Sandbox/Editor/LyViewPaneNames.h @@ -30,7 +30,7 @@ namespace LyViewPane static const char* const EntityInspector = "Entity Inspector"; static const char* const EntityInspectorPinned = "Pinned Entity Inspector"; static const char* const LevelInspector = "Level Inspector"; - static const char* const ProjectSettingsTool = "Project Settings Tool"; + static const char* const ProjectSettingsTool = "Edit Platform Settings..."; static const char* const ErrorReport = "Error Report"; static const char* const Console = "Console"; static const char* const ConsoleMenuName = "&Console"; diff --git a/Code/Sandbox/Editor/MainWindow.cpp b/Code/Sandbox/Editor/MainWindow.cpp index 8086293207..9e983c3593 100644 --- a/Code/Sandbox/Editor/MainWindow.cpp +++ b/Code/Sandbox/Editor/MainWindow.cpp @@ -748,6 +748,9 @@ void MainWindow::InitActions() am->AddAction(ID_FILE_EXPORTOCCLUSIONMESH, tr("Export Occlusion Mesh")); am->AddAction(ID_FILE_EDITLOGFILE, tr("Show Log File")); am->AddAction(ID_FILE_RESAVESLICES, tr("Resave All Slices")); + am->AddAction(ID_FILE_PROJECT_MANAGER_SETTINGS, tr("Edit Project Settings...")); + am->AddAction(ID_FILE_PROJECT_MANAGER_NEW, tr("New Project...")); + am->AddAction(ID_FILE_PROJECT_MANAGER_OPEN, tr("Open Project...")); am->AddAction(ID_GAME_PC_ENABLEVERYHIGHSPEC, tr("Very High")).SetCheckable(true) .RegisterUpdateCallback(cryEdit, &CCryEditApp::OnUpdateGameSpec); am->AddAction(ID_GAME_PC_ENABLEHIGHSPEC, tr("High")).SetCheckable(true) diff --git a/Code/Sandbox/Editor/Resource.h b/Code/Sandbox/Editor/Resource.h index 31fc9909f1..9c50045367 100644 --- a/Code/Sandbox/Editor/Resource.h +++ b/Code/Sandbox/Editor/Resource.h @@ -313,6 +313,9 @@ #define ID_CREATE_LEVEL_FG_MODULE_FROM_SELECTION 35077 #define ID_GRAPHVIEW_ADD_BLACK_BOX 35078 #define ID_GRAPHVIEW_UNGROUP 35079 +#define ID_FILE_PROJECT_MANAGER_NEW 35080 +#define ID_FILE_PROJECT_MANAGER_OPEN 35081 +#define ID_FILE_PROJECT_MANAGER_SETTINGS 35082 #define ID_TV_TRACKS_TOOLBAR_BASE 35083 // range between ID_TV_TRACKS_TOOLBAR_BASE to ID_TV_TRACKS_TOOLBAR_LAST reserved #define ID_TV_TRACKS_TOOLBAR_LAST 35183 // for up to 100 "Add Tracks..." dynamically added Track View Track buttons #define ID_OPEN_TERRAIN_EDITOR 36007 diff --git a/Code/Tools/ProjectManager/Source/ProjectManagerWindow.cpp b/Code/Tools/ProjectManager/Source/ProjectManagerWindow.cpp index 76bcc2eb99..cb1398cc61 100644 --- a/Code/Tools/ProjectManager/Source/ProjectManagerWindow.cpp +++ b/Code/Tools/ProjectManager/Source/ProjectManagerWindow.cpp @@ -14,13 +14,16 @@ #include #include +#include #include +#include +#include #include namespace O3DE::ProjectManager { - ProjectManagerWindow::ProjectManagerWindow(QWidget* parent, const AZ::IO::PathView& engineRootPath) + ProjectManagerWindow::ProjectManagerWindow(QWidget* parent, const AZ::IO::PathView& engineRootPath, const AZ::IO::PathView& projectPath, ProjectManagerScreen startScreen) : QMainWindow(parent) { m_pythonBindings = AZStd::make_unique(engineRootPath); @@ -50,7 +53,18 @@ namespace O3DE::ProjectManager // set stylesheet after creating the screens or their styles won't get updated AzQtComponents::StyleManager::setStyleSheet(this, QStringLiteral("style:ProjectManager.qss")); - screensCtrl->ForceChangeToScreen(ProjectManagerScreen::Projects, false); + // always push the projects screen first so we have something to come back to + if (startScreen != ProjectManagerScreen::Projects) + { + screensCtrl->ForceChangeToScreen(ProjectManagerScreen::Projects); + } + screensCtrl->ForceChangeToScreen(startScreen); + + if (!projectPath.empty()) + { + const QString path = QString::fromUtf8(projectPath.Native().data(), aznumeric_cast(projectPath.Native().size())); + emit screensCtrl->NotifyCurrentProject(path); + } } ProjectManagerWindow::~ProjectManagerWindow() diff --git a/Code/Tools/ProjectManager/Source/ProjectManagerWindow.h b/Code/Tools/ProjectManager/Source/ProjectManagerWindow.h index 74db3467c5..758af8fc00 100644 --- a/Code/Tools/ProjectManager/Source/ProjectManagerWindow.h +++ b/Code/Tools/ProjectManager/Source/ProjectManagerWindow.h @@ -14,6 +14,7 @@ #if !defined(Q_MOC_RUN) #include #include +#include #endif namespace O3DE::ProjectManager @@ -24,7 +25,8 @@ namespace O3DE::ProjectManager Q_OBJECT public: - explicit ProjectManagerWindow(QWidget* parent, const AZ::IO::PathView& engineRootPath); + explicit ProjectManagerWindow(QWidget* parent, const AZ::IO::PathView& engineRootPath, const AZ::IO::PathView& projectPath, + ProjectManagerScreen startScreen = ProjectManagerScreen::Projects); ~ProjectManagerWindow(); private: diff --git a/Code/Tools/ProjectManager/Source/ProjectUtils.cpp b/Code/Tools/ProjectManager/Source/ProjectUtils.cpp index 526e745d82..58e4c5c60f 100644 --- a/Code/Tools/ProjectManager/Source/ProjectUtils.cpp +++ b/Code/Tools/ProjectManager/Source/ProjectUtils.cpp @@ -192,5 +192,16 @@ namespace O3DE::ProjectManager return true; } + ProjectManagerScreen GetProjectManagerScreen(const QString& screen) + { + auto iter = s_ProjectManagerStringNames.find(screen); + if (iter != s_ProjectManagerStringNames.end()) + { + return iter.value(); + } + + return ProjectManagerScreen::Invalid; + } + } // namespace ProjectUtils } // namespace O3DE::ProjectManager diff --git a/Code/Tools/ProjectManager/Source/ProjectUtils.h b/Code/Tools/ProjectManager/Source/ProjectUtils.h index 5982bff634..d556d682f2 100644 --- a/Code/Tools/ProjectManager/Source/ProjectUtils.h +++ b/Code/Tools/ProjectManager/Source/ProjectUtils.h @@ -11,6 +11,7 @@ */ #pragma once +#include #include namespace O3DE::ProjectManager @@ -24,5 +25,6 @@ namespace O3DE::ProjectManager bool CopyProject(const QString& origPath, const QString& newPath); bool DeleteProjectFiles(const QString& path, bool force = false); bool MoveProject(const QString& origPath, const QString& newPath, QWidget* parent = nullptr); + ProjectManagerScreen GetProjectManagerScreen(const QString& screen); } // namespace ProjectUtils } // namespace O3DE::ProjectManager diff --git a/Code/Tools/ProjectManager/Source/ScreenDefs.h b/Code/Tools/ProjectManager/Source/ScreenDefs.h index 46d243f677..43ed303461 100644 --- a/Code/Tools/ProjectManager/Source/ScreenDefs.h +++ b/Code/Tools/ProjectManager/Source/ScreenDefs.h @@ -11,9 +11,13 @@ */ #pragma once +#include +#include +#include + namespace O3DE::ProjectManager { - enum ProjectManagerScreen + enum class ProjectManagerScreen { Invalid = -1, Empty, @@ -25,4 +29,21 @@ namespace O3DE::ProjectManager ProjectSettings, EngineSettings }; + + static QHash s_ProjectManagerStringNames = { + { "Empty", ProjectManagerScreen::Empty}, + { "CreateProject", ProjectManagerScreen::CreateProject}, + { "NewProjectSettings", ProjectManagerScreen::NewProjectSettings}, + { "GemCatalog", ProjectManagerScreen::GemCatalog}, + { "Projects", ProjectManagerScreen::Projects}, + { "UpdateProject", ProjectManagerScreen::UpdateProject}, + { "ProjectSettings", ProjectManagerScreen::ProjectSettings}, + { "EngineSettings", ProjectManagerScreen::EngineSettings} + }; + + // need to define qHash for ProjectManagerScreen when using scoped enums + inline uint qHash(ProjectManagerScreen key, uint seed) + { + return ::qHash(static_cast(key), seed); + } } // namespace O3DE::ProjectManager diff --git a/Code/Tools/ProjectManager/Source/main.cpp b/Code/Tools/ProjectManager/Source/main.cpp index cbeacbaf65..c597b8a729 100644 --- a/Code/Tools/ProjectManager/Source/main.cpp +++ b/Code/Tools/ProjectManager/Source/main.cpp @@ -15,13 +15,17 @@ #include #include #include +#include #include +#include #include #include #include +using namespace O3DE::ProjectManager; + int main(int argc, char* argv[]) { QApplication::setOrganizationName("O3DE"); @@ -51,7 +55,29 @@ int main(int argc, char* argv[]) AzQtComponents::StyleManager styleManager(&app); styleManager.initialize(&app, engineRootPath); - O3DE::ProjectManager::ProjectManagerWindow window(nullptr, engineRootPath); + // Get the initial start screen if one is provided via command line + constexpr char optionPrefix[] = "--"; + AZ::CommandLine commandLine(optionPrefix); + commandLine.Parse(argc, argv); + + ProjectManagerScreen startScreen = ProjectManagerScreen::Projects; + if(commandLine.HasSwitch("screen")) + { + QString screenOption = commandLine.GetSwitchValue("screen", 0).c_str(); + ProjectManagerScreen screen = ProjectUtils::GetProjectManagerScreen(screenOption); + if (screen != ProjectManagerScreen::Invalid) + { + startScreen = screen; + } + } + + AZ::IO::FixedMaxPath projectPath; + if (commandLine.HasSwitch("project-path")) + { + projectPath = commandLine.GetSwitchValue("project-path", 0).c_str(); + } + + ProjectManagerWindow window(nullptr, engineRootPath, projectPath, startScreen); window.show(); // somethings is preventing us from moving the window to the center of the From a3e73948c53820bc70406641eadb8f49a3234332 Mon Sep 17 00:00:00 2001 From: Alex Peterson <26804013+AMZN-alexpete@users.noreply.github.com> Date: Tue, 1 Jun 2021 15:32:57 -0700 Subject: [PATCH 14/19] Improved project creation validation No longer requires project name to be part of the project path. --- AutomatedTesting/preview.png | 4 +- .../Source/CreateProjectCtrl.cpp | 12 +++ .../ProjectManager/Source/CreateProjectCtrl.h | 1 + .../Source/FormLineEditWidget.cpp | 8 ++ .../Source/FormLineEditWidget.h | 1 + .../Source/NewProjectSettingsScreen.cpp | 81 ++++++++++++++----- .../Source/NewProjectSettingsScreen.h | 5 ++ .../ProjectManager/Source/ProjectsScreen.cpp | 10 +++ .../ProjectManager/Source/PythonBindings.cpp | 9 ++- .../ProjectManager/Source/ScreensCtrl.cpp | 27 ++++++- .../Tools/ProjectManager/Source/ScreensCtrl.h | 2 + Templates/DefaultProject/Template/preview.png | 4 +- scripts/o3de/o3de/engine_template.py | 23 ++++-- 13 files changed, 157 insertions(+), 30 deletions(-) diff --git a/AutomatedTesting/preview.png b/AutomatedTesting/preview.png index 3d4fe78063..c6928d31fc 100644 --- a/AutomatedTesting/preview.png +++ b/AutomatedTesting/preview.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:40949893ed7009eeaa90b7ce6057cb6be9dfaf7b162e3c26ba9dadf985939d7d -size 2038 +oid sha256:b9cd9d6f67440c193a85969ec5c082c6343e6d1fff3b6f209a0a6931eb22dd47 +size 2949 diff --git a/Code/Tools/ProjectManager/Source/CreateProjectCtrl.cpp b/Code/Tools/ProjectManager/Source/CreateProjectCtrl.cpp index 69f0a3983d..60e351cdb4 100644 --- a/Code/Tools/ProjectManager/Source/CreateProjectCtrl.cpp +++ b/Code/Tools/ProjectManager/Source/CreateProjectCtrl.cpp @@ -67,6 +67,15 @@ namespace O3DE::ProjectManager return ProjectManagerScreen::CreateProject; } + void CreateProjectCtrl::NotifyCurrentScreen() + { + ScreenWidget* currentScreen = reinterpret_cast(m_stack->currentWidget()); + if (currentScreen) + { + currentScreen->NotifyCurrentScreen(); + } + } + void CreateProjectCtrl::HandleBackButton() { if (m_stack->currentIndex() > 0) @@ -110,6 +119,9 @@ namespace O3DE::ProjectManager auto result = PythonBindingsInterface::Get()->CreateProject(m_projectTemplatePath, m_projectInfo); if (result.IsSuccess()) { + // automatically register the project + PythonBindingsInterface::Get()->AddProject(m_projectInfo.m_path); + // adding gems is not implemented yet because we don't know what targets to add or how to add them emit ChangeScreenRequest(ProjectManagerScreen::Projects); } diff --git a/Code/Tools/ProjectManager/Source/CreateProjectCtrl.h b/Code/Tools/ProjectManager/Source/CreateProjectCtrl.h index 01e3349b21..355ba3941d 100644 --- a/Code/Tools/ProjectManager/Source/CreateProjectCtrl.h +++ b/Code/Tools/ProjectManager/Source/CreateProjectCtrl.h @@ -31,6 +31,7 @@ namespace O3DE::ProjectManager explicit CreateProjectCtrl(QWidget* parent = nullptr); ~CreateProjectCtrl() = default; ProjectManagerScreen GetScreenEnum() override; + void NotifyCurrentScreen() override; protected slots: void HandleBackButton(); diff --git a/Code/Tools/ProjectManager/Source/FormLineEditWidget.cpp b/Code/Tools/ProjectManager/Source/FormLineEditWidget.cpp index 7ef7e3c7d8..6c08393910 100644 --- a/Code/Tools/ProjectManager/Source/FormLineEditWidget.cpp +++ b/Code/Tools/ProjectManager/Source/FormLineEditWidget.cpp @@ -78,6 +78,14 @@ namespace O3DE::ProjectManager m_errorLabel->setText(labelText); } + void FormLineEditWidget::setErrorLabelVisible(bool visible) + { + m_errorLabel->setVisible(visible); + m_frame->setProperty("Valid", !visible); + + refreshStyle(); + } + QLineEdit* FormLineEditWidget::lineEdit() const { return m_lineEdit; diff --git a/Code/Tools/ProjectManager/Source/FormLineEditWidget.h b/Code/Tools/ProjectManager/Source/FormLineEditWidget.h index 3094442cbd..76534f46f7 100644 --- a/Code/Tools/ProjectManager/Source/FormLineEditWidget.h +++ b/Code/Tools/ProjectManager/Source/FormLineEditWidget.h @@ -39,6 +39,7 @@ namespace O3DE::ProjectManager //! Set the error message for to display when invalid. void setErrorLabelText(const QString& labelText); + void setErrorLabelVisible(bool visible); //! Returns a pointer to the underlying LineEdit. QLineEdit* lineEdit() const; diff --git a/Code/Tools/ProjectManager/Source/NewProjectSettingsScreen.cpp b/Code/Tools/ProjectManager/Source/NewProjectSettingsScreen.cpp index b57a2b35b2..53400b3193 100644 --- a/Code/Tools/ProjectManager/Source/NewProjectSettingsScreen.cpp +++ b/Code/Tools/ProjectManager/Source/NewProjectSettingsScreen.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -49,16 +50,16 @@ namespace O3DE::ProjectManager vLayout->setContentsMargins(0,0,0,0); vLayout->setAlignment(Qt::AlignTop); { - m_projectName = new FormLineEditWidget(tr("Project name"), tr("New Project"), this); - m_projectName->setErrorLabelText( - tr("A project with this name already exists at this location. Please choose a new name or location.")); + const QString defaultName{ "NewProject" }; + const QString defaultPath = QDir::toNativeSeparators(GetDefaultProjectPath() + "/" + defaultName); + + m_projectName = new FormLineEditWidget(tr("Project name"), defaultName, this); + connect(m_projectName->lineEdit(), &QLineEdit::textChanged, this, &NewProjectSettingsScreen::ValidateProjectPath); vLayout->addWidget(m_projectName); - m_projectPath = - new FormBrowseEditWidget(tr("Project Location"), QStandardPaths::writableLocation(QStandardPaths::DocumentsLocation), this); + m_projectPath = new FormBrowseEditWidget(tr("Project Location"), defaultPath, this); m_projectPath->lineEdit()->setReadOnly(true); - m_projectPath->setErrorLabelText(tr("Please provide a valid path to a folder that exists")); - m_projectPath->lineEdit()->setValidator(new PathValidator(PathValidator::PathMode::ExistingFolder, this)); + connect(m_projectPath->lineEdit(), &QLineEdit::textChanged, this, &NewProjectSettingsScreen::ValidateProjectPath); vLayout->addWidget(m_projectPath); // if we don't use a QFrame we cannot "contain" the widgets inside and move them around @@ -112,17 +113,41 @@ namespace O3DE::ProjectManager this->setLayout(hLayout); } + QString NewProjectSettingsScreen::GetDefaultProjectPath() + { + QString defaultPath = QStandardPaths::writableLocation(QStandardPaths::DocumentsLocation); + AZ::Outcome engineInfoResult = PythonBindingsInterface::Get()->GetEngineInfo(); + if (engineInfoResult.IsSuccess()) + { + QDir path(QDir::toNativeSeparators(engineInfoResult.GetValue().m_defaultProjectsFolder)); + if (path.exists()) + { + defaultPath = path.absolutePath(); + } + } + return defaultPath; + } + ProjectManagerScreen NewProjectSettingsScreen::GetScreenEnum() { return ProjectManagerScreen::NewProjectSettings; } + void NewProjectSettingsScreen::ValidateProjectPath() + { + Validate(); + } + + void NewProjectSettingsScreen::NotifyCurrentScreen() + { + Validate(); + } ProjectInfo NewProjectSettingsScreen::GetProjectInfo() { ProjectInfo projectInfo; projectInfo.m_projectName = m_projectName->lineEdit()->text(); - projectInfo.m_path = QDir::toNativeSeparators(m_projectPath->lineEdit()->text() + "/" + projectInfo.m_projectName); + projectInfo.m_path = m_projectPath->lineEdit()->text(); return projectInfo; } @@ -133,24 +158,44 @@ namespace O3DE::ProjectManager bool NewProjectSettingsScreen::Validate() { - bool projectNameIsValid = true; - if (m_projectName->lineEdit()->text().isEmpty()) - { - projectNameIsValid = false; - } - bool projectPathIsValid = true; if (m_projectPath->lineEdit()->text().isEmpty()) { projectPathIsValid = false; + m_projectPath->setErrorLabelText(tr("Please provide a valid location.")); } - - QDir path(QDir::toNativeSeparators(m_projectPath->lineEdit()->text() + "/" + m_projectName->lineEdit()->text())); - if (path.exists() && !path.isEmpty()) + else { - projectPathIsValid = false; + QDir path(m_projectPath->lineEdit()->text()); + if (path.exists() && !path.isEmpty()) + { + projectPathIsValid = false; + m_projectPath->setErrorLabelText(tr("This folder exists and isn't empty. Please choose a different location.")); + } } + bool projectNameIsValid = true; + if (m_projectName->lineEdit()->text().isEmpty()) + { + projectNameIsValid = false; + m_projectName->setErrorLabelText(tr("Please provide a project name.")); + } + else + { + // this validation should roughly match the utils.validate_identifier which the cli + // uses to validate project names + QRegExp validProjectNameRegex("[A-Za-z][A-Za-z0-9_-]{0,63}"); + const bool result = validProjectNameRegex.exactMatch(m_projectName->lineEdit()->text()); + if (!result) + { + projectNameIsValid = false; + m_projectName->setErrorLabelText(tr("Project names must start with a letter and consist of up to 64 letter, number, '_' or '-' characters")); + } + + } + + m_projectName->setErrorLabelVisible(!projectNameIsValid); + m_projectPath->setErrorLabelVisible(!projectPathIsValid); return projectNameIsValid && projectPathIsValid; } } // namespace O3DE::ProjectManager diff --git a/Code/Tools/ProjectManager/Source/NewProjectSettingsScreen.h b/Code/Tools/ProjectManager/Source/NewProjectSettingsScreen.h index f0e9609fdc..0560f8728d 100644 --- a/Code/Tools/ProjectManager/Source/NewProjectSettingsScreen.h +++ b/Code/Tools/ProjectManager/Source/NewProjectSettingsScreen.h @@ -36,10 +36,15 @@ namespace O3DE::ProjectManager bool Validate(); + void NotifyCurrentScreen() override; + protected slots: void HandleBrowseButton(); + void ValidateProjectPath(); private: + QString GetDefaultProjectPath(); + FormLineEditWidget* m_projectName; FormBrowseEditWidget* m_projectPath; QButtonGroup* m_projectTemplateButtonGroup; diff --git a/Code/Tools/ProjectManager/Source/ProjectsScreen.cpp b/Code/Tools/ProjectManager/Source/ProjectsScreen.cpp index dd2e411ec5..7b9e3ecb9d 100644 --- a/Code/Tools/ProjectManager/Source/ProjectsScreen.cpp +++ b/Code/Tools/ProjectManager/Source/ProjectsScreen.cpp @@ -341,6 +341,16 @@ namespace O3DE::ProjectManager } else { + // refresh the projects content by re-creating it for now + if (m_projectsContent) + { + m_stack->removeWidget(m_projectsContent); + m_projectsContent->deleteLater(); + } + + m_projectsContent = CreateProjectsContent(); + + m_stack->addWidget(m_projectsContent); m_stack->setCurrentWidget(m_projectsContent); } } diff --git a/Code/Tools/ProjectManager/Source/PythonBindings.cpp b/Code/Tools/ProjectManager/Source/PythonBindings.cpp index 8db8492cae..c0481d6c87 100644 --- a/Code/Tools/ProjectManager/Source/PythonBindings.cpp +++ b/Code/Tools/ProjectManager/Source/PythonBindings.cpp @@ -513,10 +513,15 @@ namespace O3DE::ProjectManager { ProjectInfo createdProjectInfo; bool result = ExecuteWithLock([&] { - pybind11::str projectPath = projectInfo.m_path.toStdString(); + pybind11::str projectName = projectInfo.m_projectName.toStdString(); pybind11::str templatePath = projectTemplatePath.toStdString(); - auto createProjectResult = m_engineTemplate.attr("create_project")(projectPath, templatePath); + + auto createProjectResult = m_engineTemplate.attr("create_project")( + projectPath, + projectName, + templatePath + ); if (createProjectResult.cast() == 0) { createdProjectInfo = ProjectInfoFromPath(projectPath); diff --git a/Code/Tools/ProjectManager/Source/ScreensCtrl.cpp b/Code/Tools/ProjectManager/Source/ScreensCtrl.cpp index 7d31d02f6c..6206d4cee9 100644 --- a/Code/Tools/ProjectManager/Source/ScreensCtrl.cpp +++ b/Code/Tools/ProjectManager/Source/ScreensCtrl.cpp @@ -136,6 +136,7 @@ namespace O3DE::ProjectManager { shouldRestoreCurrentScreen = true; } + int tabIndex = GetScreenTabIndex(screen); // Delete old screen if it exists to start fresh DeleteScreen(screen); @@ -144,11 +145,19 @@ namespace O3DE::ProjectManager ScreenWidget* newScreen = BuildScreen(this, screen); if (newScreen->IsTab()) { - m_tabWidget->addTab(newScreen, newScreen->GetTabText()); + if (tabIndex > -1) + { + m_tabWidget->insertTab(tabIndex, newScreen, newScreen->GetTabText()); + } + else + { + m_tabWidget->addTab(newScreen, newScreen->GetTabText()); + } if (shouldRestoreCurrentScreen) { m_tabWidget->setCurrentWidget(newScreen); m_screenStack->setCurrentWidget(m_tabWidget); + newScreen->NotifyCurrentScreen(); } } else @@ -157,6 +166,7 @@ namespace O3DE::ProjectManager if (shouldRestoreCurrentScreen) { m_screenStack->setCurrentWidget(newScreen); + newScreen->NotifyCurrentScreen(); } } @@ -219,4 +229,19 @@ namespace O3DE::ProjectManager screen->NotifyCurrentScreen(); } } + + int ScreensCtrl::GetScreenTabIndex(ProjectManagerScreen screen) + { + const auto iter = m_screenMap.find(screen); + if (iter != m_screenMap.end()) + { + ScreenWidget* screenWidget = iter.value(); + if (screenWidget->IsTab()) + { + return m_tabWidget->indexOf(screenWidget); + } + } + + return -1; + } } // namespace O3DE::ProjectManager diff --git a/Code/Tools/ProjectManager/Source/ScreensCtrl.h b/Code/Tools/ProjectManager/Source/ScreensCtrl.h index 935fc78e25..3b51ed529a 100644 --- a/Code/Tools/ProjectManager/Source/ScreensCtrl.h +++ b/Code/Tools/ProjectManager/Source/ScreensCtrl.h @@ -51,6 +51,8 @@ namespace O3DE::ProjectManager void TabChanged(int index); private: + int GetScreenTabIndex(ProjectManagerScreen screen); + QStackedWidget* m_screenStack; QHash m_screenMap; QStack m_screenVisitOrder; diff --git a/Templates/DefaultProject/Template/preview.png b/Templates/DefaultProject/Template/preview.png index 3d4fe78063..a3e13481c9 100644 --- a/Templates/DefaultProject/Template/preview.png +++ b/Templates/DefaultProject/Template/preview.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:40949893ed7009eeaa90b7ce6057cb6be9dfaf7b162e3c26ba9dadf985939d7d -size 2038 +oid sha256:4a5881b8d6cfbc4ceefb14ab96844484fe19407ee030824768f9fcce2f729d35 +size 2949 diff --git a/scripts/o3de/o3de/engine_template.py b/scripts/o3de/o3de/engine_template.py index 63dec3e765..9bb62eff35 100755 --- a/scripts/o3de/o3de/engine_template.py +++ b/scripts/o3de/o3de/engine_template.py @@ -1279,6 +1279,7 @@ def create_from_template(destination_path: str, def create_project(project_path: str, + project_name: str = None, template_path: str = None, template_name: str = None, project_restricted_path: str = None, @@ -1297,6 +1298,7 @@ def create_project(project_path: str, Template instantiation specialization that makes all default assumptions for a Project template instantiation, reducing the effort needed in instancing a project :param project_path: the project path, can be absolute or relative to default projects path + :param project_name: the project name, defaults to project_path basename if not provided :param template_path: the path to the template you want to instance, can be absolute or relative to default templates path :param template_name: the name the registered template you want to instance, defaults to DefaultProject, resolves template_path :param project_restricted_path: path to the projects restricted folder, can be absolute or relative to the restricted='projects' @@ -1489,12 +1491,17 @@ def create_project(project_path: str, elif not os.path.isdir(project_path): os.makedirs(project_path) - # project name is now the last component of the project_path - project_name = os.path.basename(project_path) + if not project_name: + # project name is now the last component of the project_path + project_name = os.path.basename(project_path) + + if not utils.validate_identifier(project_name): + logger.error(f'Project name must be fewer than 64 characters, contain only alphanumeric, "_" or "-" characters, and start with a letter. {project_name}') + return 1 # project name cannot be the same as a restricted platform name if project_name in restricted_platforms: - logger.error(f'Project path cannot be a restricted name. {project_name}') + logger.error(f'Project name cannot be a restricted name. {project_name}') return 1 # project restricted name @@ -2079,6 +2086,7 @@ def _run_create_from_template(args: argparse) -> int: def _run_create_project(args: argparse) -> int: return create_project(args.project_path, + args.project_name, args.template_path, args.template_name, args.project_restricted_path, @@ -2262,10 +2270,15 @@ def add_args(subparsers) -> None: # creation of a project from a template (like create from template but makes project assumptions) create_project_subparser = subparsers.add_parser('create-project') create_project_subparser.add_argument('-pp', '--project-path', type=str, required=True, - help='The name of the project you wish to create from the template,' + help='The location of the project you wish to create from the template,' ' can be an absolute path or dev root relative.' ' Ex. C:/o3de/TestProject' - ' TestProject = ') + ' TestProject = if --project-name not provided') + create_project_subparser.add_argument('-pn', '--project-name', type=str, required=False, + help='The name of the project you wish to use, must be alphanumeric, ' + ' and can contain _ and - characters.' + ' If no name is provided, will use last component of project path.' + ' Ex. New_Project-123') group = create_project_subparser.add_mutually_exclusive_group(required=False) group.add_argument('-tp', '--template-path', type=str, required=False, From 5884fc1096f6742e956d9b309fce1f710711c436 Mon Sep 17 00:00:00 2001 From: michabr <82236305+michabr@users.noreply.github.com> Date: Tue, 1 Jun 2021 15:48:57 -0700 Subject: [PATCH 15/19] Fix LyShine instance not being initialized (#1078) --- Gems/LyShine/Code/Source/LyShineSystemComponent.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Gems/LyShine/Code/Source/LyShineSystemComponent.cpp b/Gems/LyShine/Code/Source/LyShineSystemComponent.cpp index 902752a03c..0eab7705f6 100644 --- a/Gems/LyShine/Code/Source/LyShineSystemComponent.cpp +++ b/Gems/LyShine/Code/Source/LyShineSystemComponent.cpp @@ -157,6 +157,7 @@ namespace LyShine UiSystemBus::Handler::BusConnect(); UiSystemToolsBus::Handler::BusConnect(); UiFrameworkBus::Handler::BusConnect(); + CrySystemEventBus::Handler::BusConnect(); // register all the component types internal to the LyShine module // These are registered in the order we want them to appear in the Add Component menu @@ -201,6 +202,7 @@ namespace LyShine UiSystemToolsBus::Handler::BusDisconnect(); UiFrameworkBus::Handler::BusDisconnect(); LyShineRequestBus::Handler::BusDisconnect(); + CrySystemEventBus::Handler::BusDisconnect(); LyShineAllocatorScope::DeactivateAllocators(); } From dfd63737c390090e33fcd02002fdc1f2dd04f623 Mon Sep 17 00:00:00 2001 From: Vincent Liu <5900509+onecent1101@users.noreply.github.com> Date: Tue, 1 Jun 2021 15:59:43 -0700 Subject: [PATCH 16/19] [SPEC-6720] Update session common interfaces (#956) --- .../Session/ISessionHandlingRequests.h | 22 +++++++++++++++---- .../AzFramework/Session/ISessionRequests.h | 3 +++ .../Session/SessionNotifications.h | 3 +++ 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/Code/Framework/AzFramework/AzFramework/Session/ISessionHandlingRequests.h b/Code/Framework/AzFramework/AzFramework/Session/ISessionHandlingRequests.h index 47388c56c3..a0731626ef 100644 --- a/Code/Framework/AzFramework/AzFramework/Session/ISessionHandlingRequests.h +++ b/Code/Framework/AzFramework/AzFramework/Session/ISessionHandlingRequests.h @@ -12,6 +12,7 @@ #pragma once +#include #include namespace AzFramework @@ -49,13 +50,17 @@ namespace AzFramework class ISessionHandlingClientRequests { public: - // Handle the player join session process + AZ_RTTI(ISessionHandlingClientRequests, "{41DE6BD3-72BC-4443-BFF9-5B1B9396657A}"); + ISessionHandlingClientRequests() = default; + virtual ~ISessionHandlingClientRequests() = default; + + // Request the player join session // @param sessionConnectionConfig The required properties to handle the player join session process // @return The result of player join session process - virtual bool HandlePlayerJoinSession(const SessionConnectionConfig& sessionConnectionConfig) = 0; + virtual bool RequestPlayerJoinSession(const SessionConnectionConfig& sessionConnectionConfig) = 0; - // Handle the player leave session process - virtual void HandlePlayerLeaveSession() = 0; + // Request the connected player leave session + virtual void RequestPlayerLeaveSession() = 0; }; //! ISessionHandlingServerRequests @@ -63,6 +68,10 @@ namespace AzFramework class ISessionHandlingServerRequests { public: + AZ_RTTI(ISessionHandlingServerRequests, "{4F0C17BA-F470-4242-A8CB-EC7EA805257C}"); + ISessionHandlingServerRequests() = default; + virtual ~ISessionHandlingServerRequests() = default; + // Handle the destroy session process virtual void HandleDestroySession() = 0; @@ -74,5 +83,10 @@ namespace AzFramework // Handle the player leave session process // @param playerConnectionConfig The required properties to handle the player leave session process virtual void HandlePlayerLeaveSession(const PlayerConnectionConfig& playerConnectionConfig) = 0; + + // Retrieves the file location of a pem-encoded TLS certificate + // @return If successful, returns the file location of TLS certificate file; if not successful, returns + // empty string. + virtual AZStd::string GetSessionCertificate() = 0; }; } // namespace AzFramework diff --git a/Code/Framework/AzFramework/AzFramework/Session/ISessionRequests.h b/Code/Framework/AzFramework/AzFramework/Session/ISessionRequests.h index 9d21a7f282..da65eb47f0 100644 --- a/Code/Framework/AzFramework/AzFramework/Session/ISessionRequests.h +++ b/Code/Framework/AzFramework/AzFramework/Session/ISessionRequests.h @@ -167,6 +167,9 @@ namespace AzFramework : public AZ::EBusTraits { public: + // Safeguard handler for multi-threaded use case + using MutexType = AZStd::recursive_mutex; + ////////////////////////////////////////////////////////////////////////// // EBusTraits overrides static const AZ::EBusHandlerPolicy HandlerPolicy = AZ::EBusHandlerPolicy::Multiple; diff --git a/Code/Framework/AzFramework/AzFramework/Session/SessionNotifications.h b/Code/Framework/AzFramework/AzFramework/Session/SessionNotifications.h index a61c995db7..c472fcb228 100644 --- a/Code/Framework/AzFramework/AzFramework/Session/SessionNotifications.h +++ b/Code/Framework/AzFramework/AzFramework/Session/SessionNotifications.h @@ -24,6 +24,9 @@ namespace AzFramework : public AZ::EBusTraits { public: + // Safeguard handler for multi-threaded use case + using MutexType = AZStd::recursive_mutex; + ////////////////////////////////////////////////////////////////////////// // EBusTraits overrides static const AZ::EBusHandlerPolicy HandlerPolicy = AZ::EBusHandlerPolicy::Multiple; From b3b5864f763cf5bcb9f14a9f783133d411f73f50 Mon Sep 17 00:00:00 2001 From: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com> Date: Tue, 1 Jun 2021 11:59:31 -0500 Subject: [PATCH 17/19] Added key checks for the o3de_manifest.json query functions to avoid python exceptions being raised --- scripts/o3de/o3de/manifest.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/o3de/o3de/manifest.py b/scripts/o3de/o3de/manifest.py index 9436e1dd29..2a7e5bba11 100644 --- a/scripts/o3de/o3de/manifest.py +++ b/scripts/o3de/o3de/manifest.py @@ -215,12 +215,12 @@ def get_this_engine() -> dict: def get_engines() -> list: json_data = load_o3de_manifest() - return json_data['engines'] + return json_data['engines'] if 'engines' in json_data else [] def get_projects() -> list: json_data = load_o3de_manifest() - return json_data['projects'] + return json_data['projects'] if 'projects' in json_data else [] def get_gems() -> list: @@ -233,22 +233,22 @@ def get_gems() -> list: def get_external_subdirectories() -> list: json_data = load_o3de_manifest() - return json_data['external_subdirectories'] + return json_data['external_subdirectories'] if 'external_subdirectories' in json_data else [] def get_templates() -> list: json_data = load_o3de_manifest() - return json_data['templates'] + return json_data['templates'] if 'templates' in json_data else [] def get_restricted() -> list: json_data = load_o3de_manifest() - return json_data['restricted'] + return json_data['restricted'] if 'restricted' in json_data else [] def get_repos() -> list: json_data = load_o3de_manifest() - return json_data['repos'] + return json_data['repos'] if 'repos' in json_data else [] # engine.json queries def get_engine_projects() -> list: From 810d6a8deb2da1dbdbe365e3f42cc240c2c7231e Mon Sep 17 00:00:00 2001 From: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com> Date: Tue, 1 Jun 2021 12:04:08 -0500 Subject: [PATCH 18/19] Moved the add_gem_dependency and remove_gem_dependency methods to the cmake.py file Fixed the add_gem_dependency method to append the newly added gem right after the 'set(ENABLED_GEMS...' line --- scripts/o3de/o3de/cmake.py | 91 +++++++++++++++++++++++++++++++- scripts/o3de/o3de/disable_gem.py | 36 +------------ scripts/o3de/o3de/enable_gem.py | 54 +------------------ 3 files changed, 92 insertions(+), 89 deletions(-) diff --git a/scripts/o3de/o3de/cmake.py b/scripts/o3de/o3de/cmake.py index dfcce708eb..f8e8d6ce0c 100644 --- a/scripts/o3de/o3de/cmake.py +++ b/scripts/o3de/o3de/cmake.py @@ -21,6 +21,95 @@ from o3de import manifest logger = logging.getLogger() logging.basicConfig() +enable_gem_start_marker = 'set(ENABLED_GEMS' +enable_gem_end_marker = ')' + + +def add_gem_dependency(cmake_file: pathlib.Path, + gem_name: str) -> int: + """ + adds a gem dependency to a cmake file + :param cmake_file: path to the cmake file + :param gem_name: name of the gem + :return: 0 for success or non 0 failure code + """ + if not cmake_file.is_file(): + logger.error(f'Failed to locate cmake file {str(cmake_file)}') + return 1 + + # on a line by basis, see if there already is {gem_name} + # find the first occurrence of a gem, copy its formatting and replace + # the gem name with the new one and append it + # if the gem is already present fail + t_data = [] + added = False + line_index_to_append = None + with open(cmake_file, 'r') as s: + line_index = 0 + for line in s: + if line.strip().startswith(enable_gem_start_marker): + line_index_to_append = line_index + if f'{gem_name}' == line.strip(): + logger.warning(f'{gem_name} is already enabled in file {str(cmake_file)}.') + return 0 + t_data.append(line) + line_index += 1 + + + indent = 4 + if line_index_to_append: + # Insert the gem after the 'set(ENABLED_GEMS)...` line + t_data.insert(line_index_to_append + 1, f'{" " * indent}{gem_name}\n') + added = True + + # if we didn't add, then create a new set(ENABLED_GEMS) variable + # add a new gem, if empty the correct format is 1 tab=4spaces + if not added: + t_data.append('\n') + t_data.append(f'{enable_gem_start_marker}\n') + t_data.append(f'{" " * indent}{gem_name}\n') + t_data.append(f'{enable_gem_end_marker}\n') + + # write the cmake + with open(cmake_file, 'w') as s: + s.writelines(t_data) + + return 0 + +def remove_gem_dependency(cmake_file: pathlib.Path, + gem_name: str) -> int: + """ + removes a gem dependency from a cmake file + :param cmake_file: path to the cmake file + :param gem_name: name of the gem + :return: 0 for success or non 0 failure code + """ + if not cmake_file.is_file(): + logger.error(f'Failed to locate cmake file {cmake_file}') + return 1 + + # on a line by basis, remove any line with {gem_name} + t_data = [] + # Remove the gem from the enabled_gem file by skipping the gem name entry + removed = False + with open(cmake_file, 'r') as s: + for line in s: + if gem_name == line.strip(): + removed = True + else: + t_data.append(line) + + if not removed: + logger.error(f'Failed to remove {gem_name} from cmake file {cmake_file}') + return 1 + + # write the cmake + with open(cmake_file, 'w') as s: + s.writelines(t_data) + + return 0 + + def get_project_gems(project_path: pathlib.Path, platform: str = 'Common') -> set: return get_gems_from_cmake_file(get_enabled_gem_cmake_file(project_path=project_path, platform=platform)) @@ -38,8 +127,6 @@ def get_enabled_gems(cmake_file: pathlib.Path) -> set: logger.error(f'Failed to locate cmake file {cmake_file}') return set() - enable_gem_start_marker = 'set(ENABLED_GEMS' - enable_gem_end_marker = ')' gem_target_set = set() with cmake_file.open('r') as s: in_gem_list = False diff --git a/scripts/o3de/o3de/disable_gem.py b/scripts/o3de/o3de/disable_gem.py index 61d71445f0..82fe9c8ac6 100644 --- a/scripts/o3de/o3de/disable_gem.py +++ b/scripts/o3de/o3de/disable_gem.py @@ -24,40 +24,6 @@ logger = logging.getLogger() logging.basicConfig() -def remove_gem_dependency(cmake_file: pathlib.Path, - gem_name: str) -> int: - """ - removes a gem dependency from a cmake file - :param cmake_file: path to the cmake file - :param gem_name: name of the gem - :return: 0 for success or non 0 failure code - """ - if not cmake_file.is_file(): - logger.error(f'Failed to locate cmake file {cmake_file}') - return 1 - - # on a line by basis, remove any line with {gem_name} - t_data = [] - # Remove the gem from the enabled_gem file by skipping the gem name entry - removed = False - with open(cmake_file, 'r') as s: - for line in s: - if gem_name == line.strip(): - removed = True - else: - t_data.append(line) - - if not removed: - logger.error(f'Failed to remove {gem_name} from cmake file {cmake_file}') - return 1 - - # write the cmake - with open(cmake_file, 'w') as s: - s.writelines(t_data) - - return 0 - - def disable_gem_in_project(gem_name: str = None, gem_path: pathlib.Path = None, project_name: str = None, @@ -128,7 +94,7 @@ def disable_gem_in_project(gem_name: str = None, logger.error(f'Enabled gem file {enabled_gem_file} is not present.') return 1 # remove the gem - error_code = remove_gem_dependency(enabled_gem_file, gem_json_data['gem_name']) + error_code = cmake.remove_gem_dependency(enabled_gem_file, gem_json_data['gem_name']) if error_code: ret_val = error_code diff --git a/scripts/o3de/o3de/enable_gem.py b/scripts/o3de/o3de/enable_gem.py index 0dee01e05e..0e007f6370 100644 --- a/scripts/o3de/o3de/enable_gem.py +++ b/scripts/o3de/o3de/enable_gem.py @@ -24,56 +24,6 @@ from o3de import cmake, manifest, validation logger = logging.getLogger() logging.basicConfig() -def add_gem_dependency(cmake_file: pathlib.Path, - gem_name: str) -> int: - """ - adds a gem dependency to a cmake file - :param cmake_file: path to the cmake file - :param gem_name: name of the gem - :return: 0 for success or non 0 failure code - """ - if not cmake_file.is_file(): - logger.error(f'Failed to locate cmake file {str(cmake_file)}') - return 1 - - # on a line by basis, see if there already is {gem_name} - # find the first occurrence of a gem, copy its formatting and replace - # the gem name with the new one and append it - # if the gem is already present fail - t_data = [] - added = False - line_index_to_append = None - with open(cmake_file, 'r') as s: - line_index = 0 - for line in s: - if 'ENABLED_GEMS' in line: - line_index_to_append = line_index - if f'{gem_name}' == line.strip(): - logger.warning(f'{gem_name} is already enabled in file {str(cmake_file)}.') - return 0 - t_data.append(line) - line_index += 1 - - - indent = 4 - if line_index_to_append: - t_data[line_index_to_append] = f'{" " * indent}{gem_name}\n' - added = True - - # if we didn't add, then create a new set(ENABLED_GEMS) variable - # add a new gem, if empty the correct format is 1 tab=4spaces - if not added: - t_data.append('\n') - t_data.append('set(ENABLED_GEMS\n') - t_data.append(f'{" " * indent}{gem_name}\n') - t_data.append(')\n') - - # write the cmake - with open(cmake_file, 'w') as s: - s.writelines(t_data) - - return 0 - def enable_gem_in_project(gem_name: str = None, gem_path: pathlib.Path = None, @@ -141,7 +91,7 @@ def enable_gem_in_project(gem_name: str = None, logger.error(f'Enabled gem file {enabled_gem_file} is not present.') return 1 # add the gem - ret_val = add_gem_dependency(enabled_gem_file, gem_json_data['gem_name']) + ret_val = cmake.add_gem_dependency(enabled_gem_file, gem_json_data['gem_name']) else: # Find the path to enabled gem file. @@ -150,7 +100,7 @@ def enable_gem_in_project(gem_name: str = None, if not project_enabled_gem_file.is_file(): project_enabled_gem_file.touch() # add the gem - ret_val = add_gem_dependency(project_enabled_gem_file, gem_json_data['gem_name']) + ret_val = cmake.add_gem_dependency(project_enabled_gem_file, gem_json_data['gem_name']) return ret_val From bf0df4b36962fa7dcbbe5b62d59e8e65ace2b5ca Mon Sep 17 00:00:00 2001 From: Steve Pham <82231385+spham-amzn@users.noreply.github.com> Date: Tue, 1 Jun 2021 16:37:30 -0700 Subject: [PATCH 19/19] Add Android 'gradle' job as a default job (#1082) * Add Android 'gradle' job as a default job * Replace warning about version checking type with string preprocessing of the captured version before comparisons --- cmake/Tools/common.py | 6 +++++- scripts/build/Platform/Android/build_config.json | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/cmake/Tools/common.py b/cmake/Tools/common.py index 9c0d31cd53..1990041c86 100755 --- a/cmake/Tools/common.py +++ b/cmake/Tools/common.py @@ -316,7 +316,11 @@ def verify_tool(override_tool_path, tool_name, tool_filename, argument_name, too version_match = tool_version_regex.search(version_output) if not version_match: raise RuntimeError() - result_version = LooseVersion(str(version_match.group(1)).strip()) + + + # Since we are doing a compare, strip out any non-numeric and non . character from the version otherwise we will get a TypeError on the LooseVersion comparison + result_version_str = re.sub(r"[^\.0-9]", "", str(version_match.group(1)).strip()) + result_version = LooseVersion(result_version_str) if min_version and result_version < min_version: raise LmbrCmdError(f"The {tool_desc} does not meet the minimum version of {tool_name} required ({str(min_version)}).", diff --git a/scripts/build/Platform/Android/build_config.json b/scripts/build/Platform/Android/build_config.json index b871670cd0..33248ffe26 100644 --- a/scripts/build/Platform/Android/build_config.json +++ b/scripts/build/Platform/Android/build_config.json @@ -136,6 +136,7 @@ }, "gradle": { "TAGS":[ + "default", "weekly-build-metrics" ], "COMMAND":"gradle_windows.cmd",