Merge branch 'develop' into LYN-4700

Signed-off-by: igarri <igarri@amazon.com>
This commit is contained in:
igarri
2021-08-09 09:32:42 +01:00
313 changed files with 3322 additions and 5571 deletions
@@ -0,0 +1,58 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#include <AzCore/Task/Internal/Task.h>
namespace AZ::Internal
{
Task::Task(Task&& other) noexcept
{
if (!other.m_relocator)
{
// The type-erased lambda is trivially relocatable OR, the lambda is heap allocated
memcpy(this, &other, sizeof(Task));
// Prevent deletion in the event the lambda had spilled to the heap
other.m_destroyer = nullptr;
return;
}
m_invoker = other.m_invoker;
m_relocator = other.m_relocator;
m_destroyer = other.m_destroyer;
// We now own the lambda, so clear the moved-from task's destroyer
other.m_destroyer = nullptr;
m_relocator(m_lambda, other.m_lambda);
}
Task& Task::operator=(Task&& other) noexcept
{
if (this == &other)
{
return *this;
}
this->~Task();
new (this) Task{ AZStd::move(other) };
return *this;
}
Task::~Task()
{
if (m_destroyer)
{
// The presence of m_destroyer indicates that the lambda is not trivially destructible
m_destroyer(m_lambda);
}
}
} // namespace AZ::Internal
@@ -0,0 +1,180 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#pragma once
#include <AzCore/Task/Internal/TaskConfig.h>
#include <AzCore/Task/TaskDescriptor.h>
#include <AzCore/std/containers/fixed_vector.h>
#include <AzCore/std/typetraits/is_assignable.h>
#include <AzCore/std/typetraits/is_destructible.h>
#include <AzCore/std/parallel/atomic.h>
#include <AzCore/Memory/PoolAllocator.h>
namespace AZ::Internal
{
using TaskInvoke_t = void (*)(void* lambda);
using TaskRelocate_t = void (*)(void* dst, void* src);
using TaskDestroy_t = void (*)(void* obj);
class CompiledTaskGraph;
// Lambdas are opaque types and we cannot extract any member function pointers. In order to store lambdas in a
// type erased fashion, we instead use a single function call indirection, invoking the lambda function in a
// static class function which has a stable address in memory. The Erased* methods return addresses to the
// indirect callers of the lambda copy/move assignment operators, call operator, and destructor.
//
// For lambdas that are trivially relocatable, both the returned move and copy assignment function pointers
// will be nullptr.
//
// Lambdas that are trivially destructible will result in a nullptr returned TaskDestroy_t pointer.
//
// The class will check that the lambda is copy assignable or movable.
template<typename Lambda>
class TaskTypeEraser final
{
public:
constexpr TaskInvoke_t ErasedInvoker()
{
return reinterpret_cast<TaskInvoke_t>(Invoker);
}
constexpr TaskRelocate_t ErasedRelocator()
{
if constexpr (AZStd::is_trivially_move_constructible_v<Lambda>)
{
return nullptr;
}
else if constexpr (AZStd::is_move_constructible_v<Lambda>)
{
return reinterpret_cast<TaskRelocate_t>(Mover);
}
else if constexpr (AZStd::is_copy_constructible_v<Lambda>)
{
return reinterpret_cast<TaskRelocate_t>(Copier);
}
else
{
static_assert(
AZStd::is_move_constructible_v<Lambda> || AZStd::is_copy_constructible_v<Lambda>,
"Task lambdas must be either move or copy constructible. Please verify that all captured data is move or copy "
"constructible.");
}
}
constexpr TaskDestroy_t ErasedDestroyer()
{
if constexpr (AZStd::is_trivially_destructible_v<Lambda>)
{
return nullptr;
}
else
{
return reinterpret_cast<TaskDestroy_t>(Destroyer);
}
}
private:
constexpr static void Invoker(Lambda* lambda)
{
lambda->operator()();
}
constexpr static void Mover(Lambda* dst, Lambda* src)
{
new (dst) Lambda{ AZStd::move(*src) };
}
constexpr static void Copier(Lambda* dst, Lambda* src)
{
new (dst) Lambda{ *src };
}
constexpr static void Destroyer(Lambda* lambda)
{
lambda->~Lambda();
}
};
// The Task encapsulates member function pointers to store in a homogeneously-typed container
// The function signature of all lambdas encoded in a Task is void(*)(). The lambdas can capture
// data, in which case the data is inlined in this structure. Attempting to capture more data
// will result in a compile failure, so use indirection and capture a pointer/reference to your
// data if you run into this.
class alignas(alignof(max_align_t)) Task final
{
public:
AZ_CLASS_ALLOCATOR(Task, ThreadPoolAllocator, 0);
// The inline buffer allows the Task to span two cache lines. Lambdas can capture 56
// bytes of data (7 pointers/references on a 64-bit machine).
constexpr static size_t BufferSize =
AZ_TRAIT_TASK_BYTE_SIZE - sizeof(size_t) * 5 - sizeof(uint32_t) - sizeof(TaskDescriptor) - sizeof(AZStd::atomic<uint32_t>);
Task() = default;
// Prevent binding lvalue references to lambdas
// If you are encountering a compiler error here, please either move the lambda into the AddJob function with AZStd::move
// or simply define the lambda directly as a parameter of AddJob
template<typename Lambda>
Task(TaskDescriptor const& desc, Lambda& lambda) = delete;
template<typename Lambda>
Task(TaskDescriptor const& desc, Lambda&& lambda) noexcept;
Task(Task&& other) noexcept;
Task& operator=(Task&& other) noexcept;
~Task();
void Link(Task& other);
// Indicates if this task is a root of the graph (with no dependencies)
bool IsRoot() const noexcept;
// Prepare for dispatch (reset the dependency counter to the number of inbound edges)
void Init() noexcept;
// Invoke the embedded lambda function
void Invoke();
uint8_t GetPriorityNumber() const noexcept;
private:
friend class CompiledTaskGraph;
friend class TaskWorker;
// This relocation avoids branches needed if the lambda type is unknown
template<typename Lambda>
void TypedRelocate(Lambda&& lambda, char* destination);
// Small buffer optimization for lambdas. We cover our bases here by enforcing alignment on the
// class to equal the alignment of the largest scalar type available on the system (generally
// 16 bytes).
char m_lambda[BufferSize];
AZStd::atomic<uint32_t> m_dependencyCount;
// This value is an offset in a buffer that stores dependency tracking information.
uint32_t m_successorOffset = 0;
uint32_t m_inboundLinkCount = 0;
uint32_t m_outboundLinkCount = 0;
CompiledTaskGraph* m_graph = nullptr;
TaskInvoke_t m_invoker;
// If nullptr, the lambda is trivially relocatable (via memcpy). Otherwise, it must be invoked
// when instances of this class are moved.
TaskRelocate_t m_relocator;
TaskDestroy_t m_destroyer;
TaskDescriptor m_descriptor;
};
} // namespace AZ::Internal
#include <AzCore/Task/Internal/Task.inl>
@@ -0,0 +1,84 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
namespace AZ::Internal
{
template<typename Lambda>
Task::Task(TaskDescriptor const& desc, Lambda&& lambda) noexcept
: m_descriptor{ desc }
{
static_assert(
sizeof(Lambda) <= BufferSize,
"Task lambda has too much captured data, please capture no"
"more than 56 bytes of data (likely by capturing a single reference/pointer to a container of data)");
static_assert(
alignof(Lambda) <= alignof(max_align_t),
"Task lambda has extended alignment which isn't supported."
"Please capture a reference/pointer to the data requiring an extended alignment instead");
TaskTypeEraser<Lambda> eraser;
m_invoker = eraser.ErasedInvoker();
m_relocator = eraser.ErasedRelocator();
m_destroyer = eraser.ErasedDestroyer();
// NOTE: This code is conservative in that extended alignment requirements result in a heap
// spill, even if the lambda could have occupied a portion of the inline buffer with a base
// pointer adjustment.
TypedRelocate(AZStd::forward<Lambda>(lambda), m_lambda);
}
template<typename Lambda>
void Task::TypedRelocate(Lambda&& lambda, char* destination)
{
if constexpr (AZStd::is_trivially_move_constructible_v<Lambda>)
{
memcpy(destination, reinterpret_cast<char*>(&lambda), sizeof(Lambda));
}
else if constexpr (AZStd::is_move_constructible_v<Lambda>)
{
new (destination) Lambda{ AZStd::move(lambda) };
}
else if constexpr (AZStd::is_copy_constructible_v<Lambda>)
{
new (destination) Lambda{ lambda };
}
else
{
static_assert(
AZStd::is_move_constructible_v<Lambda> || AZStd::is_copy_constructible_v<Lambda>,
"Task lambdas must be either move or copy constructible. Please verify that all captured data is move or copy "
"constructible.");
}
}
inline void Task::Init() noexcept
{
m_dependencyCount = m_inboundLinkCount;
}
inline void Task::Invoke()
{
m_invoker(m_lambda);
}
inline uint8_t Task::GetPriorityNumber() const noexcept
{
return static_cast<uint8_t>(m_descriptor.priority);
}
inline void Task::Link(Task& other)
{
++m_outboundLinkCount;
++other.m_inboundLinkCount;
}
inline bool Task::IsRoot() const noexcept
{
return m_inboundLinkCount == 0;
}
} // namespace AZ::Internal
@@ -0,0 +1,14 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#pragma once
#include <AzCore/AzCore_Traits_Platform.h>
#if !defined(AZ_TRAIT_TASK_BYTE_SIZE)
#define AZ_TRAIT_TASK_BYTE_SIZE 128
#endif
@@ -0,0 +1,49 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#pragma once
#include <AzCore/base.h>
#include <AzCore/std/limits.h>
namespace AZ
{
// Task priorities MAY be used judiciously to fine tune runtime execution, with the understanding
// that profiling is needed to understand what the critical path per frame is. Modifying
// task priorities is an EXPERT setting that should succeed a healthy dose of measurement.
enum class TaskPriority : uint8_t
{
CRITICAL = 0,
HIGH = 1,
MEDIUM = 2, // Default
LOW = 3,
PRIORITY_COUNT = 4,
};
// All submitted tasks are associated with a TaskDescriptor which defines the priority, affinitization,
// and tracking of the task resource utilization.
//
// TODO: Define various task kinds and provide a mechanism for cpuMask computation on different systems.
struct TaskDescriptor
{
// Unique task kind label (e.g. "frustum culling")
// Task names *must* be provided
const char* taskName = nullptr;
// Associates a set of task kinds together for budget tracking (e.g. "graphics")
const char* taskGroup = nullptr;
// EXPERTS ONLY. Tasks of higher priority are executed ahead of any lower priority tasks
// that were queued before it provided they had not yet started
TaskPriority priority = TaskPriority::MEDIUM;
// EXPERTS ONLY. A bitmask that restricts tasks of this kind to run only on cores
// corresponding to a set bit. 0 is synonymous with all bits set
uint32_t cpuMask = 0;
};
}
@@ -0,0 +1,362 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#include <AzCore/Task/TaskExecutor.h>
#include <AzCore/Task/TaskGraph.h>
#include <AzCore/std/containers/queue.h>
#include <AzCore/std/parallel/binary_semaphore.h>
#include <AzCore/std/parallel/exponential_backoff.h>
#include <AzCore/std/parallel/mutex.h>
#include <AzCore/std/parallel/scoped_lock.h>
#include <AzCore/std/parallel/semaphore.h>
#include <AzCore/std/parallel/thread.h>
#include <AzCore/std/string/string.h>
#include <AzCore/Module/Environment.h>
#include <random>
namespace AZ
{
namespace Internal
{
CompiledTaskGraph::CompiledTaskGraph(
AZStd::vector<Task>&& tasks,
AZStd::unordered_map<uint32_t, AZStd::vector<uint32_t>>& links,
size_t linkCount,
TaskGraph* parent)
: m_parent{ parent }
{
m_tasks = AZStd::move(tasks);
m_successors.resize(linkCount);
Task** cursor = m_successors.data();
for (uint32_t i = 0; i != m_tasks.size(); ++i)
{
Task& task = m_tasks[i];
task.m_graph = this;
task.m_successorOffset = cursor - m_successors.data();
cursor += task.m_outboundLinkCount;
AZ_Assert(task.m_outboundLinkCount == links[i].size(), "Task outbound link information mismatch");
for (uint32_t j = 0; j != task.m_outboundLinkCount; ++j)
{
m_successors[static_cast<size_t>(task.m_successorOffset) + j] = &m_tasks[links[i][j]];
}
}
// TODO: Check for dependency cycles
}
uint32_t CompiledTaskGraph::Release()
{
uint32_t remaining = --m_remaining;
if (m_parent)
{
if (remaining == 1)
{
// Allow the parent graph to be submitted again
m_parent->m_submitted = false;
}
}
else if (remaining == 0)
{
if (m_waitEvent)
{
m_waitEvent->Signal();
}
azdestroy(this);
return remaining;
}
if (m_waitEvent && remaining == (m_parent ? 1 : 0))
{
m_waitEvent->Signal();
}
return remaining;
}
struct QueueStatus
{
AZStd::atomic<uint16_t> head;
AZStd::atomic<uint16_t> tail;
AZStd::atomic<uint16_t> reserve;
};
// The Task Queue is a lock free 4-priority queue. Its basic operation is as follows:
// Each priority level is associated with a different queue, corresponding to the maximum size of a uint16_t.
// Each queue is implemented as a ring buffer, and a 64 bit atomic maintains the following state per queue:
// - offset to the "head" of the ring, from where we acquire elements
// - offset to the "tail" of the ring, which tracks where new elements should be enqueued
// - offset to a tail reservation index, which is used to reserve a slot to enqueue elements
class TaskQueue final
{
public:
// Preallocating upfront allows us to reserve slots to insert tasks without locks.
// Each thread allocated by the task manager consumes ~2 MB.
constexpr static uint16_t MaxQueueSize = 0xffff;
constexpr static uint8_t PriorityLevelCount = static_cast<uint8_t>(TaskPriority::PRIORITY_COUNT);
TaskQueue() = default;
TaskQueue(const TaskQueue&) = delete;
TaskQueue& operator=(const TaskQueue&) = delete;
void Enqueue(Task* task);
Task* TryDequeue();
private:
QueueStatus m_status[PriorityLevelCount] = {};
Task* m_queues[PriorityLevelCount][MaxQueueSize] = {};
};
void TaskQueue::Enqueue(Task* task)
{
uint8_t priority = task->GetPriorityNumber();
QueueStatus& status = m_status[priority];
AZStd::exponential_backoff backoff;
while (true)
{
uint16_t reserve = status.reserve.load();
uint16_t head = status.head.load();
// Enqueuing is done in two phases because we cannot atomically write the task to the slot we reserve
// and simulataneously publish the fact that the slot is now available.
if (reserve != head - 1)
{
// Try to reserve a slot
if (status.reserve.compare_exchange_weak(reserve, reserve + 1))
{
m_queues[priority][reserve] = task;
uint16_t expectedReserve = reserve;
// Increment the tail to advertise the new task
while (!status.tail.compare_exchange_weak(expectedReserve, reserve + 1))
{
expectedReserve = reserve;
}
return;
}
// We failed to reserve a slot, try again
}
else
{
backoff.wait();
}
}
}
Task* TaskQueue::TryDequeue()
{
for (size_t priority = 0; priority != PriorityLevelCount; ++priority)
{
QueueStatus& status = m_status[priority];
while (true)
{
uint16_t head = status.head.load();
uint16_t tail = status.tail.load();
if (head == tail)
{
// Queue empty
break;
}
else
{
Task* task = m_queues[priority][status.head];
if (status.head.compare_exchange_weak(head, head + 1))
{
return task;
}
}
}
}
return nullptr;
}
class TaskWorker
{
public:
void Spawn(::AZ::TaskExecutor& executor, size_t id, AZStd::semaphore& initSemaphore, bool affinitize)
{
m_executor = &executor;
AZStd::string threadName = AZStd::string::format("TaskWorker %zu", id);
AZStd::thread_desc desc = {};
desc.m_name = threadName.c_str();
if (affinitize)
{
desc.m_cpuId = 1 << id;
}
m_active.store(true, AZStd::memory_order_release);
m_thread = AZStd::thread{ [this, &initSemaphore]
{
initSemaphore.release();
Run();
},
&desc };
}
void Join()
{
m_active.store(false, AZStd::memory_order_release);
m_semaphore.release();
m_thread.join();
}
void Enqueue(Task* task)
{
m_queue.Enqueue(task);
if (!m_busy.exchange(true))
{
// The worker was idle prior to enqueueing the task, release the semaphore
m_semaphore.release();
}
}
private:
void Run()
{
while (m_active)
{
m_busy = false;
m_semaphore.acquire();
if (!m_active)
{
return;
}
m_busy = true;
Task* task = m_queue.TryDequeue();
while (task)
{
task->Invoke();
// Decrement counts for all task successors
for (size_t j = 0; j != task->m_outboundLinkCount; ++j)
{
Task* successor = task->m_graph->m_successors[task->m_successorOffset + j];
if (--successor->m_dependencyCount == 0)
{
m_executor->Submit(*successor);
}
}
bool isRetained = task->m_graph->m_parent != nullptr;
if (task->m_graph->Release() == (isRetained ? 1 : 0))
{
m_executor->ReleaseGraph();
}
task = m_queue.TryDequeue();
}
}
}
AZStd::thread m_thread;
AZStd::atomic<bool> m_active;
AZStd::atomic<bool> m_busy;
AZStd::binary_semaphore m_semaphore;
::AZ::TaskExecutor* m_executor;
TaskQueue m_queue;
};
} // namespace Internal
static EnvironmentVariable<TaskExecutor*> s_executor;
constexpr static const char* s_executorName = "GlobalTaskExecutor";
TaskExecutor& TaskExecutor::Instance()
{
if (!s_executor)
{
s_executor = AZ::Environment::FindVariable<TaskExecutor*>(s_executorName);
}
return **s_executor;
}
// TODO: Create the default executor as part of a component (as in TaskManagerComponent)
void TaskExecutor::SetInstance(TaskExecutor* executor)
{
AZ_Assert(!s_executor, "Attempting to set the global task executor more than once");
s_executor = AZ::Environment::CreateVariable<TaskExecutor*>("GlobalTaskExecutor");
s_executor.Set(executor);
}
TaskExecutor::TaskExecutor(uint32_t threadCount)
{
// TODO: Configure thread count + affinity based on configuration
m_threadCount = threadCount == 0 ? AZStd::thread::hardware_concurrency() : threadCount;
m_workers = reinterpret_cast<Internal::TaskWorker*>(azmalloc(m_threadCount * sizeof(Internal::TaskWorker)));
bool affinitize = m_threadCount == AZStd::thread::hardware_concurrency();
AZStd::semaphore initSemaphore;
for (size_t i = 0; i != m_threadCount; ++i)
{
new (m_workers + i) Internal::TaskWorker{};
m_workers[i].Spawn(*this, i, initSemaphore, affinitize);
}
for (size_t i = 0; i != m_threadCount; ++i)
{
initSemaphore.acquire();
}
}
TaskExecutor::~TaskExecutor()
{
for (size_t i = 0; i != m_threadCount; ++i)
{
m_workers[i].Join();
m_workers[i].~TaskWorker();
}
azfree(m_workers);
}
void TaskExecutor::Submit(Internal::CompiledTaskGraph& graph)
{
++m_graphsRemaining;
// Submit all tasks that have no inbound edges
for (Internal::Task& task : graph.Tasks())
{
if (task.IsRoot())
{
Submit(task);
}
}
}
void TaskExecutor::Submit(Internal::Task& task)
{
// TODO: Something more sophisticated is likely needed here.
// First, we are completely ignoring affinity.
// Second, some heuristics on core availability will help distribute work more effectively
m_workers[++m_lastSubmission % m_threadCount].Enqueue(&task);
}
void TaskExecutor::ReleaseGraph()
{
--m_graphsRemaining;
}
} // namespace AZ
@@ -0,0 +1,89 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#pragma once
#include <AzCore/Task/Internal/Task.h>
#include <AzCore/Task/TaskDescriptor.h>
#include <AzCore/std/containers/unordered_map.h>
#include <AzCore/std/containers/vector.h>
#include <AzCore/std/parallel/atomic.h>
#include <AzCore/std/parallel/binary_semaphore.h>
#include <AzCore/Memory/PoolAllocator.h>
namespace AZ
{
class TaskGraphEvent;
class TaskGraph;
namespace Internal
{
class CompiledTaskGraph final
{
public:
AZ_CLASS_ALLOCATOR(CompiledTaskGraph, SystemAllocator, 0)
CompiledTaskGraph(
AZStd::vector<Task>&& tasks,
AZStd::unordered_map<uint32_t, AZStd::vector<uint32_t>>& links,
size_t linkCount,
TaskGraph* parent);
AZStd::vector<Task>& Tasks() noexcept
{
return m_tasks;
}
// Indicate that a constituent task has finished and decrement a counter to determine if the
// graph should be freed (returns the value after atomic decrement)
uint32_t Release();
private:
friend class ::AZ::TaskGraph;
friend class TaskWorker;
AZStd::vector<Task> m_tasks;
AZStd::vector<Task*> m_successors;
TaskGraphEvent* m_waitEvent = nullptr;
// The pointer to the parent graph is set only if it is retained
TaskGraph* m_parent = nullptr;
AZStd::atomic<uint32_t> m_remaining;
};
class TaskWorker;
} // namespace Internal
class TaskExecutor final
{
public:
AZ_CLASS_ALLOCATOR(TaskExecutor, SystemAllocator, 0);
static TaskExecutor& Instance();
// Invoked by a system component on program launch
static void SetInstance(TaskExecutor* executor);
// Passing 0 for the threadCount requests for the thread count to match the hardware concurrency
explicit TaskExecutor(uint32_t threadCount = 0);
~TaskExecutor();
void Submit(Internal::CompiledTaskGraph& graph);
void Submit(Internal::Task& task);
private:
friend class Internal::TaskWorker;
void ReleaseGraph();
Internal::TaskWorker* m_workers;
uint32_t m_threadCount = 0;
AZStd::atomic<uint32_t> m_lastSubmission;
AZStd::atomic<uint64_t> m_graphsRemaining;
};
} // namespace AZ
@@ -0,0 +1,86 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#include <AzCore/Task/TaskGraph.h>
#include <AzCore/Task/TaskExecutor.h>
namespace AZ
{
using Internal::CompiledTaskGraph;
void TaskToken::PrecedesInternal(TaskToken& comesAfter)
{
AZ_Assert(!m_parent.m_submitted, "Cannot mutate a TaskGraph that was previously submitted.");
// Increment inbound/outbound edge counts
m_parent.m_tasks[m_index].Link(m_parent.m_tasks[comesAfter.m_index]);
m_parent.m_links[m_index].emplace_back(comesAfter.m_index);
++m_parent.m_linkCount;
}
TaskGraph::~TaskGraph()
{
if (m_retained && m_compiledTaskGraph)
{
// This job graph has already finished and we are potentially responsible for its destruction
if (m_compiledTaskGraph->Release() == 0)
{
azdestroy(m_compiledTaskGraph);
}
}
}
void TaskGraph::Reset()
{
AZ_Assert(!m_submitted, "Cannot reset a job graph while it is in flight");
if (m_compiledTaskGraph)
{
azdestroy(m_compiledTaskGraph);
m_compiledTaskGraph = nullptr;
}
m_tasks.clear();
m_links.clear();
m_linkCount = 0;
}
void TaskGraph::Submit(TaskGraphEvent* waitEvent)
{
SubmitOnExecutor(TaskExecutor::Instance(), waitEvent);
}
void TaskGraph::SubmitOnExecutor(TaskExecutor& executor, TaskGraphEvent* waitEvent)
{
if (!m_compiledTaskGraph)
{
m_compiledTaskGraph = aznew CompiledTaskGraph(AZStd::move(m_tasks), m_links, m_linkCount, m_retained ? this : nullptr);
}
m_compiledTaskGraph->m_waitEvent = waitEvent;
uint32_t taskCount = aznumeric_cast<uint32_t>(m_compiledTaskGraph->m_tasks.size());
m_compiledTaskGraph->m_remaining = taskCount + (m_retained ? 1 : 0);
for (uint32_t i = 0; i != taskCount; ++i)
{
m_compiledTaskGraph->m_tasks[i].Init();
}
executor.Submit(*m_compiledTaskGraph);
if (m_retained)
{
m_submitted = true;
}
else
{
m_compiledTaskGraph = nullptr;
Reset();
}
}
}
@@ -0,0 +1,151 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#pragma once
// NOTE: If adding additional header/symbol dependencies, consider if such additions are better
// suited in the private CompiledTaskGraph implementation instead to keep this header lean.
#include <AzCore/Task/Internal/Task.h>
#include <AzCore/Task/TaskDescriptor.h>
#include <AzCore/std/containers/array.h>
#include <AzCore/std/containers/vector.h>
#include <AzCore/std/containers/unordered_map.h>
#include <AzCore/std/parallel/binary_semaphore.h>
namespace AZ
{
namespace Internal
{
class CompiledTaskGraph;
}
class TaskExecutor;
class TaskGraph;
// A TaskToken is returned each time a Task is added to the TaskGraph. TaskTokens are used to
// express dependencies between tasks within the graph, and have no purpose after the graph
// is submitted (simply let them go out of scope)
class TaskToken final
{
public:
// Indicate that this task must finish before the task token(s) passed as the argument
template <typename... JT>
void Precedes(JT&... tokens);
// Indicate that this task must finish after the task token(s) passed as the argument
template <typename... JT>
void Follows(JT&... tokens);
private:
friend class TaskGraph;
void PrecedesInternal(TaskToken& comesAfter);
// Only the TaskGraph should be creating TaskToken
TaskToken(TaskGraph& parent, uint32_t index);
TaskGraph& m_parent;
uint32_t m_index;
};
// A TaskGraphEvent may be used to block until a task graph has finished executing. Usage
// is NOT recommended for the majority of tasks (prefer to simply containing expanding/contracting
// the graph without synchronization over the course of the frame). However, the event
// is useful for the edges of the computation graph.
//
// You are responsible for ensuring the event object lifetime exceeds the task graph lifetime.
//
// After the TaskGraphEvent is signaled, you are allowed to reuse the same TaskGraphEvent
// for a future submission.
class TaskGraphEvent
{
public:
bool IsSignaled();
void Wait();
private:
friend class ::AZ::Internal::CompiledTaskGraph;
friend class TaskGraph;
void Signal();
AZStd::binary_semaphore m_semaphore;
};
// The TaskGraph encapsulates a set of tasks and their interdependencies. After adding
// tasks, and marking dependencies as necessary, the entire graph is submitted via
// the TaskGraph::Submit method.
//
// The TaskGraph MAY be retained across multiple frames and resubmitted, provided the
// user provides some guarantees (see comments associated with TaskGraph::Retain).
class TaskGraph final
{
public:
~TaskGraph();
// Reset the state of the task graph to begin recording tasks and edges again
// NOTE: Graph must be in a "settled" state (cannot be in-flight)
void Reset();
// Add a task to the graph, retrieiving a token that can be used to express dependencies
// between tasks. The first argument specifies the TaskKind, used for tracking the task.
// NOTE: This operation is invalid if the graph is in-flight
template<typename Lambda>
TaskToken AddTask(TaskDescriptor const& descriptor, Lambda&& lambda);
template <typename... Lambdas>
AZStd::array<TaskToken, sizeof...(Lambdas)> AddTasks(TaskDescriptor const& descriptor, Lambdas&&... lambdas);
// By default, you are responsible for retaining the TaskGraph, indicating you promise that
// this TaskGraph will live as long as it takes for all constituent tasks to complete.
// Once retained, this task graph can be resubmitted after completion without any
// modifications. TaskTokens that were created as a result of adding tasks used to
// mark dependencies DO NOT need to outlive the task graph.
//
// Invoking Detach PRIOR to submission indicates you wish the tasks associated with this
// TaskGraph to deallocate upon completion. After invoking Detach, you may let this TaskGraph
// go out of scope or deallocate after submission.
//
// NOTE: The TaskGraph has no concept of resources used by design. Resubmission
// of the task graph is expected to rely on either indirection, or safe overwriting
// of previously used memory to supply new data (this can even be done as the first
// task in the graph).
// NOTE: This operation is invalid if the graph is in-flight
void Detach();
// Invoke the task graph, asserting if there are dependency violations. Note that
// submitting the same graph multiple times to process simultaneously is VALID
// behavior. This is, for example, a mechanism that allows a task graph to loop
// in perpetuity (in fact, the entire frame could be modeled as a single task graph,
// where the final task resubmits the task graph again).
//
// This API is not designed to protect against memory safety violations (nothing
// can prevent a user from incorrectly aliasing memory unsafely even without repeated
// submission). To catch memory safety violations, it is ENCOURAGED that you access
// data through TaskResource<T> handles.
void Submit(TaskGraphEvent* waitEvent = nullptr);
// Same as submit but run on a different executor than the default system executor
void SubmitOnExecutor(TaskExecutor& executor, TaskGraphEvent* waitEvent = nullptr);
private:
friend class TaskToken;
friend class Internal::CompiledTaskGraph;
Internal::CompiledTaskGraph* m_compiledTaskGraph = nullptr;
AZStd::vector<Internal::Task> m_tasks;
// Task index |-> Dependent task indices
AZStd::unordered_map<uint32_t, AZStd::vector<uint32_t>> m_links;
uint32_t m_linkCount = 0;
bool m_retained = true;
AZStd::atomic<bool> m_submitted = false;
};
} // namespace AZ
#include <AzCore/Task/TaskGraph.inl>
@@ -0,0 +1,66 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#pragma once
namespace AZ
{
inline TaskToken::TaskToken(TaskGraph& parent, uint32_t index)
: m_parent{ parent }
, m_index{ index }
{
}
template<typename... JT>
void TaskToken::Precedes(JT&... tokens)
{
(PrecedesInternal(tokens), ...);
}
template <typename... JT>
void TaskToken::Follows(JT&... tokens)
{
(tokens.PrecedesInternal(*this), ...);
}
inline bool TaskGraphEvent::IsSignaled()
{
return m_semaphore.try_acquire_for(AZStd::chrono::milliseconds{ 0 });
}
inline void TaskGraphEvent::Wait()
{
m_semaphore.acquire();
}
inline void TaskGraphEvent::Signal()
{
m_semaphore.release();
}
template<typename Lambda>
TaskToken TaskGraph::AddTask(TaskDescriptor const& desc, Lambda&& lambda)
{
AZ_Assert(!m_submitted, "Cannot mutate a TaskGraph that was previously submitted or in flight.");
m_tasks.emplace_back(desc, AZStd::forward<Lambda>(lambda));
return { *this, aznumeric_cast<uint32_t>(m_tasks.size() - 1) };
}
template <typename... Lambdas>
AZStd::array<TaskToken, sizeof...(Lambdas)> TaskGraph::AddTasks(TaskDescriptor const& descriptor, Lambdas&&... lambdas)
{
return { AddTask(descriptor, AZStd::forward<Lambdas>(lambdas))... };
}
inline void TaskGraph::Detach()
{
m_retained = false;
}
} // namespace AZ
@@ -616,6 +616,16 @@ set(FILES
Socket/AzSocket_fwd.h
Socket/AzSocket.cpp
Socket/AzSocket.h
Task/Internal/Task.cpp
Task/Internal/Task.inl
Task/Internal/Task.h
Task/Internal/TaskConfig.h
Task/TaskDescriptor.h
Task/TaskExecutor.cpp
Task/TaskExecutor.h
Task/TaskGraph.cpp
Task/TaskGraph.h
Task/TaskGraph.inl
Threading/ThreadSafeDeque.h
Threading/ThreadSafeDeque.inl
Threading/ThreadSafeObject.h
@@ -69,8 +69,7 @@ namespace AZStd
int m_priority{ -100000 };
//! The CPU ids (as a bitfield) that this thread will be running on, see \ref AZStd::thread_desc::m_cpuId.
//! Windows: This parameter is ignored.
//! On other platforms, each bit maps directly to the core numbers [0-n], default is 0
//! Each bit maps directly to the core numbers [0-n], default is 0
int m_cpuId{ AFFINITY_MASK_ALL };
//! If we can join the thread.
+642
View File
@@ -0,0 +1,642 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#include <AzCore/Task/TaskGraph.h>
#include <AzCore/Task/TaskExecutor.h>
#include <AzCore/Memory/PoolAllocator.h>
#include <AzCore/UnitTest/TestTypes.h>
#include <random>
using AZ::TaskDescriptor;
using AZ::TaskGraph;
using AZ::TaskGraphEvent;
using AZ::TaskExecutor;
using AZ::Internal::Task;
using AZ::TaskPriority;
static TaskDescriptor defaultTD{ "TaskGraphTestTask", "TaskGraphTests" };
namespace UnitTest
{
class TaskGraphTestFixture : public AllocatorsTestFixture
{
public:
void SetUp() override
{
AllocatorsTestFixture::SetUp();
AZ::AllocatorInstance<AZ::PoolAllocator>::Create();
AZ::AllocatorInstance<AZ::ThreadPoolAllocator>::Create();
m_executor = aznew TaskExecutor(4);
}
void TearDown() override
{
azdestroy(m_executor);
AZ::AllocatorInstance<AZ::ThreadPoolAllocator>::Destroy();
AZ::AllocatorInstance<AZ::PoolAllocator>::Destroy();
AllocatorsTestFixture::TearDown();
}
protected:
TaskExecutor* m_executor;
};
TEST(TaskGraphTests, TrivialTaskLambda)
{
int x = 0;
Task task(
defaultTD,
[&x]()
{
++x;
});
task.Invoke();
EXPECT_EQ(1, x);
}
TEST(TaskGraphTests, TrivialTaskLambdaMove)
{
int x = 0;
Task task(
defaultTD,
[&x]()
{
++x;
});
Task task2 = AZStd::move(task);
task2.Invoke();
EXPECT_EQ(1, x);
}
struct TrackMoves
{
TrackMoves() = default;
TrackMoves(const TrackMoves&) = delete;
TrackMoves(TrackMoves&& other)
: moveCount{other.moveCount + 1}
{
}
int moveCount = 0;
};
struct TrackCopies
{
TrackCopies() = default;
TrackCopies(TrackCopies&&) = delete;
TrackCopies(const TrackCopies& other)
: copyCount{other.copyCount + 1}
{
}
int copyCount = 0;
};
/*
TEST(TaskGraphTests, ThisShouldNotCompile)
{
auto lambda = []
{
};
Task task(defaultTD, lambda);
task.Invoke();
}
*/
TEST(TaskGraphTests, MoveOnlyTaskLambda)
{
TrackMoves tm;
int moveCount = 0;
Task task(
defaultTD,
[tm = AZStd::move(tm), &moveCount]
{
moveCount = tm.moveCount;
});
task.Invoke();
// Two moves are expected. Once into the capture body of the lambda, once to construct
// the type erased task
EXPECT_EQ(2, moveCount);
}
TEST(TaskGraphTests, MoveOnlyTaskLambdaMove)
{
TrackMoves tm;
int moveCount = 0;
Task task(
defaultTD,
[tm = AZStd::move(tm), &moveCount]
{
moveCount = tm.moveCount;
});
Task task2 = AZStd::move(task);
task2.Invoke();
EXPECT_EQ(3, moveCount);
}
TEST(TaskGraphTests, CopyOnlyTaskLambda)
{
TrackCopies tc;
int copyCount = 0;
Task task(
defaultTD,
[tc, &copyCount]
{
copyCount = tc.copyCount;
});
task.Invoke();
// Two copies are expected. Once into the capture body of the lambda, once to construct
// the type erased task
EXPECT_EQ(2, copyCount);
}
TEST(TaskGraphTests, CopyOnlyTaskLambdaMove)
{
TrackCopies tc;
int copyCount = 0;
Task task(
defaultTD,
[tc, &copyCount]
{
copyCount = tc.copyCount;
});
Task task2 = AZStd::move(task);
task2.Invoke();
EXPECT_EQ(3, copyCount);
}
TEST(TaskGraphTests, DestroyLambda)
{
// This test ensures that for a lambda with a destructor, the destructor is invoked
// exactly once on a non-moved-from object.
int x = 0;
struct TrackDestroy
{
TrackDestroy(int* px)
: count{ px }
{
}
TrackDestroy(TrackDestroy&& other)
: count{ other.count }
{
other.count = nullptr;
}
~TrackDestroy()
{
if (count)
{
++*count;
}
}
int* count = nullptr;
};
{
TrackDestroy td{ &x };
Task task(
defaultTD,
[td = AZStd::move(td)]
{
});
task.Invoke();
// Destructor should not have run yet (except on moved-from instances)
EXPECT_EQ(x, 0);
}
// Destructor should have run now
EXPECT_EQ(x, 1);
}
TEST_F(TaskGraphTestFixture, VariadicInterface)
{
int x = 0;
TaskGraph graph;
auto [a, b, c] = graph.AddTasks(
defaultTD,
[&]
{
x += 3;
},
[&]
{
x = 4 * x;
},
[&]
{
x -= 1;
});
a.Precedes(b);
b.Precedes(c);
TaskGraphEvent ev;
graph.SubmitOnExecutor(*m_executor, &ev);
ev.Wait();
EXPECT_EQ(11, x);
}
TEST_F(TaskGraphTestFixture, SerialGraph)
{
int x = 0;
TaskGraph graph;
auto a = graph.AddTask(
defaultTD,
[&]
{
x += 3;
});
auto b = graph.AddTask(
defaultTD,
[&]
{
x = 4 * x;
});
auto c = graph.AddTask(
defaultTD,
[&]
{
x -= 1;
});
a.Precedes(b);
b.Precedes(c);
TaskGraphEvent ev;
graph.SubmitOnExecutor(*m_executor, &ev);
ev.Wait();
EXPECT_EQ(11, x);
}
TEST_F(TaskGraphTestFixture, DetachedGraph)
{
int x = 0;
TaskGraphEvent ev;
{
TaskGraph graph;
auto a = graph.AddTask(
defaultTD,
[&]
{
x += 3;
});
auto b = graph.AddTask(
defaultTD,
[&]
{
x = 4 * x;
});
auto c = graph.AddTask(
defaultTD,
[&]
{
x -= 1;
});
a.Precedes(b);
b.Precedes(c);
graph.Detach();
graph.SubmitOnExecutor(*m_executor, &ev);
}
ev.Wait();
EXPECT_EQ(11, x);
}
TEST_F(TaskGraphTestFixture, ForkJoin)
{
AZStd::atomic<int> x = 0;
// Task a initializes x to 3
// Task b and c toggles the lowest two bits atomically
// Task d decrements x
TaskGraph graph;
auto a = graph.AddTask(
defaultTD,
[&]
{
x = 0b111;
});
auto b = graph.AddTask(
defaultTD,
[&]
{
x ^= 1;
});
auto c = graph.AddTask(
defaultTD,
[&]
{
x ^= 2;
});
auto d = graph.AddTask(
defaultTD,
[&]
{
x -= 1;
});
// a <-- Root
// / \
// b c
// \ /
// d
a.Precedes(b, c);
d.Follows(b, c);
TaskGraphEvent ev;
graph.SubmitOnExecutor(*m_executor, &ev);
ev.Wait();
EXPECT_EQ(3, x);
}
TEST_F(TaskGraphTestFixture, SpawnSubgraph)
{
AZStd::atomic<int> x = 0;
TaskGraph graph;
auto a = graph.AddTask(
defaultTD,
[&]
{
x = 0b111;
});
auto b = graph.AddTask(
defaultTD,
[&]
{
x ^= 1;
});
auto c = graph.AddTask(
defaultTD,
[&]
{
x ^= 2;
TaskGraph subgraph;
auto e = subgraph.AddTask(
defaultTD,
[&]
{
x ^= 0b1000;
});
auto f = subgraph.AddTask(
defaultTD,
[&]
{
x ^= 0b10000;
});
auto g = subgraph.AddTask(
defaultTD,
[&]
{
x += 0b1000;
});
e.Precedes(g);
f.Precedes(g);
TaskGraphEvent ev;
subgraph.SubmitOnExecutor(*m_executor, &ev);
ev.Wait();
});
auto d = graph.AddTask(
defaultTD,
[&]
{
x -= 1;
});
// NOTE: The ideal way to express this topology is without the wait on the subgraph
// at task g, but this is more an illustrative test. Better is to express the entire
// graph in a single larger graph.
// a <-- Root
// / \
// b c - f
// \ \ \
// \ e - g
// \ /
// \ /
// \ /
// d
a.Precedes(b);
a.Precedes(c);
b.Precedes(d);
c.Precedes(d);
TaskGraphEvent ev;
graph.SubmitOnExecutor(*m_executor, &ev);
ev.Wait();
EXPECT_EQ(3 | 0b100000, x);
}
TEST_F(TaskGraphTestFixture, RetainedGraph)
{
AZStd::atomic<int> x = 0;
TaskGraph graph;
auto a = graph.AddTask(
defaultTD,
[&]
{
x = 0b111;
});
auto b = graph.AddTask(
defaultTD,
[&]
{
x ^= 1;
});
auto c = graph.AddTask(
defaultTD,
[&]
{
x ^= 2;
});
auto d = graph.AddTask(
defaultTD,
[&]
{
x -= 1;
});
auto e = graph.AddTask(
defaultTD,
[&]
{
x ^= 0b1000;
});
auto f = graph.AddTask(
defaultTD,
[&]
{
x ^= 0b10000;
});
auto g = graph.AddTask(
defaultTD,
[&]
{
x += 0b1000;
});
// a <-- Root
// / \
// b c - f
// \ \ \
// \ e - g
// \ /
// \ /
// \ /
// d
a.Precedes(b, c);
b.Precedes(d);
c.Precedes(e, f);
g.Follows(e, f);
g.Precedes(d);
TaskGraphEvent ev;
graph.SubmitOnExecutor(*m_executor, &ev);
ev.Wait();
EXPECT_EQ(3 | 0b100000, x);
x = 0;
graph.SubmitOnExecutor(*m_executor, &ev);
ev.Wait();
EXPECT_EQ(3 | 0b100000, x);
}
} // namespace UnitTest
#if defined(HAVE_BENCHMARK)
namespace Benchmark
{
class TaskGraphBenchmarkFixture : public ::benchmark::Fixture
{
public:
void SetUp(benchmark::State&) override
{
executor = new TaskExecutor;
graph = new TaskGraph;
}
void TearDown(benchmark::State&) override
{
delete graph;
delete executor;
}
TaskDescriptor descriptors[4] = { { "critical", "benchmark", TaskPriority::CRITICAL },
{ "high", "benchmark", TaskPriority::HIGH },
{ "medium", "benchmark", TaskPriority::MEDIUM },
{ "low", "benchmark", TaskPriority::LOW } };
TaskGraph* graph;
TaskExecutor* executor;
};
BENCHMARK_F(TaskGraphBenchmarkFixture, QueueToDequeue)(benchmark::State& state)
{
graph->AddTask(
descriptors[2],
[]
{
});
for (auto _ : state)
{
TaskGraphEvent ev;
graph->SubmitOnExecutor(*executor, &ev);
ev.Wait();
}
}
BENCHMARK_F(TaskGraphBenchmarkFixture, OneAfterAnother)(benchmark::State& state)
{
auto a = graph->AddTask(
descriptors[2],
[]
{
});
auto b = graph->AddTask(
descriptors[2],
[]
{
});
a.Precedes(b);
for (auto _ : state)
{
TaskGraphEvent ev;
graph->SubmitOnExecutor(*executor, &ev);
ev.Wait();
}
}
BENCHMARK_F(TaskGraphBenchmarkFixture, FourToOneJoin)(benchmark::State& state)
{
auto [a, b, c, d, e] = graph->AddTasks(
descriptors[2],
[]
{
},
[]
{
},
[]
{
},
[]
{
},
[]
{
});
e.Follows(a, b, c, d);
for (auto _ : state)
{
TaskGraphEvent ev;
graph->SubmitOnExecutor(*executor, &ev);
ev.Wait();
}
}
} // namespace Benchmark
#endif
@@ -65,6 +65,7 @@ set(FILES
StreamerTests.cpp
StringFunc.cpp
SystemFile.cpp
TaskTests.cpp
TickBusTest.cpp
TimeDataStatistics.cpp
UUIDTests.cpp
@@ -8,72 +8,100 @@
#include "Utils.h"
#include <AzCore/IO/Path/Path.h>
#include <AzCore/std/functional.h>
#include <AzCore/StringFunc/StringFunc.h>
UnitTest::ScopedTemporaryDirectory::ScopedTemporaryDirectory()
namespace UnitTest
{
constexpr int MaxAttempts = 255;
void DeleteFolderRecursive(const AZ::IO::PathView& path)
{
auto callback = [&path](AZStd::string_view filename, bool isFile) -> bool
{
if (isFile)
{
auto filePath = AZ::IO::FixedMaxPath(path) / filename;
AZ::IO::SystemFile::Delete(filePath.c_str());
}
else
{
if (filename != "." && filename != "..")
{
auto folderPath = AZ::IO::FixedMaxPath(path) / filename;
DeleteFolderRecursive(folderPath);
}
}
return true;
};
auto searchPath = AZ::IO::FixedMaxPath(path) / "*";
AZ::IO::SystemFile::FindFiles(searchPath.c_str(), callback);
AZ::IO::SystemFile::DeleteDir(AZ::IO::FixedMaxPathString(path.Native()).c_str());
}
ScopedTemporaryDirectory::ScopedTemporaryDirectory()
{
constexpr int MaxAttempts = 255;
#if !AZ_TRAIT_USE_POSIX_TEMP_FOLDER
const auto userTempFolder = std::filesystem::temp_directory_path();
const auto userTempFolder = std::filesystem::temp_directory_path();
#else
AZ::IO::Path userTempFolder("/tmp");
AZ::IO::Path userTempFolder("/tmp");
#endif
for (int i = 0; i < MaxAttempts; ++i)
{
auto randomFolder = AZ::Uuid::CreateRandom().ToString<AZStd::fixed_string<512>>(false, false);
AZ::IO::FixedMaxPath testPath;
#if !AZ_TRAIT_USE_POSIX_TEMP_FOLDER
auto path = userTempFolder / ("UnitTest-" + randomFolder).c_str();
testPath = path.string().c_str();
#else
userTempFolder /= ("UnitTest-" + randomFolder).c_str();
testPath = userTempFolder.c_str();
#endif
if (!AZ::IO::SystemFile::Exists(testPath.c_str()))
for (int i = 0; i < MaxAttempts; ++i)
{
#if !AZ_TRAIT_USE_POSIX_TEMP_FOLDER
m_path = path;
m_tempDirectory = m_path.string().c_str();
auto randomFolder = AZ::Uuid::CreateRandom().ToString<AZStd::fixed_string<512>>(false, false);
AZ::IO::FixedMaxPath testPath;
#if !AZ_TRAIT_USE_POSIX_TEMP_FOLDER
auto path = userTempFolder / ("UnitTest-" + randomFolder).c_str();
testPath = path.string().c_str();
#else
m_tempDirectory = testPath;
userTempFolder /= ("UnitTest-" + randomFolder).c_str();
testPath = userTempFolder.c_str();
#endif
m_directoryExists = AZ::IO::SystemFile::CreateDir(m_tempDirectory.c_str());
break;
if (!AZ::IO::SystemFile::Exists(testPath.c_str()))
{
#if !AZ_TRAIT_USE_POSIX_TEMP_FOLDER
m_path = path;
m_tempDirectory = m_path.string().c_str();
#else
m_tempDirectory = testPath;
#endif
m_directoryExists = AZ::IO::SystemFile::CreateDir(m_tempDirectory.c_str());
break;
}
}
AZ_Error("ScopedTemporaryDirectory", !m_tempDirectory.empty(), "Failed to create unique temporary directory after attempting %d random folder names", MaxAttempts);
}
ScopedTemporaryDirectory::~ScopedTemporaryDirectory()
{
if (m_directoryExists)
{
DeleteFolderRecursive(m_tempDirectory);
}
}
AZ_Error("ScopedTemporaryDirectory", !m_tempDirectory.empty(), "Failed to create unique temporary directory after attempting %d random folder names", MaxAttempts);
}
UnitTest::ScopedTemporaryDirectory::~ScopedTemporaryDirectory()
{
if (m_directoryExists)
bool ScopedTemporaryDirectory::IsValid() const
{
AZ::IO::SystemFile::DeleteDir(m_tempDirectory.c_str());
return m_directoryExists;
}
}
bool UnitTest::ScopedTemporaryDirectory::IsValid() const
{
return m_directoryExists;
}
const char* UnitTest::ScopedTemporaryDirectory::GetDirectory() const
{
return m_tempDirectory.c_str();
}
const char* ScopedTemporaryDirectory::GetDirectory() const
{
return m_tempDirectory.c_str();
}
#if !AZ_TRAIT_USE_POSIX_TEMP_FOLDER
const std::filesystem::path& UnitTest::ScopedTemporaryDirectory::GetPath() const
{
return m_path;
}
std::filesystem::path UnitTest::ScopedTemporaryDirectory::operator/(const std::filesystem::path& rhs) const
{
return m_path / rhs;
}
const std::filesystem::path& ScopedTemporaryDirectory::GetPath() const
{
return m_path;
}
std::filesystem::path ScopedTemporaryDirectory::operator/(const std::filesystem::path& rhs) const
{
return m_path / rhs;
}
#endif // !AZ_TRAIT_USE_POSIX_TEMP_FOLDER
}
@@ -18,6 +18,9 @@
namespace UnitTest
{
//! Deletes a folder hierarchy from the supplied path
void DeleteFolderRecursive(const AZ::IO::PathView& path);
// Creates a randomly named folder inside the user's temporary directory.
// The folder and all contents will be destroyed when the object goes out of scope
struct ScopedTemporaryDirectory
@@ -352,8 +352,20 @@ namespace UnitTest
AzToolsFramework::EditorEntityContextRequestBus::BroadcastResult(
entityId, &AzToolsFramework::EditorEntityContextRequestBus::Events::CreateNewEditorEntity, name);
if (!entityId.IsValid())
{
AZ_Error("CreateDefaultEditorEntity", false, "Failed to create editor entity '%s'", name);
return AZ::EntityId();
}
AZ::Entity* entity = GetEntityById(entityId);
if (!entity)
{
AZ_Error("CreateDefaultEditorEntity", false, "Invalid entity obtained from Id %s", entityId.ToString().c_str());
return AZ::EntityId();
}
entity->Deactivate();
// add required components for the Editor entity
@@ -11,7 +11,12 @@
namespace UnitTest
{
ToolsTestApplication::ToolsTestApplication(AZStd::string applicationName)
: ToolsApplication()
:ToolsTestApplication(AZStd::move(applicationName), 0, nullptr)
{
}
ToolsTestApplication::ToolsTestApplication(AZStd::string applicationName, int argc, char** argv)
: AzToolsFramework::ToolsApplication(&argc, &argv)
, m_applicationName(AZStd::move(applicationName))
{
}
@@ -18,6 +18,7 @@ namespace UnitTest
{
public:
explicit ToolsTestApplication(AZStd::string applicationName);
ToolsTestApplication(AZStd::string applicationName, int argc, char** argv);
void SetSettingsRegistrySpecializations(AZ::SettingsRegistryInterface::Specializations& specializations) override;
protected:
@@ -50,11 +50,19 @@ namespace AzToolsFramework
AzFramework::ViewportDebugDisplayEventBus::Handler::BusConnect(GetEntityContextId());
}
m_entityDataCache = AZStd::make_unique<EditorVisibleEntityDataCache>();
// temporarily disconnect from EditorInteractionSystemViewportSelectionRequestBus in case during the creation of
// m_interactionRequests (see interactionRequestsBuilder below) an event is propagated to the handler, if this happens then
// m_interactionRequests will be null as it will not have finished being created yet so we ensure no events are forwarded to it
EditorInteractionSystemViewportSelectionRequestBus::Handler::BusDisconnect();
m_interactionRequests.reset(); // BusConnect/Disconnect in constructor/destructor,
// so have to reset before assigning the new one
m_interactionRequests = interactionRequestsBuilder(m_entityDataCache.get());
{
m_entityDataCache = AZStd::make_unique<EditorVisibleEntityDataCache>();
m_interactionRequests.reset(); // BusConnect/Disconnect in constructor/destructor,
// so have to reset before assigning the new one
m_interactionRequests = interactionRequestsBuilder(m_entityDataCache.get());
}
EditorInteractionSystemViewportSelectionRequestBus::Handler::BusConnect(GetEntityContextId());
}
void EditorInteractionSystemComponent::SetDefaultHandler()
@@ -380,7 +380,6 @@ namespace AzToolsFramework::ViewportUi::Internal
}
PrepareWidgetForViewportUi(widget);
m_renderOverlay->setFocus();
}
void ViewportUiDisplay::SetUiOverlayContentsAnchored(QPointer<QWidget> widget, const Qt::Alignment alignment)
@@ -392,7 +391,6 @@ namespace AzToolsFramework::ViewportUi::Internal
PrepareWidgetForViewportUi(widget);
m_uiOverlayLayout.AddAnchoredWidget(widget, alignment);
m_renderOverlay->setFocus();
}
void ViewportUiDisplay::UpdateUiOverlayGeometry()
@@ -7,6 +7,7 @@
*/
#include <AzCore/UnitTest/TestTypes.h>
#include <AzCore/Settings/SettingsRegistryMergeUtils.h>
#include <AzToolsFramework/Asset/AssetSeedManager.h>
#include <AzFramework/Asset/AssetRegistry.h>
#include <AzCore/IO/FileIO.h>
@@ -49,19 +50,22 @@ namespace UnitTest
void SetUp() override
{
using namespace AZ::Data;
m_application = new ToolsTestApplication("AssetFileInfoListComparisonTest");
constexpr size_t MaxCommandArgsCount = 128;
using FixedValueString = AZ::SettingsRegistryInterface::FixedValueString;
using ArgumentContainer = AZStd::fixed_vector<char*, MaxCommandArgsCount>;
// The first command line argument is assumed to be the executable name so add a blank entry for it
ArgumentContainer argContainer{ {} };
// Append Command Line override for the Project Cache Path
auto projectCachePathOverride = FixedValueString::format(R"(--project-cache-path="%s")", m_tempDir.GetDirectory());
auto projectPathOverride = FixedValueString{ R"(--project-path=AutomatedTesting)" };
argContainer.push_back(projectCachePathOverride.data());
argContainer.push_back(projectPathOverride.data());
m_application = new ToolsTestApplication("AssetFileInfoListComparisonTest", aznumeric_caster(argContainer.size()), argContainer.data());
AzToolsFramework::AssetSeedManager assetSeedManager;
AzFramework::AssetRegistry assetRegistry;
m_localFileIO = aznew AZ::IO::LocalFileIO();
m_priorFileIO = AZ::IO::FileIOBase::GetInstance();
AZ::IO::FileIOBase::SetInstance(nullptr);
AZ::IO::FileIOBase::SetInstance(m_localFileIO);
AZ::IO::FileIOBase::GetInstance()->SetAlias("@assets@", m_tempDir.GetDirectory());
AZStd::string assetRoot = AzToolsFramework::PlatformAddressedAssetCatalog::GetAssetRootForPlatform(AzFramework::PlatformId::PC);
const AZStd::string assetRoot = AzToolsFramework::PlatformAddressedAssetCatalog::GetAssetRootForPlatform(AzFramework::PlatformId::PC);
for (int idx = 0; idx < TotalAssets; idx++)
{
@@ -75,7 +79,8 @@ namespace UnitTest
AZ_TEST_START_TRACE_SUPPRESSION;
if (m_fileStreams[idx].Open(m_assetsPath[idx].c_str(), AZ::IO::OpenMode::ModeWrite | AZ::IO::OpenMode::ModeBinary | AZ::IO::OpenMode::ModeCreatePath))
{
m_fileStreams[idx].Write(info.m_relativePath.size(), info.m_relativePath.data());
AZ::IO::SizeType bytesWritten = m_fileStreams[idx].Write(info.m_relativePath.size(), info.m_relativePath.data());
EXPECT_EQ(bytesWritten, info.m_relativePath.size());
AZ_TEST_STOP_TRACE_SUPPRESSION(1); // writing to asset cache folder
}
else
@@ -92,6 +97,7 @@ namespace UnitTest
assetRegistry.RegisterAssetDependency(m_assets[3], AZ::Data::ProductDependency(m_assets[4], 0));
m_application->Start(AzFramework::Application::Descriptor());
// Without this, the user settings component would attempt to save on finalize/shutdown. Since the file is
// shared across the whole engine, if multiple tests are run in parallel, the saving could cause a crash
// in the unit tests.
@@ -109,14 +115,16 @@ namespace UnitTest
AZStd::string pcCatalogFile = AzToolsFramework::PlatformAddressedAssetCatalog::GetCatalogRegistryPathForPlatform(AzFramework::PlatformId::PC);
ASSERT_TRUE(AzFramework::AssetCatalog::SaveCatalog(pcCatalogFile.c_str(), &assetRegistry)) << "Unable to save the asset catalog file.\n";
bool catalogSaved = AzFramework::AssetCatalog::SaveCatalog(pcCatalogFile.c_str(), &assetRegistry);
EXPECT_TRUE(catalogSaved) << "Unable to save the asset catalog file.\n";
m_pcCatalog = new AzToolsFramework::PlatformAddressedAssetCatalog(AzFramework::PlatformId::PC);
assetSeedManager.AddSeedAsset(m_assets[0], AzFramework::PlatformFlags::Platform_PC);
assetSeedManager.AddSeedAsset(m_assets[1], AzFramework::PlatformFlags::Platform_PC);
assetSeedManager.SaveAssetFileInfo(TempFiles[FileIndex::FirstAssetFileInfoList], AzFramework::PlatformFlags::Platform_PC, {});
bool firstAssetFileInfoListSaved = assetSeedManager.SaveAssetFileInfo(TempFiles[FileIndex::FirstAssetFileInfoList], AzFramework::PlatformFlags::Platform_PC, {});
EXPECT_TRUE(firstAssetFileInfoListSaved);
// Modify contents of asset2
int fileIndex = 2;
@@ -124,7 +132,8 @@ namespace UnitTest
if (m_fileStreams[fileIndex].Open(m_assetsPath[fileIndex].c_str(), AZ::IO::OpenMode::ModeWrite | AZ::IO::OpenMode::ModeBinary | AZ::IO::OpenMode::ModeCreatePath))
{
AZStd::string fileContent = AZStd::string::format("new Asset%d.txt", fileIndex);// changing file content
m_fileStreams[fileIndex].Write(fileContent.size(), fileContent.c_str());
AZ::IO::SizeType bytesWritten = m_fileStreams[fileIndex].Write(fileContent.size(), fileContent.c_str());
EXPECT_EQ(bytesWritten, fileContent.size());
AZ_TEST_STOP_TRACE_SUPPRESSION(1); // writing to asset cache folder
}
else
@@ -138,7 +147,8 @@ namespace UnitTest
if (m_fileStreams[fileIndex].Open(m_assetsPath[fileIndex].c_str(), AZ::IO::OpenMode::ModeWrite | AZ::IO::OpenMode::ModeBinary | AZ::IO::OpenMode::ModeCreatePath))
{
AZStd::string fileContent = AZStd::string::format("new Asset%d.txt", fileIndex);// changing file content
m_fileStreams[fileIndex].Write(fileContent.size(), fileContent.c_str());
AZ::IO::SizeType bytesWritten = m_fileStreams[fileIndex].Write(fileContent.size(), fileContent.c_str());
EXPECT_EQ(bytesWritten, fileContent.size());
AZ_TEST_STOP_TRACE_SUPPRESSION(1); // writing to asset cache folder
}
else
@@ -149,7 +159,8 @@ namespace UnitTest
assetSeedManager.RemoveSeedAsset(m_assets[0], AzFramework::PlatformFlags::Platform_PC);
assetSeedManager.AddSeedAsset(m_assets[5], AzFramework::PlatformFlags::Platform_PC);
assetSeedManager.SaveAssetFileInfo(TempFiles[FileIndex::SecondAssetFileInfoList], AzFramework::PlatformFlags::Platform_PC, {});
bool secondAssetFileInfoListSaved = assetSeedManager.SaveAssetFileInfo(TempFiles[FileIndex::SecondAssetFileInfoList], AzFramework::PlatformFlags::Platform_PC, {});
EXPECT_TRUE(secondAssetFileInfoListSaved);
}
void TearDown() override
@@ -162,7 +173,8 @@ namespace UnitTest
if (fileIO->Exists(TempFiles[idx]))
{
AZ_TEST_START_TRACE_SUPPRESSION;
fileIO->Remove(TempFiles[idx]);
AZ::IO::Result result = fileIO->Remove(TempFiles[idx]);
EXPECT_EQ(result.GetResultCode(), AZ::IO::ResultCode::Success);
AZ_TEST_STOP_TRACE_SUPPRESSION(1); // deleting from asset cache folder
}
}
@@ -175,7 +187,8 @@ namespace UnitTest
if (fileIO->Exists(m_assetsPath[idx].c_str()))
{
AZ_TEST_START_TRACE_SUPPRESSION;
fileIO->Remove(m_assetsPath[idx].c_str());
AZ::IO::Result result = fileIO->Remove(m_assetsPath[idx].c_str());
EXPECT_EQ(result.GetResultCode(), AZ::IO::ResultCode::Success);
AZ_TEST_STOP_TRACE_SUPPRESSION(1); // deleting from asset cache folder
}
}
@@ -184,15 +197,12 @@ namespace UnitTest
if (fileIO->Exists(pcCatalogFile.c_str()))
{
AZ_TEST_START_TRACE_SUPPRESSION;
fileIO->Remove(pcCatalogFile.c_str());
AZ::IO::Result result = fileIO->Remove(pcCatalogFile.c_str());
EXPECT_EQ(result.GetResultCode(), AZ::IO::ResultCode::Success);
AZ_TEST_STOP_TRACE_SUPPRESSION(1); // deleting from asset cache folder
}
delete m_pcCatalog;
delete m_localFileIO;
m_localFileIO = nullptr;
AZ::IO::FileIOBase::SetInstance(nullptr);
AZ::IO::FileIOBase::SetInstance(m_priorFileIO);
m_application->Stop();
delete m_application;
@@ -742,11 +752,9 @@ namespace UnitTest
}
ToolsTestApplication* m_application;
ToolsTestApplication* m_application = nullptr;
UnitTest::ScopedTemporaryDirectory m_tempDir;
AzToolsFramework::PlatformAddressedAssetCatalog* m_pcCatalog;
AZ::IO::FileIOBase* m_priorFileIO = nullptr;
AZ::IO::FileIOBase* m_localFileIO = nullptr;
AzToolsFramework::PlatformAddressedAssetCatalog* m_pcCatalog = nullptr;
AZ::IO::FileIOStream m_fileStreams[TotalAssets];
AZ::Data::AssetId m_assets[TotalAssets];
AZStd::string m_assetsPath[TotalAssets];
@@ -23,11 +23,12 @@
#include <AzCore/UserSettings/UserSettingsComponent.h>
#include <AzCore/Utils/Utils.h>
#include <AzTest/Utils.h>
#include <Utils/Utils.h>
namespace // anonymous
{
static const int s_totalAssets = 12;
static const int s_totalTestPlatforms = 2;
const char* s_catalogFile = "AssetCatalog.xml";
AZ::Data::AssetId assets[s_totalAssets];
const char TestSliceAssetPath[] = "test.slice";
@@ -55,18 +56,30 @@ namespace UnitTest
void SetUp() override
{
using namespace AZ::Data;
m_application = new ToolsTestApplication("AssetSeedManagerTest");
constexpr size_t MaxCommandArgsCount = 128;
using FixedValueString = AZ::SettingsRegistryInterface::FixedValueString;
using ArgumentContainer = AZStd::fixed_vector<char*, MaxCommandArgsCount>;
// The first command line argument is assumed to be the executable name so add a blank entry for it
ArgumentContainer argContainer{ {} };
// Append Command Line override for the Project Cache Path
AZ::IO::Path cacheProjectRootFolder{ m_tempDir.GetDirectory() };
auto projectCachePathOverride = FixedValueString::format(R"(--project-cache-path="%s")", cacheProjectRootFolder.c_str());
auto projectPathOverride = FixedValueString{ R"(--project-path=AutomatedTesting)" };
argContainer.push_back(projectCachePathOverride.data());
argContainer.push_back(projectPathOverride.data());
m_application = new ToolsTestApplication("AssetSeedManagerTest", aznumeric_caster(argContainer.size()), argContainer.data());
m_assetSeedManager = new AzToolsFramework::AssetSeedManager();
m_assetRegistry = new AzFramework::AssetRegistry();
AZ::SettingsRegistryInterface* registry = AZ::SettingsRegistry::Get();
auto projectPathKey =
AZ::SettingsRegistryInterface::FixedValueString(AZ::SettingsRegistryMergeUtils::BootstrapSettingsRootKey) + "/project_path";
registry->Set(projectPathKey, "AutomatedTesting");
AZ::SettingsRegistryMergeUtils::MergeSettingsToRegistry_AddRuntimeFilePaths(*registry);
m_application->Start(AzFramework::Application::Descriptor());
// By default @assets@ is setup to include the platform at the end. But this test is going to
// loop over platforms and it will be included as part of the relative path of the file.
// So the asset folder for these tests have to point to the cache project root folder, which
// doesn't include the platform.
AZ::IO::FileIOBase::GetInstance()->SetAlias("@assets@", cacheProjectRootFolder.c_str());
for (int idx = 0; idx < s_totalAssets; idx++)
{
assets[idx] = AssetId(AZ::Uuid::CreateRandom(), 0);
@@ -83,17 +96,18 @@ namespace UnitTest
int platformCount = 0;
for(auto thisPlatform : m_testPlatforms)
{
AZStd::string assetRoot = AzToolsFramework::PlatformAddressedAssetCatalog::GetAssetRootForPlatform(thisPlatform);
AZ::IO::Path assetRoot = AzToolsFramework::PlatformAddressedAssetCatalog::GetAssetRootForPlatform(thisPlatform);
for (int idx = 0; idx < s_totalAssets; idx++)
{
AzFramework::StringFunc::Path::Join(assetRoot.c_str(), m_assetsPath[idx].c_str(), m_assetsPathFull[platformCount][idx]);
m_assetsPathFull[platformCount][idx] = (assetRoot / m_assetsPath[idx]).Native();
AZ_TEST_START_TRACE_SUPPRESSION;
if (m_fileStreams[platformCount][idx].Open(m_assetsPathFull[platformCount][idx].c_str(), AZ::IO::OpenMode::ModeWrite | AZ::IO::OpenMode::ModeBinary | AZ::IO::OpenMode::ModeCreatePath))
{
m_fileStreams[platformCount][idx].Write(m_assetsPath[idx].size(), m_assetsPath[idx].data());
AZ::IO::SizeType bytesWritten = m_fileStreams[platformCount][idx].Write(m_assetsPath[idx].size(), m_assetsPath[idx].data());
EXPECT_EQ(bytesWritten, m_assetsPath[idx].size());
m_fileStreams[platformCount][idx].Close();
AZ_TEST_STOP_TRACE_SUPPRESSION_NO_COUNT; // writing to asset cache folder, only invalid for PC, not invalid in Jenkins
AZ_TEST_STOP_TRACE_SUPPRESSION(1); // writing to asset cache folder
}
else
{
@@ -117,7 +131,7 @@ namespace UnitTest
AZ_TEST_START_TRACE_SUPPRESSION;
AZ::IO::FileIOStream dynamicSliceFileIOStream(TestDynamicSliceAssetPath, AZ::IO::OpenMode::ModeWrite | AZ::IO::OpenMode::ModeText);
AZ_TEST_STOP_TRACE_SUPPRESSION_NO_COUNT; // writing to asset cache folder, not invalid in Jenkins
AZ_TEST_STOP_TRACE_SUPPRESSION(1); // writing to asset cache folder
AZ::Data::AssetInfo sliceAssetInfo;
sliceAssetInfo.m_relativePath = TestSliceAssetPath;
@@ -131,7 +145,7 @@ namespace UnitTest
AZ_TEST_START_TRACE_SUPPRESSION;
AZ::IO::FileIOStream sliceFileIOStream(TestSliceAssetPath, AZ::IO::OpenMode::ModeWrite | AZ::IO::OpenMode::ModeText);
AZ_TEST_STOP_TRACE_SUPPRESSION_NO_COUNT; // writing to asset cache folder, not invalid in Jenkins
AZ_TEST_STOP_TRACE_SUPPRESSION(1); // writing to asset cache folder
// asset0 -> asset1 -> asset2 -> asset4
// --> asset3
@@ -197,58 +211,6 @@ namespace UnitTest
void TearDown() override
{
AZ::IO::FileIOBase* fileIO = AZ::IO::FileIOBase::GetInstance();
if (fileIO->Exists(s_catalogFile))
{
AZ_TEST_START_TRACE_SUPPRESSION;
fileIO->Remove(s_catalogFile);
AZ_TEST_STOP_TRACE_SUPPRESSION_NO_COUNT; // deleting from asset cache folder, not invalid in Jenkins
}
for (size_t platformCount = 0; platformCount < s_totalTestPlatforms; ++platformCount)
{
// Deleting all the temporary files
for (int idx = 0; idx < s_totalAssets; idx++)
{
// we need to close the handle before we try to remove the file
if (fileIO->Exists(m_assetsPathFull[platformCount][idx].c_str()))
{
AZ_TEST_START_TRACE_SUPPRESSION;
fileIO->Remove(m_assetsPathFull[platformCount][idx].c_str());
AZ_TEST_STOP_TRACE_SUPPRESSION_NO_COUNT; // deleting from asset cache folder, not invalid in Jenkins
}
}
}
if (fileIO->Exists(TestSliceAssetPath))
{
AZ_TEST_START_TRACE_SUPPRESSION;
fileIO->Remove(TestSliceAssetPath);
AZ_TEST_STOP_TRACE_SUPPRESSION_NO_COUNT; // deleting from asset cache folder, not invalid in Jenkins
}
if (fileIO->Exists(TestDynamicSliceAssetPath))
{
AZ_TEST_START_TRACE_SUPPRESSION;
fileIO->Remove(TestDynamicSliceAssetPath);
AZ_TEST_STOP_TRACE_SUPPRESSION_NO_COUNT; // deleting from asset cache folder, not invalid in Jenkins
}
auto pcCatalogFile = AzToolsFramework::PlatformAddressedAssetCatalog::GetCatalogRegistryPathForPlatform(AzFramework::PlatformId::PC);
auto androidCatalogFile = AzToolsFramework::PlatformAddressedAssetCatalog::GetCatalogRegistryPathForPlatform(AzFramework::PlatformId::ANDROID_ID);
if (fileIO->Exists(pcCatalogFile.c_str()))
{
AZ_TEST_START_TRACE_SUPPRESSION;
fileIO->Remove(pcCatalogFile.c_str());
AZ_TEST_STOP_TRACE_SUPPRESSION_NO_COUNT; // deleting from asset cache folder, not invalid in Jenkins
}
if (fileIO->Exists(androidCatalogFile.c_str()))
{
fileIO->Remove(androidCatalogFile.c_str());
}
delete m_assetSeedManager;
delete m_assetRegistry;
delete m_pcCatalog;
@@ -284,7 +246,7 @@ namespace UnitTest
// Attempt to save to the same file. Should not be allowed.
AZ_TEST_START_TRACE_SUPPRESSION;
EXPECT_FALSE(m_assetSeedManager->Save(filePath));
AZ_TEST_STOP_TRACE_SUPPRESSION_NO_COUNT; // writing to asset cache folder, not invalid in Jenkins
AZ_TEST_STOP_TRACE_SUPPRESSION(1); // One error expected
// Clean up the test environment
AZ::IO::SystemFile::SetWritable(filePath.c_str(), true);
@@ -310,7 +272,7 @@ namespace UnitTest
// Attempt to save to the same file. Should not be allowed.
AZ_TEST_START_TRACE_SUPPRESSION;
EXPECT_FALSE(m_assetSeedManager->SaveAssetFileInfo(filePath, AzFramework::PlatformFlags::Platform_PC, {}));
AZ_TEST_STOP_TRACE_SUPPRESSION_NO_COUNT; // writing to asset cache folder, not invalid in Jenkins
AZ_TEST_STOP_TRACE_SUPPRESSION(1); // One error expected
// Clean up the test environment
AZ::IO::SystemFile::SetWritable(filePath.c_str(), true);
@@ -379,7 +341,7 @@ namespace UnitTest
// Step we are testing
AZ_TEST_START_TRACE_SUPPRESSION;
m_assetSeedManager->AddPlatformToAllSeeds(AzFramework::PlatformId::ANDROID_ID);
AZ_TEST_STOP_TRACE_SUPPRESSION_NO_COUNT; // writing to asset cache folder, not invalid in Jenkins
AZ_TEST_STOP_TRACE_SUPPRESSION(1); // One error expected
// Verification
AzFramework::PlatformFlags expectedPlatformFlags = AzFramework::PlatformFlags::Platform_PC | AzFramework::PlatformFlags::Platform_ANDROID;
@@ -649,9 +611,10 @@ namespace UnitTest
if (m_fileStreams[0][fileIndex].Open(m_assetsPathFull[0][fileIndex].c_str(), AZ::IO::OpenMode::ModeWrite | AZ::IO::OpenMode::ModeBinary | AZ::IO::OpenMode::ModeCreatePath))
{
AZStd::string fileContent = AZStd::string::format("asset%d.txt", fileIndex);
m_fileStreams[0][fileIndex].Write(fileContent.size(), fileContent.c_str());
AZ::IO::SizeType bytesWritten = m_fileStreams[0][fileIndex].Write(fileContent.size(), fileContent.c_str());
EXPECT_EQ(bytesWritten, fileContent.size());
m_fileStreams[0][fileIndex].Close();
AZ_TEST_STOP_TRACE_SUPPRESSION_NO_COUNT; // writing to asset cache folder, not invalid in Jenkins
AZ_TEST_STOP_TRACE_SUPPRESSION(1); // writing to asset cache folder
}
AzToolsFramework::AssetFileInfoList assetList2 = m_assetSeedManager->GetDependencyList(AzFramework::PlatformId::PC);
@@ -682,9 +645,10 @@ namespace UnitTest
if (m_fileStreams[0][fileIndex].Open(m_assetsPathFull[0][fileIndex].c_str(), AZ::IO::OpenMode::ModeWrite | AZ::IO::OpenMode::ModeBinary | AZ::IO::OpenMode::ModeCreatePath))
{
AZStd::string fileContent = AZStd::string::format("asset%d.txt", fileIndex + 1);// changing file content
m_fileStreams[0][fileIndex].Write(fileContent.size(), fileContent.c_str());
AZ::IO::SizeType bytesWritten = m_fileStreams[0][fileIndex].Write(fileContent.size(), fileContent.c_str());
EXPECT_EQ(bytesWritten, fileContent.size());
m_fileStreams[0][fileIndex].Close();
AZ_TEST_STOP_TRACE_SUPPRESSION_NO_COUNT; // writing to asset cache folder, not invalid in Jenkins
AZ_TEST_STOP_TRACE_SUPPRESSION(1); // writing to asset cache folder
}
AzToolsFramework::AssetFileInfoList assetList2 = m_assetSeedManager->GetDependencyList(AzFramework::PlatformId::PC);
@@ -790,16 +754,17 @@ namespace UnitTest
}
AzToolsFramework::AssetSeedManager* m_assetSeedManager;
AzFramework::AssetRegistry* m_assetRegistry;
ToolsTestApplication* m_application;
AzToolsFramework::PlatformAddressedAssetCatalog* m_pcCatalog;
AzToolsFramework::PlatformAddressedAssetCatalog* m_androidCatalog;
AzToolsFramework::AssetSeedManager* m_assetSeedManager = nullptr;
AzFramework::AssetRegistry* m_assetRegistry = nullptr;
ToolsTestApplication* m_application = nullptr;
AzToolsFramework::PlatformAddressedAssetCatalog* m_pcCatalog = nullptr;
AzToolsFramework::PlatformAddressedAssetCatalog* m_androidCatalog = nullptr;
AZ::IO::FileIOStream m_fileStreams[s_totalTestPlatforms][s_totalAssets];
AzFramework::PlatformId m_testPlatforms[s_totalTestPlatforms];
AZStd::string m_assetsPath[s_totalAssets];
AZStd::string m_assetsPathFull[s_totalTestPlatforms][s_totalAssets];
AZ::Data::AssetId m_testDynamicSliceAssetId;
UnitTest::ScopedTemporaryDirectory m_tempDir;
};
TEST_F(AssetSeedManagerTest, AssetSeedManager_SaveSeedListFile_FileIsReadOnly)
@@ -1285,7 +1285,7 @@ namespace UnitTest
Crc32 uiHandler = 0;
EXPECT_EQ(it->ReadAttribute(AZ::Edit::UIHandlers::Handler, uiHandler), true);
EXPECT_EQ(uiHandler, AZ_CRC("TestHandler"));
EXPECT_EQ(it->GetElementMetadata()->m_name, "UIElement");
EXPECT_STREQ(it->GetElementMetadata()->m_name, "UIElement");
EXPECT_EQ(it->GetElementMetadata()->m_nameCrc, AZ_CRC("UIElement"));
uiHandler = 0;
@@ -1293,7 +1293,7 @@ namespace UnitTest
++it;
EXPECT_EQ(it->ReadAttribute(AZ::Edit::UIHandlers::Handler, uiHandler), true);
EXPECT_EQ(uiHandler, AZ_CRC("TestHandler2"));
EXPECT_EQ(it->GetElementMetadata()->m_name, "UIElement2");
EXPECT_STREQ(it->GetElementMetadata()->m_name, "UIElement2");
EXPECT_EQ(it->GetElementMetadata()->m_nameCrc, AZ_CRC("UIElement2"));
}
};
@@ -1356,21 +1356,21 @@ namespace UnitTest
auto it = children.begin();
EXPECT_EQ(it->GetElementMetadata()->m_name, "aggregatedDataElement");
EXPECT_STREQ(it->GetElementMetadata()->m_name, "aggregatedDataElement");
++it;
if (i == 0)
{
EXPECT_EQ(it->GetElementMetadata()->m_name, "notAggregatedDataElement");
EXPECT_STREQ(it->GetElementMetadata()->m_name, "notAggregatedDataElement");
++it;
}
EXPECT_EQ(it->GetElementMetadata()->m_name, "aggregatedUIElement");
EXPECT_STREQ(it->GetElementMetadata()->m_name, "aggregatedUIElement");
++it;
if (i == 0)
{
EXPECT_EQ(it->GetElementMetadata()->m_name, "notAggregatedUIElement");
EXPECT_STREQ(it->GetElementMetadata()->m_name, "notAggregatedUIElement");
++it;
}
}
@@ -1505,11 +1505,11 @@ namespace UnitTest
AZStd::string childName(child.GetElementMetadata()->m_name);
if (childName.compare("GroupFloat") == 0)
{
EXPECT_EQ(child.GetGroupElementMetadata()->m_description, "Normal Group");
EXPECT_STREQ(child.GetGroupElementMetadata()->m_description, "Normal Group");
}
if (childName.compare("ToggleGroupInt") == 0)
{
EXPECT_EQ(child.GetGroupElementMetadata()->m_description, "Group Toggle");
EXPECT_STREQ(child.GetGroupElementMetadata()->m_description, "Group Toggle");
}
if ((childName.compare("SubDataNormal") == 0) || (childName.compare("SubDataToggle") == 0))
{
@@ -1518,11 +1518,11 @@ namespace UnitTest
childName = subChild.GetElementMetadata()->m_name;
if (childName.compare("SubInt") == 0)
{
EXPECT_EQ(subChild.GetGroupElementMetadata()->m_description, "Normal SubGroup");
EXPECT_STREQ(subChild.GetGroupElementMetadata()->m_description, "Normal SubGroup");
}
if (childName.compare("SubFloat") == 0)
{
EXPECT_EQ(subChild.GetGroupElementMetadata()->m_description, "SubGroup Toggle");
EXPECT_STREQ(subChild.GetGroupElementMetadata()->m_description, "SubGroup Toggle");
}
}
}
@@ -1552,7 +1552,7 @@ namespace UnitTest
AZStd::string childName(child.GetElementMetadata()->m_name);
if (childName.compare(paramName) == 0)
{
EXPECT_EQ(child.GetParent()->GetClassMetadata()->m_name, "GroupTestComponent");
EXPECT_STREQ(child.GetParent()->GetClassMetadata()->m_name, "GroupTestComponent");
}
if ((childName.compare("SubDataNormal") == 0) || (childName.compare("SubDataToggle") == 0))
{
@@ -1561,7 +1561,7 @@ namespace UnitTest
childName = subChild.GetElementMetadata()->m_name;
if (childName.compare(paramName) == 0)
{
EXPECT_EQ(subChild.GetParent()->GetClassMetadata()->m_name, "SubData");
EXPECT_STREQ(subChild.GetParent()->GetClassMetadata()->m_name, "SubData");
}
}
}
@@ -19,9 +19,8 @@
#include <AzFramework/Platform/PlatformDefaults.h>
#include <AzFramework/StringFunc/StringFunc.h>
#include <AzTest/AzTest.h>
#include <QTemporaryDir>
#include <QDir>
#include <AzCore/UserSettings/UserSettingsComponent.h>
#include <Utils/Utils.h>
namespace
{
@@ -35,25 +34,22 @@ namespace UnitTest
{
public:
AZStd::string GetTempFolder()
{
QTemporaryDir dir;
QDir tempPath(dir.path());
return tempPath.absolutePath().toUtf8().data();
}
void SetUp() override
{
using namespace AZ::Data;
m_application = new ToolsTestApplication("AddressedAssetCatalogManager"); // Shorter name because Setting Registry
// specialization are 32 characters max.
constexpr size_t MaxCommandArgsCount = 128;
using FixedValueString = AZ::SettingsRegistryInterface::FixedValueString;
using ArgumentContainer = AZStd::fixed_vector<char*, MaxCommandArgsCount>;
// The first command line argument is assumed to be the executable name so add a blank entry for it
ArgumentContainer argContainer{ {} };
AZ::SettingsRegistryInterface* registry = AZ::SettingsRegistry::Get();
auto projectPathKey =
AZ::SettingsRegistryInterface::FixedValueString(AZ::SettingsRegistryMergeUtils::BootstrapSettingsRootKey) + "/project_path";
registry->Set(projectPathKey, "AutomatedTesting");
AZ::SettingsRegistryMergeUtils::MergeSettingsToRegistry_AddRuntimeFilePaths(*registry);
// Append Command Line override for the Project Cache Path
AZ::IO::Path cacheProjectRootFolder{ m_tempDir.GetDirectory() };
auto projectCachePathOverride = FixedValueString::format(R"(--project-cache-path="%s")", cacheProjectRootFolder.c_str());
auto projectPathOverride = FixedValueString{ R"(--project-path=AutomatedTesting)" };
argContainer.push_back(projectCachePathOverride.data());
argContainer.push_back(projectPathOverride.data());
m_application = new ToolsTestApplication("AddressedAssetCatalogManager", aznumeric_caster(argContainer.size()), argContainer.data());
m_application->Start(AzFramework::Application::Descriptor());
// Without this, the user settings component would attempt to save on finalize/shutdown. Since the file is
@@ -61,33 +57,36 @@ namespace UnitTest
// in the unit tests.
AZ::UserSettingsComponentRequestBus::Broadcast(&AZ::UserSettingsComponentRequests::DisableSaveOnFinalize);
AZStd::string cacheFolder;
AzFramework::StringFunc::Path::Join(GetTempFolder().c_str(), "testplatform", cacheFolder);
AzFramework::StringFunc::Path::Join(cacheFolder.c_str(), "testproject", cacheFolder);
AZ::IO::FileIOBase::GetInstance()->SetAlias("@assets@", cacheFolder.c_str());
// By default @assets@ is setup to include the platform at the end. But this test is going to
// loop over all platforms and it will be included as part of the relative path of the file.
// So the asset folder for these tests have to point to the cache project root folder, which
// doesn't include the platform.
AZ::IO::FileIOBase::GetInstance()->SetAlias("@assets@", cacheProjectRootFolder.c_str());
for (int platformNum = AzFramework::PlatformId::PC; platformNum < AzFramework::PlatformId::NumPlatformIds; ++platformNum)
{
AZStd::string platformName{ AzFramework::PlatformHelper::GetPlatformName(static_cast<AzFramework::PlatformId>(platformNum)) };
const AZStd::string platformName{ AzFramework::PlatformHelper::GetPlatformName(static_cast<AzFramework::PlatformId>(platformNum)) };
if (!platformName.length())
{
// Do not test disabled platforms
continue;
}
AZStd::unique_ptr<AzFramework::AssetRegistry> assetRegistry = AZStd::make_unique<AzFramework::AssetRegistry>();
for (int idx = 0; idx < s_totalAssets; idx++)
{
m_assets[platformNum][idx] = AssetId(AZ::Uuid::CreateRandom(), 0);
AZ::Data::AssetInfo info;
info.m_relativePath = AZStd::string::format("%s%sAsset%d_%s.txt", cacheFolder.c_str(), AZ_CORRECT_FILESYSTEM_SEPARATOR_STRING, idx, platformName.c_str());
info.m_relativePath = AZStd::move((AZ::IO::Path(platformName) / AZStd::string::format("Asset%d.txt", idx)).Native());
info.m_assetId = m_assets[platformNum][idx];
assetRegistry->RegisterAsset(m_assets[platformNum][idx], info);
m_assetsPath[platformNum][idx] = info.m_relativePath;
m_assetsPath[platformNum][idx] = AZStd::move((cacheProjectRootFolder / info.m_relativePath).Native());
AZ_TEST_START_TRACE_SUPPRESSION;
if (m_fileStreams[platformNum][idx].Open(m_assetsPath[platformNum][idx].c_str(), AZ::IO::OpenMode::ModeWrite | AZ::IO::OpenMode::ModeBinary | AZ::IO::OpenMode::ModeCreatePath))
{
m_fileStreams[platformNum][idx].Write(info.m_relativePath.size(), info.m_relativePath.data());
AZ::IO::SizeType bytesWritten = m_fileStreams[platformNum][idx].Write(info.m_relativePath.size(), info.m_relativePath.data());
EXPECT_EQ(bytesWritten, info.m_relativePath.size());
m_fileStreams[platformNum][idx].Close();
AZ_TEST_STOP_TRACE_SUPPRESSION(1); // writing to asset cache folder
}
else
@@ -112,48 +111,15 @@ namespace UnitTest
void TearDown() override
{
AZ::IO::FileIOBase* fileIO = AZ::IO::FileIOBase::GetInstance();
for (int platformNum = AzFramework::PlatformId::PC; platformNum < AzFramework::PlatformId::NumPlatformIds; ++platformNum)
{
AZStd::string platformName{ AzFramework::PlatformHelper::GetPlatformName(static_cast<AzFramework::PlatformId>(platformNum)) };
if (!platformName.length())
{
// Do not test disabled platforms
continue;
}
AZStd::string catalogPath = AzToolsFramework::PlatformAddressedAssetCatalog::GetCatalogRegistryPathForPlatform(static_cast<AzFramework::PlatformId>(platformNum));
if (fileIO->Exists(catalogPath.c_str()))
{
fileIO->Remove(catalogPath.c_str());
}
// Deleting all the temporary files
for (int idx = 0; idx < s_totalAssets; idx++)
{
// we need to close the handle before we try to remove the file
m_fileStreams[platformNum][idx].Close();
if (fileIO->Exists(m_assetsPath[platformNum][idx].c_str()))
{
AZ_TEST_START_TRACE_SUPPRESSION;
fileIO->Remove(m_assetsPath[platformNum][idx].c_str());
AZ_TEST_STOP_TRACE_SUPPRESSION(1); // removing from asset cache folder
}
}
}
delete m_localFileIO;
m_localFileIO = nullptr;
AZ::IO::FileIOBase::SetInstance(m_priorFileIO);
delete m_PlatformAddressedAssetCatalogManager;
m_application->Stop();
delete m_application;
}
AzToolsFramework::PlatformAddressedAssetCatalogManager* m_PlatformAddressedAssetCatalogManager;
ToolsTestApplication* m_application;
AZ::IO::FileIOBase* m_priorFileIO = nullptr;
AZ::IO::FileIOBase* m_localFileIO = nullptr;
AzToolsFramework::PlatformAddressedAssetCatalogManager* m_PlatformAddressedAssetCatalogManager = nullptr;
ToolsTestApplication* m_application = nullptr;
UnitTest::ScopedTemporaryDirectory m_tempDir;
AZ::IO::FileIOStream m_fileStreams[AzFramework::PlatformId::NumPlatformIds][s_totalAssets];
AZ::Data::AssetId m_assets[AzFramework::PlatformId::NumPlatformIds][s_totalAssets];
@@ -183,12 +149,14 @@ namespace UnitTest
TEST_F(PlatformAddressedAssetCatalogManagerTest, PlatformAddressedAssetCatalogManager_CatalogExistsChecks_Success)
{
EXPECT_EQ(AzToolsFramework::PlatformAddressedAssetCatalog::CatalogExists(AzFramework::PlatformId::ANDROID_ID), true);
AZStd::string androidCatalogPath = AzToolsFramework::PlatformAddressedAssetCatalog::GetCatalogRegistryPathForPlatform(AzFramework::PlatformId::ANDROID_ID);
if (AZ::IO::FileIOBase::GetInstance()->Exists(androidCatalogPath.c_str()))
{
AZ::IO::FileIOBase::GetInstance()->Remove(androidCatalogPath.c_str());
AZ_TEST_START_TRACE_SUPPRESSION;
AZ::IO::Result result = AZ::IO::FileIOBase::GetInstance()->Remove(androidCatalogPath.c_str());
EXPECT_EQ(result.GetResultCode(), AZ::IO::ResultCode::Success);
AZ_TEST_STOP_TRACE_SUPPRESSION(1); // removing from asset cache folder
}
EXPECT_EQ(AzToolsFramework::PlatformAddressedAssetCatalog::CatalogExists(AzFramework::PlatformId::ANDROID_ID), false);
}
@@ -218,31 +186,32 @@ namespace UnitTest
: public AllocatorsFixture
{
public:
AZStd::string GetTempFolder()
{
QTemporaryDir dir;
QDir tempPath(dir.path());
return tempPath.absolutePath().toUtf8().data();
}
void SetUp() override
{
AZ::IO::FileIOBase::SetInstance(nullptr); // The API requires the old instance to be destroyed first
AZ::IO::FileIOBase::SetInstance(new AZ::IO::LocalFileIO());
constexpr size_t MaxCommandArgsCount = 128;
using FixedValueString = AZ::SettingsRegistryInterface::FixedValueString;
using ArgumentContainer = AZStd::fixed_vector<char*, MaxCommandArgsCount>;
// The first command line argument is assumed to be the executable name so add a blank entry for it
ArgumentContainer argContainer{ {} };
AZStd::string cacheFolder;
AzFramework::StringFunc::Path::Join(GetTempFolder().c_str(), "testplatform", cacheFolder);
AzFramework::StringFunc::Path::Join(cacheFolder.c_str(), "testproject", cacheFolder);
AZ::IO::FileIOBase::GetInstance()->SetAlias("@assets@", cacheFolder.c_str());
// Append Command Line override for the Project Cache Path
AZ::IO::Path cacheProjectRootFolder{ m_tempDir.GetDirectory() };
auto projectCachePathOverride = FixedValueString::format(R"(--project-cache-path="%s")", cacheProjectRootFolder.c_str());
auto projectPathOverride = FixedValueString{ R"(--project-path=AutomatedTesting)" };
argContainer.push_back(projectCachePathOverride.data());
argContainer.push_back(projectPathOverride.data());
m_application = new ToolsTestApplication("MessageTest", aznumeric_caster(argContainer.size()), argContainer.data());
m_platformAddressedAssetCatalogManager = AZStd::make_unique<AzToolsFramework::PlatformAddressedAssetCatalogManager>(AzFramework::PlatformId::Invalid);
}
void TearDown() override
{
m_platformAddressedAssetCatalogManager.reset();
delete m_application;
}
ToolsTestApplication* m_application = nullptr;
AZStd::unique_ptr<AzToolsFramework::PlatformAddressedAssetCatalogManager> m_platformAddressedAssetCatalogManager;
UnitTest::ScopedTemporaryDirectory m_tempDir;
};
TEST_F(MessageTest, PlatformAddressedAssetCatalogManagerMessageTest_MessagesForwarded_CountsMatch)
@@ -253,7 +222,7 @@ namespace UnitTest
AZ_TEST_START_TRACE_SUPPRESSION;
auto* mockCatalog = new ::testing::NiceMock<PlatformAddressedAssetCatalogMessageTest>(AzFramework::PlatformId::ANDROID_ID);
AZ_TEST_STOP_TRACE_SUPPRESSION(1);
AZ_TEST_STOP_TRACE_SUPPRESSION(1); // Expected error not finding catalog
AZStd::unique_ptr< ::testing::NiceMock<PlatformAddressedAssetCatalogMessageTest>> catalogHolder;
catalogHolder.reset(mockCatalog);