Merge remote-tracking branch 'upstream/development' into nvsickle/DomPatch

This commit is contained in:
Nicholas Van Sickle
2022-02-07 10:51:43 -08:00
2456 changed files with 259183 additions and 105897 deletions
@@ -17,6 +17,11 @@
#include <AzCore/Module/Environment.h>
#include <AzCore/std/parallel/shared_mutex.h>
namespace AZStd
{
class any;
}
namespace AZ
{
namespace Data
@@ -13,7 +13,6 @@ set(FILES
Instance/InstanceData.h
Instance/InstanceData.cpp
Instance/InstanceDatabase.h
std/containers/array_view.h
std/containers/fixed_vector_set.h
std/containers/lru_cache.h
std/containers/vector_set.h
@@ -1,156 +0,0 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#pragma once
#include <AzCore/std/containers/vector.h>
#include <AzCore/std/containers/fixed_vector.h>
#include <AzCore/std/containers/array.h>
namespace AZStd
{
/**
* Immutable wrapper for an array of data. It does not maintain storage for the data,
* but just holds pointers to mark the beginning and end of the array. It can be
* conveniently constructed from a variety of other container types like array,
* vector, and fixed_vector.
*
* Example:
* Given "void Func(AZStd::array_view<int> a) {...}" you can call...
* - Func({1,2,3});
* - AZStd::array<int,3> a = {1,2,3};
* Func(a);
* - AZStd::vector<int> v = {1,2,3};
* Func(v);
* - AZStd::fixed_vector<int,10> fv = {1,2,3};
* Func(fv);
*
* Since the array_view does not copy and store any data, it is only valid as long as the data used to create it is valid.
*/
template <class Element>
class array_view final
{
public:
using value_type = Element;
using pointer = value_type*;
using const_pointer = const value_type*;
using reference = value_type&;
using const_reference = const value_type&;
using size_type = AZStd::size_t;
using difference_type = AZStd::ptrdiff_t;
using iterator = const value_type*;
using const_iterator = const value_type*;
using reverse_iterator = AZStd::reverse_iterator<iterator>;
using const_reverse_iterator = AZStd::reverse_iterator<const_iterator>;
array_view()
: m_begin(nullptr)
, m_end(nullptr)
{ }
~array_view() = default;
array_view(const_pointer s, size_type length)
: m_begin(s)
, m_end(m_begin + length)
{
if (length == 0) erase();
}
array_view(const_pointer first, const_pointer last)
: m_begin(first)
, m_end(last)
{ }
// We explicitly delete this constructor because it's too easy to accidentally
// create an array_view to just the first element instead of an entire array.
array_view(const_pointer s) = delete;
template<AZStd::size_t N>
array_view(const AZStd::array<value_type, N>& data)
: m_begin(data.data())
, m_end(m_begin + data.size())
{ }
array_view(const AZStd::vector<value_type>& data)
: m_begin(data.data())
, m_end(m_begin + data.size())
{ }
template<AZStd::size_t N>
array_view(const AZStd::fixed_vector<value_type, N>& data)
: m_begin(data.data())
, m_end(m_begin + data.size())
{ }
array_view(const array_view&) = default;
array_view(array_view&& other)
: array_view(other.m_begin, other.m_end)
{
#if AZ_DEBUG_BUILD // Clearing the original pointers isn't necessary, but is good for debugging
other.m_begin = nullptr;
other.m_end = nullptr;
#endif
}
array_view& operator=(const array_view& other) = default;
array_view& operator=(array_view&& other)
{
m_begin = other.m_begin;
m_end = other.m_end;
#if AZ_DEBUG_BUILD // Clearing the original pointers isn't necessary, but is good for debugging
other.m_begin = nullptr;
other.m_end = nullptr;
#endif
return *this;
}
size_type size() const { return m_end - m_begin; }
bool empty() const { return m_end == m_begin; }
const_pointer data() const { return m_begin; }
const_reference operator[](size_type index) const
{
AZ_Assert(index < size(), "index value is out of range");
return m_begin[index];
}
void erase() { m_begin = m_end = nullptr; }
iterator begin() const { return m_begin; }
iterator end() const { return m_end; }
const_iterator cbegin() const { return m_begin; }
const_iterator cend() const { return m_end; }
reverse_iterator rbegin() const { return reverse_iterator(m_end); }
reverse_iterator rend() const { return reverse_iterator(m_begin); }
const_reverse_iterator crbegin() const { return const_reverse_iterator(cend()); }
const_reverse_iterator crend() const { return const_reverse_iterator(cbegin()); }
friend bool operator==(array_view lhs, array_view rhs)
{
return lhs.m_begin == rhs.m_begin && lhs.m_end == rhs.m_end;
}
friend bool operator!=(array_view lhs, array_view rhs) { return !(lhs == rhs); }
friend bool operator< (array_view lhs, array_view rhs) { return lhs.m_begin < rhs.m_begin || lhs.m_begin == rhs.m_begin && lhs.m_end < rhs.m_end; }
friend bool operator> (array_view lhs, array_view rhs) { return lhs.m_begin > rhs.m_begin || lhs.m_begin == rhs.m_begin && lhs.m_end > rhs.m_end; }
friend bool operator<=(array_view lhs, array_view rhs) { return lhs == rhs || lhs < rhs; }
friend bool operator>=(array_view lhs, array_view rhs) { return lhs == rhs || lhs > rhs; }
private:
const_pointer m_begin;
const_pointer m_end;
};
} // namespace AZStd
-300
View File
@@ -1,300 +0,0 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#include <AtomCore/std/containers/array_view.h>
#include <AzCore/std/containers/array.h>
#include <AzCore/UnitTest/TestTypes.h>
namespace UnitTest
{
using namespace AZStd;
class ArrayView : public AllocatorsTestFixture
{
protected:
template<typename T>
void ExpectEqual(initializer_list<T> expectedValues, array_view<T> arrayView)
{
EXPECT_EQ(false, arrayView.empty());
EXPECT_EQ(expectedValues.size(), arrayView.size());
typename AZStd::vector<T>::const_iterator iterator = arrayView.begin();
for (int i = 0; i < expectedValues.size(); ++i, ++iterator)
{
EXPECT_EQ(expectedValues.begin()[i], arrayView[i]);
EXPECT_EQ(expectedValues.begin()[i], *iterator);
}
EXPECT_EQ(iterator, arrayView.end());
}
};
TEST_F(ArrayView, DefaultConstructor)
{
array_view<bool> defaultView;
EXPECT_EQ(nullptr, defaultView.begin());
EXPECT_EQ(nullptr, defaultView.end());
EXPECT_EQ(0, defaultView.size());
EXPECT_EQ(true, defaultView.empty());
}
TEST_F(ArrayView, PointerConstructor1)
{
int originalValues[4] = { 2,3,4,5 };
array_view<int> view(originalValues, AZ_ARRAY_SIZE(originalValues));
ExpectEqual({ 2,3,4,5 }, view);
EXPECT_EQ(originalValues, view.begin());
EXPECT_EQ(&originalValues[4], view.end());
}
TEST_F(ArrayView, PointerConstructor2)
{
int originalValues[3] = { 6,7,8 };
array_view<int> view(originalValues, &originalValues[3]);
ExpectEqual({ 6,7,8 }, view);
EXPECT_EQ(originalValues, view.begin());
EXPECT_EQ(&originalValues[3], view.end());
}
TEST_F(ArrayView, ArrayConstructor)
{
array<int, 4> originalValues = { 9,10,11,12 };
array_view<int> view(originalValues);
ExpectEqual({ 9,10,11,12 }, view);
EXPECT_EQ(originalValues.begin(), view.begin());
EXPECT_EQ(originalValues.end(), view.end());
}
TEST_F(ArrayView, VectorConstructor)
{
vector<int> originalValues = { 13,14,15,16,17,18 };
array_view<int> view(originalValues);
ExpectEqual({ 13,14,15,16,17,18 }, view);
EXPECT_EQ(originalValues.begin(), view.begin());
EXPECT_EQ(originalValues.end(), view.end());
}
TEST_F(ArrayView, FixedVectorConstructor)
{
fixed_vector<int, 10> originalValues = { 17,18,19 }; // Note that even though the fixed_vector capacity is 10, it's size is 3, so the view size will be 3 as well
array_view<int> view(originalValues);
ExpectEqual({ 17,18,19 }, view);
EXPECT_EQ(originalValues.begin(), view.begin());
EXPECT_EQ(originalValues.end(), view.end());
}
TEST_F(ArrayView, CopyConstructor)
{
fixed_vector<int, 2> originalValues = { 27,28 };
array_view<int> view1(originalValues);
array_view<int> view2(view1);
ExpectEqual({ 27,28 }, view2);
EXPECT_EQ(view1.begin(), view2.begin());
EXPECT_EQ(view1.end(), view2.end());
}
TEST_F(ArrayView, MoveConstructor)
{
int originalValues[] = { 29,30,31 };
array_view<int> view1(originalValues, AZ_ARRAY_SIZE(originalValues));
array_view<int> view2(AZStd::move(view1));
ExpectEqual({ 29,30,31 }, view2);
EXPECT_EQ(originalValues, view2.begin());
EXPECT_EQ(&originalValues[3], view2.end());
// This isn't strictly necessary but is a good way to make sure the move
// constructor actually exists and it itn't just calling the copy constructor
#if AZ_DEBUG_BUILD // The pointers are only cleared in debug
EXPECT_EQ(nullptr, view1.begin());
EXPECT_EQ(nullptr, view1.end());
#endif
}
TEST_F(ArrayView, AssignmentOperator)
{
fixed_vector<int, 4> originalValues = { 32,33,34,35 };
array_view<int> view1(originalValues);
array_view<int> view2;
view2 = view1;
ExpectEqual({ 32,33,34,35 }, view2);
EXPECT_EQ(view1.begin(), view2.begin());
EXPECT_EQ(view1.end(), view2.end());
}
TEST_F(ArrayView, MoveAssignmentOperator)
{
int originalValues[] = { 36,37,38,39,40 };
array_view<int> view1(originalValues, AZ_ARRAY_SIZE(originalValues));
array_view<int> view2;
view2 = AZStd::move(view1);
ExpectEqual({ 36,37,38,39,40 }, view2);
EXPECT_EQ(originalValues, view2.begin());
EXPECT_EQ(&originalValues[5], view2.end());
// This isn't strictly necessary but is a good way to make sure the move
// assignment operator actually exists and it itn't just calling the norm
// assignment operator
#if AZ_DEBUG_BUILD // The pointers are only cleared in debug
EXPECT_EQ(nullptr, view1.begin());
EXPECT_EQ(nullptr, view1.end());
#endif
}
TEST_F(ArrayView, Erase)
{
fixed_vector<int, 4> originalValues = { 1,2,3,4 };
array_view<int> view(originalValues);
view.erase();
EXPECT_EQ(nullptr, view.begin());
EXPECT_EQ(nullptr, view.end());
EXPECT_EQ(0, view.size());
EXPECT_EQ(true, view.empty());
}
TEST_F(ArrayView, BeginAndEnd)
{
fixed_vector<int, 4> originalValues = { 1,2,3,4 };
array_view<int> view(originalValues);
EXPECT_EQ(1, view.begin()[0]);
EXPECT_EQ(4, view.end()[-1]);
EXPECT_EQ(1, view.cbegin()[0]);
EXPECT_EQ(4, view.cend()[-1]);
EXPECT_EQ(4, view.rbegin()[0]);
EXPECT_EQ(1, view.rend()[-1]);
EXPECT_EQ(4, view.crbegin()[0]);
EXPECT_EQ(1, view.crend()[-1]);
}
TEST_F(ArrayView, ImplicitConstruction)
{
// This test verifies that we can pass in various non-array_view types
// into functions that take an array_view
// The compile cannot detect the correct template type so that has to be specified explicitly
ExpectEqual<int>({ 1,2,3 }, vector<int>({ 1,2,3 }));
ExpectEqual<int>({ 1,2,3 }, fixed_vector<int, 3>({ 1,2,3 }));
ExpectEqual<int>({ 1,2,3 }, array<int, 3>({ 1,2,3 }));
}
void CheckComparisonOperators(bool areEqual, array_view<int> a, array_view<int> b)
{
EXPECT_EQ(areEqual, a == b);
// For less/greater operators, the exact order doesn't really matter;
// We just check for internal consistency
if (areEqual)
{
EXPECT_EQ(false, a != b);
EXPECT_EQ(false, a < b);
EXPECT_EQ(false, a > b);
EXPECT_EQ(true, a <= b);
EXPECT_EQ(true, a >= b);
}
else
{
EXPECT_EQ(true, a != b);
EXPECT_EQ(a > b, a >= b);
EXPECT_EQ(a < b, a <= b);
EXPECT_NE(a > b, a < b);
EXPECT_NE(a >= b, a <= b);
EXPECT_NE(a >= b, a < b);
EXPECT_NE(a > b, a <= b);
EXPECT_NE(a <= b, a > b);
EXPECT_NE(a < b, a >= b);
}
}
TEST_F(ArrayView, ComparisonOperators)
{
int arrayA[] = { 1,2,3 };
int arrayB[] = { 1,2,3 };
array_view<int> arrayA_view(arrayA, 3);
array_view<int> arrayB_view(arrayB, 3);
array_view<int> arrayA_otherView(arrayA, 3);
// view of a sub-array aligned to the beginning of the array
array_view<int> arrayA_headView(arrayA, 2);
array_view<int> arrayB_headView(arrayB, 2);
// view of a sub-array aligned to the end of the array
array_view<int> arrayA_tailView(&arrayA[1], 2);
array_view<int> arrayB_tailView(&arrayB[1], 2);
// view of a sub-array in the middle of the array
array_view<int> arrayA_centerView(&arrayA[1], 1);
array_view<int> arrayB_centerView(&arrayB[1], 1);
// Same view
CheckComparisonOperators(true, arrayA_view, arrayA_view);
// Different view, same array
CheckComparisonOperators(true, arrayA_view, arrayA_otherView);
CheckComparisonOperators(true, arrayA_otherView, arrayA_view);
// Different arrays
CheckComparisonOperators(false, arrayA_view, arrayB_view);
CheckComparisonOperators(false, arrayB_view, arrayA_view);
// Same arrays, but one is a just a subset of the array
CheckComparisonOperators(false, arrayA_view, arrayA_headView);
CheckComparisonOperators(false, arrayA_view, arrayA_tailView);
CheckComparisonOperators(false, arrayA_view, arrayA_centerView);
CheckComparisonOperators(false, arrayA_headView, arrayA_view);
CheckComparisonOperators(false, arrayA_tailView, arrayA_view);
CheckComparisonOperators(false, arrayA_centerView, arrayA_view);
// Different arrays, different lengths
CheckComparisonOperators(false, arrayA_view, arrayB_headView);
CheckComparisonOperators(false, arrayB_view, arrayA_headView);
CheckComparisonOperators(false, arrayB_headView, arrayA_view);
CheckComparisonOperators(false, arrayA_headView, arrayB_view);
}
TEST_F(ArrayView, AssertOutOfBounds)
{
array_view<int> view({ 1,2,3,4 });
AZ_TEST_START_TRACE_SUPPRESSION;
view[4];
view[5];
AZ_TEST_STOP_TRACE_SUPPRESSION(2);
}
}
@@ -7,7 +7,6 @@
#
set(FILES
ArrayView.cpp
ConcurrencyCheckerTests.cpp
InstanceDatabase.cpp
lru_cache.cpp
@@ -64,7 +64,7 @@ public class LumberyardActivity extends NativeActivity
////////////////////////////////////////////////////////////////
// called from the native to get the application package name
// e.g. com.lumberyard.samples for SamplesProject
// e.g. org.o3de.samples for SamplesProject
public String GetPackageName()
{
return getApplicationContext().getPackageName();
@@ -19,7 +19,7 @@
#include <AzCore/std/string/string.h>
#include <AzCore/std/string/string_view.h>
#include <AzCore/std/typetraits/is_base_of.h>
#include <AzCore/IO/Streamer/FileRequest.h>
#include <AzCore/IO/IStreamerTypes.h>
namespace AZ
{
@@ -7,11 +7,62 @@
*/
#include <AzCore/Asset/AssetDataStream.h>
#include <AzCore/Memory/Memory.h>
#include <AzCore/std/containers/vector.h>
#include <AzCore/Asset/AssetCommon.h>
#include <AzCore/Debug/Profiler.h>
#include <AzCore/IO/IStreamer.h>
#include <AzCore/IO/Streamer/FileRequest.h>
#include <AzCore/std/parallel/condition_variable.h>
namespace AZ::Data
{
namespace DataStreamInternal
{
struct AssetDataStreamPrivate
{
//! Optional data buffer that's been directly passed in through Open(), instead of reading data from a file.
AZStd::vector<AZ::u8> m_preloadedData;
//! The current active streamer read request - tracked in case we need to cancel it prematurely
AZ::IO::FileRequestPtr m_curReadRequest{ nullptr };
//! Synchronization for the read request, so that it's possible to block until completion.
AZStd::mutex m_readRequestMutex;
AZStd::condition_variable m_readRequestActive;
void SetReadRequest(AZ::IO::FileRequestPtr&& req)
{
AZStd::scoped_lock<AZStd::mutex> lock(m_readRequestMutex);
// The read request finished, so stop tracking it.
m_curReadRequest = AZStd::move(req);
}
void BlockUntilReadComplete()
{
AZStd::unique_lock lock(m_readRequestMutex);
m_readRequestActive.wait(
lock,
[this]
{
return m_curReadRequest == nullptr;
});
lock.unlock();
}
void CancelRequest()
{
AZStd::scoped_lock<AZStd::mutex> lock(m_readRequestMutex);
if (m_curReadRequest)
{
auto streamer = Interface<IO::IStreamer>::Get();
m_curReadRequest = streamer->Cancel(m_curReadRequest);
}
}
};
} // namespace Internal
AssetDataStream::AssetDataStream(AZ::IO::IStreamerTypes::RequestMemoryAllocator* bufferAllocator)
: m_bufferAllocator(bufferAllocator ? bufferAllocator : &m_defaultAllocator)
: m_privateData(AZStd::make_unique<DataStreamInternal::AssetDataStreamPrivate>())
, m_bufferAllocator(bufferAllocator ? bufferAllocator : &m_defaultAllocator)
{
ClearInternalStateData();
}
@@ -53,9 +104,9 @@ namespace AZ::Data
OpenInternal(data.size(), "(mem buffer)");
// Directly take ownership of the provided buffer
m_preloadedData = AZStd::move(data);
m_buffer = m_preloadedData.data();
m_loadedSize = m_preloadedData.size();
m_privateData->m_preloadedData = AZStd::move(data);
m_buffer = m_privateData->m_preloadedData.data();
m_loadedSize = m_privateData->m_preloadedData.size();
}
void AssetDataStream::Open(const AZStd::string& filePath, size_t fileOffset, size_t assetSize,
@@ -65,7 +116,7 @@ namespace AZ::Data
AZ_PROFILE_FUNCTION(AzCore);
AZ_Assert(!m_isOpen, "Attempting to open the stream when it is already open.");
AZ_Assert(!m_curReadRequest, "Queueing an asset stream load while one is still in progress.");
AZ_Assert(!m_privateData->m_curReadRequest, "Queueing an asset stream load while one is still in progress.");
AZ_Assert(!filePath.empty(), "AssetDataStream::Open called without a valid file name.");
// Initialize the state variables and start tracking the overall load timings
@@ -97,11 +148,8 @@ namespace AZ::Data
"Buffer for %s was expected to be %zu bytes, but is %zu bytes.",
m_filePath.c_str(), m_requestedAssetSize, m_loadedSize);
{
AZStd::scoped_lock<AZStd::mutex> lock(m_readRequestMutex);
// The read request finished, so stop tracking it.
m_curReadRequest = nullptr;
}
// The read request finished, so stop tracking it.
m_privateData->SetReadRequest(nullptr);
// Call the load callback to start processing the loaded data.
if (loadCallback)
@@ -115,21 +163,22 @@ namespace AZ::Data
}
// Notify that the load is complete, in case anyone is using BlockUntilLoadComplete to block.
m_readRequestActive.notify_one();
m_privateData->m_readRequestActive.notify_one();
};
// Queue the raw file load with the file streamer.
auto streamer = AZ::Interface<AZ::IO::IStreamer>::Get();
m_curReadRequest = streamer->Read(
m_privateData->m_curReadRequest =
streamer->Read(
m_filePath,
*m_bufferAllocator,
m_requestedAssetSize,
deadline, priority, m_fileOffset);
m_curDeadline = deadline;
m_curPriority = priority;
streamer->SetRequestCompleteCallback(m_curReadRequest, streamerCallback);
streamer->SetRequestCompleteCallback(m_privateData->m_curReadRequest, streamerCallback);
streamer->QueueRequest(m_curReadRequest);
streamer->QueueRequest(m_privateData->m_curReadRequest);
}
else
{
@@ -139,19 +188,19 @@ namespace AZ::Data
loadCallback(AZ::IO::IStreamerTypes::RequestStatus::Completed);
}
m_readRequestActive.notify_one();
m_privateData->m_readRequestActive.notify_one();
}
}
void AssetDataStream::Reschedule(AZStd::chrono::milliseconds newDeadline, AZ::IO::IStreamerTypes::Priority newPriority)
{
if (m_curReadRequest && (newDeadline < m_curDeadline || newPriority > m_curPriority))
if (m_privateData->m_curReadRequest && (newDeadline < m_curDeadline || newPriority > m_curPriority))
{
auto deadline = AZStd::GetMin(m_curDeadline, newDeadline);
auto priority = AZStd::GetMax(m_curPriority, newPriority);
auto streamer = Interface<IO::IStreamer>::Get();
m_curReadRequest = streamer->RescheduleRequest(m_curReadRequest, deadline, priority);
m_privateData->m_curReadRequest = streamer->RescheduleRequest(m_privateData->m_curReadRequest, deadline, priority);
m_curDeadline = deadline;
m_curPriority = priority;
}
@@ -159,15 +208,13 @@ namespace AZ::Data
void AssetDataStream::BlockUntilLoadComplete()
{
AZStd::unique_lock<AZStd::mutex> lock(m_readRequestMutex);
m_readRequestActive.wait(lock, [this] { return m_curReadRequest == nullptr; });
lock.unlock();
m_privateData->BlockUntilReadComplete();
}
void AssetDataStream::ClearInternalStateData()
{
// Clear all our internal state data.
m_preloadedData.resize(0);
m_privateData->m_preloadedData.resize(0);
m_buffer = nullptr;
m_loadedSize = 0;
m_requestedAssetSize = 0;
@@ -204,10 +251,10 @@ namespace AZ::Data
void AssetDataStream::Close()
{
AZ_Assert(m_isOpen, "Attempting to close a stream that hasn't been opened.");
AZ_Assert(m_curReadRequest == nullptr, "Attempting to close a stream with a read request in flight.");
AZ_Assert(m_privateData->m_curReadRequest == nullptr, "Attempting to close a stream with a read request in flight.");
// Destroy the asset buffer and unlock the allocator, so the allocator itself knows that it is no longer needed.
if (m_buffer != m_preloadedData.data())
if (m_buffer != m_privateData->m_preloadedData.data())
{
m_bufferAllocator->Release(m_buffer);
}
@@ -221,12 +268,7 @@ namespace AZ::Data
void AssetDataStream::RequestCancel()
{
AZStd::scoped_lock<AZStd::mutex> lock(m_readRequestMutex);
if (m_curReadRequest)
{
auto streamer = Interface<IO::IStreamer>::Get();
m_curReadRequest = streamer->Cancel(m_curReadRequest);
}
m_privateData->CancelRequest();
}
void AssetDataStream::Seek(AZ::IO::OffsetType bytes, AZ::IO::GenericStream::SeekMode mode)
@@ -9,17 +9,26 @@
#include <AzCore/IO/GenericStreams.h>
#include <AzCore/IO/IStreamerTypes.h>
#include <AzCore/IO/IStreamer.h>
#include <AzCore/std/containers/vector.h>
#include <AzCore/std/parallel/condition_variable.h>
#include <AzCore/Asset/AssetCommon.h>
#include <AzCore/Debug/Profiler.h>
#include <AzCore/std/functional.h>
#include <AzCore/std/smart_ptr/unique_ptr.h>
namespace AZStd
{
template<class T, class Allocator>
class vector;
}
namespace AZ::Data
{
namespace DataStreamInternal
{
struct AssetDataStreamPrivate;
}
class AssetDataStream : public AZ::IO::GenericStream
{
public:
using VectorDataSource = AZStd::vector<AZ::u8, AZStd::allocator>;
// The default Generic Stream APIs in this class will only allow for a single sequential pass
// through the data, no seeking. Reads will block when pages aren't available yet, and
// pages will be marked for recycling once reading has progressed beyond them.
@@ -29,10 +38,10 @@ namespace AZ::Data
~AssetDataStream() override;
// Open the AssetDataStream and make a copy of the provided memory buffer.
void Open(const AZStd::vector<AZ::u8>& data);
void Open(const VectorDataSource& data);
// Open the AssetDataStream and directly take ownership of a pre-populated memory buffer.
void Open(AZStd::vector<AZ::u8>&& data);
void Open(VectorDataSource&& data);
// Open the AssetDataStream and load it via file streaming
using OnCompleteCallback = AZStd::function<void(AZ::IO::IStreamerTypes::RequestStatus)>;
@@ -91,6 +100,8 @@ namespace AZ::Data
void ClearInternalStateData();
AZStd::unique_ptr<DataStreamInternal::AssetDataStreamPrivate> m_privateData;
//! The allocator to use for allocating / deallocating asset buffers
AZ::IO::IStreamerTypes::RequestMemoryAllocator* m_bufferAllocator{ nullptr };
@@ -106,9 +117,6 @@ namespace AZ::Data
//! The amount of data that's expected to be loaded.
size_t m_requestedAssetSize{ 0 };
//! Optional data buffer that's been directly passed in through Open(), instead of reading data from a file.
AZStd::vector<AZ::u8> m_preloadedData;
//! The buffer that will hold the raw data after it's loaded from the file.
void* m_buffer{ nullptr };
@@ -119,19 +127,12 @@ namespace AZ::Data
//! The current offset representing how far we've read into the buffer.
size_t m_curOffset{ 0 };
//! The current active streamer read request - tracked in case we need to cancel it prematurely
AZ::IO::FileRequestPtr m_curReadRequest{ nullptr };
//! The current request deadline. Used to avoid requesting a reschedule to the same (current) deadline.
AZStd::chrono::milliseconds m_curDeadline{ AZ::IO::IStreamerTypes::s_noDeadline };
//! The current request priority. Used to avoid requesting a reschedule to the same (current) priority.
AZ::IO::IStreamerTypes::Priority m_curPriority{ AZ::IO::IStreamerTypes::s_priorityMedium };
//! Synchronization for the read request, so that it's possible to block until completion.
AZStd::mutex m_readRequestMutex;
AZStd::condition_variable m_readRequestActive;
//! Track whether or not the stream is currently open
bool m_isOpen{ false };
@@ -92,6 +92,12 @@ namespace AZ::Data
result.Combine(resultHint);
}
if (SerializedAssetTracker* assetTracker = context.GetMetadata().Find<SerializedAssetTracker>();
assetTracker != nullptr && result.GetProcessing() == JSR::Processing::Completed)
{
assetTracker->AddAsset(*instance);
}
return context.Report(result,
result.GetProcessing() == JSR::Processing::Completed ? "Successfully stored Asset<T>." : "Failed to store Asset<T>.");
}
@@ -10,6 +10,7 @@
#include <AzCore/Asset/AssetManager_private.h>
#include <AzCore/Asset/AssetDataStream.h>
#include <AzCore/Casting/numeric_cast.h>
#include <AzCore/Debug/Profiler.h>
#include <AzCore/IO/IStreamer.h>
#include <AzCore/Math/Crc.h>
#include <AzCore/Math/MathUtils.h>
@@ -12,7 +12,6 @@
#include <AzCore/Asset/AssetContainer.h>
#include <AzCore/Asset/AssetDataStream.h>
#include <AzCore/Asset/AssetManagerBus.h>
#include <AzCore/IO/Streamer/FileRequest.h>
#include <AzCore/Memory/Memory.h>
#include <AzCore/Memory/SystemAllocator.h> // used as allocator for most components
#include <AzCore/std/parallel/mutex.h>
-11
View File
@@ -1,11 +0,0 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#define AZCORE_BUILD_NUMBER 368
#define AZCORE_BUILD_DATE "Thu 10/10/2013"
#define AZCORE_BUILD_TIME "19:42:16.96"
#define AZCORE_SOURCE_CHANGELIST 2992189
@@ -49,7 +49,7 @@ namespace AZ
return m_entity->GetId();
}
AZ_Warning("System", false, "Can't get component %p entity ID as it is not attached to an entity yet!", this);
AZ_Warning("System", false, "Can't get component (type: %s, addr: %p) entity ID as it is not attached to an entity yet!", RTTI_GetTypeName(), this);
return EntityId();
}
@@ -60,7 +60,7 @@ namespace AZ
return NamedEntityId(m_entity->GetId(), m_entity->GetName());
}
AZ_Warning("System", false, "Can't get component %p entity ID as it is not attached to an entity yet!", this);
AZ_Warning("System", false, "Can't get component (type: %s, addr: %p) entity ID as it is not attached to an entity yet!", RTTI_GetTypeName(), this);
return NamedEntityId();
}
@@ -152,8 +152,6 @@ namespace AZ
m_reservedDebug = 0;
m_recordingMode = Debug::AllocationRecords::RECORD_STACK_IF_NO_FILE_LINE;
m_stackRecordLevels = 5;
m_useOverrunDetection = false;
m_useMalloc = false;
}
bool AppDescriptorConverter(SerializeContext& serialize, SerializeContext::DataElementNode& node)
@@ -323,9 +321,6 @@ namespace AZ
->Field("blockSize", &Descriptor::m_memoryBlocksByteSize)
->Field("reservedOS", &Descriptor::m_reservedOS)
->Field("reservedDebug", &Descriptor::m_reservedDebug)
->Field("useOverrunDetection", &Descriptor::m_useOverrunDetection)
->Field("useMalloc", &Descriptor::m_useMalloc)
->Field("allocatorRemappings", &Descriptor::m_allocatorRemappings)
->Field("modules", &Descriptor::m_modules)
;
@@ -361,8 +356,6 @@ namespace AZ
->Attribute(Edit::Attributes::Step, &Descriptor::m_pageSize)
->DataElement(Edit::UIHandlers::SpinBox, &Descriptor::m_reservedOS, "OS reserved memory", "System memory reserved for OS (used only when 'Allocate all memory at startup' is true)")
->DataElement(Edit::UIHandlers::SpinBox, &Descriptor::m_reservedDebug, "Memory reserved for debugger", "System memory reserved for Debug allocator, like memory tracking (used only when 'Allocate all memory at startup' is true)")
->DataElement(Edit::UIHandlers::CheckBox, &Descriptor::m_useOverrunDetection, "Use Overrun Detection", "Use the overrun detection memory manager (only available on some platforms, ignored in Release builds)")
->DataElement(Edit::UIHandlers::CheckBox, &Descriptor::m_useMalloc, "Use Malloc", "Use malloc for memory allocations (for memory debugging only, ignored in Release builds)")
;
}
}
@@ -881,7 +874,7 @@ namespace AZ
AZ::AllocatorInstance<AZ::SystemAllocator>::Create(desc);
AZ::Debug::Trace::Instance().Init();
AZ::Debug::AllocationRecords* records = AllocatorInstance<SystemAllocator>::GetAllocator().GetRecords();
AZ::Debug::AllocationRecords* records = AllocatorInstance<SystemAllocator>::Get().GetRecords();
if (records)
{
records->SetMode(m_descriptor.m_recordingMode);
@@ -893,35 +886,6 @@ namespace AZ
m_isSystemAllocatorOwner = true;
}
#ifndef RELEASE
if (m_descriptor.m_useOverrunDetection)
{
OverrunDetectionSchema::Descriptor overrunDesc(false);
s_overrunDetectionSchema = Environment::CreateVariable<OverrunDetectionSchema>(AzTypeInfo<OverrunDetectionSchema>::Name(), overrunDesc);
OverrunDetectionSchema* schemaPtr = &s_overrunDetectionSchema.Get();
AZ::AllocatorManager::Instance().SetOverrideAllocatorSource(schemaPtr);
}
if (m_descriptor.m_useMalloc)
{
AZ_Printf("Malloc", "WARNING: Malloc override is enabled. Registered allocators will use malloc instead of their normal allocation schemas.");
s_mallocSchema = Environment::CreateVariable<MallocSchema>(AzTypeInfo<MallocSchema>::Name());
MallocSchema* schemaPtr = &s_mallocSchema.Get();
AZ::AllocatorManager::Instance().SetOverrideAllocatorSource(schemaPtr);
}
#endif
AllocatorManager& allocatorManager = AZ::AllocatorManager::Instance();
for (const auto& remapping : m_descriptor.m_allocatorRemappings)
{
allocatorManager.AddAllocatorRemapping(remapping.m_from.c_str(), remapping.m_to.c_str());
}
allocatorManager.FinalizeConfiguration();
}
void ComponentApplication::MergeSettingsToRegistry(SettingsRegistryInterface& registry)
@@ -142,10 +142,6 @@ namespace AZ
AZ::u64 m_reservedDebug; //!< Reserved memory for Debugging (allocation,etc.). Used only when m_grabAllMemory is set to true. (default: 0)
Debug::AllocationRecords::Mode m_recordingMode; //!< When to record stack traces (default: AZ::Debug::AllocationRecords::RECORD_STACK_IF_NO_FILE_LINE)
AZ::u64 m_stackRecordLevels; //!< If stack recording is enabled, how many stack levels to record. (default: 5)
bool m_useOverrunDetection; //!< True to use the overrun detection memory management scheme. Only available on some platforms; greatly increases memory consumption.
bool m_useMalloc; //!< True to use malloc instead of the internal memory manager. Intended for debugging purposes only.
AllocatorRemappings m_allocatorRemappings; //!< List of remappings of allocators to perform, so that they can alias each other.
ModuleDescriptorList m_modules; //!< Dynamic modules used by the application.
//!< These will be loaded on startup.
@@ -159,7 +155,7 @@ namespace AZ
//! If set, this allocator is used to allocate the temporary bootstrap memory, as well as the main \ref SystemAllocator heap.
//! If it's left nullptr (default), the \ref OSAllocator will be used.
IAllocatorAllocate* m_allocator = nullptr;
IAllocator* m_allocator = nullptr;
//! Callback to create AZ::Modules for the static libraries linked by this application.
//! Leave null if the application uses no static AZ::Modules.
@@ -372,7 +368,7 @@ namespace AZ
bool m_isOSAllocatorOwner{ false };
bool m_ownsConsole{};
void* m_fixedMemoryBlock{ nullptr }; //!< Pointer to the memory block allocator, so we can free it OnDestroy.
IAllocatorAllocate* m_osAllocator{ nullptr };
IAllocator* m_osAllocator{ nullptr };
EntitySetType m_entities;
AZ::SettingsRegistryInterface::NotifyEventHandler m_projectPathChangedHandler;
@@ -230,7 +230,7 @@ namespace AZ
EBUS_EVENT_ID(m_id, EntityBus, OnEntityDeactivated, m_id);
EBUS_EVENT(EntitySystemBus, OnEntityDeactivated, m_id);
AZ_Assert(m_state == State::Active, "Component should be in Active state to br Deactivated!");
AZ_Assert(m_state == State::Active, "Component should be in Active state to be Deactivated!");
SetState(State::Deactivating);
for (ComponentArrayType::reverse_iterator it = m_components.rbegin(); it != m_components.rend(); ++it)
@@ -15,7 +15,7 @@ struct z_stream_s;
namespace AZ
{
class IAllocator;
class IAllocatorAllocate;
class IAllocatorSchema;
/**
* The most well known and used compression algorithm. It gives the best compression ratios even on level 1,
@@ -90,7 +90,7 @@ namespace AZ
z_stream_s* m_strDeflate;
z_stream_s* m_strInflate;
IAllocatorAllocate* m_workMemoryAllocator;
IAllocatorSchema* m_workMemoryAllocator;
};
}
@@ -26,7 +26,7 @@ ZLib::ZLib(IAllocator* workMemAllocator)
: m_strDeflate(nullptr)
, m_strInflate(nullptr)
{
m_workMemoryAllocator = workMemAllocator ? workMemAllocator->GetAllocationSource() : nullptr;
m_workMemoryAllocator = workMemAllocator->GetSchema();
if (!m_workMemoryAllocator)
{
m_workMemoryAllocator = &AllocatorInstance<SystemAllocator>::Get();
@@ -55,7 +55,7 @@ ZLib::~ZLib()
//=========================================================================
void* ZLib::AllocateMem(void* userData, unsigned int items, unsigned int size)
{
IAllocatorAllocate* allocator = reinterpret_cast<IAllocatorAllocate*>(userData);
IAllocator* allocator = reinterpret_cast<IAllocator*>(userData);
return allocator->Allocate(items * size, 4, 0, "ZLib", __FILE__, __LINE__);
}
@@ -65,7 +65,7 @@ void* ZLib::AllocateMem(void* userData, unsigned int items, unsigned int size)
//=========================================================================
void ZLib::FreeMem(void* userData, void* address)
{
IAllocatorAllocate* allocator = reinterpret_cast<IAllocatorAllocate*>(userData);
IAllocator* allocator = reinterpret_cast<IAllocator*>(userData);
allocator->DeAllocate(address);
}
@@ -16,7 +16,7 @@
using namespace AZ;
ZStd::ZStd(IAllocatorAllocate* workMemAllocator)
ZStd::ZStd(IAllocator* workMemAllocator)
{
m_workMemoryAllocator = workMemAllocator;
if (!m_workMemoryAllocator)
@@ -41,13 +41,13 @@ ZStd::~ZStd()
void* ZStd::AllocateMem(void* userData, size_t size)
{
IAllocatorAllocate* allocator = reinterpret_cast<IAllocatorAllocate*>(userData);
IAllocator* allocator = reinterpret_cast<IAllocator*>(userData);
return allocator->Allocate(size, 4, 0, "ZStandard", __FILE__, __LINE__);
}
void ZStd::FreeMem(void* userData, void* address)
{
IAllocatorAllocate* allocator = reinterpret_cast<IAllocatorAllocate*>(userData);
IAllocator* allocator = reinterpret_cast<IAllocator*>(userData);
allocator->DeAllocate(address);
}
@@ -17,12 +17,11 @@
namespace AZ
{
class IAllocator;
class IAllocatorAllocate;
class ZStd
{
public:
ZStd(IAllocatorAllocate* workMemAllocator = 0);
ZStd(IAllocator* workMemAllocator = 0);
~ZStd();
enum FlushType
@@ -77,7 +76,7 @@ namespace AZ
ZSTD_CStream* m_streamCompression;
ZSTD_DStream* m_streamDecompression;
IAllocatorAllocate* m_workMemoryAllocator;
IAllocator* m_workMemoryAllocator;
ZSTD_inBuffer m_inBuffer;
ZSTD_outBuffer m_outBuffer;
size_t m_nextBlockSize;
@@ -53,7 +53,6 @@ namespace AZ::Dom
ValueAllocator()
: Base("DomValueAllocator", "Allocator for AZ::Dom::Value")
{
DisableOverriding();
}
};
+4 -11
View File
@@ -10,11 +10,6 @@
#include <AzCore/Debug/Budget.h>
#include <AzCore/Statistics/StatisticalProfilerProxy.h>
#ifdef USE_PIX
#include <AzCore/PlatformIncl.h>
#include <WinPixEventRuntime/pix3.h>
#endif
#if defined(AZ_PROFILER_MACRO_DISABLE) // by default we never disable the profiler registers as their overhead should be minimal, you can
// still do that for your code though.
#define AZ_PROFILE_SCOPE(...)
@@ -72,8 +67,7 @@ namespace AZ::Debug
Profiler() = default;
virtual ~Profiler() = default;
// support for the extra macro args (e.g. format strings) will come in a later PR
virtual void BeginRegion(const Budget* budget, const char* eventName) = 0;
virtual void BeginRegion(const Budget* budget, const char* eventName, size_t eventNameArgCount, ...) = 0;
virtual void EndRegion(const Budget* budget) = 0;
};
@@ -81,12 +75,11 @@ namespace AZ::Debug
{
public:
template<typename... T>
static void BeginRegion([[maybe_unused]] Budget* budget, [[maybe_unused]] const char* eventName, [[maybe_unused]] T const&... args);
static void EndRegion([[maybe_unused]] Budget* budget);
static void BeginRegion(Budget* budget, const char* eventName, T const&... args);
static void EndRegion(Budget* budget);
template<typename... T>
ProfileScope(Budget* budget, char const* eventName, T const&... args);
ProfileScope(Budget* budget, const char* eventName, T const&... args);
~ProfileScope();
+32 -26
View File
@@ -10,48 +10,52 @@
namespace AZ::Debug
{
namespace Platform
{
template<typename... T>
void BeginProfileRegion(Budget* budget, const char* eventName, T const&... args);
void BeginProfileRegion(Budget* budget, const char* eventName);
void EndProfileRegion(Budget* budget);
} // namespace Platform
template<typename... T>
void ProfileScope::BeginRegion(
[[maybe_unused]] Budget* budget, [[maybe_unused]] const char* eventName, [[maybe_unused]] T const&... args)
{
if (!budget)
#if !defined(_RELEASE)
if (budget)
{
return;
}
#if !defined(_RELEASE)
// TODO: Verification that the supplied system name corresponds to a known budget
#if defined(USE_PIX)
PIXBeginEvent(PIX_COLOR_INDEX(budget->Crc() & 0xff), eventName, args...);
#endif
budget->BeginProfileRegion();
Platform::BeginProfileRegion(budget, eventName, args...);
if (auto profiler = AZ::Interface<Profiler>::Get(); profiler)
{
profiler->BeginRegion(budget, eventName);
budget->BeginProfileRegion();
if (auto profiler = AZ::Interface<Profiler>::Get(); profiler)
{
profiler->BeginRegion(budget, eventName, sizeof...(T), args...);
}
}
#endif
#endif // !defined(_RELEASE)
}
inline void ProfileScope::EndRegion([[maybe_unused]] Budget* budget)
{
if (!budget)
#if !defined(_RELEASE)
if (budget)
{
return;
budget->EndProfileRegion();
if (auto profiler = AZ::Interface<Profiler>::Get(); profiler)
{
profiler->EndRegion(budget);
}
Platform::EndProfileRegion(budget);
}
#if !defined(_RELEASE)
budget->EndProfileRegion();
#if defined(USE_PIX)
PIXEndEvent();
#endif
if (auto profiler = AZ::Interface<Profiler>::Get(); profiler)
{
profiler->EndRegion(budget);
}
#endif
#endif // !defined(_RELEASE)
}
template<typename... T>
ProfileScope::ProfileScope(Budget* budget, char const* eventName, T const&... args)
ProfileScope::ProfileScope(Budget* budget, const char* eventName, T const&... args)
: m_budget{ budget }
{
BeginRegion(budget, eventName, args...);
@@ -63,3 +67,5 @@ namespace AZ::Debug
}
} // namespace AZ::Debug
#include <AzCore/Debug/Profiler_Platform.inl>
@@ -48,7 +48,7 @@ namespace AZ
public:
AZ_CLASS_ALLOCATOR(CompressorZStdData, AZ::SystemAllocator, 0);
CompressorZStdData(IAllocatorAllocate* zstdMemAllocator = 0)
CompressorZStdData(IAllocator* zstdMemAllocator = 0)
{
m_zstd = zstdMemAllocator;
}
+5 -1
View File
@@ -15,14 +15,18 @@
#include <AzCore/std/smart_ptr/shared_ptr.h>
#include <AzCore/std/string/string.h>
#include <AzCore/std/string/string_view.h>
#include <AzCore/std/any.h>
// These Streamer includes need to be moved to Streamer internals/implementation,
// and pull out only what we need for visibility at IStreamer.h interface declaration.
#include <AzCore/IO/Streamer/Statistics.h>
#include <AzCore/IO/Streamer/FileRequest.h>
namespace AZ::IO
{
class ExternalFileRequest;
class FileRequestHandle;
using FileRequestPtr = AZStd::intrusive_ptr<ExternalFileRequest>;
/**
* Data Streamer Interface
*/
@@ -21,7 +21,7 @@ namespace AZ::IO::IStreamerTypes
: m_allocator(AZ::AllocatorInstance<AZ::SystemAllocator>::Get())
{}
DefaultRequestMemoryAllocator::DefaultRequestMemoryAllocator(AZ::IAllocatorAllocate& allocator)
DefaultRequestMemoryAllocator::DefaultRequestMemoryAllocator(AZ::IAllocator& allocator)
: m_allocator(allocator)
{}
@@ -137,7 +137,7 @@ namespace AZ::IO::IStreamerTypes
public:
//! DefaultRequestMemoryAllocator wraps around the AZ::SystemAllocator by default.
DefaultRequestMemoryAllocator();
explicit DefaultRequestMemoryAllocator(AZ::IAllocatorAllocate& allocator);
explicit DefaultRequestMemoryAllocator(AZ::IAllocator& allocator);
~DefaultRequestMemoryAllocator() override;
void LockAllocator() override;
@@ -151,7 +151,7 @@ namespace AZ::IO::IStreamerTypes
private:
AZStd::atomic_int m_lockCounter{ 0 };
AZStd::atomic_int m_allocationCounter{ 0 };
AZ::IAllocatorAllocate& m_allocator;
AZ::IAllocator& m_allocator;
};
// The following alignment functions are put here until they're available in AzCore's math library.
+15 -15
View File
@@ -15,9 +15,9 @@ namespace AZ::IO
// Class template instantations
template class BasicPath<AZStd::string>;
template class BasicPath<FixedMaxPathString>;
template class PathIterator<PathView>;
template class PathIterator<Path>;
template class PathIterator<FixedMaxPath>;
template class PathIterator<const PathView>;
template class PathIterator<const Path>;
template class PathIterator<const FixedMaxPath>;
// Swap function instantiations
template void swap<AZStd::string>(Path& lhs, Path& rhs) noexcept;
@@ -38,16 +38,16 @@ namespace AZ::IO
const typename BasicPath<FixedMaxPathString>::value_type* rhs);
// Iterator compare instantiations
template bool operator==<PathView>(const PathIterator<PathView>& lhs,
const PathIterator<PathView>& rhs);
template bool operator==<Path>(const PathIterator<Path>& lhs,
const PathIterator<Path>& rhs);
template bool operator==<FixedMaxPath>(const PathIterator<FixedMaxPath>& lhs,
const PathIterator<FixedMaxPath>& rhs);
template bool operator!=<PathView>(const PathIterator<PathView>& lhs,
const PathIterator<PathView>& rhs);
template bool operator!=<Path>(const PathIterator<Path>& lhs,
const PathIterator<Path>& rhs);
template bool operator!=<FixedMaxPath>(const PathIterator<FixedMaxPath>& lhs,
const PathIterator<FixedMaxPath>& rhs);
template bool operator==<const PathView>(const PathIterator<const PathView>& lhs,
const PathIterator<const PathView>& rhs);
template bool operator==<const Path>(const PathIterator<const Path>& lhs,
const PathIterator<const Path>& rhs);
template bool operator==<const FixedMaxPath>(const PathIterator<const FixedMaxPath>& lhs,
const PathIterator<const FixedMaxPath>& rhs);
template bool operator!=<const PathView>(const PathIterator<const PathView>& lhs,
const PathIterator<const PathView>& rhs);
template bool operator!=<const Path>(const PathIterator<const Path>& lhs,
const PathIterator<const Path>& rhs);
template bool operator!=<const FixedMaxPath>(const PathIterator<const FixedMaxPath>& lhs,
const PathIterator<const FixedMaxPath>& rhs);
}
+11 -10
View File
@@ -43,9 +43,9 @@ namespace AZ::IO
public:
using string_view_type = AZStd::string_view;
using value_type = char;
using const_iterator = const PathIterator<PathView>;
using const_iterator = PathIterator<const PathView>;
using iterator = const_iterator;
friend PathIterator<PathView>;
friend const_iterator;
// constructors and destructor
constexpr PathView() = default;
@@ -319,9 +319,9 @@ namespace AZ::IO
using value_type = typename StringType::value_type;
using traits_type = typename StringType::traits_type;
using string_view_type = AZStd::string_view;
using const_iterator = const PathIterator<BasicPath>;
using const_iterator = PathIterator<const BasicPath>;
using iterator = const_iterator;
friend PathIterator<BasicPath>;
friend const_iterator;
// constructors and destructor
constexpr BasicPath() = default;
@@ -692,7 +692,7 @@ namespace AZ::IO
friend PathType;
using iterator_category = AZStd::bidirectional_iterator_tag;
using value_type = PathType;
using value_type = AZStd::remove_cv_t<PathType>;
using difference_type = ptrdiff_t;
using pointer = const value_type*;
using reference = const value_type&;
@@ -703,8 +703,9 @@ namespace AZ::IO
constexpr PathIterator() = default;
constexpr PathIterator(const PathIterator&) = default;
constexpr PathIterator(PathIterator&&) noexcept = default;
constexpr PathIterator& operator=(const PathIterator&) = default;
constexpr PathIterator& operator=(PathIterator&&) noexcept = default;
constexpr reference operator*() const;
@@ -733,10 +734,10 @@ namespace AZ::IO
ParserState m_state{ Singular };
};
template <typename PathType1>
constexpr bool operator==(const PathIterator<PathType1>& lhs, const PathIterator<PathType1>& rhs);
template <typename PathType1>
constexpr bool operator!=(const PathIterator<PathType1>& lhs, const PathIterator<PathType1>& rhs);
template <typename PathType>
constexpr bool operator==(const PathIterator<PathType>& lhs, const PathIterator<PathType>& rhs);
template <typename PathType>
constexpr bool operator!=(const PathIterator<PathType>& lhs, const PathIterator<PathType>& rhs);
}
#include <AzCore/IO/Path/Path.inl>
+16 -16
View File
@@ -399,7 +399,7 @@ namespace AZ::IO
constexpr auto PathView::begin() const -> const_iterator
{
auto pathParser = parser::PathParser::CreateBegin(m_path, m_preferred_separator);
PathIterator<PathView> it;
const_iterator it;
it.m_path_ref = this;
it.m_state = static_cast<typename const_iterator::ParserState>(pathParser.m_parser_state);
it.m_path_entry_view = pathParser.m_path_raw_entry;
@@ -409,7 +409,7 @@ namespace AZ::IO
constexpr auto PathView::end() const -> const_iterator
{
PathIterator<PathView> it;
const_iterator it;
it.m_state = const_iterator::AtEnd;
it.m_path_ref = this;
return it;
@@ -1262,7 +1262,7 @@ namespace AZ::IO
constexpr auto BasicPath<StringType>::begin() const -> const_iterator
{
auto pathParser = parser::PathParser::CreateBegin(m_path, m_preferred_separator);
PathIterator<BasicPath> it;
const_iterator it;
it.m_path_ref = this;
it.m_state = static_cast<typename const_iterator::ParserState>(pathParser.m_parser_state);
it.m_path_entry_view = pathParser.m_path_raw_entry;
@@ -1273,7 +1273,7 @@ namespace AZ::IO
template <typename StringType>
constexpr auto BasicPath<StringType>::end() const -> const_iterator
{
PathIterator<BasicPath> it;
const_iterator it;
it.m_state = const_iterator::AtEnd;
it.m_path_ref = this;
return it;
@@ -1529,16 +1529,16 @@ namespace AZ::IO
const typename BasicPath<FixedMaxPathString>::value_type* rhs);
// Iterator compare explicit declarations
extern template bool operator==<PathView>(const PathIterator<PathView>& lhs,
const PathIterator<PathView>& rhs);
extern template bool operator==<Path>(const PathIterator<Path>& lhs,
const PathIterator<Path>& rhs);
extern template bool operator==<FixedMaxPath>(const PathIterator<FixedMaxPath>& lhs,
const PathIterator<FixedMaxPath>& rhs);
extern template bool operator!=<PathView>(const PathIterator<PathView>& lhs,
const PathIterator<PathView>& rhs);
extern template bool operator!=<Path>(const PathIterator<Path>& lhs,
const PathIterator<Path>& rhs);
extern template bool operator!=<FixedMaxPath>(const PathIterator<FixedMaxPath>& lhs,
const PathIterator<FixedMaxPath>& rhs);
extern template bool operator==<const PathView>(const PathIterator<const PathView>& lhs,
const PathIterator<const PathView>& rhs);
extern template bool operator==<const Path>(const PathIterator<const Path>& lhs,
const PathIterator<const Path>& rhs);
extern template bool operator==<const FixedMaxPath>(const PathIterator<const FixedMaxPath>& lhs,
const PathIterator<const FixedMaxPath>& rhs);
extern template bool operator!=<const PathView>(const PathIterator<const PathView>& lhs,
const PathIterator<const PathView>& rhs);
extern template bool operator!=<const Path>(const PathIterator<const Path>& lhs,
const PathIterator<const Path>& rhs);
extern template bool operator!=<const FixedMaxPath>(const PathIterator<const FixedMaxPath>& lhs,
const PathIterator<const FixedMaxPath>& rhs);
}
@@ -10,6 +10,7 @@
#include <AzCore/AzCore_Traits_Platform.h>
#include <AzCore/Casting/numeric_cast.h>
#include <AzCore/std/concepts/concepts.h>
namespace AZ::IO::Internal
{
@@ -17,7 +18,7 @@ namespace AZ::IO::Internal
{
return elem == '/' || elem == '\\';
}
template <typename InputIt, typename EndIt, typename = AZStd::enable_if_t<AZStd::Internal::is_input_iterator_v<InputIt>>>
template <typename InputIt, typename EndIt, typename = AZStd::enable_if_t<AZStd::input_iterator<InputIt>>>
static constexpr bool HasDrivePrefix(InputIt first, EndIt last)
{
size_t prefixSize = AZStd::distance(first, last);
@@ -46,7 +47,7 @@ namespace AZ::IO::Internal
//! Windows root names can have include drive letter within them
template <typename InputIt>
constexpr auto ConsumeRootName(InputIt entryBeginIter, InputIt entryEndIter, const char preferredSeparator)
-> AZStd::enable_if_t<AZStd::Internal::is_forward_iterator_v<InputIt>, InputIt>
-> AZStd::enable_if_t<AZStd::forward_iterator<InputIt>, InputIt>
{
if (preferredSeparator == PosixPathSeparator)
{
@@ -147,7 +148,7 @@ namespace AZ::IO::Internal
//! If the preferred separator is '/' just checks if the path starts with a '/
//! Otherwise a check for a Windows absolute path occurs
//! Windows absolute paths can include a RootName
template <typename InputIt, typename EndIt, typename = AZStd::enable_if_t<AZStd::Internal::is_input_iterator_v<InputIt>>>
template <typename InputIt, typename EndIt, typename = AZStd::enable_if_t<AZStd::input_iterator<InputIt>>>
static constexpr bool IsAbsolute(InputIt first, EndIt last, const char preferredSeparator)
{
size_t pathSize = AZStd::distance(first, last);
@@ -208,11 +209,11 @@ namespace AZ::IO::parser
enum ParserState : uint8_t
{
// Zero is a special sentinel value used by default constructed iterators.
PS_BeforeBegin = PathIterator<PathView>::BeforeBegin,
PS_InRootName = PathIterator<PathView>::InRootName,
PS_InRootDir = PathIterator<PathView>::InRootDir,
PS_InFilenames = PathIterator<PathView>::InFilenames,
PS_AtEnd = PathIterator<PathView>::AtEnd
PS_BeforeBegin = PathView::const_iterator::BeforeBegin,
PS_InRootName = PathView::const_iterator::InRootName,
PS_InRootDir = PathView::const_iterator::InRootDir,
PS_InFilenames = PathView::const_iterator::InFilenames,
PS_AtEnd = PathView::const_iterator::AtEnd
};
struct PathParser
@@ -137,18 +137,18 @@ namespace AZ::IO
AZStd::visit([this, request](auto&& args)
{
using Command = AZStd::decay_t<decltype(args)>;
if constexpr (AZStd::is_same_v<Command, FileRequest::ReadData>)
if constexpr (AZStd::is_same_v<Command, Requests::ReadData>)
{
ReadFile(request, args);
return;
}
else
{
if constexpr (AZStd::is_same_v<Command, FileRequest::FlushData>)
if constexpr (AZStd::is_same_v<Command, Requests::FlushData>)
{
FlushCache(args.m_path);
}
else if constexpr (AZStd::is_same_v<Command, FileRequest::FlushAllData>)
else if constexpr (AZStd::is_same_v<Command, Requests::FlushAllData>)
{
FlushEntireCache();
}
@@ -166,7 +166,7 @@ namespace AZ::IO
{
Section& delayed = m_delayedSections.front();
AZ_Assert(delayed.m_parent, "Delayed section doesn't have a reference to the original request.");
auto data = AZStd::get_if<FileRequest::ReadData>(&delayed.m_parent->GetCommand());
auto data = AZStd::get_if<Requests::ReadData>(&delayed.m_parent->GetCommand());
AZ_Assert(data, "A request in the delayed queue of the BlockCache didn't have a parent with read data.");
// This call can add the same section to the back of the queue if there's not
// enough space. Because of this the entry needs to be removed from the delayed
@@ -233,7 +233,7 @@ namespace AZ::IO
}
}
void BlockCache::ReadFile(FileRequest* request, FileRequest::ReadData& data)
void BlockCache::ReadFile(FileRequest* request, Requests::ReadData& data)
{
if (!m_next)
{
@@ -250,7 +250,7 @@ namespace AZ::IO
m_numMetaDataRetrievalInProgress--;
if (fileSizeRequest.GetStatus() == IStreamerTypes::RequestStatus::Completed)
{
auto& requestInfo = AZStd::get<FileRequest::FileMetaDataRetrievalData>(fileSizeRequest.GetCommand());
auto& requestInfo = AZStd::get<Requests::FileMetaDataRetrievalData>(fileSizeRequest.GetCommand());
if (requestInfo.m_found)
{
ContinueReadFile(request, requestInfo.m_fileSize);
@@ -272,7 +272,7 @@ namespace AZ::IO
Section main;
Section epilog;
auto& data = AZStd::get<FileRequest::ReadData>(request->GetCommand());
auto& data = AZStd::get<Requests::ReadData>(request->GetCommand());
if (!SplitRequest(prolog, main, epilog, data.m_path, fileLength, data.m_offset, data.m_size,
reinterpret_cast<u8*>(data.m_output)))
@@ -21,6 +21,12 @@
namespace AZ::IO
{
class RequestPath;
namespace Requests
{
struct ReadData;
}
struct BlockCacheConfig final :
public IStreamerStackConfig
{
@@ -109,7 +115,7 @@ namespace AZ::IO
using TimePoint = AZStd::chrono::system_clock::time_point;
void ReadFile(FileRequest* request, FileRequest::ReadData& data);
void ReadFile(FileRequest* request, Requests::ReadData& data);
void ContinueReadFile(FileRequest* request, u64 fileLength);
CacheResult ReadFromCache(FileRequest* request, Section& section, const RequestPath& filePath);
CacheResult ReadFromCache(FileRequest* request, Section& section, u32 cacheBlock);
@@ -101,12 +101,12 @@ namespace AZ::IO
AZStd::visit([this, request](auto&& args)
{
using Command = AZStd::decay_t<decltype(args)>;
if constexpr (AZStd::is_same_v<Command, FileRequest::CreateDedicatedCacheData>)
if constexpr (AZStd::is_same_v<Command, Requests::CreateDedicatedCacheData>)
{
args.m_range = FileRange::CreateRangeForEntireFile();
m_context->PushPreparedRequest(request);
}
else if constexpr (AZStd::is_same_v<Command, FileRequest::DestroyDedicatedCacheData>)
else if constexpr (AZStd::is_same_v<Command, Requests::DestroyDedicatedCacheData>)
{
args.m_range = FileRange::CreateRangeForEntireFile();
m_context->PushPreparedRequest(request);
@@ -125,28 +125,28 @@ namespace AZ::IO
AZStd::visit([this, request](auto&& args)
{
using Command = AZStd::decay_t<decltype(args)>;
if constexpr (AZStd::is_same_v<Command, FileRequest::ReadData>)
if constexpr (AZStd::is_same_v<Command, Requests::ReadData>)
{
ReadFile(request, args);
return;
}
else if constexpr (AZStd::is_same_v<Command, FileRequest::CreateDedicatedCacheData>)
else if constexpr (AZStd::is_same_v<Command, Requests::CreateDedicatedCacheData>)
{
CreateDedicatedCache(request, args);
return;
}
else if constexpr (AZStd::is_same_v<Command, FileRequest::DestroyDedicatedCacheData>)
else if constexpr (AZStd::is_same_v<Command, Requests::DestroyDedicatedCacheData>)
{
DestroyDedicatedCache(request, args);
return;
}
else
{
if constexpr (AZStd::is_same_v<Command, FileRequest::FlushData>)
if constexpr (AZStd::is_same_v<Command, Requests::FlushData>)
{
FlushCache(args.m_path);
}
else if constexpr (AZStd::is_same_v<Command, FileRequest::FlushAllData>)
else if constexpr (AZStd::is_same_v<Command, Requests::FlushAllData>)
{
FlushEntireCache();
}
@@ -200,7 +200,7 @@ namespace AZ::IO
}
}
void DedicatedCache::ReadFile(FileRequest* request, FileRequest::ReadData& data)
void DedicatedCache::ReadFile(FileRequest* request, Requests::ReadData& data)
{
size_t index = FindCache(data.m_path, data.m_offset);
if (index == s_fileNotFound)
@@ -255,7 +255,7 @@ namespace AZ::IO
StreamStackEntry::CollectStatistics(statistics);
}
void DedicatedCache::CreateDedicatedCache(FileRequest* request, FileRequest::CreateDedicatedCacheData& data)
void DedicatedCache::CreateDedicatedCache(FileRequest* request, Requests::CreateDedicatedCacheData& data)
{
size_t index = FindCache(data.m_path, data.m_range);
if (index == s_fileNotFound)
@@ -276,7 +276,7 @@ namespace AZ::IO
m_context->MarkRequestAsCompleted(request);
}
void DedicatedCache::DestroyDedicatedCache(FileRequest* request, FileRequest::DestroyDedicatedCacheData& data)
void DedicatedCache::DestroyDedicatedCache(FileRequest* request, Requests::DestroyDedicatedCacheData& data)
{
size_t index = FindCache(data.m_path, data.m_range);
if (index != s_fileNotFound)
@@ -11,15 +11,21 @@
#include <AzCore/IO/Streamer/BlockCache.h>
#include <AzCore/IO/Streamer/FileRange.h>
#include <AzCore/IO/Streamer/Statistics.h>
#include <AzCore/IO/Streamer/StreamerConfiguration.h>
#include <AzCore/IO/Streamer/StreamStackEntry.h>
#include <AzCore/IO/Streamer/StreamerConfiguration.h>
#include <AzCore/Memory/SystemAllocator.h>
#include <AzCore/std/limits.h>
#include <AzCore/std/containers/vector.h>
#include <AzCore/std/limits.h>
#include <AzCore/std/smart_ptr/unique_ptr.h>
namespace AZ::IO
{
namespace Requests
{
struct CreateDedicatedCacheData;
struct DestroyDedicatedCacheData;
} // namespace Requests
struct DedicatedCacheConfig final :
public IStreamerStackConfig
{
@@ -56,16 +62,19 @@ namespace AZ::IO
void UpdateStatus(Status& status) const override;
void UpdateCompletionEstimates(AZStd::chrono::system_clock::time_point now, AZStd::vector<FileRequest*>& internalPending,
StreamerContext::PreparedQueue::iterator pendingBegin, StreamerContext::PreparedQueue::iterator pendingEnd) override;
void UpdateCompletionEstimates(
AZStd::chrono::system_clock::time_point now,
AZStd::vector<FileRequest*>& internalPending,
StreamerContext::PreparedQueue::iterator pendingBegin,
StreamerContext::PreparedQueue::iterator pendingEnd) override;
void CollectStatistics(AZStd::vector<Statistic>& statistics) const override;
private:
void CreateDedicatedCache(FileRequest* request, FileRequest::CreateDedicatedCacheData& data);
void DestroyDedicatedCache(FileRequest* request, FileRequest::DestroyDedicatedCacheData& data);
void CreateDedicatedCache(FileRequest* request, Requests::CreateDedicatedCacheData& data);
void DestroyDedicatedCache(FileRequest* request, Requests::DestroyDedicatedCacheData& data);
void ReadFile(FileRequest* request, FileRequest::ReadData& data);
void ReadFile(FileRequest* request, Requests::ReadData& data);
size_t FindCache(const RequestPath& filename, FileRange range);
size_t FindCache(const RequestPath& filename, u64 offset);
@@ -12,22 +12,30 @@
#include <AzCore/IO/Streamer/FileRequest.h>
#include <AzCore/IO/Streamer/StreamerContext.h>
namespace AZ::IO
//
// Command structures.
//
namespace AZ::IO::Requests
{
//
// Command structures.
//
ReadData::ReadData(void* output, u64 outputSize, const RequestPath& path, u64 offset, u64 size, bool sharedRead)
: m_path(path)
, m_output(output)
, m_outputSize(outputSize)
, m_offset(offset)
, m_size(size)
, m_sharedRead(sharedRead)
{
}
FileRequest::ExternalRequestData::ExternalRequestData(FileRequestPtr&& request)
: m_request(AZStd::move(request))
{}
FileRequest::RequestPathStoreData::RequestPathStoreData(RequestPath path)
: m_path(AZStd::move(path))
{}
FileRequest::ReadRequestData::ReadRequestData(RequestPath path, void* output, u64 outputSize, u64 offset, u64 size,
AZStd::chrono::system_clock::time_point deadline, IStreamerTypes::Priority priority)
ReadRequestData::ReadRequestData(
RequestPath path,
void* output,
u64 outputSize,
u64 offset,
u64 size,
AZStd::chrono::system_clock::time_point deadline,
IStreamerTypes::Priority priority)
: m_path(AZStd::move(path))
, m_allocator(nullptr)
, m_deadline(deadline)
@@ -37,10 +45,16 @@ namespace AZ::IO
, m_size(size)
, m_priority(priority)
, m_memoryType(IStreamerTypes::MemoryType::ReadWrite) // Only generic memory can be assigned externally.
{}
{
}
FileRequest::ReadRequestData::ReadRequestData(RequestPath path, IStreamerTypes::RequestMemoryAllocator* allocator,
u64 offset, u64 size, AZStd::chrono::system_clock::time_point deadline, IStreamerTypes::Priority priority)
ReadRequestData::ReadRequestData(
RequestPath path,
IStreamerTypes::RequestMemoryAllocator* allocator,
u64 offset,
u64 size,
AZStd::chrono::system_clock::time_point deadline,
IStreamerTypes::Priority priority)
: m_path(AZStd::move(path))
, m_allocator(allocator)
, m_deadline(deadline)
@@ -50,9 +64,10 @@ namespace AZ::IO
, m_size(size)
, m_priority(priority)
, m_memoryType(IStreamerTypes::MemoryType::ReadWrite) // Only generic memory can be assigned externally.
{}
{
}
FileRequest::ReadRequestData::~ReadRequestData()
ReadRequestData::~ReadRequestData()
{
if (m_allocator != nullptr)
{
@@ -64,65 +79,80 @@ namespace AZ::IO
}
}
FileRequest::ReadData::ReadData(void* output, u64 outputSize, const RequestPath& path, u64 offset, u64 size, bool sharedRead)
: m_output(output)
, m_outputSize(outputSize)
, m_path(path)
, m_offset(offset)
, m_size(size)
, m_sharedRead(sharedRead)
{}
CreateDedicatedCacheData::CreateDedicatedCacheData(RequestPath path, const FileRange& range)
: m_path(AZStd::move(path))
, m_range(range)
{
}
FileRequest::CompressedReadData::CompressedReadData(CompressionInfo&& compressionInfo, void* output, u64 readOffset, u64 readSize)
DestroyDedicatedCacheData::DestroyDedicatedCacheData(RequestPath path, const FileRange& range)
: m_path(AZStd::move(path))
, m_range(range)
{
}
ExternalRequestData::ExternalRequestData(FileRequestPtr&& request)
: m_request(AZStd::move(request))
{
}
RequestPathStoreData::RequestPathStoreData(RequestPath path)
: m_path(AZStd::move(path))
{
}
CompressedReadData::CompressedReadData(CompressionInfo&& compressionInfo, void* output, u64 readOffset, u64 readSize)
: m_compressionInfo(AZStd::move(compressionInfo))
, m_output(output)
, m_readOffset(readOffset)
, m_readSize(readSize)
{}
{
}
FileRequest::FileExistsCheckData::FileExistsCheckData(const RequestPath& path)
FileExistsCheckData::FileExistsCheckData(const RequestPath& path)
: m_path(path)
{}
{
}
FileRequest::FileMetaDataRetrievalData::FileMetaDataRetrievalData(const RequestPath& path)
FileMetaDataRetrievalData::FileMetaDataRetrievalData(const RequestPath& path)
: m_path(path)
{}
{
}
FileRequest::CancelData::CancelData(FileRequestPtr target)
CancelData::CancelData(FileRequestPtr target)
: m_target(AZStd::move(target))
{}
{
}
FileRequest::FlushData::FlushData(RequestPath path)
FlushData::FlushData(RequestPath path)
: m_path(AZStd::move(path))
{}
{
}
FileRequest::RescheduleData::RescheduleData(FileRequestPtr target, AZStd::chrono::system_clock::time_point newDeadline,
RescheduleData::RescheduleData(
FileRequestPtr target,
AZStd::chrono::system_clock::time_point newDeadline,
IStreamerTypes::Priority newPriority)
: m_target(AZStd::move(target))
, m_newDeadline(newDeadline)
, m_newPriority(newPriority)
{}
{
}
FileRequest::CreateDedicatedCacheData::CreateDedicatedCacheData(RequestPath path, const FileRange& range)
: m_path(AZStd::move(path))
, m_range(range)
{}
FileRequest::DestroyDedicatedCacheData::DestroyDedicatedCacheData(RequestPath path, const FileRange& range)
: m_path(AZStd::move(path))
, m_range(range)
{}
FileRequest::ReportData::ReportData(ReportType reportType)
ReportData::ReportData(ReportType reportType)
: m_reportType(reportType)
{}
{
}
FileRequest::CustomData::CustomData(AZStd::any data, bool failWhenUnhandled)
CustomData::CustomData(AZStd::any data, bool failWhenUnhandled)
: m_data(AZStd::move(data))
, m_failWhenUnhandled(failWhenUnhandled)
{}
{
}
} // namespace AZ::IO::Requests
namespace AZ::IO
{
//
// FileRequest
//
@@ -145,14 +175,14 @@ namespace AZ::IO
m_parent = request->m_request.m_parent;
request->m_request.m_parent = this;
m_dependencies++;
m_command.emplace<ExternalRequestData>(AZStd::move(request));
m_command.emplace<Requests::ExternalRequestData>(AZStd::move(request));
}
void FileRequest::CreateRequestPathStore(FileRequest* parent, RequestPath path)
{
AZ_Assert(AZStd::holds_alternative<AZStd::monostate>(m_command),
"Attempting to set FileRequest to 'CreateRequestPathStore', but another task was already assigned.");
m_command.emplace<RequestPathStoreData>(AZStd::move(path));
m_command.emplace<Requests::RequestPathStoreData>(AZStd::move(path));
SetOptionalParent(parent);
}
@@ -161,7 +191,7 @@ namespace AZ::IO
{
AZ_Assert(AZStd::holds_alternative<AZStd::monostate>(m_command),
"Attempting to set FileRequest to 'ReadRequest', but another task was already assigned.");
m_command.emplace<ReadRequestData>(AZStd::move(path), output, outputSize, offset, size, deadline, priority);
m_command.emplace<Requests::ReadRequestData>(AZStd::move(path), output, outputSize, offset, size, deadline, priority);
}
void FileRequest::CreateReadRequest(RequestPath path, IStreamerTypes::RequestMemoryAllocator* allocator, u64 offset, u64 size,
@@ -169,7 +199,7 @@ namespace AZ::IO
{
AZ_Assert(AZStd::holds_alternative<AZStd::monostate>(m_command),
"Attempting to set FileRequest to 'ReadRequest', but another task was already assigned.");
m_command.emplace<ReadRequestData>(AZStd::move(path), allocator, offset, size, deadline, priority);
m_command.emplace<Requests::ReadRequestData>(AZStd::move(path), allocator, offset, size, deadline, priority);
}
void FileRequest::CreateRead(FileRequest* parent, void* output, u64 outputSize, const RequestPath& path,
@@ -177,7 +207,7 @@ namespace AZ::IO
{
AZ_Assert(AZStd::holds_alternative<AZStd::monostate>(m_command),
"Attempting to set FileRequest to 'Read', but another task was already assigned.");
m_command.emplace<ReadData>(output, outputSize, AZStd::move(path), offset, size, sharedRead);
m_command.emplace<Requests::ReadData>(output, outputSize, AZStd::move(path), offset, size, sharedRead);
SetOptionalParent(parent);
}
@@ -192,7 +222,7 @@ namespace AZ::IO
{
AZ_Assert(AZStd::holds_alternative<AZStd::monostate>(m_command),
"Attempting to set FileRequest to 'CompressedRead', but another task was already assigned.");
m_command.emplace<CompressedReadData>(AZStd::move(compressionInfo), output, readOffset, readSize);
m_command.emplace<Requests::CompressedReadData>(AZStd::move(compressionInfo), output, readOffset, readSize);
SetOptionalParent(parent);
}
@@ -200,7 +230,7 @@ namespace AZ::IO
{
AZ_Assert(AZStd::holds_alternative<AZStd::monostate>(m_command),
"Attempting to set FileRequest to 'Wait', but another task was already assigned.");
m_command.emplace<WaitData>();
m_command.emplace<Requests::WaitData>();
SetOptionalParent(parent);
}
@@ -208,21 +238,21 @@ namespace AZ::IO
{
AZ_Assert(AZStd::holds_alternative<AZStd::monostate>(m_command),
"Attempting to set FileRequest to 'FileExistsCheck', but another task was already assigned.");
m_command.emplace<FileExistsCheckData>(path);
m_command.emplace<Requests::FileExistsCheckData>(path);
}
void FileRequest::CreateFileMetaDataRetrieval(const RequestPath& path)
{
AZ_Assert(AZStd::holds_alternative<AZStd::monostate>(m_command),
"Attempting to set FileRequest to 'FileMetaDataRetrieval', but another task was already assigned.");
m_command.emplace<FileMetaDataRetrievalData>(path);
m_command.emplace<Requests::FileMetaDataRetrievalData>(path);
}
void FileRequest::CreateCancel(FileRequestPtr target)
{
AZ_Assert(AZStd::holds_alternative<AZStd::monostate>(m_command),
"Attempting to set FileRequest to 'Cancel', but another task was already assigned.");
m_command.emplace<CancelData>(AZStd::move(target));
m_command.emplace<Requests::CancelData>(AZStd::move(target));
}
void FileRequest::CreateReschedule(FileRequestPtr target, AZStd::chrono::system_clock::time_point newDeadline,
@@ -230,28 +260,28 @@ namespace AZ::IO
{
AZ_Assert(AZStd::holds_alternative<AZStd::monostate>(m_command),
"Attempting to set FileRequest to 'Reschedule', but another task was already assigned.");
m_command.emplace<RescheduleData>(AZStd::move(target), newDeadline, newPriority);
m_command.emplace<Requests::RescheduleData>(AZStd::move(target), newDeadline, newPriority);
}
void FileRequest::CreateFlush(RequestPath path)
{
AZ_Assert(AZStd::holds_alternative<AZStd::monostate>(m_command),
"Attempting to set FileRequest to 'Flush', but another task was already assigned.");
m_command.emplace<FlushData>(AZStd::move(path));
m_command.emplace<Requests::FlushData>(AZStd::move(path));
}
void FileRequest::CreateFlushAll()
{
AZ_Assert(AZStd::holds_alternative<AZStd::monostate>(m_command),
"Attempting to set FileRequest to 'FlushAll', but another task was already assigned.");
m_command.emplace<FlushAllData>();
m_command.emplace<Requests::FlushAllData>();
}
void FileRequest::CreateDedicatedCacheCreation(RequestPath path, const FileRange& range, FileRequest* parent)
{
AZ_Assert(AZStd::holds_alternative<AZStd::monostate>(m_command),
"Attempting to set FileRequest to 'CreateDedicateCache', but another task was already assigned.");
m_command.emplace<CreateDedicatedCacheData>(AZStd::move(path), range);
m_command.emplace<Requests::CreateDedicatedCacheData>(AZStd::move(path), range);
SetOptionalParent(parent);
}
@@ -259,22 +289,22 @@ namespace AZ::IO
{
AZ_Assert(AZStd::holds_alternative<AZStd::monostate>(m_command),
"Attempting to set FileRequest to 'DestroyDedicateCache', but another task was already assigned.");
m_command.emplace<DestroyDedicatedCacheData>(AZStd::move(path), range);
m_command.emplace<Requests::DestroyDedicatedCacheData>(AZStd::move(path), range);
SetOptionalParent(parent);
}
void FileRequest::CreateReport(ReportData::ReportType reportType)
void FileRequest::CreateReport(Requests::ReportType reportType)
{
AZ_Assert(AZStd::holds_alternative<AZStd::monostate>(m_command),
"Attempting to set FileRequest to 'Report', but another task was already assigned.");
m_command.emplace<ReportData>(reportType);
m_command.emplace<Requests::ReportData>(reportType);
}
void FileRequest::CreateCustom(AZStd::any data, bool failWhenUnhandled, FileRequest* parent)
{
AZ_Assert(AZStd::holds_alternative<AZStd::monostate>(m_command),
"Attempting to set FileRequest to 'Custom', but another task was already assigned.");
m_command.emplace<CustomData>(AZStd::move(data), failWhenUnhandled);
m_command.emplace<Requests::CustomData>(AZStd::move(data), failWhenUnhandled);
SetOptionalParent(parent);
}
@@ -361,7 +391,7 @@ namespace AZ::IO
"Request does not contain a valid command. It may have been reset already or was never assigned a command.");
return true;
}
else if constexpr (AZStd::is_same_v<Command, FileRequest::CustomData>)
else if constexpr (AZStd::is_same_v<Command, Requests::CustomData>)
{
return args.m_failWhenUnhandled;
}
@@ -398,7 +428,7 @@ namespace AZ::IO
const FileRequest* current = this;
while (current)
{
auto* link = AZStd::get_if<ExternalRequestData>(&current->m_command);
auto* link = AZStd::get_if<Requests::ExternalRequestData>(&current->m_command);
if (!link)
{
current = current->m_parent;
@@ -27,7 +27,252 @@ namespace AZ::IO
class ExternalFileRequest;
using FileRequestPtr = AZStd::intrusive_ptr<ExternalFileRequest>;
} // namespace AZ::IO
namespace AZ::IO::Requests
{
//! Request to read data. This is a translated request and holds an absolute path and has been
//! resolved to the archive file if needed.
struct ReadData
{
inline constexpr static IStreamerTypes::Priority s_orderPriority = IStreamerTypes::s_priorityMedium;
inline constexpr static bool s_failWhenUnhandled = true;
ReadData(void* output, u64 outputSize, const RequestPath& path, u64 offset, u64 size, bool sharedRead);
const RequestPath& m_path; //!< The path to the file that contains the requested data.
void* m_output; //!< Target output to write the read data to.
u64 m_outputSize; //!< Size of memory m_output points to. This needs to be at least as big as m_size, but can be bigger.
u64 m_offset; //!< The offset in bytes into the file.
u64 m_size; //!< The number of bytes to read from the file.
bool m_sharedRead; //!< True if other code will be reading from the file or the stack entry can exclusively lock.
};
//! Request to read data. This is an untranslated request and holds a relative path. The Scheduler
//! will translate this to the appropriate ReadData or CompressedReadData.
struct ReadRequestData
{
inline constexpr static IStreamerTypes::Priority s_orderPriority = IStreamerTypes::s_priorityMedium;
inline constexpr static bool s_failWhenUnhandled = true;
ReadRequestData(
RequestPath path,
void* output,
u64 outputSize,
u64 offset,
u64 size,
AZStd::chrono::system_clock::time_point deadline,
IStreamerTypes::Priority priority);
ReadRequestData(
RequestPath path,
IStreamerTypes::RequestMemoryAllocator* allocator,
u64 offset,
u64 size,
AZStd::chrono::system_clock::time_point deadline,
IStreamerTypes::Priority priority);
~ReadRequestData();
RequestPath m_path; //!< Relative path to the target file.
IStreamerTypes::RequestMemoryAllocator* m_allocator; //!< Allocator used to manage the memory for this request.
AZStd::chrono::system_clock::time_point m_deadline; //!< Time by which this request should have been completed.
void* m_output; //!< The memory address assigned (during processing) to store the read data to.
u64 m_outputSize; //!< The memory size of the addressed used to store the read data.
u64 m_offset; //!< The offset in bytes into the file.
u64 m_size; //!< The number of bytes to read from the file.
IStreamerTypes::Priority m_priority; //!< Priority used for ordering requests. This is used when requests have the same deadline.
IStreamerTypes::MemoryType m_memoryType; //!< The type of memory provided by the allocator if used.
};
//! Creates a cache dedicated to a single file. This is best used for files where blocks are read from
//! periodically such as audio banks of video files.
struct CreateDedicatedCacheData
{
inline constexpr static IStreamerTypes::Priority s_orderPriority = IStreamerTypes::s_priorityHigh;
inline constexpr static bool s_failWhenUnhandled = false;
CreateDedicatedCacheData(RequestPath path, const FileRange& range);
RequestPath m_path;
FileRange m_range;
};
//! Destroys a cache dedicated to a single file that was previously created by CreateDedicatedCache
struct DestroyDedicatedCacheData
{
inline constexpr static IStreamerTypes::Priority s_orderPriority = IStreamerTypes::s_priorityHigh;
inline constexpr static bool s_failWhenUnhandled = false;
DestroyDedicatedCacheData(RequestPath path, const FileRange& range);
RequestPath m_path;
FileRange m_range;
};
enum class ReportType : int8_t
{
FileLocks
};
struct ReportData
{
inline constexpr static IStreamerTypes::Priority s_orderPriority = IStreamerTypes::s_priorityLow;
inline constexpr static bool s_failWhenUnhandled = false;
explicit ReportData(ReportType reportType);
ReportType m_reportType;
};
//! Stores a reference to the external request so it stays alive while the request is being processed.
//! This is needed because Streamer supports fire-and-forget requests since completion can be handled by
//! registering a callback.
struct ExternalRequestData
{
inline constexpr static IStreamerTypes::Priority s_orderPriority = IStreamerTypes::s_priorityMedium;
inline constexpr static bool s_failWhenUnhandled = true;
explicit ExternalRequestData(FileRequestPtr&& request);
FileRequestPtr m_request; //!< The request that was send to Streamer.
};
//! Stores an instance of a RequestPath. To reduce copying instances of a RequestPath functions that
//! need a path take them by reference to the original request. In some cases a path originates from
//! within in the stack and temporary storage is needed. This struct allows for that temporary storage
//! so it can be safely referenced later.
struct RequestPathStoreData
{
inline constexpr static IStreamerTypes::Priority s_orderPriority = IStreamerTypes::s_priorityMedium;
inline constexpr static bool s_failWhenUnhandled = true;
explicit RequestPathStoreData(RequestPath path);
RequestPath m_path;
};
//! Request to read and decompress data.
struct CompressedReadData
{
inline constexpr static IStreamerTypes::Priority s_orderPriority = IStreamerTypes::s_priorityMedium;
inline constexpr static bool s_failWhenUnhandled = true;
CompressedReadData(CompressionInfo&& compressionInfo, void* output, u64 readOffset, u64 readSize);
CompressionInfo m_compressionInfo;
void* m_output; //!< Target output to write the read data to.
u64 m_readOffset; //!< The offset into the decompressed to start copying from.
u64 m_readSize; //!< Number of bytes to read from the decompressed file.
};
//! Holds the progress of an operation chain until this request is explicitly completed.
struct WaitData
{
inline constexpr static IStreamerTypes::Priority s_orderPriority = IStreamerTypes::s_priorityMedium;
inline constexpr static bool s_failWhenUnhandled = true;
};
//! Checks to see if any node in the stack can find a file at the provided path.
struct FileExistsCheckData
{
inline constexpr static IStreamerTypes::Priority s_orderPriority = IStreamerTypes::s_priorityHigh;
inline constexpr static bool s_failWhenUnhandled = false;
explicit FileExistsCheckData(const RequestPath& path);
const RequestPath& m_path;
bool m_found{ false };
};
//! Searches for a file in the stack and retrieves the meta data. This may be slower than a file exists
//! check.
struct FileMetaDataRetrievalData
{
inline constexpr static IStreamerTypes::Priority s_orderPriority = IStreamerTypes::s_priorityHigh;
inline constexpr static bool s_failWhenUnhandled = false;
explicit FileMetaDataRetrievalData(const RequestPath& path);
const RequestPath& m_path;
u64 m_fileSize{ 0 };
bool m_found{ false };
};
//! Cancels a request in the stream stack, if possible.
struct CancelData
{
inline constexpr static IStreamerTypes::Priority s_orderPriority = IStreamerTypes::s_priorityHighest;
inline constexpr static bool s_failWhenUnhandled = false;
explicit CancelData(FileRequestPtr target);
FileRequestPtr m_target; //!< The request that will be canceled.
};
//! Updates the priority and deadline of a request that has not been queued yet.
struct RescheduleData
{
inline constexpr static IStreamerTypes::Priority s_orderPriority = IStreamerTypes::s_priorityHigh;
inline constexpr static bool s_failWhenUnhandled = false;
RescheduleData(FileRequestPtr target, AZStd::chrono::system_clock::time_point newDeadline, IStreamerTypes::Priority newPriority);
FileRequestPtr m_target; //!< The request that will be rescheduled.
AZStd::chrono::system_clock::time_point m_newDeadline; //!< The new deadline for the request.
IStreamerTypes::Priority m_newPriority; //!< The new priority for the request.
};
//! Flushes all references to the provided file in the streaming stack.
struct FlushData
{
inline constexpr static IStreamerTypes::Priority s_orderPriority = IStreamerTypes::s_priorityHigh;
inline constexpr static bool s_failWhenUnhandled = false;
explicit FlushData(RequestPath path);
RequestPath m_path;
};
//! Flushes all caches in the streaming stack.
struct FlushAllData
{
inline constexpr static IStreamerTypes::Priority s_orderPriority = IStreamerTypes::s_priorityHigh;
inline constexpr static bool s_failWhenUnhandled = false;
};
//! Data for a custom command. This can be used by nodes added extensions that need data that can't be stored
//! in the already provided data.
struct CustomData
{
inline constexpr static IStreamerTypes::Priority s_orderPriority = IStreamerTypes::s_priorityMedium;
CustomData(AZStd::any data, bool failWhenUnhandled);
AZStd::any m_data; //!< The data for the custom request.
bool m_failWhenUnhandled; //!< Whether or not the request is marked as failed or success when no node process it.
};
using CommandVariant = AZStd::variant<
AZStd::monostate,
ExternalRequestData,
RequestPathStoreData,
ReadRequestData,
ReadData,
CompressedReadData,
WaitData,
FileExistsCheckData,
FileMetaDataRetrievalData,
CancelData,
RescheduleData,
FlushData,
FlushAllData,
CreateDedicatedCacheData,
DestroyDedicatedCacheData,
ReportData,
CustomData>;
} // namespace AZ::IO::Requests
namespace AZ::IO
{
class FileRequest final
{
public:
@@ -36,218 +281,7 @@ namespace AZ::IO
friend class StreamerContext;
friend class ExternalFileRequest;
//! Stores a reference to the external request so it stays alive while the request is being processed.
//! This is needed because Streamer supports fire-and-forget requests since completion can be handled by
//! registering a callback.
struct ExternalRequestData
{
inline constexpr static IStreamerTypes::Priority s_orderPriority = IStreamerTypes::s_priorityMedium;
inline constexpr static bool s_failWhenUnhandled = true;
explicit ExternalRequestData(FileRequestPtr&& request);
FileRequestPtr m_request; //!< The request that was send to Streamer.
};
//! Stores an instance of a RequestPath. To reduce copying instances of a RequestPath functions that
//! need a path take them by reference to the original request. In some cases a path originates from
//! within in the stack and temporary storage is needed. This struct allows for that temporary storage
//! so it can be safely referenced later.
struct RequestPathStoreData
{
inline constexpr static IStreamerTypes::Priority s_orderPriority = IStreamerTypes::s_priorityMedium;
inline constexpr static bool s_failWhenUnhandled = true;
explicit RequestPathStoreData(RequestPath path);
RequestPath m_path;
};
//! Request to read data. This is an untranslated request and holds a relative path. The Scheduler
//! will translate this to the appropriate ReadData or CompressedReadData.
struct ReadRequestData
{
inline constexpr static IStreamerTypes::Priority s_orderPriority = IStreamerTypes::s_priorityMedium;
inline constexpr static bool s_failWhenUnhandled = true;
ReadRequestData(RequestPath path, void* output, u64 outputSize, u64 offset, u64 size,
AZStd::chrono::system_clock::time_point deadline, IStreamerTypes::Priority priority);
ReadRequestData(RequestPath path, IStreamerTypes::RequestMemoryAllocator* allocator, u64 offset, u64 size,
AZStd::chrono::system_clock::time_point deadline, IStreamerTypes::Priority priority);
~ReadRequestData();
RequestPath m_path; //!< Relative path to the target file.
IStreamerTypes::RequestMemoryAllocator* m_allocator; //!< Allocator used to manage the memory for this request.
AZStd::chrono::system_clock::time_point m_deadline; //!< Time by which this request should have been completed.
void* m_output; //!< The memory address assigned (during processing) to store the read data to.
u64 m_outputSize; //!< The memory size of the addressed used to store the read data.
u64 m_offset; //!< The offset in bytes into the file.
u64 m_size; //!< The number of bytes to read from the file.
IStreamerTypes::Priority m_priority; //!< Priority used for ordering requests. This is used when requests have the same deadline.
IStreamerTypes::MemoryType m_memoryType; //!< The type of memory provided by the allocator if used.
};
//! Request to read data. This is a translated request and holds an absolute path and has been
//! resolved to the archive file if needed.
struct ReadData
{
inline constexpr static IStreamerTypes::Priority s_orderPriority = IStreamerTypes::s_priorityMedium;
inline constexpr static bool s_failWhenUnhandled = true;
ReadData(void* output, u64 outputSize, const RequestPath& path, u64 offset, u64 size, bool sharedRead);
const RequestPath& m_path; //!< The path to the file that contains the requested data.
void* m_output; //!< Target output to write the read data to.
u64 m_outputSize; //!< Size of memory m_output points to. This needs to be at least as big as m_size, but can be bigger.
u64 m_offset; //!< The offset in bytes into the file.
u64 m_size; //!< The number of bytes to read from the file.
bool m_sharedRead; //!< True if other code will be reading from the file or the stack entry can exclusively lock.
};
//! Request to read and decompress data.
struct CompressedReadData
{
inline constexpr static IStreamerTypes::Priority s_orderPriority = IStreamerTypes::s_priorityMedium;
inline constexpr static bool s_failWhenUnhandled = true;
CompressedReadData(CompressionInfo&& compressionInfo, void* output, u64 readOffset, u64 readSize);
CompressionInfo m_compressionInfo;
void* m_output; //!< Target output to write the read data to.
u64 m_readOffset; //!< The offset into the decompressed to start copying from.
u64 m_readSize; //!< Number of bytes to read from the decompressed file.
};
//! Holds the progress of an operation chain until this request is explicitly completed.
struct WaitData
{
inline constexpr static IStreamerTypes::Priority s_orderPriority = IStreamerTypes::s_priorityMedium;
inline constexpr static bool s_failWhenUnhandled = true;
};
//! Checks to see if any node in the stack can find a file at the provided path.
struct FileExistsCheckData
{
inline constexpr static IStreamerTypes::Priority s_orderPriority = IStreamerTypes::s_priorityHigh;
inline constexpr static bool s_failWhenUnhandled = false;
explicit FileExistsCheckData(const RequestPath& path);
const RequestPath& m_path;
bool m_found{ false };
};
//! Searches for a file in the stack and retrieves the meta data. This may be slower than a file exists
//! check.
struct FileMetaDataRetrievalData
{
inline constexpr static IStreamerTypes::Priority s_orderPriority = IStreamerTypes::s_priorityHigh;
inline constexpr static bool s_failWhenUnhandled = false;
explicit FileMetaDataRetrievalData(const RequestPath& path);
const RequestPath& m_path;
u64 m_fileSize{ 0 };
bool m_found{ false };
};
//! Cancels a request in the stream stack, if possible.
struct CancelData
{
inline constexpr static IStreamerTypes::Priority s_orderPriority = IStreamerTypes::s_priorityHighest;
inline constexpr static bool s_failWhenUnhandled = false;
explicit CancelData(FileRequestPtr target);
FileRequestPtr m_target; //!< The request that will be canceled.
};
//! Updates the priority and deadline of a request that has not been queued yet.
struct RescheduleData
{
inline constexpr static IStreamerTypes::Priority s_orderPriority = IStreamerTypes::s_priorityHigh;
inline constexpr static bool s_failWhenUnhandled = false;
RescheduleData(FileRequestPtr target, AZStd::chrono::system_clock::time_point newDeadline, IStreamerTypes::Priority newPriority);
FileRequestPtr m_target; //!< The request that will be rescheduled.
AZStd::chrono::system_clock::time_point m_newDeadline; //!< The new deadline for the request.
IStreamerTypes::Priority m_newPriority; //!< The new priority for the request.
};
//! Flushes all references to the provided file in the streaming stack.
struct FlushData
{
inline constexpr static IStreamerTypes::Priority s_orderPriority = IStreamerTypes::s_priorityHigh;
inline constexpr static bool s_failWhenUnhandled = false;
explicit FlushData(RequestPath path);
RequestPath m_path;
};
//! Flushes all caches in the streaming stack.
struct FlushAllData
{
inline constexpr static IStreamerTypes::Priority s_orderPriority = IStreamerTypes::s_priorityHigh;
inline constexpr static bool s_failWhenUnhandled = false;
};
//! Creates a cache dedicated to a single file. This is best used for files where blocks are read from
//! periodically such as audio banks of video files.
struct CreateDedicatedCacheData
{
inline constexpr static IStreamerTypes::Priority s_orderPriority = IStreamerTypes::s_priorityHigh;
inline constexpr static bool s_failWhenUnhandled = false;
CreateDedicatedCacheData(RequestPath path, const FileRange& range);
RequestPath m_path;
FileRange m_range;
};
//! Destroys a cache dedicated to a single file that was previously created by CreateDedicatedCache
struct DestroyDedicatedCacheData
{
inline constexpr static IStreamerTypes::Priority s_orderPriority = IStreamerTypes::s_priorityHigh;
inline constexpr static bool s_failWhenUnhandled = false;
DestroyDedicatedCacheData(RequestPath path, const FileRange& range);
RequestPath m_path;
FileRange m_range;
};
struct ReportData
{
inline constexpr static IStreamerTypes::Priority s_orderPriority = IStreamerTypes::s_priorityLow;
inline constexpr static bool s_failWhenUnhandled = false;
enum class ReportType
{
FileLocks
};
explicit ReportData(ReportType reportType);
ReportType m_reportType;
};
//! Data for a custom command. This can be used by nodes added extensions that need data that can't be stored
//! in the already provided data.
struct CustomData
{
inline constexpr static IStreamerTypes::Priority s_orderPriority = IStreamerTypes::s_priorityMedium;
CustomData(AZStd::any data, bool failWhenUnhandled);
AZStd::any m_data; //!< The data for the custom request.
bool m_failWhenUnhandled; //!< Whether or not the request is marked as failed or success when no node process it.
};
using CommandVariant = AZStd::variant<AZStd::monostate, ExternalRequestData, RequestPathStoreData, ReadRequestData, ReadData,
CompressedReadData, WaitData, FileExistsCheckData, FileMetaDataRetrievalData, CancelData, RescheduleData, FlushData,
FlushAllData, CreateDedicatedCacheData, DestroyDedicatedCacheData, ReportData, CustomData>;
using CommandVariant = Requests::CommandVariant;
using OnCompletionCallback = AZStd::function<void(FileRequest& request)>;
AZ_CLASS_ALLOCATOR(FileRequest, SystemAllocator, 0);
@@ -278,7 +312,7 @@ namespace AZ::IO
void CreateFlushAll();
void CreateDedicatedCacheCreation(RequestPath path, const FileRange& range = {}, FileRequest* parent = nullptr);
void CreateDedicatedCacheDestruction(RequestPath path, const FileRange& range = {}, FileRequest* parent = nullptr);
void CreateReport(ReportData::ReportType reportType);
void CreateReport(Requests::ReportType reportType);
void CreateCustom(AZStd::any data, bool failWhenUnhandled = true, FileRequest* parent = nullptr);
void SetCompletionCallback(OnCompletionCallback callback);
@@ -325,8 +359,17 @@ namespace AZ::IO
//! Command and parameters for the request.
CommandVariant m_command;
//! Status of the request.
AZStd::atomic<IStreamerTypes::RequestStatus> m_status{ IStreamerTypes::RequestStatus::Pending };
//! Estimated time this request will complete. This is an estimation and depends on many
//! factors which can cause it to change drastically from moment to moment.
AZStd::chrono::system_clock::time_point m_estimatedCompletion;
//! The file request that has a dependency on this one. This can be null if there are no
//! other request depending on this one to complete.
FileRequest* m_parent{ nullptr };
//! Id assigned when the request is added to the pending queue.
size_t m_pendingId{ 0 };
//! Called once the request has completed. This will always be called from the Streamer thread
//! and thread safety is the responsibility of called function. When assigning a lambda avoid
@@ -336,16 +379,8 @@ namespace AZ::IO
//! a longer running task is needed consider using a job to do the work.
OnCompletionCallback m_onCompletion;
//! Estimated time this request will complete. This is an estimation and depends on many
//! factors which can cause it to change drastically from moment to moment.
AZStd::chrono::system_clock::time_point m_estimatedCompletion;
//! The file request that has a dependency on this one. This can be null if there are no
//! other request depending on this one to complete.
FileRequest* m_parent{ nullptr };
//! Id assigned when the request is added to the pending queue.
size_t m_pendingId{ 0 };
//! Status of the request.
AZStd::atomic<IStreamerTypes::RequestStatus> m_status{ IStreamerTypes::RequestStatus::Pending };
//! The number of dependent file request that need to complete before this one is done.
u16 m_dependencies{ 0 };
@@ -91,12 +91,12 @@ namespace AZ::IO
AZStd::visit([this, request](auto&& args)
{
using Command = AZStd::decay_t<decltype(args)>;
if constexpr (AZStd::is_same_v<Command, FileRequest::ReadRequestData>)
if constexpr (AZStd::is_same_v<Command, Requests::ReadRequestData>)
{
PrepareReadRequest(request, args);
}
else if constexpr (AZStd::is_same_v<Command, FileRequest::CreateDedicatedCacheData> ||
AZStd::is_same_v<Command, FileRequest::DestroyDedicatedCacheData>)
else if constexpr (AZStd::is_same_v<Command, Requests::CreateDedicatedCacheData> ||
AZStd::is_same_v<Command, Requests::DestroyDedicatedCacheData>)
{
PrepareDedicatedCache(request, args.m_path);
}
@@ -114,11 +114,11 @@ namespace AZ::IO
AZStd::visit([this, request](auto&& args)
{
using Command = AZStd::decay_t<decltype(args)>;
if constexpr (AZStd::is_same_v<Command, FileRequest::CompressedReadData>)
if constexpr (AZStd::is_same_v<Command, Requests::CompressedReadData>)
{
m_pendingReads.push_back(request);
}
else if constexpr (AZStd::is_same_v<Command, FileRequest::FileExistsCheckData>)
else if constexpr (AZStd::is_same_v<Command, Requests::FileExistsCheckData>)
{
m_pendingFileExistChecks.push_back(request);
}
@@ -203,7 +203,7 @@ namespace AZ::IO
{
FileRequest* compressedRequest = m_processingJobs[i].m_waitRequest->GetParent();
AZ_Assert(compressedRequest, "A wait request attached to FullFileDecompressor was completed but didn't have a parent compressed request.");
auto data = AZStd::get_if<FileRequest::CompressedReadData>(&compressedRequest->GetCommand());
auto data = AZStd::get_if<Requests::CompressedReadData>(&compressedRequest->GetCommand());
AZ_Assert(data, "Compressed request in the decompression queue in FullFileDecompressor didn't contain compression read data.");
size_t bytesToDecompress = data->m_compressionInfo.m_compressedSize;
@@ -255,7 +255,7 @@ namespace AZ::IO
// Calculate the amount of time it will take to decompress the data.
FileRequest* compressedRequest = m_readRequests[i]->GetParent();
auto data = AZStd::get_if<FileRequest::CompressedReadData>(&compressedRequest->GetCommand());
auto data = AZStd::get_if<Requests::CompressedReadData>(&compressedRequest->GetCommand());
size_t bytesToDecompress = data->m_compressionInfo.m_compressedSize;
auto decompressionDuration = AZStd::chrono::microseconds(
@@ -290,7 +290,7 @@ namespace AZ::IO
void FullFileDecompressor::EstimateCompressedReadRequest(FileRequest* request, AZStd::chrono::microseconds& cumulativeDelay,
AZStd::chrono::microseconds decompressionDelay, double totalDecompressionDurationUs, double totalBytesDecompressed) const
{
auto data = AZStd::get_if<FileRequest::CompressedReadData>(&request->GetCommand());
auto data = AZStd::get_if<Requests::CompressedReadData>(&request->GetCommand());
if (data)
{
AZStd::chrono::microseconds processingTime = decompressionDelay;
@@ -343,7 +343,7 @@ namespace AZ::IO
m_numRunningJobs == 0;
}
void FullFileDecompressor::PrepareReadRequest(FileRequest* request, FileRequest::ReadRequestData& data)
void FullFileDecompressor::PrepareReadRequest(FileRequest* request, Requests::ReadRequestData& data)
{
CompressionInfo info;
if (CompressionUtils::FindCompressionInfo(info, data.m_path.GetRelativePath()))
@@ -359,7 +359,7 @@ namespace AZ::IO
{
FileRequest* pathStorageRequest = m_context->GetNewInternalRequest();
pathStorageRequest->CreateRequestPathStore(request, AZStd::move(info.m_archiveFilename));
auto& pathStorage = AZStd::get<FileRequest::RequestPathStoreData>(pathStorageRequest->GetCommand());
auto& pathStorage = AZStd::get<Requests::RequestPathStoreData>(pathStorageRequest->GetCommand());
nextRequest->CreateRead(pathStorageRequest, data.m_output, data.m_outputSize, pathStorage.m_path,
info.m_offset + data.m_offset, data.m_size, info.m_isSharedPak);
@@ -370,13 +370,13 @@ namespace AZ::IO
auto callback = [this, nextRequest](const FileRequest& checkRequest)
{
AZ_PROFILE_FUNCTION(AzCore);
auto check = AZStd::get_if<FileRequest::FileExistsCheckData>(&checkRequest.GetCommand());
auto check = AZStd::get_if<Requests::FileExistsCheckData>(&checkRequest.GetCommand());
AZ_Assert(check,
"Callback in FullFileDecompressor::PrepareReadRequest expected FileExistsCheck but got another command.");
if (check->m_found)
{
FileRequest* originalRequest = m_context->RejectRequest(nextRequest);
if (AZStd::holds_alternative<FileRequest::RequestPathStoreData>(originalRequest->GetCommand()))
if (AZStd::holds_alternative<Requests::RequestPathStoreData>(originalRequest->GetCommand()))
{
originalRequest = m_context->RejectRequest(originalRequest);
}
@@ -412,12 +412,12 @@ namespace AZ::IO
AZStd::visit([request, &info, nextRequest](auto&& args)
{
using Command = AZStd::decay_t<decltype(args)>;
if constexpr (AZStd::is_same_v<Command, FileRequest::CreateDedicatedCacheData>)
if constexpr (AZStd::is_same_v<Command, Requests::CreateDedicatedCacheData>)
{
nextRequest->CreateDedicatedCacheCreation(AZStd::move(info.m_archiveFilename),
FileRange::CreateRange(info.m_offset, info.m_compressedSize), request);
}
else if constexpr (AZStd::is_same_v<Command, FileRequest::DestroyDedicatedCacheData>)
else if constexpr (AZStd::is_same_v<Command, Requests::DestroyDedicatedCacheData>)
{
nextRequest->CreateDedicatedCacheDestruction(AZStd::move(info.m_archiveFilename),
FileRange::CreateRange(info.m_offset, info.m_compressedSize), request);
@@ -429,7 +429,7 @@ namespace AZ::IO
auto callback = [this, nextRequest](const FileRequest& checkRequest)
{
AZ_PROFILE_FUNCTION(AzCore);
auto check = AZStd::get_if<FileRequest::FileExistsCheckData>(&checkRequest.GetCommand());
auto check = AZStd::get_if<Requests::FileExistsCheckData>(&checkRequest.GetCommand());
AZ_Assert(check,
"Callback in FullFileDecompressor::PrepareDedicatedCache expected FileExistsCheck but got another command.");
if (check->m_found)
@@ -461,7 +461,7 @@ namespace AZ::IO
void FullFileDecompressor::FileExistsCheck(FileRequest* checkRequest)
{
auto& fileCheckRequest = AZStd::get<FileRequest::FileExistsCheckData>(checkRequest->GetCommand());
auto& fileCheckRequest = AZStd::get<Requests::FileExistsCheckData>(checkRequest->GetCommand());
CompressionInfo info;
if (CompressionUtils::FindCompressionInfo(info, fileCheckRequest.m_path.GetRelativePath()))
{
@@ -487,7 +487,7 @@ namespace AZ::IO
{
if (m_readBufferStatus[i] == ReadBufferStatus::Unused)
{
auto data = AZStd::get_if<FileRequest::CompressedReadData>(&compressedReadRequest->GetCommand());
auto data = AZStd::get_if<Requests::CompressedReadData>(&compressedReadRequest->GetCommand());
AZ_Assert(data, "Compressed request that's starting a read in FullFileDecompressor didn't contain compression read data.");
AZ_Assert(data->m_compressionInfo.m_decompressor,
"FileRequest for FullFileDecompressor is missing a decompression callback.");
@@ -549,7 +549,7 @@ namespace AZ::IO
}
else
{
auto data = AZStd::get_if<FileRequest::CompressedReadData>(&compressedRequest->GetCommand());
auto data = AZStd::get_if<Requests::CompressedReadData>(&compressedRequest->GetCommand());
AZ_Assert(data, "Compressed request in FullFileDecompressor that finished unsuccessfully didn't contain compression read data.");
CompressionInfo& info = data->m_compressionInfo;
size_t offsetAdjustment = info.m_offset - AZ_SIZE_ALIGN_DOWN(info.m_offset, aznumeric_cast<size_t>(m_alignment));
@@ -591,7 +591,7 @@ namespace AZ::IO
}
FileRequest* waitRequest = m_readRequests[readSlot];
AZ_Assert(AZStd::holds_alternative<FileRequest::WaitData>(waitRequest->GetCommand()),
AZ_Assert(AZStd::holds_alternative<Requests::WaitData>(waitRequest->GetCommand()),
"File request waiting for decompression wasn't marked as being a wait operation.");
FileRequest* compressedRequest = waitRequest->GetParent();
AZ_Assert(compressedRequest, "Read requests started by FullFileDecompressor is missing a parent request.");
@@ -610,7 +610,7 @@ namespace AZ::IO
m_readBuffers[readSlot] = nullptr;
AZ::Job* decompressionJob;
auto data = AZStd::get_if<FileRequest::CompressedReadData>(&compressedRequest->GetCommand());
auto data = AZStd::get_if<Requests::CompressedReadData>(&compressedRequest->GetCommand());
AZ_Assert(data, "Compressed request in FullFileDecompressor that's starting decompression didn't contain compression read data.");
AZ_Assert(data->m_compressionInfo.m_decompressor, "FullFileDecompressor is queuing a decompression job but couldn't find a decompressor.");
@@ -664,7 +664,7 @@ namespace AZ::IO
FileRequest* compressedRequest = jobInfo.m_waitRequest->GetParent();
AZ_Assert(compressedRequest, "A wait request attached to FullFileDecompressor was completed but didn't have a parent compressed request.");
auto data = AZStd::get_if<FileRequest::CompressedReadData>(&compressedRequest->GetCommand());
auto data = AZStd::get_if<Requests::CompressedReadData>(&compressedRequest->GetCommand());
AZ_Assert(data, "Compressed request in FullFileDecompressor that completed decompression didn't contain compression read data.");
CompressionInfo& info = data->m_compressionInfo;
size_t offsetAdjustment = info.m_offset - AZ_SIZE_ALIGN_DOWN(info.m_offset, aznumeric_cast<size_t>(m_alignment));
@@ -694,7 +694,7 @@ namespace AZ::IO
FileRequest* compressedRequest = info.m_waitRequest->GetParent();
AZ_Assert(compressedRequest, "A wait request attached to FullFileDecompressor was completed but didn't have a parent compressed request.");
auto request = AZStd::get_if<FileRequest::CompressedReadData>(&compressedRequest->GetCommand());
auto request = AZStd::get_if<Requests::CompressedReadData>(&compressedRequest->GetCommand());
AZ_Assert(request, "Compressed request in FullFileDecompressor that's running full decompression didn't contain compression read data.");
CompressionInfo& compressionInfo = request->m_compressionInfo;
AZ_Assert(compressionInfo.m_decompressor, "Full decompressor job started, but there's no decompressor callback assigned.");
@@ -719,7 +719,7 @@ namespace AZ::IO
FileRequest* compressedRequest = info.m_waitRequest->GetParent();
AZ_Assert(compressedRequest, "A wait request attached to FullFileDecompressor was completed but didn't have a parent compressed request.");
auto request = AZStd::get_if<FileRequest::CompressedReadData>(&compressedRequest->GetCommand());
auto request = AZStd::get_if<Requests::CompressedReadData>(&compressedRequest->GetCommand());
AZ_Assert(request, "Compressed request in FullFileDecompressor that's running partial decompression didn't contain compression read data.");
CompressionInfo& compressionInfo = request->m_compressionInfo;
AZ_Assert(compressionInfo.m_decompressor, "Partial decompressor job started, but there's no decompressor callback assigned.");
@@ -21,6 +21,11 @@
namespace AZ::IO
{
namespace Requests
{
struct ReadRequestData;
}
struct FullFileDecompressorConfig final :
public IStreamerStackConfig
{
@@ -87,7 +92,7 @@ namespace AZ::IO
bool IsIdle() const;
void PrepareReadRequest(FileRequest* request, FileRequest::ReadRequestData& data);
void PrepareReadRequest(FileRequest* request, Requests::ReadRequestData& data);
void PrepareDedicatedCache(FileRequest* request, const RequestPath& path);
void FileExistsCheck(FileRequest* checkRequest);
@@ -118,7 +118,7 @@ namespace AZ::IO
return;
}
auto data = AZStd::get_if<FileRequest::ReadData>(&request->GetCommand());
auto data = AZStd::get_if<Requests::ReadData>(&request->GetCommand());
if (data == nullptr)
{
StreamStackEntry::QueueRequest(request);
@@ -156,7 +156,7 @@ namespace AZ::IO
void ReadSplitter::QueueAlignedRead(FileRequest* request)
{
auto data = AZStd::get_if<FileRequest::ReadData>(&request->GetCommand());
auto data = AZStd::get_if<Requests::ReadData>(&request->GetCommand());
AZ_Assert(data != nullptr, "Provided request to queue by the Read Splitter did not contain a read command.");
if (data->m_size <= m_maxReadSize)
@@ -187,7 +187,7 @@ namespace AZ::IO
bool ReadSplitter::QueueAlignedRead(PendingRead& pending)
{
auto data = AZStd::get_if<FileRequest::ReadData>(&pending.m_request->GetCommand());
auto data = AZStd::get_if<Requests::ReadData>(&pending.m_request->GetCommand());
AZ_Assert(data != nullptr, "Provided request to queue by the Read Splitter did not contain a read command.");
while (pending.m_readSize > 0)
@@ -237,7 +237,7 @@ namespace AZ::IO
void ReadSplitter::QueueBufferedRead(FileRequest* request)
{
auto data = AZStd::get_if<FileRequest::ReadData>(&request->GetCommand());
auto data = AZStd::get_if<Requests::ReadData>(&request->GetCommand());
AZ_Assert(data != nullptr, "Provided request to queue by the Read Splitter did not contain a read command.");
PendingRead pendingRead;
@@ -262,7 +262,7 @@ namespace AZ::IO
bool ReadSplitter::QueueBufferedRead(PendingRead& pending)
{
auto data = AZStd::get_if<FileRequest::ReadData>(&pending.m_request->GetCommand());
auto data = AZStd::get_if<Requests::ReadData>(&pending.m_request->GetCommand());
AZ_Assert(data != nullptr, "Provided request to queue by the Read Splitter did not contain a read command.");
while (pending.m_readSize > 0)
@@ -6,9 +6,11 @@
*
*/
#include <AzCore/IO/Streamer/Scheduler.h>
#include <AzCore/Casting/numeric_cast.h>
#include <AzCore/Debug/Profiler.h>
#include <AzCore/IO/Streamer/Scheduler.h>
#include <AzCore/IO/Streamer/FileRequest.h>
#include <AzCore/std/containers/deque.h>
#include <AzCore/std/sort.h>
@@ -35,6 +37,8 @@ namespace AZ::IO
m_threadData.m_streamStack = AZStd::move(streamStack);
}
Scheduler::~Scheduler() = default;
void Scheduler::Start(const AZStd::thread_desc& threadDesc)
{
if (!m_isRunning)
@@ -222,10 +226,10 @@ namespace AZ::IO
{
using Command = AZStd::decay_t<decltype(args)>;
if constexpr (
AZStd::is_same_v<Command, FileRequest::ReadData> ||
AZStd::is_same_v<Command, FileRequest::CompressedReadData>)
AZStd::is_same_v<Command, Requests::ReadData> ||
AZStd::is_same_v<Command, Requests::CompressedReadData>)
{
auto parentReadRequest = next->GetCommandFromChain<FileRequest::ReadRequestData>();
auto parentReadRequest = next->GetCommandFromChain<Requests::ReadRequestData>();
AZ_Assert(parentReadRequest != nullptr, "The issued read request can't be found for the (compressed) read command.");
size_t size = parentReadRequest->m_size;
@@ -234,7 +238,7 @@ namespace AZ::IO
AZ_Assert(parentReadRequest->m_allocator,
"The read request was issued without a memory allocator or valid output address.");
u64 recommendedSize = size;
if constexpr (AZStd::is_same_v<Command, FileRequest::ReadData>)
if constexpr (AZStd::is_same_v<Command, Requests::ReadData>)
{
recommendedSize = m_recommendations.CalculateRecommendedMemorySize(size, parentReadRequest->m_offset);
}
@@ -249,12 +253,12 @@ namespace AZ::IO
parentReadRequest->m_output = allocation.m_address;
parentReadRequest->m_outputSize = allocation.m_size;
parentReadRequest->m_memoryType = allocation.m_type;
if constexpr (AZStd::is_same_v<Command, FileRequest::ReadData>)
if constexpr (AZStd::is_same_v<Command, Requests::ReadData>)
{
args.m_output = parentReadRequest->m_output;
args.m_outputSize = allocation.m_size;
}
else if constexpr (AZStd::is_same_v<Command, FileRequest::CompressedReadData>)
else if constexpr (AZStd::is_same_v<Command, Requests::CompressedReadData>)
{
args.m_output = parentReadRequest->m_output;
}
@@ -267,7 +271,7 @@ namespace AZ::IO
}
#endif
if constexpr (AZStd::is_same_v<Command, FileRequest::ReadData>)
if constexpr (AZStd::is_same_v<Command, Requests::ReadData>)
{
m_threadData.m_lastFilePath = args.m_path;
m_threadData.m_lastFileOffset = args.m_offset + args.m_size;
@@ -275,7 +279,7 @@ namespace AZ::IO
m_processingSize += args.m_size;
#endif
}
else if constexpr (AZStd::is_same_v<Command, FileRequest::CompressedReadData>)
else if constexpr (AZStd::is_same_v<Command, Requests::CompressedReadData>)
{
const CompressionInfo& info = args.m_compressionInfo;
m_threadData.m_lastFilePath = info.m_archiveFilename;
@@ -288,15 +292,15 @@ namespace AZ::IO
"Streamer queued %zu: %s", next->GetCommand().index(), parentReadRequest->m_path.GetRelativePath());
m_threadData.m_streamStack->QueueRequest(next);
}
else if constexpr (AZStd::is_same_v<Command, FileRequest::CancelData>)
else if constexpr (AZStd::is_same_v<Command, Requests::CancelData>)
{
return Thread_ProcessCancelRequest(next, args);
}
else if constexpr (AZStd::is_same_v<Command, FileRequest::RescheduleData>)
else if constexpr (AZStd::is_same_v<Command, Requests::RescheduleData>)
{
return Thread_ProcessRescheduleRequest(next, args);
}
else if constexpr (AZStd::is_same_v<Command, FileRequest::FlushData> || AZStd::is_same_v<Command, FileRequest::FlushAllData>)
else if constexpr (AZStd::is_same_v<Command, Requests::FlushData> || AZStd::is_same_v<Command, Requests::FlushAllData>)
{
AZ_PROFILE_INTERVAL_START_COLORED(AzCore, next, ProfilerColor,
"Streamer queued %zu", next->GetCommand().index());
@@ -345,7 +349,7 @@ namespace AZ::IO
#endif
{
using Command = AZStd::decay_t<decltype(args)>;
if constexpr (AZStd::is_same_v<Command, FileRequest::ReadRequestData>)
if constexpr (AZStd::is_same_v<Command, Requests::ReadRequestData>)
{
if (args.m_output == nullptr && args.m_allocator != nullptr)
{
@@ -393,7 +397,7 @@ namespace AZ::IO
}
}
void Scheduler::Thread_ProcessCancelRequest(FileRequest* request, FileRequest::CancelData& data)
void Scheduler::Thread_ProcessCancelRequest(FileRequest* request, Requests::CancelData& data)
{
AZ_PROFILE_INTERVAL_START_COLORED(AzCore, request, ProfilerColor, "Streamer queued cancel");
auto& pending = m_context.GetPreparedRequests();
@@ -415,7 +419,7 @@ namespace AZ::IO
m_threadData.m_streamStack->QueueRequest(request);
}
void Scheduler::Thread_ProcessRescheduleRequest(FileRequest* request, FileRequest::RescheduleData& data)
void Scheduler::Thread_ProcessRescheduleRequest(FileRequest* request, Requests::RescheduleData& data)
{
AZ_PROFILE_INTERVAL_START_COLORED(AzCore, request, ProfilerColor, "Streamer queued reschedule");
auto& pendingRequests = m_context.GetPreparedRequests();
@@ -424,7 +428,7 @@ namespace AZ::IO
if (pending->WorksOn(data.m_target))
{
// Read requests are the only requests that use deadlines and dynamic priorities.
auto readRequest = pending->GetCommandFromChain<FileRequest::ReadRequestData>();
auto readRequest = pending->GetCommandFromChain<Requests::ReadRequestData>();
if (readRequest)
{
readRequest->m_deadline = data.m_newDeadline;
@@ -463,8 +467,8 @@ namespace AZ::IO
// Order is the same for both requests, so prioritize the request that are at risk of missing
// it's deadline.
const FileRequest::ReadRequestData* firstRead = first->GetCommandFromChain<FileRequest::ReadRequestData>();
const FileRequest::ReadRequestData* secondRead = second->GetCommandFromChain<FileRequest::ReadRequestData>();
const Requests::ReadRequestData* firstRead = first->GetCommandFromChain<Requests::ReadRequestData>();
const Requests::ReadRequestData* secondRead = second->GetCommandFromChain<Requests::ReadRequestData>();
if (firstRead == nullptr || secondRead == nullptr)
{
@@ -496,11 +500,11 @@ namespace AZ::IO
auto sameFile = [this](auto&& args)
{
using Command = AZStd::decay_t<decltype(args)>;
if constexpr (AZStd::is_same_v<Command, FileRequest::ReadData>)
if constexpr (AZStd::is_same_v<Command, Requests::ReadData>)
{
return m_threadData.m_lastFilePath == args.m_path;
}
else if constexpr (AZStd::is_same_v<Command, FileRequest::CompressedReadData>)
else if constexpr (AZStd::is_same_v<Command, Requests::CompressedReadData>)
{
return m_threadData.m_lastFilePath == args.m_compressionInfo.m_archiveFilename;
}
@@ -517,11 +521,11 @@ namespace AZ::IO
auto offset = [](auto&& args) -> s64
{
using Command = AZStd::decay_t<decltype(args)>;
if constexpr (AZStd::is_same_v<Command, FileRequest::ReadData>)
if constexpr (AZStd::is_same_v<Command, Requests::ReadData>)
{
return aznumeric_caster(args.m_offset);
}
else if constexpr (AZStd::is_same_v<Command, FileRequest::CompressedReadData>)
else if constexpr (AZStd::is_same_v<Command, Requests::CompressedReadData>)
{
return aznumeric_caster(args.m_compressionInfo.m_offset);
}
@@ -8,6 +8,7 @@
#pragma once
#include <AzCore/IO/Streamer/RequestPath.h>
#include <AzCore/IO/IStreamerTypes.h>
#include <AzCore/IO/Streamer/Statistics.h>
#include <AzCore/IO/Streamer/StreamerConfiguration.h>
@@ -24,11 +25,19 @@ namespace AZ::IO
{
class FileRequest;
namespace Requests
{
struct CancelData;
struct RescheduleData;
} // namespace Requests
class Scheduler final
{
public:
explicit Scheduler(AZStd::shared_ptr<StreamStackEntry> streamStack, u64 memoryAlignment = AZCORE_GLOBAL_NEW_ALIGNMENT,
u64 sizeAlignment = 1, u64 granularity = 1_mib);
~Scheduler();
void Start(const AZStd::thread_desc& threadDesc);
void Stop();
@@ -61,14 +70,14 @@ namespace AZ::IO
bool Thread_ExecuteRequests();
bool Thread_PrepareRequests(AZStd::vector<FileRequestPtr>& outstandingRequests);
void Thread_ProcessTillIdle();
void Thread_ProcessCancelRequest(FileRequest* request, FileRequest::CancelData& data);
void Thread_ProcessRescheduleRequest(FileRequest* request, FileRequest::RescheduleData& data);
void Thread_ProcessCancelRequest(FileRequest* request, Requests::CancelData& data);
void Thread_ProcessRescheduleRequest(FileRequest* request, Requests::RescheduleData& data);
enum class Order
{
FirstRequest, //< The first request is the most important to process next.
SecondRequest, //< The second request is the most important to process next.
Equal //< Both requests are equally important.
FirstRequest, //!< The first request is the most important to process next.
SecondRequest, //!< The second request is the most important to process next.
Equal //!< Both requests are equally important.
};
//! Determine which of the two provided requests is more important to process next.
Order Thread_PrioritizeRequests(const FileRequest* first, const FileRequest* second) const;
@@ -60,13 +60,13 @@ namespace AZ::IO
AZ_PROFILE_FUNCTION(AzCore);
AZ_Assert(request, "PrepareRequest was provided a null request.");
if (AZStd::holds_alternative<FileRequest::ReadRequestData>(request->GetCommand()))
if (AZStd::holds_alternative<Requests::ReadRequestData>(request->GetCommand()))
{
auto& readRequest = AZStd::get<FileRequest::ReadRequestData>(request->GetCommand());
auto& readRequest = AZStd::get<Requests::ReadRequestData>(request->GetCommand());
FileRequest* read = m_context->GetNewInternalRequest();
read->CreateRead(request, readRequest.m_output, readRequest.m_outputSize, readRequest.m_path,
readRequest.m_offset, readRequest.m_size);
read->CreateRead(
request, readRequest.m_output, readRequest.m_outputSize, readRequest.m_path, readRequest.m_offset, readRequest.m_size);
m_context->PushPreparedRequest(read);
return;
}
@@ -79,29 +79,29 @@ namespace AZ::IO
AZStd::visit([this, request](auto&& args)
{
using Command = AZStd::decay_t<decltype(args)>;
if constexpr (AZStd::is_same_v<Command, FileRequest::ReadData> ||
AZStd::is_same_v<Command, FileRequest::FileExistsCheckData> ||
AZStd::is_same_v<Command, FileRequest::FileMetaDataRetrievalData>)
if constexpr (AZStd::is_same_v<Command, Requests::ReadData> ||
AZStd::is_same_v<Command, Requests::FileExistsCheckData> ||
AZStd::is_same_v<Command, Requests::FileMetaDataRetrievalData>)
{
m_pendingRequests.push_back(request);
return;
}
else if constexpr (AZStd::is_same_v<Command, FileRequest::CancelData>)
else if constexpr (AZStd::is_same_v<Command, Requests::CancelData>)
{
CancelRequest(request, args.m_target);
return;
}
else
{
if constexpr (AZStd::is_same_v<Command, FileRequest::FlushData>)
if constexpr (AZStd::is_same_v<Command, Requests::FlushData>)
{
FlushCache(args.m_path);
}
else if constexpr (AZStd::is_same_v<Command, FileRequest::FlushAllData>)
else if constexpr (AZStd::is_same_v<Command, Requests::FlushAllData>)
{
FlushEntireCache();
}
else if constexpr (AZStd::is_same_v<Command, FileRequest::ReportData>)
else if constexpr (AZStd::is_same_v<Command, Requests::ReportData>)
{
Report(args);
}
@@ -118,15 +118,15 @@ namespace AZ::IO
AZStd::visit([this, request](auto&& args)
{
using Command = AZStd::decay_t<decltype(args)>;
if constexpr (AZStd::is_same_v<Command, FileRequest::ReadData>)
if constexpr (AZStd::is_same_v<Command, Requests::ReadData>)
{
ReadFile(request);
}
else if constexpr (AZStd::is_same_v<Command, FileRequest::FileExistsCheckData>)
else if constexpr (AZStd::is_same_v<Command, Requests::FileExistsCheckData>)
{
FileExistsRequest(request);
}
else if constexpr (AZStd::is_same_v<Command, FileRequest::FileMetaDataRetrievalData>)
else if constexpr (AZStd::is_same_v<Command, Requests::FileMetaDataRetrievalData>)
{
FileMetaDataRetrievalRequest(request);
}
@@ -199,25 +199,25 @@ namespace AZ::IO
AZStd::visit([&](auto&& args)
{
using Command = AZStd::decay_t<decltype(args)>;
if constexpr (AZStd::is_same_v<Command, FileRequest::ReadData>)
if constexpr (AZStd::is_same_v<Command, Requests::ReadData>)
{
targetFile = &args.m_path;
readSize = args.m_size;
offset = args.m_offset;
}
else if constexpr (AZStd::is_same_v<Command, FileRequest::CompressedReadData>)
else if constexpr (AZStd::is_same_v<Command, Requests::CompressedReadData>)
{
targetFile = &args.m_compressionInfo.m_archiveFilename;
readSize = args.m_compressionInfo.m_compressedSize;
offset = args.m_compressionInfo.m_offset;
}
else if constexpr (AZStd::is_same_v<Command, FileRequest::FileExistsCheckData>)
else if constexpr (AZStd::is_same_v<Command, Requests::FileExistsCheckData>)
{
readSize = 0;
AZStd::chrono::microseconds averageTime = m_getFileExistsTimeAverage.CalculateAverage();
startTime += averageTime;
}
else if constexpr (AZStd::is_same_v<Command, FileRequest::FileMetaDataRetrievalData>)
else if constexpr (AZStd::is_same_v<Command, Requests::FileMetaDataRetrievalData>)
{
readSize = 0;
AZStd::chrono::microseconds averageTime = m_getFileMetaDataTimeAverage.CalculateAverage();
@@ -254,7 +254,7 @@ namespace AZ::IO
{
AZ_PROFILE_FUNCTION(AzCore);
auto data = AZStd::get_if<FileRequest::ReadData>(&request->GetCommand());
auto data = AZStd::get_if<Requests::ReadData>(&request->GetCommand());
AZ_Assert(data, "FileRequest queued on StorageDrive to be read didn't contain read data.");
SystemFile* file = nullptr;
@@ -342,7 +342,7 @@ namespace AZ::IO
AZ_PROFILE_FUNCTION(AzCore);
TIMED_AVERAGE_WINDOW_SCOPE(m_getFileExistsTimeAverage);
auto& fileExists = AZStd::get<FileRequest::FileExistsCheckData>(request->GetCommand());
auto& fileExists = AZStd::get<Requests::FileExistsCheckData>(request->GetCommand());
size_t cacheIndex = FindFileInCache(fileExists.m_path);
if (cacheIndex != s_fileNotFound)
{
@@ -360,7 +360,7 @@ namespace AZ::IO
AZ_PROFILE_FUNCTION(AzCore);
TIMED_AVERAGE_WINDOW_SCOPE(m_getFileMetaDataTimeAverage);
auto& command = AZStd::get<FileRequest::FileMetaDataRetrievalData>(request->GetCommand());
auto& command = AZStd::get<Requests::FileMetaDataRetrievalData>(request->GetCommand());
// If the file is already open, use the file handle which usually is cheaper than asking for the file by name.
size_t cacheIndex = FindFileInCache(command.m_path);
if (cacheIndex != s_fileNotFound)
@@ -446,11 +446,11 @@ namespace AZ::IO
}
}
void StorageDrive::Report(const FileRequest::ReportData& data) const
void StorageDrive::Report(const Requests::ReportData& data) const
{
switch (data.m_reportType)
{
case FileRequest::ReportData::ReportType::FileLocks:
case Requests::ReportType::FileLocks:
for (u32 i = 0; i < m_fileHandles.size(); ++i)
{
if (m_fileHandles[i] != nullptr)
@@ -8,6 +8,7 @@
#pragma once
#include <AzCore/IO/Streamer/RequestPath.h>
#include <AzCore/IO/Streamer/Statistics.h>
#include <AzCore/IO/Streamer/StreamerConfiguration.h>
#include <AzCore/IO/Streamer/StreamStackEntry.h>
@@ -16,6 +17,11 @@
#include <AzCore/std/containers/vector.h>
#include <AzCore/std/chrono/clocks.h>
namespace AZ::IO::Requests
{
struct ReportData;
}
namespace AZ::IO
{
struct StorageDriveConfig final :
@@ -72,7 +78,7 @@ namespace AZ::IO
void EstimateCompletionTimeForRequest(FileRequest* request, AZStd::chrono::system_clock::time_point& startTime,
const RequestPath*& activeFile, u64& activeOffset) const;
void Report(const FileRequest::ReportData& data) const;
void Report(const Requests::ReportData& data) const;
TimedAverageWindow<s_statisticsWindowSize> m_fileOpenCloseTimeAverage;
TimedAverageWindow<s_statisticsWindowSize> m_getFileExistsTimeAverage;
@@ -9,6 +9,7 @@
#include <AzCore/Casting/numeric_cast.h>
#include <AzCore/Debug/Profiler.h>
#include <AzCore/IO/CompressionBus.h>
#include <AzCore/IO/Streamer/FileRequest.h>
#include <AzCore/IO/Streamer/Streamer.h>
#include <AzCore/IO/Streamer/StreamerConfiguration.h>
#include <AzCore/IO/Streamer/StreamStackEntry.h>
@@ -210,7 +211,7 @@ namespace AZ::IO
IStreamerTypes::ClaimMemory claimMemory) const
{
AZ_Assert(request.m_request, "The request handle provided to Streamer::GetReadRequestResult is invalid.");
auto readRequest = AZStd::get_if<FileRequest::ReadRequestData>(&request.m_request->GetCommand());
auto readRequest = AZStd::get_if<Requests::ReadRequestData>(&request.m_request->GetCommand());
if (readRequest != nullptr)
{
buffer = readRequest->m_output;
@@ -281,14 +282,14 @@ namespace AZ::IO
}
}
FileRequestPtr Streamer::Report(FileRequest::ReportData::ReportType reportType)
FileRequestPtr Streamer::Report(Requests::ReportType reportType)
{
FileRequestPtr result = CreateRequest();
Report(result, reportType);
return result;
}
FileRequestPtr& Streamer::Report(FileRequestPtr& request, FileRequest::ReportData::ReportType reportType)
FileRequestPtr& Streamer::Report(FileRequestPtr& request, Requests::ReportType reportType)
{
request->m_request.CreateReport(reportType);
return request;
@@ -20,6 +20,10 @@ namespace AZStd
struct thread_desc;
}
namespace AZ::IO::Requests
{
enum class ReportType : int8_t;
}
namespace AZ::IO
{
@@ -185,9 +189,9 @@ namespace AZ::IO
void RecordStatistics();
//! Tells AZ::IO::Streamer the report the information for the report to the output.
FileRequestPtr Report(FileRequest::ReportData::ReportType reportType);
FileRequestPtr Report(Requests::ReportType reportType);
//! Tells AZ::IO::Streamer the report the information for the report to the output.
FileRequestPtr& Report(FileRequestPtr& request, FileRequest::ReportData::ReportType reportType);
FileRequestPtr& Report(FileRequestPtr& request, Requests::ReportType reportType);
Streamer(const AZStd::thread_desc& threadDesc, AZStd::unique_ptr<Scheduler> streamStack);
@@ -14,6 +14,7 @@
#include <AzCore/IO/Streamer/BlockCache.h>
#include <AzCore/IO/Streamer/DedicatedCache.h>
#include <AzCore/IO/Streamer/FullFileDecompressor.h>
#include <AzCore/IO/Streamer/FileRequest.h>
#include <AzCore/IO/Streamer/Scheduler.h>
#include <AzCore/IO/Streamer/StreamerComponent.h>
#include <AzCore/IO/Streamer/StreamerConfiguration.h>
@@ -207,7 +208,7 @@ namespace AZ
{
if (m_streamer)
{
m_streamer->QueueRequest(m_streamer->Report(AZ::IO::FileRequest::ReportData::ReportType::FileLocks));
m_streamer->QueueRequest(m_streamer->Report(AZ::IO::Requests::ReportType::FileLocks));
}
}
@@ -22,6 +22,9 @@ namespace AZ
static constexpr char LatePredictionName[] = "Early completions";
static constexpr char MissedDeadlinesName[] = "Missed deadlines";
#endif // AZ_STREAMER_ADD_EXTRA_PROFILING_INFO
StreamerContext::StreamerContext() = default;
StreamerContext::~StreamerContext()
{
for (FileRequest* entry : m_internalRecycleBin)
@@ -204,7 +207,7 @@ namespace AZ
m_latePredictionsPercentageStat.GetMostRecentSample());
}
}
auto readRequest = AZStd::get_if<FileRequest::ReadRequestData>(&top->GetCommand());
auto readRequest = AZStd::get_if<Requests::ReadRequestData>(&top->GetCommand());
if (readRequest != nullptr)
{
m_missedDeadlinePercentageStat.PushSample(now < readRequest->m_deadline ? 0.0 : 1.0);
@@ -224,7 +227,7 @@ namespace AZ
top->m_onCompletion(*top);
AZ_PROFILE_INTERVAL_END(AzCore, top);
}
if (parent)
{
AZ_Assert(parent->m_dependencies > 0,
@@ -8,22 +8,27 @@
#pragma once
#include <AzCore/base.h>
#include <AzCore/IO/Streamer/FileRequest.h>
#include <AzCore/IO/Streamer/Statistics.h>
#include <AzCore/IO/Streamer/StreamerConfiguration.h>
#include <AzCore/IO/Streamer/StreamerContext_Platform.h>
#include <AzCore/std/containers/deque.h>
#include <AzCore/std/containers/vector.h>
#include <AzCore/std/containers/queue.h>
#include <AzCore/Statistics/RunningStatistic.h>
#include <AzCore/std/containers/deque.h>
#include <AzCore/std/containers/queue.h>
#include <AzCore/std/containers/vector.h>
namespace AZ::IO
{
class FileRequest;
class ExternalFileRequest;
using FileRequestPtr = AZStd::intrusive_ptr<ExternalFileRequest>;
class StreamerContext
{
public:
using PreparedQueue = AZStd::deque<FileRequest*>;
StreamerContext();
~StreamerContext();
//! Gets a new file request, either by creating a new instance or
@@ -1,197 +0,0 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#ifndef AZCORE_JOBS_JOBEXECUTOR_H
#define AZCORE_JOBS_JOBEXECUTOR_H
#pragma once
#include <AzCore/Debug/Profiler.h>
#include <AzCore/Jobs/JobFunction.h>
#include <AzCore/std/parallel/condition_variable.h>
#include <AzCore/std/smart_ptr/unique_ptr.h>
namespace AZ
{
/**
* Helper for porting legacy jobs that allows Starting and Waiting for multiple jobs asynchronously
*/
class LegacyJobExecutor final
{
public:
LegacyJobExecutor() = default;
LegacyJobExecutor(const LegacyJobExecutor&) = delete;
~LegacyJobExecutor()
{
WaitForCompletion();
}
template <class Function>
inline void StartJob(const Function& processFunction, JobContext* context = nullptr)
{
Job * job = aznew JobFunctionExecutorHelper<Function>(processFunction, *this, context);
StartJobInternal(job);
}
// SetPostJob - This API exists to support backwards compatibility and is not a recommended pattern to be copied.
// Instead, create AZ::Jobs with appropriate dependencies on each other
template <class Function>
inline void SetPostJob(LegacyJobExecutor& postJobExecutor, const Function& processFunction, JobContext* context = nullptr)
{
AZStd::unique_ptr<JobExecutorHelper> postJob(aznew JobFunctionExecutorHelper<Function>(processFunction, postJobExecutor, context)); // Allocate outside the lock
{
LockGuard lockGuard(m_conditionLock);
AZ_Assert(!m_postJob, "Post already set");
AZ_Assert(!m_running, "LegacyJobExecutor::SetPostJob() must be called before starting any jobs");
m_postJob = std::move(postJob);
// Note: m_jobCount is not incremented until we push the post job
}
}
inline void ClearPostJob()
{
LockGuard lockGuard(m_conditionLock);
m_postJob.reset();
}
inline void Reset()
{
AZ_Assert(!IsRunning(), "LegacyJobExecutor::Reset() called while jobs in flight");
}
inline void WaitForCompletion()
{
AZStd::unique_lock<decltype(m_conditionLock)> uniqueLock(m_conditionLock);
while (m_running)
{
AZ_PROFILE_FUNCTION(AzCore);
m_completionCondition.wait(uniqueLock, [this] { return !this->m_running; });
}
}
// Push a logical fence that will cause WaitForCompletion to wait until PopCompletionFence is called and all jobs are complete. Analogue to the legacy API SJobState::SetStarted()
// Note: this does NOT fence execution of jobs in relation to each other
inline void PushCompletionFence()
{
IncJobCount();
}
// Pop a logical completion fence. Analogue to the legacy API SJobState::SetStopped()
inline void PopCompletionFence()
{
JobCompleteUpdate();
}
// Are there presently jobs in-flight (queued or running)?
inline bool IsRunning()
{
return m_running;
}
private:
void JobCompleteUpdate()
{
AZ_Assert(m_jobCount, "Invalid LegacyJobExecutor::m_jobCount.");
if (--m_jobCount == 0) // note: m_jobCount is atomic, so only the last completing job will take the count to zero
{
JobExecutorHelper* postJob = nullptr;
{
// All state transitions to and from running must be serialized through the condition lock
LockGuard lockGuard(m_conditionLock);
// Test count again as another job may have started before we got the lock
if (!m_jobCount)
{
m_running = false;
postJob = m_postJob.release();
m_completionCondition.notify_all();
}
}
// outside the lock (this pointer is no longer valid)...
if (postJob)
{
postJob->StartOnExecutor();
}
}
}
void StartJobInternal(Job * job)
{
IncJobCount();
job->Start();
}
void IncJobCount()
{
if (m_jobCount++ == 0)
{
// All state transitions to and from running must be serialized through the condition lock (Even though m_running is atomic)
LockGuard lockGuard(m_conditionLock);
m_running = true;
}
}
class JobExecutorHelper
{
public:
virtual ~JobExecutorHelper() = default;
virtual void StartOnExecutor() = 0;
};
/**
* Private Job type that notifies the owning LegacyJobExecutor of completion
*/
template<class Function>
class JobFunctionExecutorHelper : public JobFunction<Function>, public JobExecutorHelper
{
using Base = JobFunction<Function>;
public:
AZ_CLASS_ALLOCATOR(JobFunctionExecutorHelper, ThreadPoolAllocator, 0)
JobFunctionExecutorHelper(typename JobFunction<Function>::FunctionCRef processFunction, LegacyJobExecutor& executor, JobContext* context)
: JobFunction<Function>(processFunction, true /* isAutoDelete */, context)
, m_executor(executor)
{
}
void StartOnExecutor() override
{
m_executor.StartJobInternal(this);
}
void Process() override
{
Base::Process();
m_executor.JobCompleteUpdate();
}
private:
LegacyJobExecutor& m_executor;
};
template<class Function>
friend class JobFunctionExecutorHelper; // For JobCompleteUpdate, StartJobInternal
using Lock = AZStd::mutex;
using LockGuard = AZStd::lock_guard<Lock>;
AZStd::condition_variable m_completionCondition;
Lock m_conditionLock;
AZStd::unique_ptr<JobExecutorHelper> m_postJob;
AZStd::atomic_uint m_jobCount{0};
AZStd::atomic_bool m_running{false};
};
}
#endif
@@ -25,7 +25,7 @@ namespace AZ
void Matrix3x4SetRowGeneric(Matrix3x4* thisPtr, ScriptDataContext& dc)
{
bool rowIsSet = false;
[[maybe_unused]] bool rowIsSet = false;
if (dc.GetNumArguments() >= 5)
{
if (dc.IsNumber(0))
@@ -88,7 +88,7 @@ namespace AZ
void Matrix3x4SetColumnGeneric(Matrix3x4* thisPtr, ScriptDataContext& dc)
{
bool columnIsSet = false;
[[maybe_unused]] bool columnIsSet = false;
if (dc.GetNumArguments() >= 4)
{
if (dc.IsNumber(0))
@@ -133,7 +133,7 @@ namespace AZ
void Matrix3x4SetTranslationGeneric(Matrix3x4* thisPtr, ScriptDataContext& dc)
{
bool translationIsSet = false;
[[maybe_unused]] bool translationIsSet = false;
if (dc.GetNumArguments() == 3 &&
dc.IsNumber(0) &&
+1 -1
View File
@@ -77,7 +77,7 @@ namespace AZ
// check open brace
char c = *current++;
bool has_open_brace = false;
[[maybe_unused]] bool has_open_brace = false;
if (c == '{')
{
c = *current++;
@@ -9,11 +9,12 @@
#include <AzCore/Memory/Memory.h>
#include <AzCore/Memory/AllocatorManager.h>
#define RECORDING_ENABLED 0
// Only used to create recordings of memory operations to use for memory benchmarks
#define O3DE_RECORDING_ENABLED 0
#if RECORDING_ENABLED
#if O3DE_RECORDING_ENABLED
#include <AzCore/std/containers/unordered_map.h>
#include <AzCore/std/containers/map.h>
#include <AzCore/IO/SystemFile.h>
#include <AzCore/std/parallel/mutex.h>
#include <AzCore/std/parallel/scoped_lock.h>
@@ -62,23 +63,24 @@ namespace
static constexpr size_t s_maxNumberOfAllocationsToRecord = 16384;
static size_t s_numberOfAllocationsRecorded = 0;
static constexpr size_t s_allocationOperationCount = 5 * 1024;
static constexpr size_t s_allocationOperationCount = 8 * 1024;
static AZStd::array<AllocatorOperation, s_allocationOperationCount> s_operations = {};
static uint64_t s_operationCounter = 0;
static unsigned int s_nextRecordId = 1;
using AllocatorOperationByAddress = AZStd::unordered_map<void*, AllocatorOperation, AZStd::less<void*>, DebugAllocator>;
using AllocatorOperationByAddress = AZStd::map<void*, AllocatorOperation, AZStd::less<void*>, DebugAllocator>;
static AllocatorOperationByAddress s_allocatorOperationByAddress;
using AvailableRecordIds = AZStd::vector<unsigned int, DebugAllocator>;
AvailableRecordIds s_availableRecordIds;
void RecordAllocatorOperation(AllocatorOperation::OperationType type, void* ptr, size_t size = 0, size_t alignment = 0)
{
AZStd::scoped_lock<AZStd::mutex> lock(s_operationsMutex);
AZStd::scoped_lock lock(s_operationsMutex);
if (s_operationCounter == s_allocationOperationCount)
{
AZ::IO::SystemFile file;
int mode = AZ::IO::SystemFile::OpenMode::SF_OPEN_APPEND | AZ::IO::SystemFile::OpenMode::SF_OPEN_WRITE_ONLY;
// memoryrecordings.bin is being output to the current working directory
if (!file.Exists("memoryrecordings.bin"))
{
mode |= AZ::IO::SystemFile::OpenMode::SF_OPEN_CREATE;
@@ -158,8 +160,8 @@ namespace
namespace AZ
{
AllocatorBase::AllocatorBase(IAllocatorAllocate* allocationSource, const char* name, const char* desc)
: IAllocator(allocationSource)
AllocatorBase::AllocatorBase(IAllocatorSchema* allocationSchema, const char* name, const char* desc)
: IAllocator(allocationSchema)
, m_name(name)
, m_desc(desc)
{
@@ -184,11 +186,6 @@ namespace AZ
return m_desc;
}
IAllocatorAllocate* AllocatorBase::GetSchema()
{
return nullptr;
}
Debug::AllocationRecords* AllocatorBase::GetRecords()
{
return m_records;
@@ -205,11 +202,6 @@ namespace AZ
return m_isReady;
}
bool AllocatorBase::CanBeOverridden() const
{
return m_canBeOverridden;
}
void AllocatorBase::PostCreate()
{
if (m_registrationEnabled)
@@ -272,11 +264,6 @@ namespace AZ
return m_isProfilingActive;
}
void AllocatorBase::DisableOverriding()
{
m_canBeOverridden = false;
}
void AllocatorBase::DisableRegistration()
{
m_registrationEnabled = false;
@@ -285,10 +272,6 @@ namespace AZ
void AllocatorBase::ProfileAllocation(
void* ptr, size_t byteSize, size_t alignment, const char* name, const char* fileName, int lineNum, int suppressStackRecord)
{
#if defined(AZ_HAS_VARIADIC_TEMPLATES) && defined(AZ_DEBUG_BUILD)
++suppressStackRecord; // one more for the fact the ebus is a function
#endif // AZ_HAS_VARIADIC_TEMPLATES
if (m_isProfilingActive)
{
auto records = GetRecords();
@@ -298,7 +281,7 @@ namespace AZ
}
}
#if RECORDING_ENABLED
#if O3DE_RECORDING_ENABLED
RecordAllocatorOperation(AllocatorOperation::ALLOCATE, ptr, byteSize, alignment);
#endif
}
@@ -313,7 +296,7 @@ namespace AZ
records->UnregisterAllocation(ptr, byteSize, alignment, info);
}
}
#if RECORDING_ENABLED
#if O3DE_RECORDING_ENABLED
RecordAllocatorOperation(AllocatorOperation::DEALLOCATE, ptr, byteSize, alignment);
#endif
}
@@ -330,7 +313,7 @@ namespace AZ
ProfileDeallocation(ptr, 0, 0, &info);
ProfileAllocation(newPtr, newSize, newAlignment, info.m_name, info.m_fileName, info.m_lineNum, 0);
}
#if RECORDING_ENABLED
#if O3DE_RECORDING_ENABLED
RecordAllocatorOperation(AllocatorOperation::DEALLOCATE, ptr);
RecordAllocatorOperation(AllocatorOperation::ALLOCATE, newPtr, newSize, newAlignment);
#endif
@@ -351,7 +334,7 @@ namespace AZ
records->ResizeAllocation(ptr, newSize);
}
}
#if RECORDING_ENABLED
#if O3DE_RECORDING_ENABLED
RecordAllocatorOperation(AllocatorOperation::ALLOCATE, ptr, newSize);
#endif
}
@@ -22,7 +22,7 @@ namespace AZ
class AllocatorBase : public IAllocator
{
protected:
AllocatorBase(IAllocatorAllocate* allocationSource, const char* name, const char* desc);
AllocatorBase(IAllocatorSchema* allocationSchema, const char* name, const char* desc);
~AllocatorBase();
public:
@@ -32,11 +32,9 @@ namespace AZ
//---------------------------------------------------------------------
const char* GetName() const override;
const char* GetDescription() const override;
IAllocatorAllocate* GetSchema() override;
Debug::AllocationRecords* GetRecords() final;
void SetRecords(Debug::AllocationRecords* records) final;
bool IsReady() const final;
bool CanBeOverridden() const final;
void PostCreate() override;
void PreDestroy() final;
void SetLazilyCreated(bool lazy) final;
@@ -68,10 +66,6 @@ namespace AZ
return byteSize;
}
/// Call to disallow this allocator from being overridden.
/// Only kernel-level allocators where it would be especially problematic for them to be overridden should do this.
void DisableOverriding();
/// Call to disallow this allocator from being registered with the AllocatorManager.
/// Only kernel-level allocators where it would be especially problematic for them to be registered with the AllocatorManager should do this.
void DisableRegistration();
@@ -107,7 +101,6 @@ namespace AZ
bool m_isLazilyCreated = false;
bool m_isProfilingActive = false;
bool m_isReady = false;
bool m_canBeOverridden = true;
bool m_registrationEnabled = true;
};
@@ -12,17 +12,12 @@
#include <AzCore/Memory/OSAllocator.h>
#include <AzCore/Memory/AllocationRecords.h>
#include <AzCore/Memory/AllocatorOverrideShim.h>
#include <AzCore/Memory/MallocSchema.h>
#include <AzCore/std/parallel/lock.h>
#include <AzCore/std/smart_ptr/make_shared.h>
#include <AzCore/std/containers/array.h>
#if !defined(RELEASE) && !defined(AZCORE_MEMORY_ENABLE_OVERRIDES)
# define AZCORE_MEMORY_ENABLE_OVERRIDES
#endif
namespace AZ::Internal
{
struct AMStringHasher
@@ -54,18 +49,6 @@ namespace AZ::Internal
namespace AZ
{
struct AllocatorManager::InternalData
{
explicit InternalData(const AZStdIAllocator& alloc)
: m_allocatorMap(alloc)
, m_remappings(alloc)
, m_remappingsReverse(alloc)
{}
Internal::AllocatorNameMap m_allocatorMap;
Internal::AllocatorRemappings m_remappings;
Internal::AllocatorRemappings m_remappingsReverse;
};
static EnvironmentVariable<AllocatorManager> s_allocManager = nullptr;
static AllocatorManager* s_allocManagerDebug = nullptr; // For easier viewing in crash dumps
@@ -81,16 +64,6 @@ static Internal::PreEnvironmentAttachData& GetPreEnvironmentAttachData()
void AllocatorManager::PreRegisterAllocator(IAllocator* allocator)
{
auto& data = GetPreEnvironmentAttachData();
#ifdef AZCORE_MEMORY_ENABLE_OVERRIDES
// All allocators must switch to an OverrideEnabledAllocationSource proxy if they are to support allocator overriding.
if (allocator->CanBeOverridden())
{
auto shim = Internal::AllocatorOverrideShim::Create(allocator, &data.m_mallocSchema);
allocator->SetAllocationSource(shim);
}
#endif
{
AZStd::lock_guard<AZStd::mutex> lock(data.m_mutex);
AZ_Assert(data.m_unregisteredAllocatorCount < Internal::PreEnvironmentAttachData::MAX_UNREGISTERED_ALLOCATORS, "Too many allocators trying to register before environment attached!");
@@ -175,12 +148,9 @@ AllocatorManager::AllocatorManager()
}
)
{
m_overrideSource = nullptr;
m_numAllocators = 0;
m_isAllocatorLeaking = false;
m_configurationFinalized = false;
m_defaultTrackingRecordMode = Debug::AllocationRecords::RECORD_NO_RECORDS;
m_data = new (m_mallocSchema->Allocate(sizeof(InternalData), AZStd::alignment_of<InternalData>::value, 0)) InternalData(AZStdIAllocator(m_mallocSchema.get()));
}
//=========================================================================
@@ -210,10 +180,6 @@ AllocatorManager::RegisterAllocator(class IAllocator* alloc)
alloc->SetProfilingActive(m_profilingRefcount.load() > 0);
m_allocators[m_numAllocators++] = alloc;
#ifdef AZCORE_MEMORY_ENABLE_OVERRIDES
ConfigureAllocatorOverrides(alloc);
#endif
}
//=========================================================================
@@ -232,81 +198,12 @@ AllocatorManager::InternalDestroy()
// Do not actually destroy the lazy allocator as it may have work to do during non-deterministic shutdown
}
if (m_data)
{
m_data->~InternalData();
m_mallocSchema->DeAllocate(m_data);
m_data = nullptr;
}
if (!m_isAllocatorLeaking)
{
AZ_Assert(m_numAllocators == 0, "There are still %d registered allocators!", m_numAllocators);
}
}
//=========================================================================
// ConfigureAllocatorOverrides
// [10/14/2018]
//=========================================================================
void
AllocatorManager::ConfigureAllocatorOverrides(IAllocator* alloc)
{
auto record = m_data->m_allocatorMap.emplace(AZStd::piecewise_construct, AZStd::forward_as_tuple(alloc->GetName(), AZStdIAllocator(m_mallocSchema.get())), AZStd::forward_as_tuple(alloc));
// We only need to keep going if the allocator supports overrides.
if (!alloc->CanBeOverridden())
{
return;
}
if (!alloc->IsAllocationSourceChanged())
{
// All allocators must switch to an OverrideEnabledAllocationSource proxy if they are to support allocator overriding.
auto overrideEnabled = Internal::AllocatorOverrideShim::Create(alloc, m_mallocSchema.get());
alloc->SetAllocationSource(overrideEnabled);
}
auto itr = m_data->m_remappings.find(record.first->first);
if (itr != m_data->m_remappings.end())
{
auto remapTo = m_data->m_allocatorMap.find(itr->second);
if (remapTo != m_data->m_allocatorMap.end())
{
static_cast<Internal::AllocatorOverrideShim*>(alloc->GetAllocationSource())->SetOverride(remapTo->second->GetOriginalAllocationSource());
}
}
itr = m_data->m_remappingsReverse.find(record.first->first);
if (itr != m_data->m_remappingsReverse.end())
{
auto remapFrom = m_data->m_allocatorMap.find(itr->second);
if (remapFrom != m_data->m_allocatorMap.end())
{
AZ_Assert(!m_configurationFinalized, "Allocators may only remap to allocators that have been created before configuration finalization");
static_cast<Internal::AllocatorOverrideShim*>(remapFrom->second->GetAllocationSource())->SetOverride(alloc->GetOriginalAllocationSource());
}
}
if (m_overrideSource)
{
static_cast<Internal::AllocatorOverrideShim*>(alloc->GetAllocationSource())->SetOverride(m_overrideSource);
}
if (m_configurationFinalized)
{
// We can get rid of the intermediary if configuration won't be changing any further.
// (The creation of it at the top of this function was superflous, but it made it easier to set things up going through a single code path.)
auto shim = static_cast<Internal::AllocatorOverrideShim*>(alloc->GetAllocationSource());
alloc->SetAllocationSource(shim->GetOverride());
Internal::AllocatorOverrideShim::Destroy(shim);
}
}
//=========================================================================
// UnRegisterAllocator
// [9/17/2009]
@@ -365,7 +262,7 @@ AllocatorManager::GarbageCollect()
for (int i = 0; i < m_numAllocators; ++i)
{
m_allocators[i]->GetAllocationSource()->GarbageCollect();
m_allocators[i]->GetSchema()->GarbageCollect();
}
}
@@ -414,94 +311,6 @@ AllocatorManager::SetTrackingMode(Debug::AllocationRecords::Mode mode)
}
}
//=========================================================================
// SetOverrideSchema
// [8/17/2018]
//=========================================================================
void
AllocatorManager::SetOverrideAllocatorSource(IAllocatorAllocate* source, bool overrideExistingAllocators)
{
(void)source;
(void)overrideExistingAllocators;
#ifdef AZCORE_MEMORY_ENABLE_OVERRIDES
AZ_Assert(!m_configurationFinalized, "You cannot set an allocator source after FinalizeConfiguration() has been called.");
m_overrideSource = source;
if (overrideExistingAllocators)
{
AZStd::lock_guard<AZStd::mutex> lock(m_allocatorListMutex);
for (int i = 0; i < m_numAllocators; ++i)
{
if (m_allocators[i]->CanBeOverridden())
{
auto shim = static_cast<Internal::AllocatorOverrideShim*>(m_allocators[i]->GetAllocationSource());
shim->SetOverride(source);
}
}
}
#endif
}
//=========================================================================
// AddAllocatorRemapping
// [8/27/2018]
//=========================================================================
void
AllocatorManager::AddAllocatorRemapping(const char* fromName, const char* toName)
{
(void)fromName;
(void)toName;
#ifdef AZCORE_MEMORY_ENABLE_OVERRIDES
AZ_Assert(!m_configurationFinalized, "You cannot set an allocator remapping after FinalizeConfiguration() has been called.");
m_data->m_remappings.emplace(AZStd::piecewise_construct, AZStd::forward_as_tuple(fromName, m_mallocSchema.get()), AZStd::forward_as_tuple(toName, m_mallocSchema.get()));
m_data->m_remappingsReverse.emplace(AZStd::piecewise_construct, AZStd::forward_as_tuple(toName, m_mallocSchema.get()), AZStd::forward_as_tuple(fromName, m_mallocSchema.get()));
#endif
}
void
AllocatorManager::FinalizeConfiguration()
{
if (m_configurationFinalized)
{
return;
}
#ifdef AZCORE_MEMORY_ENABLE_OVERRIDES
{
AZStd::lock_guard<AZStd::mutex> lock(m_allocatorListMutex);
for (int i = 0; i < m_numAllocators; ++i)
{
if (!m_allocators[i]->CanBeOverridden())
{
continue;
}
auto shim = static_cast<Internal::AllocatorOverrideShim*>(m_allocators[i]->GetAllocationSource());
if (!shim->IsOverridden())
{
m_allocators[i]->ResetAllocationSource();
Internal::AllocatorOverrideShim::Destroy(shim);
}
else if (!shim->HasOrphanedAllocations())
{
m_allocators[i]->SetAllocationSource(shim->GetOverride());
Internal::AllocatorOverrideShim::Destroy(shim);
}
else
{
shim->SetFinalizedConfiguration();
}
}
}
#endif
m_configurationFinalized = true;
}
void
AllocatorManager::EnterProfilingMode()
{
@@ -545,27 +354,18 @@ AllocatorManager::DumpAllocators()
size_t totalConsumedBytes = 0;
memset(m_dumpInfo, 0, sizeof(m_dumpInfo));
void* sourceList[m_maxNumAllocators];
AZ_Printf(TAG, "%d allocators active\n", m_numAllocators);
AZ_Printf(TAG, "Index,Name,Used kb,Reserved kb,Consumed kb\n");
for (int i = 0; i < m_numAllocators; i++)
{
auto allocator = m_allocators[i];
auto source = allocator->GetAllocationSource();
IAllocator* allocator = GetAllocator(i);
const char* name = allocator->GetName();
size_t usedBytes = source->NumAllocatedBytes();
size_t reservedBytes = source->Capacity();
size_t usedBytes = allocator->NumAllocatedBytes();
size_t reservedBytes = allocator->Capacity();
size_t consumedBytes = reservedBytes;
// Very hacky and inefficient check to see if this allocator obtains its memory from another allocator
sourceList[i] = source;
if (AZStd::find(sourceList, sourceList + i, allocator->GetSchema()) != sourceList + i)
{
consumedBytes = 0;
}
totalUsedBytes += usedBytes;
totalReservedBytes += reservedBytes;
totalConsumedBytes += consumedBytes;
@@ -585,61 +385,21 @@ void AllocatorManager::GetAllocatorStats(size_t& allocatedBytes, size_t& capacit
AZStd::lock_guard<AZStd::mutex> lock(m_allocatorListMutex);
const int allocatorCount = GetNumAllocators();
AZStd::unordered_map<IAllocatorAllocate*, IAllocator*> existingAllocators;
AZStd::unordered_map<IAllocatorAllocate*, IAllocator*> sourcesToAllocators;
// Build a mapping of original allocator sources to their allocators
for (int i = 0; i < allocatorCount; ++i)
{
IAllocator* allocator = GetAllocator(i);
sourcesToAllocators.emplace(allocator->GetOriginalAllocationSource(), allocator);
}
for (int i = 0; i < allocatorCount; ++i)
{
IAllocator* allocator = GetAllocator(i);
IAllocatorAllocate* source = allocator->GetAllocationSource();
IAllocatorAllocate* originalSource = allocator->GetOriginalAllocationSource();
IAllocatorAllocate* schema = allocator->GetSchema();
IAllocator* alias = (source != originalSource) ? sourcesToAllocators[source] : nullptr;
if (schema && !alias)
{
// Check to see if this allocator's source maps to another allocator
// Need to check both the schema and the allocator itself, as either one might be used as the alias depending on how it's implemented
AZStd::array<IAllocatorAllocate*, 2> checkAllocators = { { schema, allocator->GetAllocationSource() } };
for (IAllocatorAllocate* check : checkAllocators)
{
auto existing = existingAllocators.emplace(check, allocator);
if (!existing.second)
{
alias = existing.first->second;
// Do not break out of the loop as we need to add to the map for all entries
}
}
}
static const IAllocator* OS_ALLOCATOR = &AllocatorInstance<OSAllocator>::GetAllocator();
size_t sourceAllocatedBytes = source->NumAllocatedBytes();
size_t sourceCapacityBytes = source->Capacity();
if (allocator == OS_ALLOCATOR)
{
// Need to special case the OS allocator because its capacity is a made-up number. Better to just use the allocated amount, it will hopefully be small anyway.
sourceCapacityBytes = sourceAllocatedBytes;
}
allocatedBytes += allocator->NumAllocatedBytes();
capacityBytes += allocator->Capacity();
if (outStats)
{
outStats->emplace(outStats->end(), allocator->GetName(), alias ? alias->GetName() : allocator->GetDescription(), sourceAllocatedBytes, sourceCapacityBytes, alias != nullptr);
}
if (!alias)
{
allocatedBytes += sourceAllocatedBytes;
capacityBytes += sourceCapacityBytes;
outStats->emplace(outStats->end(),
allocator->GetName(),
allocator->GetDescription(),
allocator->NumAllocatedBytes(),
allocator->Capacity());
}
}
}
@@ -87,17 +87,6 @@ namespace AZ
/// Especially for great code and engines...
void SetAllocatorLeaking(bool allowLeaking) { m_isAllocatorLeaking = allowLeaking; }
/// Set an override allocator
/// All allocators registered with the AllocatorManager will automatically redirect to this allocator
/// if set.
void SetOverrideAllocatorSource(IAllocatorAllocate* source, bool overrideExistingAllocators = true);
/// Retrieve the override schema
IAllocatorAllocate* GetOverrideAllocatorSource() const { return m_overrideSource; }
void AddAllocatorRemapping(const char* fromName, const char* toName);
void FinalizeConfiguration();
/// Enter or exit profiling mode; calls to Enter must be matched with calls to Exit
void EnterProfilingMode();
void ExitProfilingMode();
@@ -116,19 +105,17 @@ namespace AZ
struct AllocatorStats
{
AllocatorStats(const char* name, const char* aliasOrDescription, size_t allocatedBytes, size_t capacityBytes, bool isAlias)
AllocatorStats(const char* name, const char* aliasOrDescription, size_t allocatedBytes, size_t capacityBytes)
: m_name(name)
, m_aliasOrDescription(aliasOrDescription)
, m_allocatedBytes(allocatedBytes)
, m_capacityBytes(capacityBytes)
, m_isAlias(isAlias)
{}
AZStd::string m_name;
AZStd::string m_aliasOrDescription;
size_t m_allocatedBytes;
size_t m_capacityBytes;
bool m_isAlias;
};
void GetAllocatorStats(size_t& usedBytes, size_t& reservedBytes, AZStd::vector<AllocatorStats>* outStats = nullptr);
@@ -160,7 +147,6 @@ namespace AZ
private:
void InternalDestroy();
void ConfigureAllocatorOverrides(IAllocator* alloc);
void DebugBreak(void* address, const Debug::AllocationInfo& info);
AZ::MallocSchema* CreateMallocSchema();
@@ -175,14 +161,9 @@ namespace AZ
MemoryBreak m_memoryBreak[MaxNumMemoryBreaks];
char m_activeBreaks;
AZStd::mutex m_allocatorListMutex;
IAllocatorAllocate* m_overrideSource;
DumpInfo m_dumpInfo[m_maxNumAllocators];
struct InternalData;
InternalData* m_data;
bool m_configurationFinalized;
AZStd::atomic<int> m_profilingRefcount;
AZ::Debug::AllocationRecords::Mode m_defaultTrackingRecordMode;
@@ -1,227 +0,0 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#include <AzCore/Memory/AllocatorOverrideShim.h>
namespace AZ::Internal
{
AllocatorOverrideShim* AllocatorOverrideShim::Create(IAllocator* owningAllocator, IAllocatorAllocate* shimAllocationSource)
{
void* memory = shimAllocationSource->Allocate(sizeof(AllocatorOverrideShim), AZStd::alignment_of<AllocatorOverrideShim>::value, 0);
auto result = new (memory) AllocatorOverrideShim(owningAllocator, shimAllocationSource);
return result;
}
void AllocatorOverrideShim::Destroy(AllocatorOverrideShim* source)
{
auto shimAllocationSource = source->m_shimAllocationSource;
source->~AllocatorOverrideShim();
shimAllocationSource->DeAllocate(source);
}
AllocatorOverrideShim::AllocatorOverrideShim(IAllocator* owningAllocator, IAllocatorAllocate* shimAllocationSource)
: m_owningAllocator(owningAllocator)
, m_source(owningAllocator->GetOriginalAllocationSource())
, m_overridingSource(owningAllocator->GetOriginalAllocationSource())
, m_shimAllocationSource(shimAllocationSource)
, m_records(typename AllocationSet::hasher(), typename AllocationSet::key_eq(), StdAllocationSrc(shimAllocationSource))
{
}
void AllocatorOverrideShim::SetOverride(IAllocatorAllocate* source)
{
m_overridingSource = source;
}
IAllocatorAllocate* AllocatorOverrideShim::GetOverride() const
{
return m_overridingSource;
}
bool AllocatorOverrideShim::IsOverridden() const
{
return m_source != m_overridingSource;
}
bool AllocatorOverrideShim::HasOrphanedAllocations() const
{
return !m_records.empty();
}
void AllocatorOverrideShim::SetFinalizedConfiguration()
{
m_finalizedConfiguration = true;
}
typename AllocatorOverrideShim::pointer_type AllocatorOverrideShim::Allocate(size_type byteSize, size_type alignment, int flags, const char* name, const char* fileName, int lineNum, unsigned int suppressStackRecord)
{
pointer_type ptr = m_overridingSource->Allocate(byteSize, alignment, flags, name, fileName, lineNum, suppressStackRecord);
if (!IsOverridden())
{
lock_type lock(m_mutex);
m_records.insert(ptr); // Record in case we need to orphan this allocation later
}
return ptr;
}
void AllocatorOverrideShim::DeAllocate(pointer_type ptr, size_type byteSize, size_type alignment)
{
IAllocatorAllocate* source = m_overridingSource;
bool destroy = false;
{
lock_type lock(m_mutex);
// Check to see if this came from a prior allocation source
if (m_records.erase(ptr) && IsOverridden())
{
source = m_source;
if (m_records.empty() && m_finalizedConfiguration)
{
// All orphaned records are gone; we are no longer needed
m_owningAllocator->SetAllocationSource(m_overridingSource);
destroy = true; // Must destroy outside the lock
}
}
}
source->DeAllocate(ptr, byteSize, alignment);
if (destroy)
{
Destroy(this);
}
}
typename AllocatorOverrideShim::size_type AllocatorOverrideShim::Resize(pointer_type ptr, size_type newSize)
{
IAllocatorAllocate* source = m_overridingSource;
if (IsOverridden())
{
// Determine who owns the allocation
lock_type lock(m_mutex);
if (m_records.count(ptr))
{
source = m_source;
}
}
size_t result = source->Resize(ptr, newSize);
return result;
}
typename AllocatorOverrideShim::pointer_type AllocatorOverrideShim::ReAllocate(pointer_type ptr, size_type newSize, size_type newAlignment)
{
pointer_type newPtr = nullptr;
bool useOverride = true;
bool destroy = false;
if (IsOverridden())
{
lock_type lock(m_mutex);
if (m_records.erase(ptr))
{
// An old allocation needs to be transferred to the new, overriding allocator.
useOverride = false; // We'll do the reallocation here
size_t oldSize = m_source->AllocationSize(ptr);
if (newSize)
{
newPtr = m_overridingSource->Allocate(newSize, newAlignment, 0);
memcpy(newPtr, ptr, AZStd::min(newSize, oldSize));
}
m_source->DeAllocate(ptr, oldSize);
if (m_records.empty() && m_finalizedConfiguration)
{
// All orphaned records are gone; we are no longer needed
m_owningAllocator->SetAllocationSource(m_overridingSource);
destroy = true; // Must destroy outside the lock
}
}
}
if (useOverride)
{
// Default behavior, we weren't deleting an old allocation
newPtr = m_overridingSource->ReAllocate(ptr, newSize, newAlignment);
if (!IsOverridden())
{
// Still need to do bookkeeping if we haven't been overridden yet
lock_type lock(m_mutex);
m_records.erase(ptr);
m_records.insert(newPtr);
}
}
if (destroy)
{
Destroy(this);
}
return newPtr;
}
typename AllocatorOverrideShim::size_type AllocatorOverrideShim::AllocationSize(pointer_type ptr)
{
IAllocatorAllocate* source = m_overridingSource;
if (IsOverridden())
{
// Determine who owns the allocation
lock_type lock(m_mutex);
if (m_records.count(ptr))
{
source = m_source;
}
}
return source->AllocationSize(ptr);
}
void AllocatorOverrideShim::GarbageCollect()
{
m_source->GarbageCollect();
}
typename AllocatorOverrideShim::size_type AllocatorOverrideShim::NumAllocatedBytes() const
{
return m_source->NumAllocatedBytes();
}
typename AllocatorOverrideShim::size_type AllocatorOverrideShim::Capacity() const
{
return m_source->Capacity();
}
typename AllocatorOverrideShim::size_type AllocatorOverrideShim::GetMaxAllocationSize() const
{
return m_source->GetMaxAllocationSize();
}
auto AllocatorOverrideShim::GetMaxContiguousAllocationSize() const -> size_type
{
return m_source->GetMaxContiguousAllocationSize();
}
IAllocatorAllocate* AllocatorOverrideShim::GetSubAllocator()
{
return m_source->GetSubAllocator();
}
} // namespace AZ::Internal
@@ -1,102 +0,0 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#pragma once
#include <AzCore/Memory/Memory.h>
#include <AzCore/std/containers/unordered_set.h>
#include <AzCore/std/parallel/mutex.h>
namespace AZ
{
class AllocatorManager;
namespace Internal
{
/**
* A shim schema that solves the problem of overriding lazily-created allocators especially, that perform allocations before the override happens.
*
* Any allocator that *might* be overridden at some point must have this shim installed as its allocation source. This is done automatically by
* the AllocationManager; you generally do not have to interact with this shim directly at all.
*
* The shim will keep track of any allocations that occur, and if the allocator gets overridden it will ensure that prior allocations are
* deallocated with the old schema rather than the new one.
*
* There is some performance cost to this intrusion, however, in most cases it's only temporary:
* * Once the application calls FinalizeConfiguration(), any non-overridden allocators will have their shims destroyed.
* * Any overridden allocators that do not have prior allocations will have their shims destroyed.
* * Any overridden allocator will automatically destroy its shim once the last of the prior allocations has been deallocated.
*
* Note that an allocator that gets overridden but has prior allocations that it never intends to deallocate (such as file-level statics that never
* get changed) will keep its shim indefinitely. This is an unfortunate cost but only affects those allocators if they are being overridden.
*/
class AllocatorOverrideShim
: public IAllocatorAllocate
{
friend AllocatorManager;
public:
//---------------------------------------------------------------------
// IAllocator implementation
//---------------------------------------------------------------------
pointer_type Allocate(size_type byteSize, size_type alignment, int flags = 0, const char* name = nullptr, const char* fileName = nullptr, int lineNum = 0, unsigned int suppressStackRecord = 0) override;
void DeAllocate(pointer_type ptr, size_type byteSize = 0, size_type alignment = 0) override;
size_type Resize(pointer_type ptr, size_type newSize) override;
pointer_type ReAllocate(pointer_type ptr, size_type newSize, size_type newAlignment) override;
size_type AllocationSize(pointer_type ptr) override;
void GarbageCollect() override;
size_type NumAllocatedBytes() const override;
size_type Capacity() const override;
size_type GetMaxAllocationSize() const override;
size_type GetMaxContiguousAllocationSize() const override;
IAllocatorAllocate* GetSubAllocator() override;
private:
/// Creates a shim using a custom memory source
static AllocatorOverrideShim* Create(IAllocator* owningAllocator, IAllocatorAllocate* shimAllocationSource);
static void Destroy(AllocatorOverrideShim* source);
AllocatorOverrideShim(IAllocator* owningAllocator, IAllocatorAllocate* shimAllocationSource);
/// Overrides the shim's memory source with a different memory source.
void SetOverride(IAllocatorAllocate* source);
/// Returns the override source.
IAllocatorAllocate* GetOverride() const;
/// Returns true if the shim has an override source set on it.
bool IsOverridden() const;
/// Returns true if there are orphaned allocations from before the shim had its override set.
bool HasOrphanedAllocations() const;
/// Called by the AllocatorManager to signify that the configuration has been finalized by the application.
void SetFinalizedConfiguration();
private:
class StdAllocationSrc : public AZStdIAllocator
{
public:
StdAllocationSrc(IAllocatorAllocate* schema = nullptr) : AZStdIAllocator(schema)
{
}
};
typedef AZStd::mutex mutex_type;
typedef AZStd::lock_guard<mutex_type> lock_type;
typedef AZStd::unordered_set<void*, AZStd::hash<void*>, AZStd::equal_to<void*>, StdAllocationSrc> AllocationSet;
IAllocator* m_owningAllocator;
IAllocatorAllocate* m_source;
IAllocatorAllocate* m_overridingSource;
IAllocatorAllocate* m_shimAllocationSource;
AllocationSet m_records;
mutex_type m_mutex;
bool m_finalizedConfiguration = false;
};
}
}
@@ -20,8 +20,7 @@ namespace AZ
// [1/28/2011]
//=========================================================================
BestFitExternalMapAllocator::BestFitExternalMapAllocator()
: AllocatorBase(this, "BestFitExternalMapAllocator", "Best fit allocator with external tracking storage!")
, m_schema(nullptr)
: AllocatorBase(nullptr, "BestFitExternalMapAllocator", "Best fit allocator with external tracking storage!")
{
}
@@ -182,13 +181,4 @@ namespace AZ
return m_schema->GetMaxContiguousAllocationSize();
}
//=========================================================================
// GetSubAllocator
// [1/28/2011]
//=========================================================================
IAllocatorAllocate* BestFitExternalMapAllocator::GetSubAllocator()
{
return m_schema->GetSubAllocator();
}
} // namespace AZ
@@ -19,7 +19,6 @@ namespace AZ
*/
class BestFitExternalMapAllocator
: public AllocatorBase
, public IAllocatorAllocate
{
public:
AZ_TYPE_INFO(BestFitExternalMapAllocator, "{36266C8B-9A2C-4E3E-9812-3DB260868A2B}")
@@ -37,7 +36,7 @@ namespace AZ
static const int m_memoryBlockAlignment = 16;
void* m_memoryBlock; ///< Pointer to memory to allocate from. Can be uncached.
unsigned int m_memoryBlockByteSize; ///< Sizes if the memory block.
IAllocatorAllocate* m_mapAllocator; ///< Allocator for the free chunks map. If null the SystemAllocator will be used.
IAllocator* m_mapAllocator; ///< Allocator for the free chunks map. If null the SystemAllocator will be used.
bool m_allocationRecords; ///< True if we want to track memory allocations, otherwise false.
unsigned char m_stackRecordLevels; ///< If stack recording is enabled, how many stack levels to record.
@@ -52,7 +51,7 @@ namespace AZ
AllocatorDebugConfig GetDebugConfig() override;
//////////////////////////////////////////////////////////////////////////
// IAllocatorAllocate
// IAllocatorSchema
pointer_type Allocate(size_type byteSize, size_type alignment, int flags = 0, const char* name = 0, const char* fileName = 0, int lineNum = 0, unsigned int suppressStackRecord = 0) override;
void DeAllocate(pointer_type ptr, size_type byteSize = 0, size_type alignment = 0) override;
size_type Resize(pointer_type ptr, size_type newSize) override;
@@ -63,7 +62,6 @@ namespace AZ
size_type Capacity() const override;
size_type GetMaxAllocationSize() const override;
size_type GetMaxContiguousAllocationSize() const override;
IAllocatorAllocate* GetSubAllocator() override;
//////////////////////////////////////////////////////////////////////////
protected:
@@ -71,7 +69,6 @@ namespace AZ
BestFitExternalMapAllocator& operator=(const BestFitExternalMapAllocator&);
Descriptor m_desc;
BestFitExternalMapSchema* m_schema;
};
}
@@ -42,9 +42,15 @@ namespace AZ
// Allocate
// [1/28/2011]
//=========================================================================
BestFitExternalMapSchema::pointer_type BestFitExternalMapSchema::Allocate(size_type byteSize, size_type alignment, int flags)
BestFitExternalMapSchema::pointer_type BestFitExternalMapSchema::Allocate(
size_type byteSize,
size_type alignment,
[[maybe_unused]] int flags,
[[maybe_unused]] const char* name,
[[maybe_unused]] const char* fileName,
[[maybe_unused]] int lineNum,
[[maybe_unused]] unsigned int suppressStackRecord)
{
(void)flags;
char* address = nullptr;
AZ_Assert(alignment > 0 && (alignment & (alignment - 1)) == 0, "Alignment must be >0 and power of 2!");
for (int i = 0; i < 2; ++i) // max 2 attempts to allocate
@@ -96,7 +102,7 @@ namespace AZ
// DeAllocate
// [1/28/2011]
//=========================================================================
void BestFitExternalMapSchema::DeAllocate(pointer_type ptr)
void BestFitExternalMapSchema::DeAllocate(pointer_type ptr, [[maybe_unused]] size_type byteSize, [[maybe_unused]] size_type alignment)
{
if (ptr == nullptr)
{
@@ -125,6 +131,18 @@ namespace AZ
return 0;
}
BestFitExternalMapSchema::size_type BestFitExternalMapSchema::Resize(pointer_type, size_type)
{
AZ_Assert(false, "%s unsupported", AZ_FUNCTION_SIGNATURE);
return 0;
}
BestFitExternalMapSchema::pointer_type BestFitExternalMapSchema::ReAllocate(pointer_type, size_type, size_type)
{
AZ_Assert(false, "%s unsupported", AZ_FUNCTION_SIGNATURE);
return nullptr;
}
//=========================================================================
// GetMaxAllocationSize
// [1/28/2011]
@@ -20,7 +20,7 @@ namespace AZ
* External map allows us to use this allocator with uncached memory,
* because the tracking node is stored outside the main chunk.
*/
class BestFitExternalMapSchema
class BestFitExternalMapSchema : public IAllocatorSchema
{
public:
typedef void* pointer_type;
@@ -44,26 +44,27 @@ namespace AZ
static const int m_memoryBlockAlignment = 16;
void* m_memoryBlock; ///< Pointer to memory to allocate from. Can be uncached.
unsigned int m_memoryBlockByteSize; ///< Sizes if the memory block.
IAllocatorAllocate* m_mapAllocator; ///< Allocator for the free chunks map. If null the SystemAllocator will be used.
IAllocator* m_mapAllocator; ///< Allocator for the free chunks map. If null the SystemAllocator will be used.
};
BestFitExternalMapSchema(const Descriptor& desc);
pointer_type Allocate(size_type byteSize, size_type alignment, int flags);
void DeAllocate(pointer_type ptr);
size_type AllocationSize(pointer_type ptr);
pointer_type Allocate(size_type byteSize, size_type alignment, int flags = 0, const char* name = nullptr, const char* fileName = nullptr, int lineNum = 0, unsigned int suppressStackRecord = 0) override;
void DeAllocate(pointer_type ptr, size_type byteSize = 0, size_type alignment = 0) override;
size_type Resize(pointer_type ptr, size_type newSize) override;
pointer_type ReAllocate(pointer_type ptr, size_type newSize, size_type newAlignment) override;
size_type AllocationSize(pointer_type ptr) override;
AZ_FORCE_INLINE size_type NumAllocatedBytes() const { return m_used; }
AZ_FORCE_INLINE size_type Capacity() const { return m_desc.m_memoryBlockByteSize; }
size_type GetMaxAllocationSize() const;
size_type GetMaxContiguousAllocationSize() const;
AZ_FORCE_INLINE IAllocatorAllocate* GetSubAllocator() const { return m_desc.m_mapAllocator; }
AZ_FORCE_INLINE size_type NumAllocatedBytes() const override { return m_used; }
AZ_FORCE_INLINE size_type Capacity() const override { return m_desc.m_memoryBlockByteSize; }
size_type GetMaxAllocationSize() const override;
size_type GetMaxContiguousAllocationSize() const override;
/**
* Since we don't consolidate chucnks at free time (too expensive) we will do it as we need or when we can't
* Since we don't consolidate chunks at free time (too expensive) we will do it as we need or when we can't
* allocate memory. This function is at least O(nlogn) where 'n' are the free chunks.
*/
void GarbageCollect();
void GarbageCollect() override;
private:
AZ_FORCE_INLINE size_type ChunckSize(pointer_type ptr);
@@ -107,7 +107,6 @@ namespace AZ
m_used = 0;
m_desc = desc;
m_subAllocator = nullptr;
for (int i = 0; i < Descriptor::m_maxNumBlocks; ++i)
{
@@ -17,7 +17,7 @@ namespace AZ
* Internally uses use dlmalloc or version of it (nedmalloc, ptmalloc3).
*/
class HeapSchema
: public IAllocatorAllocate
: public IAllocatorSchema
{
public:
typedef void* pointer_type;
@@ -52,7 +52,6 @@ namespace AZ
size_type Capacity() const override { return m_capacity; }
size_type GetMaxAllocationSize() const override;
size_type GetMaxContiguousAllocationSize() const override;
IAllocatorAllocate* GetSubAllocator() override { return m_subAllocator; }
void GarbageCollect() override {}
private:
@@ -62,7 +61,7 @@ namespace AZ
Descriptor m_desc;
size_type m_capacity; ///< Capacity in bytes.
size_type m_used; ///< Number of bytes in use.
IAllocatorAllocate* m_subAllocator;
IAllocatorSchema* m_subAllocator;
bool m_ownMemoryBlock[Descriptor::m_maxNumBlocks];
};
}
@@ -719,8 +719,12 @@ namespace AZ {
#endif // DEBUG_ALLOCATOR
size_t mTotalAllocatedSizeBuckets = 0;
size_t mTotalCapacitySizeBuckets = 0;
// Bucket-dependent counters need to atomic since the locks that protect bucket allocations are per bucket
// So multiple threads could be updating these counters
AZStd::atomic<size_t> mTotalAllocatedSizeBuckets = 0;
AZStd::atomic<size_t> mTotalCapacitySizeBuckets = 0;
// In the case of tree allocations, there is a lock on the tree, so these counters are protected from multiple
// threads through that lock
size_t mTotalAllocatedSizeTree = 0;
size_t mTotalCapacitySizeTree = 0;
public:
@@ -1081,7 +1085,7 @@ namespace AZ {
const size_t m_treePageAlignment;
const size_t m_poolPageSize;
bool m_isPoolAllocations;
IAllocatorAllocate* m_subAllocator;
IAllocatorSchema* m_subAllocator;
#if !defined (USE_MUTEX_PER_BUCKET)
mutable AZStd::mutex m_mutex;
@@ -19,7 +19,7 @@ namespace AZ
* Heap allocator schema, based on Dimitar Lazarov "High Performance Heap Allocator".
*/
class HphaSchema
: public IAllocatorAllocate
: public IAllocatorSchema
{
public:
/**
@@ -47,7 +47,7 @@ namespace AZ
unsigned int m_isPoolAllocations : 1; ///< True to allow allocations from pools, otherwise false.
size_t m_fixedMemoryBlockByteSize; ///< Memory block size, if 0 we use the OS memory allocation functions.
void* m_fixedMemoryBlock; ///< Can be NULL if so the we will allocate memory from the subAllocator if m_memoryBlocksByteSize is != 0.
IAllocatorAllocate* m_subAllocator; ///< Allocator that m_memoryBlocks memory was allocated from or should be allocated (if NULL).
IAllocatorSchema* m_subAllocator; ///< Allocator that m_memoryBlocks memory was allocated from or should be allocated (if NULL).
size_t m_systemChunkSize; ///< Size of chunk to request from the OS when more memory is needed (defaults to m_pageSize)
size_t m_capacity; ///< Max size this allocator can grow to
};
@@ -68,7 +68,6 @@ namespace AZ
size_type GetMaxAllocationSize() const override;
size_type GetMaxContiguousAllocationSize() const override;
size_type GetUnAllocatedMemory(bool isPrint = false) const override;
IAllocatorAllocate* GetSubAllocator() override { return m_desc.m_subAllocator; }
/// Return unused memory to the OS (if we don't use fixed block). Don't call this unless you really need free memory, it is slow.
void GarbageCollect() override;
@@ -9,23 +9,12 @@
namespace AZ
{
IAllocator::IAllocator(IAllocatorAllocate* allocationSource)
: m_allocationSource(allocationSource)
, m_originalAllocationSource(allocationSource)
IAllocator::IAllocator(IAllocatorSchema* schema)
: m_schema(schema)
{
}
IAllocator::~IAllocator()
{
}
void IAllocator::SetAllocationSource(IAllocatorAllocate* allocationSource)
{
m_allocationSource = allocationSource;
}
void IAllocator::ResetAllocationSource()
{
m_allocationSource = m_originalAllocationSource;
}
}
@@ -28,19 +28,18 @@ namespace AZ
class AllocatorManager;
/**
* Allocator alloc/free basic interface. It is separate because it can be used
* for user provided allocators overrides
* Allocator schema interface
*/
class IAllocatorAllocate
class IAllocatorSchema
{
public:
typedef void* pointer_type;
typedef size_t size_type;
typedef ptrdiff_t difference_type;
virtual ~IAllocatorAllocate() {}
virtual ~IAllocatorSchema() = default;
virtual pointer_type Allocate(size_type byteSize, size_type alignment, int flags = 0, const char* name = 0, const char* fileName = 0, int lineNum = 0, unsigned int suppressStackRecord = 0) = 0;
virtual pointer_type Allocate(size_type byteSize, size_type alignment, int flags = 0, const char* name = nullptr, const char* fileName = nullptr, int lineNum = 0, unsigned int suppressStackRecord = 0) = 0;
virtual void DeAllocate(pointer_type ptr, size_type byteSize = 0, size_type alignment = 0) = 0;
/// Resize an allocated memory block. Returns the new adjusted size (as close as possible or equal to the requested one) or 0 (if you don't support resize at all).
virtual size_type Resize(pointer_type ptr, size_type newSize) = 0;
@@ -70,8 +69,6 @@ namespace AZ
* that will be reported.
*/
virtual size_type GetUnAllocatedMemory(bool isPrint = false) const { (void)isPrint; return 0; }
/// Returns a pointer to a sub-allocator or NULL.
virtual IAllocatorAllocate* GetSubAllocator() = 0;
};
/**
@@ -100,56 +97,19 @@ namespace AZ
/**
* Interface class for all allocators.
*/
class IAllocator
class IAllocator : public IAllocatorSchema
{
public:
IAllocator(IAllocatorAllocate* allocationSource);
IAllocator(IAllocatorSchema* schema = nullptr);
virtual ~IAllocator();
// @{ Every system allocator is required to provide name this is how
// Every system allocator is required to provide name this is how
// it will be registered with the allocator manager.
virtual const char* GetName() const = 0;
virtual const char* GetDescription() const = 0;
// @}
//---------------------------------------------------------------------
// Code releating to the allocation source is made concrete within this
// interface as a performance optimization.
//---------------------------------------------------------------------
/// Returns the current allocation source, which may be used to perform memory allocations.
AZ_FORCE_INLINE IAllocatorAllocate* GetAllocationSource() const
{
return m_allocationSource;
}
/// Returns the original allocation source. Generally only used for debugging purposes.
AZ_FORCE_INLINE IAllocatorAllocate* GetOriginalAllocationSource() const
{
return m_originalAllocationSource;
}
/// Returns true if the allocation source has changed from its original value.
AZ_FORCE_INLINE bool IsAllocationSourceChanged() const
{
return m_allocationSource != m_originalAllocationSource;
}
/// Sets the allocation source, effectively overriding the allocator.
/// Be very careful doing this, as existing allocations will be deallocated through the new source,
/// typically leading to unwanted effects (such as crashes).
void SetAllocationSource(IAllocatorAllocate* allocationSource);
/// Restores the allocation source to its original value.
/// Be very careful doing this, as allocations that came from the new source will now be deallocated
/// through the original source, typically leading to unwanted effects (such as crashes).
void ResetAllocationSource();
//---------------------------------------------------------------------
/// Returns the schema, if the allocator uses one. Returns nullptr if the allocator does not use a schema.
/// This is mainly used when debugging to determine if allocators alias each other under the hood.
virtual IAllocatorAllocate* GetSchema() = 0;
/// Returns the schema
AZ_FORCE_INLINE IAllocatorSchema* GetSchema() const { return m_schema; };
/// Returns the debug configuration for this allocator.
virtual AllocatorDebugConfig GetDebugConfig() = 0;
@@ -163,11 +123,6 @@ namespace AZ
/// Returns true if this allocator is ready to use.
virtual bool IsReady() const = 0;
/// Returns true if this allocator can be overridden with a different source.
/// Almost all allocators should return true. There are very few minor exceptions, such as the OS Allocator, that are required for direct
/// interfacing with the kernel and must never be overridden under any circumstances.
virtual bool CanBeOverridden() const = 0;
/// Returns true if the allocator was lazily created. Exposed primarily for testing systems that need to verify the state of allocators.
virtual bool IsLazilyCreated() const = 0;
@@ -195,9 +150,7 @@ namespace AZ
virtual void Destroy() = 0;
protected:
// The allocation source is made a direct member of the interface as a performance optimization.
IAllocatorAllocate * m_allocationSource;
IAllocatorAllocate* m_originalAllocationSource;
IAllocatorSchema* m_schema;
template<class Allocator>
friend class AllocatorStorage::StoragePolicyBase;
@@ -22,31 +22,15 @@ namespace AZ::Internal
namespace AZ
{
static constexpr size_t DEFAULT_ALIGNMENT = sizeof(void*) * 2; // Default malloc alignment
//---------------------------------------------------------------------
// MallocSchema methods
//---------------------------------------------------------------------
MallocSchema::MallocSchema(const Descriptor& desc)
MallocSchema::MallocSchema(const Descriptor&)
: m_bytesAllocated(0)
{
if (desc.m_useAZMalloc)
{
static const int DEFAULT_ALIGNMENT = sizeof(void*) * 2; // Default malloc alignment
m_mallocFn = [](size_t byteSize)
{
return AZ_OS_MALLOC(byteSize, DEFAULT_ALIGNMENT);
};
m_freeFn = [](void* ptr)
{
AZ_OS_FREE(ptr);
};
}
else
{
m_mallocFn = &malloc;
m_freeFn = &free;
}
}
MallocSchema::~MallocSchema()
@@ -84,7 +68,7 @@ namespace AZ
((alignment > sizeof(double))
? alignment
: 0); // Malloc will align to a minimum boundary for native objects, so we only pad if aligning to a large value
void* data = (*m_mallocFn)(required);
void* data = AZ_OS_MALLOC(required, DEFAULT_ALIGNMENT);
void* result = PointerAlignUp(reinterpret_cast<void*>(reinterpret_cast<size_t>(data) + sizeof(Internal::Header)), alignment);
Internal::Header* header = PointerAlignDown<Internal::Header>(
(Internal::Header*)(reinterpret_cast<size_t>(result) - sizeof(Internal::Header)), AZStd::alignment_of<Internal::Header>::value);
@@ -112,7 +96,7 @@ namespace AZ
void* freePtr = reinterpret_cast<void*>(reinterpret_cast<size_t>(ptr) - static_cast<size_t>(header->offset));
m_bytesAllocated -= header->size;
(*m_freeFn)(freePtr);
AZ_OS_FREE(freePtr);
}
MallocSchema::pointer_type MallocSchema::ReAllocate(pointer_type ptr, size_type newSize, size_type newAlignment)
@@ -166,11 +150,6 @@ namespace AZ
return AZ_CORE_MAX_ALLOCATOR_SIZE;
}
IAllocatorAllocate* MallocSchema::GetSubAllocator()
{
return nullptr;
}
void MallocSchema::GarbageCollect()
{
}
@@ -16,7 +16,7 @@ namespace AZ
* Uses malloc internally. Mainly intended for debugging using host operating system features.
*/
class MallocSchema
: public IAllocatorAllocate
: public IAllocatorSchema
{
public:
AZ_TYPE_INFO("MallocSchema", "{2A21D120-A42A-484C-997C-5735DCCA5FE9}");
@@ -25,21 +25,13 @@ namespace AZ
typedef size_t size_type;
typedef ptrdiff_t difference_type;
struct Descriptor
{
Descriptor(bool useAZMalloc = true)
: m_useAZMalloc(useAZMalloc)
{
}
bool m_useAZMalloc;
};
struct Descriptor {};
MallocSchema(const Descriptor& desc = Descriptor());
virtual ~MallocSchema();
//---------------------------------------------------------------------
// IAllocatorAllocate
// IAllocatorSchema
//---------------------------------------------------------------------
pointer_type Allocate(size_type byteSize, size_type alignment, int flags, const char* name = 0, const char* fileName = 0, int lineNum = 0, unsigned int suppressStackRecord = 0) override;
void DeAllocate(pointer_type ptr, size_type byteSize = 0, size_type alignment = 0) override;
@@ -51,15 +43,9 @@ namespace AZ
size_type Capacity() const override;
size_type GetMaxAllocationSize() const override;
size_type GetMaxContiguousAllocationSize() const override;
IAllocatorAllocate* GetSubAllocator() override;
void GarbageCollect() override;
private:
typedef void* (*MallocFn)(size_t);
typedef void (*FreeFn)(void*);
AZStd::atomic<size_t> m_bytesAllocated;
MallocFn m_mallocFn;
FreeFn m_freeFn;
};
}
@@ -7,22 +7,8 @@
*/
#include <AzCore/Memory/Memory.h>
#include <AzCore/Memory/OSAllocator.h>
#include <AzCore/Memory/SystemAllocator.h>
#include <AzCore/std/parallel/containers/concurrent_fixed_unordered_set.h>
#include <AzCore/Memory/AllocatorManager.h>
AZ::AllocatorStorage::LazyAllocatorRef::~LazyAllocatorRef()
{
m_destructor(*m_allocator);
}
void AZ::AllocatorStorage::LazyAllocatorRef::Init(size_t size, size_t alignment, CreationFn creationFn, DestructionFn destructionFn)
{
m_allocator = AZ::AllocatorManager::CreateLazyAllocator(size, alignment, creationFn);
m_destructor = destructionFn;
}
//////////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////////
// New overloads
+15 -111
View File
@@ -141,9 +141,9 @@ void* operator new[](std::size_t, const AZ::Internal::AllocatorDummy*);
*/
#define azfree(...) AZ_MACRO_SPECIALIZE(azfree_, AZ_VA_NUM_ARGS(__VA_ARGS__), (__VA_ARGS__))
/// Returns allocation size, based on it's pointer \ref AZ::IAllocatorAllocate::AllocationSize.
/// Returns allocation size, based on it's pointer \ref AZ::IAllocatorSchema::AllocationSize.
#define azallocsize(_Ptr, _Allocator) AZ::AllocatorInstance< _Allocator >::Get().AllocationSize(_Ptr)
/// Returns the new expanded size or 0 if NOT supported by the allocator \ref AZ::IAllocatorAllocate::Resize.
/// Returns the new expanded size or 0 if NOT supported by the allocator \ref AZ::IAllocatorSchema::Resize.
#define azallocresize(_Ptr, _NewSize, _Allocator) AZ::AllocatorInstance< _Allocator >::Get().Resize(_Ptr, _NewSize)
namespace AZ {
@@ -521,19 +521,6 @@ namespace AZ
{
namespace AllocatorStorage
{
/// A private structure to create heap-storage for an allocator that won't expire until other static module members are destructed.
struct LazyAllocatorRef
{
using CreationFn = IAllocator*(*)(void*);
using DestructionFn = void(*)(IAllocator&);
~LazyAllocatorRef();
void Init(size_t size, size_t alignment, CreationFn creationFn, DestructionFn destructionFn);
IAllocator* m_allocator = nullptr;
DestructionFn m_destructor = nullptr;
};
/**
* A base class for all storage policies. This exists to provide access to private IAllocator methods via template friends.
*/
@@ -640,87 +627,6 @@ namespace AZ
template<class Allocator>
EnvironmentVariable<Allocator> EnvironmentStoragePolicy<Allocator>::s_allocator;
/**
* ModuleStoragePolicy stores the allocator in a static variable that is local to the module using it.
* This forces separate instances of the allocator to exist in each module, and permits lazy instantiation.
* We only tolerate this for some special allocators, primarily to maintain backwards compatibility with CryEngine,
* since it still allocates outside of code in the data section.
*
* It has two ways of storing its allocator: either on the heap, which is the preferred way, since it guarantees
* the memory for the allocator won't be deallocated (such as in a DLL) before anyone that's using it. If disabled
* the allocator is stored in a static variable, which should only be used where this isn't a problem a shut-down
* time, such as on a console.
*/
template<class Allocator, bool StoreAllocatorOnHeap>
struct ModuleStoragePolicyBase;
template<class Allocator>
struct ModuleStoragePolicyBase<Allocator, false>: public StoragePolicyBase<Allocator>
{
protected:
// Use a static instance to store the allocator. This is not recommended when the order of shut-down with the module matters, as the allocator could have its memory destroyed
// before the users of it are destroyed. The primary use case for this is allocators that need to support the CRT, as they cannot allocate from the heap.
static Allocator& GetModuleAllocatorInstance()
{
static Allocator* s_allocator = nullptr;
static typename AZStd::aligned_storage<sizeof(Allocator), AZStd::alignment_of<Allocator>::value>::type s_storage;
if (!s_allocator)
{
s_allocator = new (&s_storage) Allocator;
StoragePolicyBase<Allocator>::Create(*s_allocator, typename Allocator::Descriptor(), true);
}
return *s_allocator;
}
};
template<class Allocator>
struct ModuleStoragePolicyBase<Allocator, true> : public StoragePolicyBase<Allocator>
{
protected:
// Store-on-heap implementation uses the LazyAllocatorRef to create and destroy an allocator using heap-space so there isn't a problem with destruction order within the module.
static Allocator& GetModuleAllocatorInstance()
{
static LazyAllocatorRef s_allocator;
if (!s_allocator.m_allocator)
{
s_allocator.Init(sizeof(Allocator), AZStd::alignment_of<Allocator>::value, [](void* mem) -> IAllocator* { return new (mem) Allocator; }, &StoragePolicyBase<Allocator>::Destroy);
StoragePolicyBase<Allocator>::Create(*static_cast<Allocator*>(s_allocator.m_allocator), typename Allocator::Descriptor(), true);
}
return *static_cast<Allocator*>(s_allocator.m_allocator);
}
};
template<class Allocator, bool StoreAllocatorOnHeap = true>
class ModuleStoragePolicy : public ModuleStoragePolicyBase<Allocator, StoreAllocatorOnHeap>
{
public:
using Base = ModuleStoragePolicyBase<Allocator, StoreAllocatorOnHeap>;
static IAllocator& GetAllocator()
{
return Base::GetModuleAllocatorInstance();
}
static void Create(const typename Allocator::Descriptor& desc = typename Allocator::Descriptor())
{
StoragePolicyBase<Allocator>::Create(Base::GetModuleAllocatorInstance(), desc, true);
}
static void Destroy()
{
StoragePolicyBase<Allocator>::Destroy(Base::GetModuleAllocatorInstance());
}
static bool IsReady()
{
return Base::GetModuleAllocatorInstance().IsReady();
}
};
}
namespace Internal
@@ -734,12 +640,15 @@ namespace AZ
public:
typedef typename Allocator::Descriptor Descriptor;
AZ_FORCE_INLINE static IAllocatorAllocate& Get()
// Maintained for backwards compatibility, prefer to use Get() instead.
// Get was previously used to get the the schema, however, that bypasses what the allocators are doing.
// If the schema is needed, call Get().GetSchema()
AZ_FORCE_INLINE static IAllocator& GetAllocator()
{
return *GetAllocator().GetAllocationSource();
return StoragePolicy::GetAllocator();
}
AZ_FORCE_INLINE static IAllocator& GetAllocator()
AZ_FORCE_INLINE static IAllocator& Get()
{
return StoragePolicy::GetAllocator();
}
@@ -781,7 +690,7 @@ namespace AZ
// structure of another allocator
template <class ParentAllocator>
class ChildAllocatorSchema
: public IAllocatorAllocate
: public IAllocatorSchema
{
public:
// No descriptor is necessary, as the parent allocator is expected to already
@@ -792,7 +701,7 @@ namespace AZ
ChildAllocatorSchema(const Descriptor&) {}
//---------------------------------------------------------------------
// IAllocatorAllocate
// IAllocatorSchema
//---------------------------------------------------------------------
pointer_type Allocate(size_type byteSize, size_type alignment, int flags = 0, const char* name = 0, const char* fileName = 0, int lineNum = 0, unsigned int suppressStackRecord = 0) override
{
@@ -848,11 +757,6 @@ namespace AZ
{
return AZ::AllocatorInstance<Parent>::Get().GetUnAllocatedMemory(isPrint);
}
IAllocatorAllocate* GetSubAllocator() override
{
return AZ::AllocatorInstance<Parent>::Get().GetSubAllocator();
}
};
/**
@@ -873,7 +777,7 @@ namespace AZ
{
if (AllocatorInstance<Allocator>::IsReady())
{
m_name = AllocatorInstance<Allocator>::GetAllocator().GetName();
m_name = AllocatorInstance<Allocator>::Get().GetName();
}
else
{
@@ -932,7 +836,7 @@ namespace AZ
typedef AZStd::ptrdiff_t difference_type;
typedef AZStd::false_type allow_memory_leaks; ///< Regular allocators should not leak.
AZ_FORCE_INLINE AZStdIAllocator(IAllocatorAllocate* allocator, const char* name = "AZ::AZStdIAllocator")
AZ_FORCE_INLINE AZStdIAllocator(IAllocator* allocator, const char* name = "AZ::AZStdIAllocator")
: m_allocator(allocator)
, m_name(name)
{
@@ -965,7 +869,7 @@ namespace AZ
AZ_FORCE_INLINE bool operator==(const AZStdIAllocator& rhs) const { return m_allocator == rhs.m_allocator; }
AZ_FORCE_INLINE bool operator!=(const AZStdIAllocator& rhs) const { return m_allocator != rhs.m_allocator; }
private:
IAllocatorAllocate* m_allocator;
IAllocator* m_allocator;
const char* m_name;
};
@@ -982,8 +886,8 @@ namespace AZ
using size_type = AZStd::size_t;
using difference_type = AZStd::ptrdiff_t;
using allow_memory_leaks = AZStd::false_type; ///< Regular allocators should not leak.
using functor_type = IAllocatorAllocate&(*)(); ///< Function Pointer must return IAllocatorAllocate&.
///< function pointers do not support covariant return types
using functor_type = IAllocator&(*)(); ///< Function Pointer must return IAllocator&.
///< function pointers do not support covariant return types
constexpr AZStdFunctorAllocator(functor_type allocatorFunctor, const char* name = "AZ::AZStdFunctorAllocator")
: m_allocatorFunctor(allocatorFunctor)
@@ -19,7 +19,6 @@ namespace AZ
, m_custom(nullptr)
, m_numAllocatedBytes(0)
{
DisableOverriding();
}
//=========================================================================
@@ -24,7 +24,6 @@ namespace AZ
*/
class OSAllocator
: public AllocatorBase
, public IAllocatorAllocate
{
public:
AZ_TYPE_INFO(OSAllocator, "{9F835EE3-F23C-454E-B4E3-011E2F3C8118}")
@@ -39,7 +38,7 @@ namespace AZ
{
Descriptor()
: m_custom(0) {}
IAllocatorAllocate* m_custom; ///< You can provide our own allocation scheme. If NULL a HeapScheme will be used with the provided Descriptor.
IAllocatorSchema* m_custom; ///< You can provide our own allocation scheme. If NULL a HeapScheme will be used with the provided Descriptor.
};
bool Create(const Descriptor& desc);
@@ -51,7 +50,7 @@ namespace AZ
AllocatorDebugConfig GetDebugConfig() override;
//////////////////////////////////////////////////////////////////////////
// IAllocatorAllocate
// IAllocatorSchema
pointer_type Allocate(size_type byteSize, size_type alignment, int flags = 0, const char* name = 0, const char* fileName = 0, int lineNum = 0, unsigned int suppressStackRecord = 0) override;
void DeAllocate(pointer_type ptr, size_type byteSize = 0, size_type alignment = 0) override;
size_type Resize(pointer_type ptr, size_type newSize) override { return m_custom ? m_custom->Resize(ptr, newSize) : 0; }
@@ -62,13 +61,12 @@ namespace AZ
size_type Capacity() const override { return m_custom ? m_custom->Capacity() : AZ_CORE_MAX_ALLOCATOR_SIZE; } // custom size or unlimited
size_type GetMaxAllocationSize() const override { return m_custom ? m_custom->GetMaxAllocationSize() : AZ_CORE_MAX_ALLOCATOR_SIZE; } // custom size or unlimited
size_type GetMaxContiguousAllocationSize() const override { return m_custom ? m_custom->GetMaxContiguousAllocationSize() : AZ_CORE_MAX_ALLOCATOR_SIZE; } // custom size or unlimited
IAllocatorAllocate* GetSubAllocator() override { return m_custom ? m_custom : NULL; }
protected:
OSAllocator(const OSAllocator&);
OSAllocator& operator=(const OSAllocator&);
IAllocatorAllocate* m_custom;
IAllocatorSchema* m_custom;
size_type m_numAllocatedBytes;
};
@@ -233,7 +233,6 @@ namespace AZ
size_type Capacity() const;
size_type GetMaxAllocationSize() const;
size_type GetMaxContiguousAllocationSize() const;
IAllocatorAllocate* GetSubAllocator();
void GarbageCollect();
private:
@@ -680,11 +679,6 @@ auto AZ::OverrunDetectionSchemaImpl::GetMaxContiguousAllocationSize() const -> s
return 0;
}
AZ::IAllocatorAllocate* AZ::OverrunDetectionSchemaImpl::GetSubAllocator()
{
return nullptr;
}
void AZ::OverrunDetectionSchemaImpl::GarbageCollect()
{
}
@@ -810,11 +804,6 @@ auto AZ::OverrunDetectionSchema::GetMaxContiguousAllocationSize() const -> size_
return m_impl->GetMaxContiguousAllocationSize();
}
AZ::IAllocatorAllocate* AZ::OverrunDetectionSchema::GetSubAllocator()
{
return m_impl->GetSubAllocator();
}
void AZ::OverrunDetectionSchema::GarbageCollect()
{
m_impl->GarbageCollect();
@@ -27,7 +27,7 @@ namespace AZ
* the requested memory, plus the trap page). On most platforms this is 8kb (4kb * 2 pages).
*/
class OverrunDetectionSchema
: public IAllocatorAllocate
: public IAllocatorSchema
{
public:
AZ_TYPE_INFO("OverrunDetectionSchema", "{0DF781AC-1615-40AE-81F7-6CA5841E2914}");
@@ -75,7 +75,7 @@ namespace AZ
virtual ~OverrunDetectionSchema();
//---------------------------------------------------------------------
// IAllocatorAllocate
// IAllocatorSchema
//---------------------------------------------------------------------
pointer_type Allocate(size_type byteSize, size_type alignment, int flags, const char* name = 0, const char* fileName = 0, int lineNum = 0, unsigned int suppressStackRecord = 0) override;
void DeAllocate(pointer_type ptr, size_type byteSize = 0, size_type alignment = 0) override;
@@ -87,7 +87,6 @@ namespace AZ
size_type Capacity() const override;
size_type GetMaxAllocationSize() const override;
size_type GetMaxContiguousAllocationSize() const override;
IAllocatorAllocate* GetSubAllocator() override;
void GarbageCollect() override;
private:
@@ -102,7 +102,7 @@ namespace AZ
}
//////////////////////////////////////////////////////////////////////////
// IAllocatorAllocate
// IAllocatorSchema
pointer_type ReAllocate(pointer_type ptr, size_type newSize, size_type newAlignment) override
{
(void)ptr;
@@ -163,7 +163,7 @@ namespace AZ
}
using AllocatorType = PoolAllocation<PoolSchemaImpl>;
IAllocatorAllocate* m_pageAllocator;
IAllocatorSchema* m_pageAllocator;
AllocatorType m_allocator;
void* m_staticDataBlock;
unsigned int m_numStaticPages;
@@ -301,7 +301,7 @@ namespace AZ
FreePagesType m_freePages;
AZStd::vector<ThreadPoolData*> m_threads; ///< Array with all separate thread data. Used to traverse end free elements.
IAllocatorAllocate* m_pageAllocator;
IAllocatorSchema* m_pageAllocator;
void* m_staticDataBlock;
size_t m_numStaticPages;
size_t m_pageSize;
@@ -744,15 +744,6 @@ namespace AZ
return m_impl->m_numStaticPages * m_impl->m_pageSize;
}
//=========================================================================
// GetPageAllocator
// [11/17/2010]
//=========================================================================
IAllocatorAllocate* PoolSchema::GetSubAllocator()
{
return m_impl->m_pageAllocator;
}
//////////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////////
// PollAllocator Implementation
@@ -1095,15 +1086,6 @@ namespace AZ
return m_impl->m_numStaticPages * m_impl->m_pageSize;
}
//=========================================================================
// GetPageAllocator
// [11/17/2010]
//=========================================================================
IAllocatorAllocate* ThreadPoolSchema::GetSubAllocator()
{
return m_impl->m_pageAllocator;
}
//=========================================================================
// ThreadPoolSchemaImpl
// [9/15/2009]
@@ -21,7 +21,7 @@ namespace AZ
* use ThreadPool Schema or do the sync yourself.
*/
class PoolSchema
: public IAllocatorAllocate
: public IAllocatorSchema
{
public:
/**
@@ -51,7 +51,7 @@ namespace AZ
* this is the minimum number of pages we will have allocated at all times, otherwise the total number of pages supported.
*/
unsigned int m_numStaticPages;
IAllocatorAllocate* m_pageAllocator; ///< If you provide this interface we will use it for page allocations, otherwise SystemAllocator will be used.
IAllocatorSchema* m_pageAllocator; ///< If you provide this interface we will use it for page allocations, otherwise SystemAllocator will be used.
};
PoolSchema(const Descriptor& desc = Descriptor());
@@ -72,7 +72,6 @@ namespace AZ
size_type GetMaxContiguousAllocationSize() const override;
size_type NumAllocatedBytes() const override;
size_type Capacity() const override;
IAllocatorAllocate* GetSubAllocator() override;
protected:
PoolSchema(const PoolSchema&);
@@ -89,7 +88,7 @@ namespace AZ
* for each thread. So there will be some memory overhead, especially if you use fixed pool sizes.
*/
class ThreadPoolSchema
: public IAllocatorAllocate
: public IAllocatorSchema
{
public:
// Functions for getting an instance of a ThreadPoolData when using thread local storage
@@ -118,7 +117,6 @@ namespace AZ
size_type GetMaxContiguousAllocationSize() const override;
size_type NumAllocatedBytes() const override;
size_type Capacity() const override;
IAllocatorAllocate* GetSubAllocator() override;
protected:
ThreadPoolSchema(const ThreadPoolSchema&);
@@ -22,17 +22,15 @@ namespace AZ
template <class Schema, class DescriptorType=typename Schema::Descriptor, bool ProfileAllocations=true, bool ReportOutOfMemory=true>
class SimpleSchemaAllocator
: public AllocatorBase
, public IAllocatorAllocate
{
public:
using Descriptor = DescriptorType;
using pointer_type = typename IAllocatorAllocate::pointer_type;
using size_type = typename IAllocatorAllocate::size_type;
using difference_type = typename IAllocatorAllocate::difference_type;
using pointer_type = typename Schema::pointer_type;
using size_type = typename Schema::size_type;
using difference_type = typename Schema::difference_type;
SimpleSchemaAllocator(const char* name, const char* desc)
: AllocatorBase(this, name, desc)
, m_schema(nullptr)
: AllocatorBase(nullptr, name, desc)
{
}
@@ -65,13 +63,8 @@ namespace AZ
return AllocatorDebugConfig();
}
IAllocatorAllocate* GetSchema() override
{
return m_schema;
}
//---------------------------------------------------------------------
// IAllocatorAllocate
// IAllocatorSchema
//---------------------------------------------------------------------
pointer_type Allocate(size_type byteSize, size_type alignment, int flags = 0, const char* name = nullptr, const char* fileName = nullptr, int lineNum = 0, unsigned int suppressStackRecord = 0) override
{
@@ -188,14 +181,6 @@ namespace AZ
{
return m_schema->GetUnAllocatedMemory(isPrint);
}
IAllocatorAllocate* GetSubAllocator() override
{
return m_schema->GetSubAllocator();
}
protected:
IAllocatorAllocate* m_schema;
private:
typename AZStd::aligned_storage<sizeof(Schema), AZStd::alignment_of<Schema>::value>::type m_schemaStorage;
@@ -50,9 +50,8 @@ namespace AZ
// [9/2/2009]
//=========================================================================
SystemAllocator::SystemAllocator()
: AllocatorBase(this, "SystemAllocator", "Fundamental generic memory allocator")
: AllocatorBase(nullptr, "SystemAllocator", "Fundamental generic memory allocator")
, m_isCustom(false)
, m_allocator(nullptr)
, m_ownsOSAllocator(false)
{
}
@@ -91,7 +90,7 @@ namespace AZ
if (desc.m_custom)
{
m_isCustom = true;
m_allocator = desc.m_custom;
m_schema = desc.m_custom;
isReady = true;
}
else
@@ -119,9 +118,9 @@ namespace AZ
AZ_Assert(!g_isSystemSchemaUsed, "AZ::SystemAllocator MUST be created first! It's the source of all allocations!");
#if AZCORE_SYSTEM_ALLOCATOR == AZCORE_SYSTEM_ALLOCATOR_HPHA
m_allocator = new (&g_systemSchema) HphaSchema(heapDesc);
m_schema = new (&g_systemSchema) HphaSchema(heapDesc);
#elif AZCORE_SYSTEM_ALLOCATOR == AZCORE_SYSTEM_ALLOCATOR_MALLOC
m_allocator = new (&g_systemSchema) MallocSchema(heapDesc);
m_schema = new (&g_systemSchema) MallocSchema(heapDesc);
#endif
g_isSystemSchemaUsed = true;
isReady = true;
@@ -134,11 +133,11 @@ namespace AZ
"System allocator must be created before any other allocator! They allocate from it.");
#if AZCORE_SYSTEM_ALLOCATOR == AZCORE_SYSTEM_ALLOCATOR_HPHA
m_allocator = azcreate(HphaSchema, (heapDesc), SystemAllocator);
m_schema = azcreate(HphaSchema, (heapDesc), SystemAllocator);
#elif AZCORE_SYSTEM_ALLOCATOR == AZCORE_SYSTEM_ALLOCATOR_MALLOC
m_allocator = azcreate(MallocSchema, (heapDesc), SystemAllocator);
m_schema = azcreate(MallocSchema, (heapDesc), SystemAllocator);
#endif
if (m_allocator == nullptr)
if (m_schema == nullptr)
{
isReady = false;
}
@@ -166,18 +165,18 @@ namespace AZ
if (!m_isCustom)
{
if ((void*)m_allocator == (void*)&g_systemSchema)
if ((void*)m_schema == (void*)&g_systemSchema)
{
#if AZCORE_SYSTEM_ALLOCATOR == AZCORE_SYSTEM_ALLOCATOR_HPHA
static_cast<HphaSchema*>(m_allocator)->~HphaSchema();
static_cast<HphaSchema*>(m_schema)->~HphaSchema();
#elif AZCORE_SYSTEM_ALLOCATOR == AZCORE_SYSTEM_ALLOCATOR_MALLOC
static_cast<MallocSchema*>(m_allocator)->~MallocSchema();
static_cast<MallocSchema*>(m_schema)->~MallocSchema();
#endif
g_isSystemSchemaUsed = false;
}
else
{
azdestroy(m_allocator);
azdestroy(m_schema);
}
}
@@ -197,11 +196,6 @@ namespace AZ
.ExcludeFromDebugging(!m_desc.m_allocationRecords);
}
IAllocatorAllocate* SystemAllocator::GetSchema()
{
return m_allocator;
}
//=========================================================================
// Allocate
// [9/2/2009]
@@ -224,14 +218,14 @@ namespace AZ
byteSize = MemorySizeAdjustedUp(byteSize);
SystemAllocator::pointer_type address =
m_allocator->Allocate(byteSize, alignment, flags, name, fileName, lineNum, suppressStackRecord + 1);
m_schema->Allocate(byteSize, alignment, flags, name, fileName, lineNum, suppressStackRecord + 1);
if (address == nullptr)
{
// Free all memory we can and try again!
AllocatorManager::Instance().GarbageCollect();
address = m_allocator->Allocate(byteSize, alignment, flags, name, fileName, lineNum, suppressStackRecord + 1);
address = m_schema->Allocate(byteSize, alignment, flags, name, fileName, lineNum, suppressStackRecord + 1);
}
if (address == nullptr)
@@ -258,7 +252,7 @@ namespace AZ
byteSize = MemorySizeAdjustedUp(byteSize);
AZ_PROFILE_MEMORY_FREE(MemoryReserved, ptr);
AZ_MEMORY_PROFILE(ProfileDeallocation(ptr, byteSize, alignment, nullptr));
m_allocator->DeAllocate(ptr, byteSize, alignment);
m_schema->DeAllocate(ptr, byteSize, alignment);
}
//=========================================================================
@@ -271,7 +265,7 @@ namespace AZ
AZ_MEMORY_PROFILE(ProfileReallocationBegin(ptr, newSize));
AZ_PROFILE_MEMORY_FREE(MemoryReserved, ptr);
pointer_type newAddress = m_allocator->ReAllocate(ptr, newSize, newAlignment);
pointer_type newAddress = m_schema->ReAllocate(ptr, newSize, newAlignment);
AZ_PROFILE_MEMORY_ALLOC(MemoryReserved, newAddress, newSize, "SystemAllocator realloc");
AZ_MEMORY_PROFILE(ProfileReallocationEnd(ptr, newAddress, newSize, newAlignment));
@@ -285,7 +279,7 @@ namespace AZ
SystemAllocator::size_type SystemAllocator::Resize(pointer_type ptr, size_type newSize)
{
newSize = MemorySizeAdjustedUp(newSize);
size_type resizedSize = m_allocator->Resize(ptr, newSize);
size_type resizedSize = m_schema->Resize(ptr, newSize);
AZ_MEMORY_PROFILE(ProfileResize(ptr, resizedSize));
@@ -298,7 +292,7 @@ namespace AZ
//=========================================================================
SystemAllocator::size_type SystemAllocator::AllocationSize(pointer_type ptr)
{
size_type allocSize = MemorySizeAdjustedDown(m_allocator->AllocationSize(ptr));
size_type allocSize = MemorySizeAdjustedDown(m_schema->AllocationSize(ptr));
return allocSize;
}
@@ -25,7 +25,6 @@ namespace AZ
*/
class SystemAllocator
: public AllocatorBase
, public IAllocatorAllocate
{
public:
AZ_TYPE_INFO(SystemAllocator, "{424C94D8-85CF-4E89-8CD6-AB5EC173E875}")
@@ -38,7 +37,7 @@ namespace AZ
* we will allocate system memory using system calls. You can
* provide arenas (spaces) with pre-allocated memory, and use the
* flag to specify which arena you want to allocate from.
* You are also allowed to supply IAllocatorAllocate, but if you do
* You are also allowed to supply IAllocatorSchema, but if you do
* so you will need to take care of all allocations, we will not use
* the default HeapSchema.
* \ref HeapSchema::Descriptor
@@ -50,7 +49,7 @@ namespace AZ
, m_allocationRecords(true)
, m_stackRecordLevels(5)
{}
IAllocatorAllocate* m_custom; ///< You can provide our own allocation scheme. If NULL a HeapScheme will be used with the provided Descriptor.
IAllocatorSchema* m_custom; ///< You can provide our own allocation scheme. If NULL a HeapScheme will be used with the provided Descriptor.
struct Heap
{
@@ -72,7 +71,7 @@ namespace AZ
int m_numFixedMemoryBlocks; ///< Number of memory blocks to use.
void* m_fixedMemoryBlocks[m_maxNumFixedBlocks]; ///< Pointers to provided memory blocks or NULL if you want the system to allocate them for you with the System Allocator.
size_t m_fixedMemoryBlocksByteSize[m_maxNumFixedBlocks]; ///< Sizes of different memory blocks (MUST be multiple of m_pageSize), if m_memoryBlock is 0 the block will be allocated for you with the System Allocator.
IAllocatorAllocate* m_subAllocator; ///< Allocator that m_memoryBlocks memory was allocated from or should be allocated (if NULL).
IAllocatorSchema* m_subAllocator; ///< Allocator that m_memoryBlocks memory was allocated from or should be allocated (if NULL).
size_t m_systemChunkSize; ///< Size of chunk to request from the OS when more memory is needed (defaults to m_pageSize)
} m_heap;
bool m_allocationRecords; ///< True if we want to track memory allocations, otherwise false.
@@ -86,25 +85,23 @@ namespace AZ
//////////////////////////////////////////////////////////////////////////
// IAllocator
AllocatorDebugConfig GetDebugConfig() override;
IAllocatorAllocate* GetSchema() override;
//////////////////////////////////////////////////////////////////////////
// IAllocatorAllocate
// IAllocatorSchema
pointer_type Allocate(size_type byteSize, size_type alignment, int flags = 0, const char* name = 0, const char* fileName = 0, int lineNum = 0, unsigned int suppressStackRecord = 0) override;
void DeAllocate(pointer_type ptr, size_type byteSize = 0, size_type alignment = 0) override;
pointer_type ReAllocate(pointer_type ptr, size_type newSize, size_type newAlignment) override;
size_type Resize(pointer_type ptr, size_type newSize) override;
size_type AllocationSize(pointer_type ptr) override;
void GarbageCollect() override { m_allocator->GarbageCollect(); }
void GarbageCollect() override { GetSchema()->GarbageCollect(); }
size_type NumAllocatedBytes() const override { return m_allocator->NumAllocatedBytes(); }
size_type Capacity() const override { return m_allocator->Capacity(); }
size_type NumAllocatedBytes() const override { return GetSchema()->NumAllocatedBytes(); }
size_type Capacity() const override { return GetSchema()->Capacity(); }
/// Keep in mind this operation will execute GarbageCollect to make sure it returns, max allocation. This function WILL be slow.
size_type GetMaxAllocationSize() const override { return m_allocator->GetMaxAllocationSize(); }
size_type GetMaxContiguousAllocationSize() const override { return m_allocator->GetMaxContiguousAllocationSize(); }
size_type GetUnAllocatedMemory(bool isPrint = false) const override { return m_allocator->GetUnAllocatedMemory(isPrint); }
IAllocatorAllocate* GetSubAllocator() override { return m_isCustom ? m_allocator : m_allocator->GetSubAllocator(); }
size_type GetMaxAllocationSize() const override { return GetSchema()->GetMaxAllocationSize(); }
size_type GetMaxContiguousAllocationSize() const override { return GetSchema()->GetMaxContiguousAllocationSize(); }
size_type GetUnAllocatedMemory(bool isPrint = false) const override { return GetSchema()->GetUnAllocatedMemory(isPrint); }
//////////////////////////////////////////////////////////////////////////
@@ -114,7 +111,6 @@ namespace AZ
Descriptor m_desc;
bool m_isCustom;
IAllocatorAllocate* m_allocator;
bool m_ownsOSAllocator;
};
}
@@ -85,7 +85,7 @@ namespace AZ
NameDictionary::~NameDictionary()
{
bool leaksDetected = false;
[[maybe_unused]] bool leaksDetected = false;
for (const auto& keyValue : m_dictionary)
{
+4 -4
View File
@@ -248,14 +248,14 @@
#if defined(__has_builtin)
#if __has_builtin(__builtin_is_constant_evaluated)
#define az_builtin_is_constant_evaluated() __builtin_is_constant_evaluated()
#define az_has_builtin_is_constant_evaluated() true
#define az_has_builtin_is_constant_evaluated true
#endif
#elif AZ_COMPILER_MSVC >= 1928
#define az_builtin_is_constant_evaluated() __builtin_is_constant_evaluated()
#define az_has_builtin_is_constant_evaluated() true
#define az_has_builtin_is_constant_evaluated true
#elif AZ_COMPILER_GCC
#define az_builtin_is_constant_evaluated() __builtin_is_constant_evaluated()
#define az_has_builtin_is_constant_evaluated() true
#define az_has_builtin_is_constant_evaluated true
#endif
#endif
@@ -271,7 +271,7 @@
}
}
#define az_builtin_is_constant_evaluated() AZ::Internal::builtin_is_constant_evaluated()
#define az_has_builtin_is_constant_evaluated() false
#define az_has_builtin_is_constant_evaluated false
#endif
// define builtin functions used by char_traits class for efficient compile time and runtime
@@ -136,7 +136,7 @@ namespace AZ::CommonOnDemandReflections
->template Constructor<typename ContainerType::value_type*>()
->Attribute(AZ::Script::Attributes::ConstructorOverride, &OnDemandLuaFunctions::ConstructStringView<ContainerType::value_type, ContainerType::traits_type>)
->Attribute(AZ::Script::Attributes::ReaderWriterOverride, ScriptContext::CustomReaderWriter(&OnDemandLuaFunctions::StringTypeToLua<ContainerType>, &OnDemandLuaFunctions::StringTypeFromLua<ContainerType>))
->Method("ToString", [](const ContainerType& stringView) { return static_cast<AZStd::string>(stringView).c_str(); }, { { { "Reference", "String view object being converted to string" } } })
->Method("ToString", [](const ContainerType& stringView) { return stringView.data(); }, { { { "Reference", "String view object being converted to string" } } })
->Attribute(AZ::Script::Attributes::ToolTip, "Converts string_view to string")
->Attribute(AZ::Script::Attributes::Operator, AZ::Script::Attributes::OperatorType::ToString)
->template WrappingMember<const char*>(&ContainerType::data)
@@ -1457,7 +1457,7 @@ namespace AZ
static void* LuaMemoryHook(void* userData, void* ptr, size_t osize, size_t nsize)
{
(void)osize;
IAllocatorAllocate* allocator = reinterpret_cast<IAllocatorAllocate*>(userData);
IAllocator* allocator = reinterpret_cast<IAllocator*>(userData);
if (nsize == 0)
{
if (ptr)
@@ -4276,7 +4276,7 @@ LUA_API const Node* lua_getDummyNode()
AZ_CLASS_ALLOCATOR(ScriptContextImpl, AZ::SystemAllocator, 0);
//////////////////////////////////////////////////////////////////////////
ScriptContextImpl(ScriptContext* owner, IAllocatorAllocate* allocator, lua_State* nativeContext)
ScriptContextImpl(ScriptContext* owner, IAllocator* allocator, lua_State* nativeContext)
: m_owner(owner)
, m_context(nullptr)
, m_debug(nullptr)
@@ -5828,7 +5828,7 @@ LUA_API const Node* lua_getDummyNode()
AZStd::thread::id m_ownerThreadId; // Check if Lua methods (including EBus handlers) are called from background threads.
};
ScriptContext::ScriptContext(ScriptContextId id, IAllocatorAllocate* allocator, lua_State* nativeContext)
ScriptContext::ScriptContext(ScriptContextId id, IAllocator* allocator, lua_State* nativeContext)
{
m_id = id;
m_impl = aznew ScriptContextImpl(this, allocator, nativeContext);
@@ -821,7 +821,7 @@ namespace AZ
CustomFromLua m_fromLua;
};
ScriptContext(ScriptContextId id = ScriptContextIds::DefaultScriptContextId, IAllocatorAllocate* allocator = nullptr, lua_State* nativeContext = nullptr);
ScriptContext(ScriptContextId id = ScriptContextIds::DefaultScriptContextId, IAllocator* allocator = nullptr, lua_State* nativeContext = nullptr);
~ScriptContext();
/// Bind LUA context (VM) a specific behaviorContext
@@ -74,7 +74,7 @@ namespace AZ
* But as the AZStdAssociativeContainer instance will not be accessed outside of the module it was
* created within then this will return this .dll/.exe module allocator
*/
classElement.m_attributes.set_allocator(AZStdFunctorAllocator([]() -> IAllocatorAllocate& { return GetCurrentSerializeContextModule().GetAllocator(); }));
classElement.m_attributes.set_allocator(AZStdFunctorAllocator([]() -> IAllocator& { return GetCurrentSerializeContextModule().GetAllocator(); }));
// Flag the field with the EnumType attribute if we're an enumeration type aliased by RemoveEnum
const bool isSpecializedEnum = AZStd::is_enum<ValueType>::value && !AzTypeInfo<ValueType>::Uuid().IsNull();
@@ -650,7 +650,7 @@ namespace AZ
* But as the AZStdAssociativeContainer instance will not be accessed outside of the module it was
* created within then this will return this .dll/.exe module allocator
*/
m_classElement.m_attributes.set_allocator(AZStdFunctorAllocator([]() -> IAllocatorAllocate& { return GetCurrentSerializeContextModule().GetAllocator(); }));
m_classElement.m_attributes.set_allocator(AZStdFunctorAllocator([]() -> IAllocator& { return GetCurrentSerializeContextModule().GetAllocator(); }));
m_classElement.m_attributes.emplace_back(AZ_CRC("KeyType", 0x15bc5303), CreateModuleAttribute(AZStd::move(uuid)));
}
@@ -203,7 +203,7 @@ namespace AZ
JsonSerializationResult::Result JsonMapSerializer::LoadElement(void* outputValue, SerializeContext::IDataContainer* container,
const SerializeContext::ClassElement* pairElement, SerializeContext::IDataContainer* pairContainer,
const SerializeContext::ClassElement* keyElement, const SerializeContext::ClassElement* valueElement,
const rapidjson::Value& key, const rapidjson::Value& value, JsonDeserializerContext& context)
const rapidjson::Value& key, const rapidjson::Value& value, JsonDeserializerContext& context, bool isMultiMap)
{
namespace JSR = JsonSerializationResult;
@@ -231,8 +231,30 @@ namespace AZ
return context.Report(keyResult, "Failed to read key for associative container.");
}
void* valueAddress = nullptr;
bool keyExists = false;
// For multimaps, we append values to keys instead updating them.
// This is to ensure legacy multimap serialization support.
if (!isMultiMap)
{
auto associativeContainer = container->GetAssociativeContainerInterface();
void* existingKeyValuePair = associativeContainer->GetElementByKey(outputValue, keyElement, keyAddress);
if (existingKeyValuePair)
{
valueAddress = pairContainer->GetElementByIndex(existingKeyValuePair, pairElement, 1);
expectedSize--;
keyExists = true;
}
}
// If the key doesn't exist or it's a multimap, we're adding the new element we reserved above.
if (!keyExists)
{
valueAddress = pairContainer->GetElementByIndex(address, pairElement, 1);
}
// Load value
void* valueAddress = pairContainer->GetElementByIndex(address, pairElement, 1);
AZ_Assert(valueAddress, "Element reserved for associative container, but unable to retrieve address of the value.");
ContinuationFlags valueLoadFlags = ContinuationFlags::LoadAsNewInstance;
if (valueElement->m_flags & SerializeContext::ClassElement::Flags::FLG_POINTER)
@@ -257,7 +279,18 @@ namespace AZ
}
else
{
container->StoreElement(outputValue, address);
// Even if the key exists, calling StoreElement will not replace the existing key
// and will free the temporary address as expected. Checking if the key already
// exists and skipping the call to StoreElement if it does, makes the intent more
// clear. The end result is the same either way.
if (!keyExists)
{
container->StoreElement(outputValue, address);
}
else
{
container->FreeReservedElement(outputValue, address, context.GetSerializeContext());
}
if (container->Size(outputValue) != expectedSize)
{
return context.Report(JSR::Tasks::ReadField, JSR::Outcomes::Unavailable,
@@ -430,7 +463,7 @@ namespace AZ
JsonSerializationResult::Result JsonUnorderedMultiMapSerializer::LoadElement(void* outputValue, SerializeContext::IDataContainer* container,
const SerializeContext::ClassElement* pairElement, SerializeContext::IDataContainer* pairContainer,
const SerializeContext::ClassElement* keyElement, const SerializeContext::ClassElement* valueElement,
const rapidjson::Value& key, const rapidjson::Value& value, JsonDeserializerContext& context)
const rapidjson::Value& key, const rapidjson::Value& value, JsonDeserializerContext& context, [[maybe_unused]] bool isMultiMap)
{
namespace JSR = JsonSerializationResult;
@@ -440,7 +473,7 @@ namespace AZ
for (auto& entry : value.GetArray())
{
result.Combine(JsonMapSerializer::LoadElement(outputValue, container, pairElement, pairContainer,
keyElement, valueElement, key, entry, context));
keyElement, valueElement, key, entry, context, true));
if (result.GetProcessing() == JSR::Processing::Halted)
{
return context.Report(result, "Unable to process the key or all values in multi-map.");
@@ -451,7 +484,7 @@ namespace AZ
else if (IsExplicitDefault(value))
{
return JsonMapSerializer::LoadElement(outputValue, container, pairElement, pairContainer,
keyElement, valueElement, key, value, context);
keyElement, valueElement, key, value, context, true);
}
else
{
@@ -32,7 +32,7 @@ namespace AZ
virtual JsonSerializationResult::Result LoadElement(void* outputValue, SerializeContext::IDataContainer* container,
const SerializeContext::ClassElement* pairElement, SerializeContext::IDataContainer* pairContainer,
const SerializeContext::ClassElement* keyElement, const SerializeContext::ClassElement* valueElement,
const rapidjson::Value& key, const rapidjson::Value& value, JsonDeserializerContext& context);
const rapidjson::Value& key, const rapidjson::Value& value, JsonDeserializerContext& context, bool isMultiMap = false);
virtual JsonSerializationResult::Result Store(rapidjson::Value& outputValue, const void* inputValue, const void* defaultValue,
const Uuid& valueTypeId, JsonSerializerContext& context, bool sortResult);
@@ -62,7 +62,7 @@ namespace AZ
JsonSerializationResult::Result LoadElement(void* outputValue, SerializeContext::IDataContainer* container,
const SerializeContext::ClassElement* pairElement, SerializeContext::IDataContainer* pairContainer,
const SerializeContext::ClassElement* keyElement, const SerializeContext::ClassElement* valueElement,
const rapidjson::Value& key, const rapidjson::Value& value, JsonDeserializerContext& context) override;
const rapidjson::Value& key, const rapidjson::Value& value, JsonDeserializerContext& context, bool isMultiMap = false) override;
using JsonMapSerializer::Store;
JsonSerializationResult::Result Store(rapidjson::Value& outputValue, const void* inputValue, const void* defaultValue,
@@ -1668,9 +1668,23 @@ namespace AZ
SerializeContext::ENUM_ACCESS_FOR_READ,
&m_errorLogger
);
if (objectStreamWriteOverrideCB.Invoke<void>(callContext, objectPtr, *classData, classElement))
if (ObjectStreamWriteOverrideResponse writeResponse;
objectStreamWriteOverrideCB.Read<ObjectStreamWriteOverrideResponse>(writeResponse, callContext, objectPtr, *classData, classElement))
{
return false;
switch (writeResponse)
{
case ObjectStreamWriteOverrideResponse::FallbackToDefaultWrite:
break;
case ObjectStreamWriteOverrideResponse::AbortWrite:
m_errorLogger.ReportError(AZStd::string::format("ObjectStream Write Element Override callback has aborted the write for class data %s",
classData->m_name).c_str());
[[fallthrough]];
case ObjectStreamWriteOverrideResponse::CompletedWrite:
return false;
default:
AZ_Error("Serialize", false, "Invalid Response %d returned from the ObjectStream Write Element Override callback", static_cast<int>(writeResponse));
return false;
}
}
else
{
@@ -49,14 +49,24 @@ namespace AZ
static const AZ::Crc32 ObjectStreamWriteElementOverride = AZ_CRC("ObjectStreamWriteElementOverride", 0x35eb659f);
}
enum class ObjectStreamWriteOverrideResponse
{
CompletedWrite,
FallbackToDefaultWrite,
AbortWrite
};
AZ_TYPE_INFO_SPECIALIZE(ObjectStreamWriteOverrideResponse, "{BDF960A8-0F18-4E9D-96DA-F800A122C42D}");
///< Callback that the object stream invokes to override saving an instance of the registered class
///< @param callContext EnumerateInstanceCallContext which contains the WriteElement BeingElemCB and the CloseElement EndElemCB
///< the callContext parameter can be passed to the SerializeContext::EnumerateInstance to continue object stream writing
///< @param classPtr class type which is of pointer to the type represented by the m_typeId value
///< @param classData reference to this instance Class Data that will be supplied to the callback
///< @param classElement class element pointer which contains information about the element being serialized.
///< root elements do not not have a valid class element pointer
using ObjectStreamWriteOverrideCB = AZStd::function<void(SerializeContext::EnumerateInstanceCallContext& callContext,
///< root elements have a nullptr classElement
///< @return enum to indicate that the override has saved the registered class and that the default writing should be skipped.
///< Returning false will have the WriteElement code fallback to using the default logic
using ObjectStreamWriteOverrideCB = AZStd::function<ObjectStreamWriteOverrideResponse(SerializeContext::EnumerateInstanceCallContext& callContext,
const void* classPtr, const SerializeContext::ClassData& classData, const SerializeContext::ClassElement* classElement)>;
AZ_TYPE_INFO_SPECIALIZE(ObjectStreamWriteOverrideCB, "{87B1A36B-8C8A-42B6-A0B5-E770D9FDBAD4}");
@@ -3245,7 +3245,7 @@ namespace AZ
return genericClassInfoFoundIt != m_moduleLocalGenericClassInfos.end() ? genericClassInfoFoundIt->second : nullptr;
}
AZ::IAllocatorAllocate& SerializeContext::PerModuleGenericClassInfo::GetAllocator()
AZ::IAllocator& SerializeContext::PerModuleGenericClassInfo::GetAllocator()
{
return m_moduleOSAllocator;
}
@@ -574,10 +574,10 @@ namespace AZ
GenericClassInfo* m_genericClassInfo = nullptr; ///< Valid when the generic class is set. So you don't search for the actual type in the class register.
Edit::ElementData* m_editData{}; ///< Pointer to edit data (generated by EditContext).
AZStd::vector<AttributeSharedPair, AZStdFunctorAllocator> m_attributes{
AZStdFunctorAllocator([]() -> IAllocatorAllocate& { return AZ::AllocatorInstance<AZ::SystemAllocator>::Get(); })
AZStdFunctorAllocator([]() -> IAllocator& { return AZ::AllocatorInstance<AZ::SystemAllocator>::Get(); })
}; ///< Attributes attached to ClassElement. Lambda is required here as AZStdFunctorAllocator expects a function pointer
///< that returns an IAllocatorAllocate& and the AZ::AllocatorInstance<AZ::SystemAllocator>::Get returns an AZ::SystemAllocator&
/// which while it inherits from IAllocatorAllocate, does not work as function pointers do not support covariant return types
///< that returns an IAllocator& and the AZ::AllocatorInstance<AZ::SystemAllocator>::Get returns an AZ::SystemAllocator&
/// which while it inherits from IAllocator, does not work as function pointers do not support covariant return types
AttributeOwnership m_attributeOwnership = AttributeOwnership::Parent;
int m_flags{}; ///<
};
@@ -639,12 +639,12 @@ namespace AZ
DataPatchUpgradeHandler m_dataPatchUpgrader;
///< Attributes for this class type. Lambda is required here as AZStdFunctorAllocator expects a function pointer
///< that returns an IAllocatorAllocate& and the AZ::AllocatorInstance<AZ::SystemAllocator>::Get returns an AZ::SystemAllocator&
/// which while it inherits from IAllocatorAllocate, does not work as function pointers do not support covariant return types
///< that returns an IAllocator& and the AZ::AllocatorInstance<AZ::SystemAllocator>::Get returns an AZ::SystemAllocator&
/// which while it inherits from IAllocator, does not work as function pointers do not support covariant return types
AZStd::vector<AttributeSharedPair, AZStdFunctorAllocator> m_attributes{AZStdFunctorAllocator(&GetSystemAllocator) };
private:
static IAllocatorAllocate& GetSystemAllocator()
static IAllocator& GetSystemAllocator()
{
return AZ::AllocatorInstance<AZ::SystemAllocator>::Get();
}
@@ -2483,7 +2483,7 @@ namespace AZ
PerModuleGenericClassInfo();
~PerModuleGenericClassInfo();
AZ::IAllocatorAllocate& GetAllocator();
AZ::IAllocator& GetAllocator();
void AddGenericClassInfo(AZ::GenericClassInfo* genericClassInfo);
void RemoveGenericClassInfo(const AZ::TypeId& canonicalTypeId);
@@ -2546,12 +2546,12 @@ namespace AZ
template <typename T, typename ContainerType>
AttributePtr CreateModuleAttribute(T&& attrValue)
{
IAllocatorAllocate& moduleAllocator = GetCurrentSerializeContextModule().GetAllocator();
IAllocator& moduleAllocator = GetCurrentSerializeContextModule().GetAllocator();
void* rawMemory = moduleAllocator.Allocate(sizeof(ContainerType), alignof(ContainerType));
new (rawMemory) ContainerType{ AZStd::forward<T>(attrValue) };
auto attributeDeleter = [](Attribute* attribute)
{
IAllocatorAllocate& moduleAllocator = GetCurrentSerializeContextModule().GetAllocator();
IAllocator& moduleAllocator = GetCurrentSerializeContextModule().GetAllocator();
attribute->~Attribute();
moduleAllocator.DeAllocate(attribute);
};
@@ -12,6 +12,8 @@
namespace AZ
{
enum class ObjectStreamWriteOverrideResponse;
namespace VariantSerializationInternal
{
template <class ValueType>
@@ -32,7 +34,7 @@ namespace AZ
* But as the AZStdAssociativeContainer instance will not be accessed outside of the module it was
* created within then this will return this .dll/.exe module allocator
*/
classElement.m_attributes.set_allocator(AZStdFunctorAllocator([]() -> IAllocatorAllocate& { return GetCurrentSerializeContextModule().GetAllocator(); }));
classElement.m_attributes.set_allocator(AZStdFunctorAllocator([]() -> IAllocator& { return GetCurrentSerializeContextModule().GetAllocator(); }));
}
template<size_t Index, size_t... Digits>
@@ -429,7 +431,7 @@ namespace AZ
// the serialize context dll module allocator has to be used to manage the lifetime of the ClassData attributes within a module
// If a module which reflects a variant is unloaded, then the dll module allocator will properly unreflect the variant type from the serialize context
// for this particular module
AZStdFunctorAllocator dllAllocator([]() -> IAllocatorAllocate& { return GetCurrentSerializeContextModule().GetAllocator(); });
AZStdFunctorAllocator dllAllocator([]() -> IAllocator& { return GetCurrentSerializeContextModule().GetAllocator(); });
m_classData.m_attributes.set_allocator(AZStd::move(dllAllocator));
// Create the ObjectStreamWriteOverrideCB in the current module
@@ -480,7 +482,7 @@ namespace AZ
}
}
private:
static void ObjectStreamWriter(SerializeContext::EnumerateInstanceCallContext& callContext, const void* variantPtr,
static ObjectStreamWriteOverrideResponse ObjectStreamWriter(SerializeContext::EnumerateInstanceCallContext& callContext, const void* variantPtr,
[[maybe_unused]] const SerializeContext::ClassData& variantClassData, const SerializeContext::ClassElement* variantClassElement)
{
auto alternativeVisitor = [&callContext, variantClassElement](auto&& elementAlt)
@@ -503,6 +505,9 @@ namespace AZ
};
AZStd::visit(AZStd::move(alternativeVisitor), *reinterpret_cast<const VariantType*>(variantPtr));
// To avoid including ObjectStream.h into this file, we static cast the value of 0
// to an AZ::ObjectStreamWriteElemntResponse which corresponds to the CompletedWrite enum value
return static_cast<AZ::ObjectStreamWriteOverrideResponse>(0);
}
VariantSerializationInternal::AZStdVariantContainer<Types...> m_variantContainer;
@@ -1,49 +0,0 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#include "TimeDataStatisticsManager.h"
namespace AZ
{
namespace Statistics
{
void TimeDataStatisticsManager::PushTimeDataSample(const char * registerName, const AZ::Debug::ProfilerRegister::TimeData& timeData)
{
const AZStd::string statName(registerName);
NamedRunningStatistic* statistic = GetStatistic(statName);
if (!statistic)
{
const AZStd::string units("us");
AddStatistic(statName, statName, units, false);
AZ::Debug::ProfilerRegister::TimeData zeroTimeData;
memset(&zeroTimeData, 0, sizeof(AZ::Debug::ProfilerRegister::TimeData));
m_previousTimeData[statName] = zeroTimeData;
statistic = GetStatistic(statName);
AZ_Assert(statistic != nullptr, "Fatal error adding a new statistic object");
}
const AZ::u64 accumulatedTime = timeData.m_time;
const AZ::s64 totalNumCalls = timeData.m_calls;
const AZ::u64 previousAccumulatedTime = m_previousTimeData[statName].m_time;
const AZ::s64 previousTotalNumCalls = m_previousTimeData[statName].m_calls;
const AZ::u64 deltaTime = accumulatedTime - previousAccumulatedTime;
const AZ::s64 deltaCalls = totalNumCalls - previousTotalNumCalls;
if (deltaCalls == 0)
{
//This is the same old data. Let's skip it
return;
}
double newSample = static_cast<double>(deltaTime) / deltaCalls;
statistic->PushSample(newSample);
m_previousTimeData[statName] = timeData;
}
} //namespace Statistics
} //namespace AZ
@@ -1,51 +0,0 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#pragma once
#include <AzCore/Debug/Profiler.h>
#include <AzCore/Statistics/StatisticsManager.h>
namespace AZ
{
namespace Statistics
{
/**
* @brief Specialization useful for data generated with AZ::Debug::FrameProfileComponent
*
* Timer based data collection using AZ_PROFILE_TIMER(...), available in
* AzCore/Debug/Profiler.h can be collected when using AZ::Debug::FrameProfilerComponent
* and AZ::Debug::FrameProfilerBus. The method PushTimeDataSample(...) is a convenience
* to convert those Timer registers into a RunningStatistic.
*
*
*/
class TimeDataStatisticsManager : public StatisticsManager<>
{
public:
TimeDataStatisticsManager() = default;
virtual ~TimeDataStatisticsManager() = default;
/**
* @brief Adds one sample data to a specific running stat by name.
*
* This method is specialized to work with ProfilerRegister::TimeData that can be intercepted
* during AZ::Debug::FrameProfilerBus::OnFrameProfilerData().
* For each @param registerName a new RunningStat object is created if it doesn't exist.
*
* Adds the TimeData as one sample for its RunningStatistic.
*/
void PushTimeDataSample(const char * registerName, const AZ::Debug::ProfilerRegister::TimeData& timeData);
protected:
///We store here the previous value from the previous timer frame data.
///This is necessary because AZ_PROFILER_TIMER is cumulative
///and we need the time spent for each call.
AZStd::unordered_map<AZStd::string, AZ::Debug::ProfilerRegister::TimeData> m_previousTimeData;
};
} //namespace Statistics
} //namespace AZ
@@ -242,7 +242,6 @@ set(FILES
Jobs/JobManagerComponent.cpp
Jobs/JobManagerComponent.h
Jobs/JobManagerDesc.h
Jobs/LegacyJobExecutor.h
Jobs/MultipleDependentJob.h
Jobs/task_group.h
Math/Aabb.cpp
@@ -371,8 +370,6 @@ set(FILES
Memory/AllocatorBase.h
Memory/AllocatorManager.cpp
Memory/AllocatorManager.h
Memory/AllocatorOverrideShim.cpp
Memory/AllocatorOverrideShim.h
Memory/AllocatorWrapper.h
Memory/AllocatorScope.h
Memory/BestFitExternalMapAllocator.cpp
@@ -19,6 +19,7 @@ set(FILES
any.h
base.h
config.h
concepts/concepts.h
createdestroy.h
docs.h
exceptions.h
@@ -27,11 +28,14 @@ set(FILES
hash.cpp
hash.h
hash_table.h
iterator/iterator_primitives.h
iterator.h
limits.h
numeric.h
math.h
optional.h
ranges/iter_move.h
ranges/ranges.h
ratio.h
reference_wrapper.h
sort.h
@@ -151,6 +155,7 @@ set(FILES
typetraits/alignment_of.h
typetraits/config.h
typetraits/common_type.h
typetraits/common_reference.h
typetraits/conjunction.h
typetraits/disjunction.h
typetraits/extent.h
@@ -217,4 +222,6 @@ set(FILES
typetraits/void_t.h
typetraits/internal/type_sequence_traits.h
typetraits/internal/is_template_copy_constructible.h
utility/declval.h
utility/move.h
)

Some files were not shown because too many files have changed in this diff Show More