Initial commit

This commit is contained in:
alexpete
2021-03-05 11:26:34 -08:00
commit a10351f38d
27091 changed files with 5521199 additions and 0 deletions
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,35 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <AzCore/Memory/Memory.h>
#include <AzCore/IO/Streamer/DedicatedCache.h>
#include <Tests/Streamer/StreamStackEntryConformityTests.h>
namespace AZ::IO
{
class DedicatedCacheTestDescription :
public StreamStackEntryConformityTestsDescriptor<DedicatedCache>
{
public:
DedicatedCache CreateInstance() override
{
return DedicatedCache(1 * 1024 * 1024, 64 * 1024, AZCORE_GLOBAL_NEW_ALIGNMENT, false);
}
bool UsesSlots() const override
{
return false;
}
};
INSTANTIATE_TYPED_TEST_CASE_P(Streamer_DedicatedCacheConformityTests, StreamStackEntryConformityTests, DedicatedCacheTestDescription);
} // namespace AZ::IO
@@ -0,0 +1,401 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <AzCore/Casting/numeric_cast.h>
#include <AzCore/UnitTest/TestTypes.h>
#include <AzTest/AzTest.h>
#include <AzCore/IO/Streamer/FileRequest.h>
#include <AzCore/IO/Streamer/FullFileDecompressor.h>
#include <AzCore/IO/Streamer/StreamerContext.h>
#include <AzCore/Memory/Memory.h>
#include <AzCore/Memory/PoolAllocator.h>
#include <AzCore/std/smart_ptr/unique_ptr.h>
#include <AzCore/std/smart_ptr/make_shared.h>
#include <Tests/Streamer/StreamStackEntryConformityTests.h>
#include <Tests/Streamer/StreamStackEntryMock.h>
namespace AZ::IO
{
class FullFileDecompressorTestDescription :
public StreamStackEntryConformityTestsDescriptor<FullFileDecompressor>
{
public:
static constexpr u32 m_arbitrarilyLargeAlignment = 4096;
FullFileDecompressor CreateInstance() override
{
return FullFileDecompressor(2, 2, m_arbitrarilyLargeAlignment);
}
void SetUp() override
{
AllocatorInstance<PoolAllocator>::Create();
AllocatorInstance<ThreadPoolAllocator>::Create();
}
void TearDown() override
{
AllocatorInstance<ThreadPoolAllocator>::Destroy();
AllocatorInstance<PoolAllocator>::Destroy();
}
};
INSTANTIATE_TYPED_TEST_CASE_P(
Streamer_FullFileDecompressorConformityTests, StreamStackEntryConformityTests, FullFileDecompressorTestDescription);
class Streamer_FullDecompressorTest
: public UnitTest::AllocatorsFixture
{
public:
enum CompressionState
{
Uncompressed,
Compressed,
Corrupted
};
enum ReadResult
{
Success,
Failed,
Canceled
};
void SetUp() override
{
UnitTest::AllocatorsFixture::SetUp();
AllocatorInstance<PoolAllocator>::Create();
AllocatorInstance<ThreadPoolAllocator>::Create();
}
void TearDown() override
{
m_decompressor.reset();
m_mock.reset();
m_decompressor = nullptr;
m_mock = nullptr;
delete[] m_buffer;
m_buffer = nullptr;
delete m_context;
m_context = nullptr;
AllocatorInstance<ThreadPoolAllocator>::Destroy();
AllocatorInstance<PoolAllocator>::Destroy();
UnitTest::AllocatorsFixture::TearDown();
}
void SetupEnvironment(u32 maxNumReads, u32 maxNumJobs)
{
m_buffer = new u32[m_fakeFileLength >> 2];
m_mock = AZStd::make_shared<StreamStackEntryMock>();
m_decompressor = AZStd::make_shared<FullFileDecompressor>(maxNumReads, maxNumJobs,
FullFileDecompressorTestDescription::m_arbitrarilyLargeAlignment);
m_context = new StreamerContext();
m_decompressor->SetContext(*m_context);
m_decompressor->SetNext(m_mock);
}
void SetupEnvironment()
{
SetupEnvironment(1, 1);
}
void MockReadCalls(ReadResult mockResult)
{
using ::testing::_;
using ::testing::AnyNumber;
using ::testing::Return;
EXPECT_CALL(*m_mock, ExecuteRequests())
.WillOnce(Return(true))
.WillRepeatedly(Return(false));
EXPECT_CALL(*m_mock, QueueRequest(_));
EXPECT_CALL(*m_mock, UpdateStatus(_)).Times(AnyNumber());
switch (mockResult)
{
case ReadResult::Success:
ON_CALL(*m_mock, QueueRequest(_))
.WillByDefault(Invoke(this, &Streamer_FullDecompressorTest::PrepareReadRequest));
break;
case ReadResult::Failed:
ON_CALL(*m_mock, QueueRequest(_))
.WillByDefault(Invoke(this, &Streamer_FullDecompressorTest::PrepareFailedReadRequest));
break;
case ReadResult::Canceled:
ON_CALL(*m_mock, QueueRequest(_))
.WillByDefault(Invoke(this, &Streamer_FullDecompressorTest::PrepareCanceledReadRequest));
break;
default:
AZ_Assert(false, "Unexpected mock result type.");
}
}
void PrepareReadRequest(FileRequest* request)
{
auto data = AZStd::get_if<FileRequest::ReadData>(&request->GetCommand());
ASSERT_NE(nullptr, data);
u64 size = data->m_size >> 2;
u32* buffer = reinterpret_cast<u32*>(data->m_output);
for (u64 i = 0; i < size; ++i)
{
buffer[i] = aznumeric_caster(data->m_offset + (i << 2));
}
request->SetStatus(IStreamerTypes::RequestStatus::Completed);
m_context->MarkRequestAsCompleted(request);
}
void PrepareFailedReadRequest(FileRequest* request)
{
request->SetStatus(IStreamerTypes::RequestStatus::Failed);
m_context->MarkRequestAsCompleted(request);
}
void PrepareCanceledReadRequest(FileRequest* request)
{
request->SetStatus(IStreamerTypes::RequestStatus::Canceled);
m_context->MarkRequestAsCompleted(request);
}
static bool Decompressor(bool sleep, const void* compressed, size_t compressedSize, void* uncompressed, [[maybe_unused]] size_t uncompressedBufferSize)
{
AZ_Assert(compressedSize == uncompressedBufferSize, "Fake decompression algorithm only supports copying data.");
if (sleep)
{
AZStd::this_thread::sleep_for(AZStd::chrono::milliseconds(80));
}
memcpy(uncompressed, compressed, compressedSize);
return true;
}
static bool CorruptedDecompressor(const CompressionInfo&, const void*, size_t, void*, size_t)
{
return false;
}
void ProcessCompressedRead(u64 offset, u64 size, CompressionState compressionState, IStreamerTypes::RequestStatus expectedResult)
{
CompressionInfo compressionInfo;
compressionInfo.m_compressedSize = m_fakeFileLength;
compressionInfo.m_isCompressed = (compressionState == CompressionState::Compressed || compressionState == CompressionState::Corrupted);
compressionInfo.m_offset = 0;
compressionInfo.m_uncompressedSize = m_fakeFileLength;
if (compressionState == CompressionState::Corrupted)
{
compressionInfo.m_decompressor = &Streamer_FullDecompressorTest::CorruptedDecompressor;
}
else
{
compressionInfo.m_decompressor = [](const CompressionInfo&, const void* compressed,
size_t compressedSize, void* uncompressed, size_t uncompressedBufferSize) -> bool
{
return Streamer_FullDecompressorTest::Decompressor(false,
compressed, compressedSize, uncompressed, uncompressedBufferSize);
};
}
FileRequest* request = m_context->GetNewInternalRequest();
request->CreateCompressedRead(nullptr, AZStd::move(compressionInfo), m_buffer, offset, size);
bool result = true;
auto completed = [&result, expectedResult](const FileRequest& request)
{
result = result && request.GetStatus() == expectedResult;
};
request->SetCompletionCallback(completed);
m_decompressor->QueueRequest(request);
bool hasCompleted = false;
while (m_decompressor->ExecuteRequests() || !hasCompleted)
{
StreamStackEntry::Status status;
m_decompressor->UpdateStatus(status);
if (status.m_isIdle)
{
hasCompleted = true;
}
m_context->FinalizeCompletedRequests();
}
EXPECT_TRUE(result);
}
void ProcessMultipleCompressedReads()
{
using ::testing::_;
using ::testing::AnyNumber;
using ::testing::Return;
static const constexpr size_t count = 16;
EXPECT_CALL(*m_mock, ExecuteRequests())
.WillOnce(Return(true))
.WillRepeatedly(Return(false));
EXPECT_CALL(*m_mock, QueueRequest(_)).Times(count);
EXPECT_CALL(*m_mock, UpdateStatus(_)).Times(AnyNumber());
ON_CALL(*m_mock, QueueRequest(_))
.WillByDefault(Invoke(this, &Streamer_FullDecompressorTest::PrepareReadRequest));
CompressionInfo compressionInfo;
compressionInfo.m_compressedSize = m_fakeFileLength;
compressionInfo.m_isCompressed = true;
compressionInfo.m_offset = 0;
compressionInfo.m_uncompressedSize = m_fakeFileLength;
compressionInfo.m_decompressor = [](const CompressionInfo&, const void* compressed,
size_t compressedSize, void* uncompressed, size_t uncompressedBufferSize) -> bool
{
return Streamer_FullDecompressorTest::Decompressor(true,
compressed, compressedSize, uncompressed, uncompressedBufferSize);
};
bool allCompleted = true;
auto completed = [&allCompleted](const FileRequest& request)
{
allCompleted = allCompleted && request.GetStatus() == IStreamerTypes::RequestStatus::Completed;
};
FileRequest* requests[count];
AZStd::unique_ptr<u32[]> buffers[count];
for (size_t i = 0; i < count; ++i)
{
buffers[i] = AZStd::unique_ptr<u32[]>(new u32[m_fakeFileLength >> 2]);
requests[i] = m_context->GetNewInternalRequest();
requests[i]->CreateCompressedRead(nullptr, compressionInfo, buffers[i].get(), 0, m_fakeFileLength);
requests[i]->SetCompletionCallback(completed);
m_decompressor->QueueRequest(requests[i]);
}
bool hasCompleted = false;
while (m_decompressor->ExecuteRequests() || !hasCompleted)
{
StreamStackEntry::Status status;
m_decompressor->UpdateStatus(status);
if (status.m_isIdle)
{
hasCompleted = true;
}
m_context->FinalizeCompletedRequests();
}
EXPECT_TRUE(allCompleted);
}
void VerifyReadBuffer(u32* buffer, u64 offset, u64 size)
{
size = size >> 2;
for (u64 i = 0; i < size; ++i)
{
// Using assert here because in case of a problem EXPECT would
// cause a large amount of log noise.
ASSERT_EQ(buffer[i], offset + (i << 2));
}
}
void VerifyReadBuffer(u64 offset, u64 size)
{
VerifyReadBuffer(m_buffer, offset, size);
}
u32* m_buffer;
StreamerContext* m_context;
AZStd::shared_ptr<FullFileDecompressor> m_decompressor;
AZStd::shared_ptr<StreamStackEntryMock> m_mock;
u64 m_fakeFileLength{ 1 * 1024 * 1024 };
};
TEST_F(Streamer_FullDecompressorTest, DecompressedRead_FullReadAndDecompressData_SuccessfullyReadData)
{
SetupEnvironment();
MockReadCalls(ReadResult::Success);
ProcessCompressedRead(0, m_fakeFileLength, CompressionState::Compressed, IStreamerTypes::RequestStatus::Completed);
VerifyReadBuffer(0, m_fakeFileLength);
}
TEST_F(Streamer_FullDecompressorTest, DecompressedRead_PartialReadAndDecompressData_SuccessfullyReadData)
{
SetupEnvironment();
MockReadCalls(ReadResult::Success);
ProcessCompressedRead(256, m_fakeFileLength-512, CompressionState::Compressed, IStreamerTypes::RequestStatus::Completed);
VerifyReadBuffer(256, m_fakeFileLength-512);
}
TEST_F(Streamer_FullDecompressorTest, DecompressedRead_FullReadFromArchive_SuccessfullyReadData)
{
SetupEnvironment();
MockReadCalls(ReadResult::Success);
ProcessCompressedRead(0, m_fakeFileLength, CompressionState::Uncompressed, IStreamerTypes::RequestStatus::Completed);
VerifyReadBuffer(0, m_fakeFileLength);
}
TEST_F(Streamer_FullDecompressorTest, DecompressedRead_PartialReadFromArchive_SuccessfullyReadData)
{
SetupEnvironment();
MockReadCalls(ReadResult::Success);
ProcessCompressedRead(256, m_fakeFileLength - 512, CompressionState::Uncompressed, IStreamerTypes::RequestStatus::Completed);
VerifyReadBuffer(256, m_fakeFileLength - 512);
}
TEST_F(Streamer_FullDecompressorTest, DecompressedRead_FailedRead_FailureIsDetectedAndReported)
{
SetupEnvironment();
MockReadCalls(ReadResult::Failed);
ProcessCompressedRead(0, m_fakeFileLength, CompressionState::Compressed, IStreamerTypes::RequestStatus::Failed);
}
TEST_F(Streamer_FullDecompressorTest, DecompressedRead_CanceledRead_CancelIsDetectedAndReported)
{
SetupEnvironment();
MockReadCalls(ReadResult::Canceled);
ProcessCompressedRead(0, m_fakeFileLength, CompressionState::Compressed, IStreamerTypes::RequestStatus::Canceled);
}
TEST_F(Streamer_FullDecompressorTest, DecompressedRead_CorruptedArchiveRead_RequestIsCompletedWithFailedState)
{
SetupEnvironment();
MockReadCalls(ReadResult::Success);
ProcessCompressedRead(0, m_fakeFileLength, CompressionState::Corrupted, IStreamerTypes::RequestStatus::Failed);
}
TEST_F(Streamer_FullDecompressorTest, DecompressedRead_MultipleRequestsWithSingleJob_AllRequestsComplete)
{
SetupEnvironment(4, 1);
ProcessMultipleCompressedReads();
}
TEST_F(Streamer_FullDecompressorTest, DecompressedRead_MultipleRequestsWithSingleRead_AllRequestsComplete)
{
SetupEnvironment(1, 4);
ProcessMultipleCompressedReads();
}
TEST_F(Streamer_FullDecompressorTest, DecompressedRead_MultipleRequestsWithSingleReadAndJob_AllRequestsComplete)
{
SetupEnvironment(1, 1);
ProcessMultipleCompressedReads();
}
TEST_F(Streamer_FullDecompressorTest, DecompressedRead_MultipleRequestsWithMultipleReadAndJobs_AllRequestsComplete)
{
SetupEnvironment(4, 4);
ProcessMultipleCompressedReads();
}
} // namespace AZ::IO
@@ -0,0 +1,59 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <gmock/gmock.h>
#include <AzCore/IO/IStreamer.h>
using namespace AZ::IO;
class StreamerMock : public AZ::IO::IStreamer
{
public:
MOCK_METHOD7(Read, FileRequestPtr(AZStd::string_view, void*, size_t, size_t,
AZStd::chrono::microseconds, IStreamerTypes::Priority, size_t));
MOCK_METHOD8(Read, FileRequestPtr& (FileRequestPtr&, AZStd::string_view, void*, size_t, size_t,
AZStd::chrono::microseconds, IStreamerTypes::Priority, size_t));
MOCK_METHOD6(Read, FileRequestPtr(AZStd::string_view, IStreamerTypes::RequestMemoryAllocator&,
size_t, AZStd::chrono::microseconds, IStreamerTypes::Priority, size_t));
MOCK_METHOD7(Read, FileRequestPtr& (FileRequestPtr&, AZStd::string_view, IStreamerTypes::RequestMemoryAllocator&,
size_t, AZStd::chrono::microseconds, IStreamerTypes::Priority, size_t));
MOCK_METHOD1(Cancel, FileRequestPtr(FileRequestPtr));
MOCK_METHOD2(Cancel, FileRequestPtr& (FileRequestPtr&, FileRequestPtr));
MOCK_METHOD3(RescheduleRequest, FileRequestPtr(FileRequestPtr, AZStd::chrono::microseconds, IStreamerTypes::Priority));
MOCK_METHOD4(RescheduleRequest, FileRequestPtr& (FileRequestPtr&, FileRequestPtr, AZStd::chrono::microseconds, IStreamerTypes::Priority));
MOCK_METHOD1(CreateDedicatedCache, FileRequestPtr(AZStd::string_view));
MOCK_METHOD2(CreateDedicatedCache, FileRequestPtr& (FileRequestPtr&, AZStd::string_view));
MOCK_METHOD1(DestroyDedicatedCache, FileRequestPtr(AZStd::string_view));
MOCK_METHOD2(DestroyDedicatedCache, FileRequestPtr&(FileRequestPtr&, AZStd::string_view));
MOCK_METHOD1(FlushCache, FileRequestPtr(AZStd::string_view));
MOCK_METHOD2(FlushCache, FileRequestPtr& (FileRequestPtr&, AZStd::string_view));
MOCK_METHOD0(FlushCaches, FileRequestPtr());
MOCK_METHOD1(FlushCaches, FileRequestPtr& (FileRequestPtr&));
MOCK_METHOD1(Custom, FileRequestPtr(AZStd::any));
MOCK_METHOD2(Custom, FileRequestPtr& (FileRequestPtr&, AZStd::any));
MOCK_METHOD2(SetRequestCompleteCallback, FileRequestPtr&(FileRequestPtr&, OnCompleteCallback));
MOCK_METHOD0(CreateRequest, FileRequestPtr());
MOCK_METHOD2(CreateRequestBatch, void(AZStd::vector<FileRequestPtr>&, size_t));
MOCK_METHOD1(QueueRequest, void(const FileRequestPtr&));
MOCK_METHOD1(QueueRequestBatch, void(const AZStd::vector<FileRequestPtr>&));
MOCK_METHOD1(QueueRequestBatch, void(AZStd::vector<FileRequestPtr>&&));
MOCK_CONST_METHOD1(HasRequestCompleted, bool(FileRequestHandle));
MOCK_CONST_METHOD1(GetRequestStatus, IStreamerTypes::RequestStatus(FileRequestHandle));
MOCK_CONST_METHOD1(GetEstimatedRequestCompletionTime, AZStd::chrono::system_clock::time_point(FileRequestHandle));
MOCK_CONST_METHOD4(GetReadRequestResult, bool(FileRequestHandle, void*&, AZ::u64&, IStreamerTypes::ClaimMemory));
MOCK_METHOD1(CollectStatistics, void(AZStd::vector<Statistic>&));
MOCK_CONST_METHOD0(GetRecommendations, const IStreamerTypes::Recommendations&());
MOCK_METHOD0(SuspendProcessing, void());
MOCK_METHOD0(ResumeProcessing, void());
};
@@ -0,0 +1,44 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <AzTest/AzTest.h>
#include <AzCore/IO/IStreamerTypes.h>
namespace AZ::IO::IStreamerTypes
{
class RequestMemoryAllocatorMock
: public RequestMemoryAllocator
{
public:
~RequestMemoryAllocatorMock() override = default;
MOCK_METHOD0(LockAllocator, void());
MOCK_METHOD0(UnlockAllocator, void());
MOCK_METHOD3(Allocate, RequestMemoryAllocatorResult(AZ::u64, AZ::u64, size_t));
MOCK_METHOD1(Release, void(void*));
inline RequestMemoryAllocatorResult ForwardAllocate(AZ::u64 minimalSize, AZ::u64 recommendedSize, size_t alignment)
{
return m_defaultAllocator.Allocate(minimalSize, recommendedSize, alignment);
}
inline void ForwardRelease(void* address)
{
return m_defaultAllocator.Release(address);
}
private:
DefaultRequestMemoryAllocator m_defaultAllocator;
};
} // namespace AZ::IO::IStreamerTypes
@@ -0,0 +1,500 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <AzCore/IO/IStreamerTypes.h>
#include <AzCore/IO/Streamer/ReadSplitter.h>
#include <AzCore/Memory/Memory.h>
#include <AzCore/UnitTest/TestTypes.h>
#include <Tests/FileIOBaseTestTypes.h>
#include <Tests/Streamer/StreamStackEntryConformityTests.h>
#include <Tests/Streamer/StreamStackEntryMock.h>
namespace AZ::IO
{
class ReadSplitterTestDescription :
public StreamStackEntryConformityTestsDescriptor<ReadSplitter>
{
public:
ReadSplitter CreateInstance() override
{
return ReadSplitter(64_kib, AZCORE_GLOBAL_NEW_ALIGNMENT, 1, 0, false, true);
}
bool UsesSlots() const override
{
return false;
}
};
class ReadSplitterWithBufferTestDescription :
public StreamStackEntryConformityTestsDescriptor<ReadSplitter>
{
public:
ReadSplitter CreateInstance() override
{
return ReadSplitter(64_kib, 4096, 512, 5_mib, true, true);
}
bool UsesSlots() const override
{
return true;
}
};
using ReadSplitterTestTypes = ::testing::Types<ReadSplitterTestDescription, ReadSplitterWithBufferTestDescription>;
INSTANTIATE_TYPED_TEST_CASE_P(Streamer_ReadSplitterConformityTests, StreamStackEntryConformityTests, ReadSplitterTestTypes);
class Streamer_ReadSplitterTest
: public UnitTest::ScopedAllocatorSetupFixture
{
public:
static constexpr u64 SplitSize = 1_kib;
static constexpr size_t MemoryAlignment = 4096;
static constexpr size_t SizeAlignment = 512;
Streamer_ReadSplitterTest()
: m_mock(AZStd::make_shared<StreamStackEntryMock>())
{
}
void SetUp() override
{
m_prevFileIO = AZ::IO::FileIOBase::GetInstance();
AZ::IO::FileIOBase::SetInstance(&m_fileIO);
}
void TearDown() override
{
if (m_readSplitter)
{
delete m_readSplitter;
m_readSplitter = nullptr;
}
AZ::IO::FileIOBase::SetInstance(m_prevFileIO);
}
void CreateReadSplitter(u64 maxReadSize, u32 memoryAlignment, u32 sizeAlignment, size_t bufferSize,
bool adjustOffset, bool splitAlignedRequests)
{
using ::testing::_;
m_readSplitter = new ReadSplitter(maxReadSize, memoryAlignment, sizeAlignment, bufferSize,
adjustOffset, splitAlignedRequests);
m_readSplitter->SetNext(m_mock);
EXPECT_CALL(*m_mock, SetContext(_));
m_readSplitter->SetContext(m_context);
}
void CreateStandardReadSplitter()
{
CreateReadSplitter(SplitSize, AZCORE_GLOBAL_NEW_ALIGNMENT, 1, 0, false, true);
}
void CreateAlignmentAwareReadSplitter(size_t bufferSize, bool adjustOffset)
{
CreateReadSplitter(SplitSize, MemoryAlignment, SizeAlignment, bufferSize, adjustOffset, true);
}
void CreatePassThroughReadSplitter()
{
// By having no buffer all requests are considered aligned. By turning off splitting
// splitting aligned request this configuration effectively records the alignment state
// and passes the request on to the next stack entry.
CreateReadSplitter(SplitSize, AZCORE_GLOBAL_NEW_ALIGNMENT, 1, 0, false, false);
}
protected:
UnitTest::TestFileIOBase m_fileIO;
FileIOBase* m_prevFileIO{};
StreamerContext m_context;
ReadSplitter* m_readSplitter{ nullptr };
AZStd::shared_ptr<StreamStackEntryMock> m_mock;
};
TEST_F(Streamer_ReadSplitterTest, QueueRequest_LessThanSplitSize_RequestIsForwarded)
{
CreateStandardReadSplitter();
FileRequest* readRequest = m_context.GetNewInternalRequest();
RequestPath path;
path.InitFromRelativePath("TestPath");
readRequest->CreateRead(nullptr, nullptr, SplitSize / 2, path, 0, SplitSize / 2);
EXPECT_CALL(*m_mock, QueueRequest(readRequest)).Times(1);
m_readSplitter->QueueRequest(readRequest);
m_context.RecycleRequest(readRequest);
}
TEST_F(Streamer_ReadSplitterTest, QueueRequest_TwiceTheSplitSize_TwoSubRequestsCreated)
{
using ::testing::_;
CreateStandardReadSplitter();
char buffer[SplitSize * 2];
FileRequest* readRequest = m_context.GetNewInternalRequest();
RequestPath path;
path.InitFromRelativePath("TestPath");
readRequest->CreateRead(nullptr, buffer, SplitSize * 2, path, 0, SplitSize * 2);
AZStd::vector<FileRequest*> subRequests;
EXPECT_CALL(*m_mock, QueueRequest(_))
.Times(2)
.WillRepeatedly([&subRequests](FileRequest* request) {subRequests.push_back(request); });
m_readSplitter->QueueRequest(readRequest);
for (size_t i=0; i<subRequests.size(); ++i)
{
EXPECT_EQ(subRequests[i]->GetParent(), readRequest);
FileRequest::ReadData* data = AZStd::get_if<FileRequest::ReadData>(&subRequests[i]->GetCommand());
ASSERT_NE(nullptr, data);
EXPECT_EQ(SplitSize, data->m_size);
EXPECT_EQ(SplitSize * i, data->m_offset);
EXPECT_EQ(buffer + (SplitSize * i), data->m_output);
EXPECT_EQ(path, data->m_path);
m_context.MarkRequestAsCompleted(subRequests[i]);
}
m_context.FinalizeCompletedRequests();
}
TEST_F(Streamer_ReadSplitterTest, QueueRequest_NoSplitOnAlignedEnabled_RequestIsForwardedWithoutChange)
{
CreatePassThroughReadSplitter();
FileRequest* readRequest = m_context.GetNewInternalRequest();
RequestPath path;
path.InitFromRelativePath("TestPath");
readRequest->CreateRead(nullptr, nullptr, SplitSize * 2, path, 0, SplitSize * 2);
EXPECT_CALL(*m_mock, QueueRequest(readRequest)).Times(1);
m_readSplitter->QueueRequest(readRequest);
m_context.RecycleRequest(readRequest);
}
TEST_F(Streamer_ReadSplitterTest, QueueRequest_MoreSubRequestsThanDepenencies_AdditionalDependenciesAreDelayed)
{
using ::testing::_;
CreateStandardReadSplitter();
constexpr size_t batchSize = FileRequest::GetMaxNumDependencies();
constexpr size_t numSubReads = batchSize + 2;
constexpr size_t size = numSubReads * SplitSize;
auto buffer = AZStd::unique_ptr<u8[]>(new u8[size]);
FileRequest* readRequest = m_context.GetNewInternalRequest();
RequestPath path;
path.InitFromRelativePath("TestPath");
readRequest->CreateRead(nullptr, buffer.get(), size, path, 0, size);
AZStd::vector<FileRequest*> subRequests;
subRequests.reserve(numSubReads);
EXPECT_CALL(*m_mock, QueueRequest(_))
.Times(batchSize)
.WillRepeatedly([&subRequests](FileRequest* request) { subRequests.push_back(request); });
m_readSplitter->QueueRequest(readRequest);
ASSERT_EQ(subRequests.size(), batchSize);
for (size_t i = 0; i < subRequests.size(); ++i)
{
EXPECT_EQ(subRequests[i]->GetParent(), readRequest);
FileRequest::ReadData* data = AZStd::get_if<FileRequest::ReadData>(&subRequests[i]->GetCommand());
ASSERT_NE(nullptr, data);
EXPECT_EQ(SplitSize, data->m_size);
EXPECT_EQ(SplitSize * i, data->m_offset);
EXPECT_EQ(buffer.get() + (SplitSize * i), data->m_output);
EXPECT_EQ(path, data->m_path);
m_context.MarkRequestAsCompleted(subRequests[i]);
}
subRequests.clear();
EXPECT_CALL(*m_mock, QueueRequest(_))
.Times(numSubReads - batchSize)
.WillRepeatedly([&subRequests](FileRequest* request) { subRequests.push_back(request); });
m_context.FinalizeCompletedRequests();
for (size_t i = 0; i < subRequests.size(); ++i)
{
EXPECT_EQ(subRequests[i]->GetParent(), readRequest);
FileRequest::ReadData* data = AZStd::get_if<FileRequest::ReadData>(&subRequests[i]->GetCommand());
ASSERT_NE(nullptr, data);
EXPECT_EQ(SplitSize, data->m_size);
EXPECT_EQ(SplitSize * (batchSize + i), data->m_offset);
EXPECT_EQ(buffer.get() + (SplitSize * (batchSize + i)), data->m_output);
EXPECT_EQ(path, data->m_path);
m_context.MarkRequestAsCompleted(subRequests[i]);
}
m_context.FinalizeCompletedRequests();
}
TEST_F(Streamer_ReadSplitterTest, QueueRequest_UnalignedMemoryAdjusted_BuffersAreUsedToReadTo)
{
using ::testing::_;
CreateAlignmentAwareReadSplitter(1_mib, false);
constexpr u64 readSize = SplitSize / 2;
u8* memory = reinterpret_cast<u8*>(azmalloc(readSize + 3, MemoryAlignment));
u8* buffer = memory + 3; // Adjust the starting address so it doesn't align
FileRequest* readRequest = m_context.GetNewInternalRequest();
RequestPath path;
path.InitFromRelativePath("TestPath");
readRequest->CreateRead(nullptr, buffer, readSize, path, 0, readSize);
FileRequest* subRequest{ nullptr };
EXPECT_CALL(*m_mock, QueueRequest(_))
.Times(1)
.WillRepeatedly([&subRequest](FileRequest* request) { subRequest = request; });
m_readSplitter->QueueRequest(readRequest);
ASSERT_NE(nullptr, subRequest);
FileRequest::ReadData* data = AZStd::get_if<FileRequest::ReadData>(&subRequest->GetCommand());
EXPECT_NE(buffer, data->m_output);
EXPECT_EQ(readSize, data->m_size);
EXPECT_EQ(0, data->m_offset);
EXPECT_EQ(path, data->m_path);
u32* subRequestBuffer = reinterpret_cast<u32*>(data->m_output);
for (u64 i = 0; i < data->m_size / sizeof(u32); ++i)
{
subRequestBuffer[i] = aznumeric_caster(i);
}
m_context.MarkRequestAsCompleted(subRequest);
m_context.FinalizeCompletedRequests();
u32* readBuffer = reinterpret_cast<u32*>(buffer);
for (u64 i = 0; i < readSize / sizeof(u32); ++i)
{
ASSERT_EQ(aznumeric_cast<u32>(i), readBuffer[i]);
}
azfree(memory);
}
TEST_F(Streamer_ReadSplitterTest, QueueRequest_UnalignedOffsetAdjusted_BuffersAreUsedToReadTo)
{
using ::testing::_;
CreateAlignmentAwareReadSplitter(1_mib, true);
constexpr u64 offsetAdjustment = sizeof(u32) * 2;
constexpr u64 readSize = SplitSize / 2;
ASSERT_GT(MemoryAlignment, offsetAdjustment);
u8* buffer = reinterpret_cast<u8*>(azmalloc(readSize, MemoryAlignment));
FileRequest* readRequest = m_context.GetNewInternalRequest();
RequestPath path;
path.InitFromRelativePath("TestPath");
readRequest->CreateRead(nullptr, buffer, readSize, path, offsetAdjustment, readSize);
FileRequest* subRequest{ nullptr };
EXPECT_CALL(*m_mock, QueueRequest(_))
.Times(1)
.WillRepeatedly([&subRequest](FileRequest* request) { subRequest = request; });
m_readSplitter->QueueRequest(readRequest);
ASSERT_NE(nullptr, subRequest);
FileRequest::ReadData* data = AZStd::get_if<FileRequest::ReadData>(&subRequest->GetCommand());
EXPECT_NE(buffer, data->m_output);
EXPECT_EQ(readSize + offsetAdjustment, data->m_size);
EXPECT_EQ(0, data->m_offset);
EXPECT_EQ(path, data->m_path);
u32* subRequestBuffer = reinterpret_cast<u32*>(data->m_output);
for (u64 i = 0; i < data->m_size / sizeof(u32); ++i)
{
subRequestBuffer[i] = aznumeric_caster(i);
}
m_context.MarkRequestAsCompleted(subRequest);
m_context.FinalizeCompletedRequests();
u32* readBuffer = reinterpret_cast<u32*>(buffer);
for (u64 i = 0; i < readSize / sizeof(u32); ++i)
{
ASSERT_EQ(aznumeric_cast<u32>(i) + (offsetAdjustment / sizeof(u32)), readBuffer[i]);
}
azfree(buffer);
}
TEST_F(Streamer_ReadSplitterTest, QueueRequest_ReadMoreThanFitsInTheCache_ReadsAreDelayedAndThenContinued)
{
using ::testing::_;
CreateAlignmentAwareReadSplitter(SplitSize * 4, false);
constexpr u64 readSize = SplitSize * 6;
u8* memory = reinterpret_cast<u8*>(azmalloc(readSize + 3, MemoryAlignment));
u8* buffer = memory + 3; // Adjust the starting address so it doesn't align
FileRequest* readRequest = m_context.GetNewInternalRequest();
RequestPath path;
path.InitFromRelativePath("TestPath");
readRequest->CreateRead(nullptr, buffer, readSize, path, 0, readSize);
EXPECT_CALL(*m_mock, QueueRequest(_))
.Times(4)
.WillRepeatedly([this](FileRequest* request) { m_context.MarkRequestAsCompleted(request); });
m_readSplitter->QueueRequest(readRequest);
EXPECT_CALL(*m_mock, QueueRequest(_))
.Times(2)
.WillRepeatedly([this](FileRequest* request) { m_context.MarkRequestAsCompleted(request); });
m_context.FinalizeCompletedRequests();
azfree(memory);
}
TEST_F(Streamer_ReadSplitterTest, QueueRequest_AlignedReadAfterDelayedRead_SecondReadIsDelayedAsWellAndBothComplete)
{
using ::testing::_;
CreateAlignmentAwareReadSplitter(SplitSize * 4, false);
constexpr u64 readSize = SplitSize * 6;
size_t completedRequests{ 0 };
u8* memory0 = reinterpret_cast<u8*>(azmalloc(readSize + 3, MemoryAlignment));
u8* buffer = memory0 + 3; // Adjust the starting address so it doesn't align
FileRequest* readRequestDelayed = m_context.GetNewInternalRequest();
RequestPath path;
path.InitFromRelativePath("TestPath");
readRequestDelayed->CreateRead(nullptr, buffer, readSize, path, 0, readSize);
readRequestDelayed->SetCompletionCallback([&completedRequests](FileRequestHandle) { ++completedRequests; });
u8* memory1 = reinterpret_cast<u8*>(azmalloc(readSize, MemoryAlignment));
FileRequest* readRequestAligned = m_context.GetNewInternalRequest();
readRequestAligned->CreateRead(nullptr, memory1, readSize, path, 0, readSize);
readRequestAligned->SetCompletionCallback([&completedRequests](FileRequestHandle) { ++completedRequests; });
EXPECT_CALL(*m_mock, QueueRequest(_))
.Times(4)
.WillRepeatedly([this, parent = readRequestDelayed](FileRequest* request)
{
EXPECT_EQ(request->GetParent(), parent);
m_context.MarkRequestAsCompleted(request);
});
m_readSplitter->QueueRequest(readRequestDelayed);
m_readSplitter->QueueRequest(readRequestAligned);
auto delayedCallback = [this, parent = readRequestDelayed](FileRequest* request)
{
EXPECT_EQ(request->GetParent(), parent);
m_context.MarkRequestAsCompleted(request);
};
auto alignedCallback = [this, parent = readRequestAligned](FileRequest* request)
{
EXPECT_EQ(request->GetParent(), parent);
m_context.MarkRequestAsCompleted(request);
};
EXPECT_CALL(*m_mock, QueueRequest(_))
.Times(2 + 6)
.WillOnce(delayedCallback)
.WillOnce(delayedCallback)
.WillRepeatedly(alignedCallback);
m_context.FinalizeCompletedRequests();
EXPECT_EQ(2, completedRequests);
azfree(memory1);
azfree(memory0);
}
TEST_F(Streamer_ReadSplitterTest, QueueRequest_BufferedReadAfterDelayedRead_SecondReadIsDelayedAsWellAndBothComplete)
{
using ::testing::_;
CreateAlignmentAwareReadSplitter(SplitSize * 4, false);
constexpr u64 readSize = SplitSize * 6;
size_t completedRequests{ 0 };
u8* memory0 = reinterpret_cast<u8*>(azmalloc(readSize + 3, MemoryAlignment));
u8* buffer0 = memory0 + 3; // Adjust the starting address so it doesn't align
FileRequest* readRequestDelayed = m_context.GetNewInternalRequest();
RequestPath path;
path.InitFromRelativePath("TestPath");
readRequestDelayed->CreateRead(nullptr, buffer0, readSize, path, 0, readSize);
readRequestDelayed->SetCompletionCallback([&completedRequests](FileRequestHandle) { ++completedRequests; });
u8* memory1 = reinterpret_cast<u8*>(azmalloc(readSize + 3, MemoryAlignment));
u8* buffer1 = memory1 + 3;
FileRequest* readRequestBuffered = m_context.GetNewInternalRequest();
readRequestBuffered->CreateRead(nullptr, buffer1, readSize, path, 0, readSize);
readRequestBuffered->SetCompletionCallback([&completedRequests](FileRequestHandle) { ++completedRequests; });
EXPECT_CALL(*m_mock, QueueRequest(_))
.Times(4)
.WillRepeatedly([this, parent = readRequestDelayed](FileRequest* request)
{
EXPECT_EQ(request->GetParent(), parent);
m_context.MarkRequestAsCompleted(request);
});
m_readSplitter->QueueRequest(readRequestDelayed);
m_readSplitter->QueueRequest(readRequestBuffered);
auto delayedCallback = [this, parent = readRequestDelayed](FileRequest* request)
{
EXPECT_EQ(request->GetParent(), parent);
m_context.MarkRequestAsCompleted(request);
};
auto alignedCallback = [this, parent = readRequestBuffered](FileRequest* request)
{
EXPECT_EQ(request->GetParent(), parent);
m_context.MarkRequestAsCompleted(request);
};
EXPECT_CALL(*m_mock, QueueRequest(_))
.Times(2 + 6)
.WillOnce(delayedCallback)
.WillOnce(delayedCallback)
.WillRepeatedly(alignedCallback);
m_context.FinalizeCompletedRequests();
EXPECT_EQ(2, completedRequests);
azfree(memory1);
azfree(memory0);
}
TEST_F(Streamer_ReadSplitterTest, CollectStatistics_StatsAreReturned_AfterCallStatsAreAdded)
{
using ::testing::_;
CreateStandardReadSplitter();
AZStd::vector<Statistic> statistics;
ASSERT_TRUE(statistics.empty());
EXPECT_CALL(*m_mock, CollectStatistics(_));
m_readSplitter->CollectStatistics(statistics);
EXPECT_GT(statistics.size(), 0);
}
} // namespace AZ::IO
@@ -0,0 +1,384 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <AzCore/Casting/lossy_cast.h>
#include <AzCore/IO/Streamer/Streamer.h>
#include <AzCore/IO/Streamer/Scheduler.h>
#include <AzCore/std/parallel/atomic.h>
#include <AzCore/std/parallel/binary_semaphore.h>
#include <AzCore/std/smart_ptr/make_shared.h>
#include <AzCore/std/smart_ptr/unique_ptr.h>
#include <AzCore/UnitTest/TestTypes.h>
#include <Tests/Streamer/IStreamerTypesMock.h>
#include <Tests/Streamer/StreamStackEntryMock.h>
namespace AZ::IO
{
class Streamer_SchedulerTest
: public UnitTest::AllocatorsFixture
{
protected:
StreamerContext* m_streamerContext{ nullptr };
public:
void SetUp() override
{
using ::testing::_;
using ::testing::AnyNumber;
UnitTest::AllocatorsFixture::SetUp();
m_mock = AZStd::make_shared<StreamStackEntryMock>();
ON_CALL(*m_mock, PrepareRequest(_)).WillByDefault([this](FileRequest* request) { m_mock->ForwardPrepareRequest(request); });
ON_CALL(*m_mock, QueueRequest(_)).WillByDefault([this](FileRequest* request) { m_mock->ForwardQueueRequest(request); });
ON_CALL(*m_mock, UpdateStatus(_)).WillByDefault([this](StreamStackEntry::Status& status)
{
status.m_numAvailableSlots = 1;
status.m_isIdle = m_isStackIdle;
});
// Expectation needs to be set before the Scheduler thread is started otherwise it may or may not hit before it's set in
// a test.
EXPECT_CALL(*m_mock, SetContext(_))
.Times(1)
.WillOnce([this](StreamerContext& context)
{
m_streamerContext = &context;
m_mock->ForwardSetContext(context);
});
EXPECT_CALL(*m_mock, UpdateStatus(_)).Times(AnyNumber());
auto isIdle = m_isStackIdle.load();
m_isStackIdle = true;
m_streamer = aznew IO::Streamer(AZStd::thread_desc{}, AZStd::make_unique<Scheduler>(m_mock));
m_isStackIdle = isIdle;
Interface<IO::IStreamer>::Register(m_streamer);
}
void TearDown() override
{
m_isStackIdle = true;
if (m_streamer)
{
Interface<IO::IStreamer>::Unregister(m_streamer);
delete m_streamer;
m_streamer = nullptr;
}
m_mock.reset();
UnitTest::AllocatorsFixture::TearDown();
}
void MockForRead()
{
using ::testing::_;
using ::testing::AtLeast;
EXPECT_CALL(*m_mock, UpdateStatus(_)).Times(AtLeast(1));
EXPECT_CALL(*m_mock, UpdateCompletionEstimates(_, _, _, _)).Times(AtLeast(1));
EXPECT_CALL(*m_mock, PrepareRequest(_))
.Times(1)
.WillOnce([this](FileRequest* request)
{
AZ_Assert(m_streamerContext, "AZ::IO::Streamer is not ready to process requests.");
auto readData = AZStd::get_if<FileRequest::ReadRequestData>(&request->GetCommand());
AZ_Assert(readData, "Test didn't pass in the correct request.");
FileRequest* read = m_streamerContext->GetNewInternalRequest();
read->CreateRead(request, readData->m_output, readData->m_outputSize, readData->m_path,
readData->m_offset, readData->m_size);
m_streamerContext->PushPreparedRequest(read);
});
EXPECT_CALL(*m_mock, ExecuteRequests()).Times(AtLeast(1));
EXPECT_CALL(*m_mock, QueueRequest(_))
.Times(1)
.WillOnce([this](FileRequest* request)
{
AZ_Assert(m_streamerContext, "AZ::IO::Streamer is not ready to process requests.");
auto readData = AZStd::get_if<FileRequest::ReadData>(&request->GetCommand());
AZ_Assert(readData, "Test didn't pass in the correct request.");
auto output = reinterpret_cast<uint8_t*>(readData->m_output);
AZ_Assert(output != nullptr, "Output buffer has not been set.");
for (size_t i = 0; i < readData->m_size; ++i)
{
output[i] = azlossy_cast<uint8_t>(readData->m_offset + i);
}
request->SetStatus(IStreamerTypes::RequestStatus::Completed);
m_streamerContext->MarkRequestAsCompleted(request);
});
}
void MockAllocatorForUnclaimedMemory(IStreamerTypes::RequestMemoryAllocatorMock& mock, AZStd::binary_semaphore& sync)
{
using ::testing::_;
EXPECT_CALL(mock, LockAllocator()).Times(1);
EXPECT_CALL(mock, UnlockAllocator())
.Times(1)
.WillOnce([&sync]()
{
sync.release();
});
EXPECT_CALL(mock, Allocate(_, _, _))
.Times(1)
.WillOnce([&mock](size_t minimalSize, size_t recommendedSize, size_t alignment)
{
return mock.ForwardAllocate(minimalSize, recommendedSize, alignment);
});
EXPECT_CALL(mock, Release(_))
.Times(1)
.WillOnce([&mock](void* address)
{
mock.ForwardRelease(address);
});
}
void MockAllocatorForClaimedMemory(IStreamerTypes::RequestMemoryAllocatorMock& mock)
{
using ::testing::_;
EXPECT_CALL(mock, LockAllocator()).Times(1);
EXPECT_CALL(mock, UnlockAllocator()).Times(1);
EXPECT_CALL(mock, Allocate(_, _, _))
.Times(1)
.WillOnce([&mock](size_t minimalSize, size_t recommendedSize, size_t alignment)
{
return mock.ForwardAllocate(minimalSize, recommendedSize, alignment);
});
EXPECT_CALL(mock, Release(_)).Times(0);
}
protected:
// Using Streamer to interact with the Scheduler as not all functionality
// is publicly exposed. Since Streamer is mostly the threaded front end for
// the Scheduler, this is fine.
Streamer* m_streamer{ nullptr };
AZStd::shared_ptr<StreamStackEntryMock> m_mock;
AZStd::atomic_bool m_isStackIdle = false;
};
TEST_F(Streamer_SchedulerTest, QueueNextRequest_QueueUnclaimedFireAndForgetReadWithAllocator_AllocatorCalledAndMemoryFreedAgain)
{
using ::testing::_;
using ::testing::AtLeast;
using ::testing::Return;
MockForRead();
AZStd::binary_semaphore allocatorSync;
IStreamerTypes::RequestMemoryAllocatorMock allocatorMock;
MockAllocatorForUnclaimedMemory(allocatorMock, allocatorSync);
AZStd::binary_semaphore readSync;
auto wait = [&readSync](FileRequestHandle)
{
readSync.release();
};
// Scoped to simulate a fire-and-forget request which should cause the request to be deleted after
// completion and free the allocated memory.
{
FileRequestPtr read = m_streamer->Read("TestPath", allocatorMock, 8);
m_streamer->SetRequestCompleteCallback(read, wait);
m_streamer->QueueRequest(read);
}
ASSERT_TRUE(readSync.try_acquire_for(AZStd::chrono::seconds(5)));
ASSERT_TRUE(allocatorSync.try_acquire_for(AZStd::chrono::seconds(5)));
}
TEST_F(Streamer_SchedulerTest, QueueNextRequest_QueueUnclaimedReadWithAllocator_AllocatorCalledAndMemoryFreedAgain)
{
using ::testing::_;
using ::testing::AtLeast;
using ::testing::Return;
MockForRead();
AZStd::binary_semaphore allocatorSync;
IStreamerTypes::RequestMemoryAllocatorMock allocatorMock;
MockAllocatorForUnclaimedMemory(allocatorMock, allocatorSync);
AZStd::binary_semaphore readSync;
auto wait = [&readSync](FileRequestHandle)
{
readSync.release();
};
// Scoped so the request goes out to scope before ending the test. This should trigger the
// memory release on this thread.
{
FileRequestPtr read = m_streamer->Read("TestPath", allocatorMock, 8);
m_streamer->SetRequestCompleteCallback(read, wait);
m_streamer->QueueRequest(read);
ASSERT_TRUE(readSync.try_acquire_for(AZStd::chrono::seconds(5)));
}
ASSERT_TRUE(allocatorSync.try_acquire_for(AZStd::chrono::seconds(5)));
}
TEST_F(Streamer_SchedulerTest, QueueNextRequest_QueueClaimedReadWithAllocator_AllocatorCalledAndMemoryFreedAgain)
{
using ::testing::_;
using ::testing::AtLeast;
using ::testing::Return;
MockForRead();
IStreamerTypes::RequestMemoryAllocatorMock allocatorMock;
MockAllocatorForClaimedMemory(allocatorMock);
AZStd::binary_semaphore readSync;
auto wait = [&readSync](FileRequestHandle)
{
readSync.release();
};
FileRequestPtr read = m_streamer->Read("TestPath", allocatorMock, 8);
m_streamer->SetRequestCompleteCallback(read, wait);
m_streamer->QueueRequest(read);
ASSERT_TRUE(readSync.try_acquire_for(AZStd::chrono::seconds(5)));
void* buffer = nullptr;
u64 readSize = 0;
EXPECT_TRUE(m_streamer->GetReadRequestResult(read, buffer, readSize, IStreamerTypes::ClaimMemory::Yes));
ASSERT_NE(nullptr, buffer);
allocatorMock.ForwardRelease(buffer);
}
TEST_F(Streamer_SchedulerTest, ProcessCancelRequest_CancelReadRequest_MockDoesNotReceiveReadRequest)
{
using ::testing::_;
using ::testing::AtLeast;
using ::testing::Return;
EXPECT_CALL(*m_mock, UpdateStatus(_)).Times(AtLeast(1));
EXPECT_CALL(*m_mock, UpdateCompletionEstimates(_, _, _, _)).Times(AtLeast(1));
EXPECT_CALL(*m_mock, PrepareRequest(_)).Times(AtLeast(1));
EXPECT_CALL(*m_mock, ExecuteRequests()).Times(AtLeast(1));
EXPECT_CALL(*m_mock, QueueRequest(_)).Times(1);
AZStd::atomic_int counter = 2;
AZStd::binary_semaphore sync;
auto wait = [&sync, &counter](FileRequestHandle)
{
if (--counter == 0)
{
sync.release();
}
};
char fakeBuffer[8];
FileRequestPtr read = m_streamer->Read("TestPath", fakeBuffer, sizeof(fakeBuffer), 8);
FileRequestPtr cancel = m_streamer->Cancel(read);
m_streamer->SetRequestCompleteCallback(read, wait);
m_streamer->SetRequestCompleteCallback(cancel, wait);
m_streamer->SuspendProcessing();
m_streamer->QueueRequest(read);
m_streamer->QueueRequest(cancel);
m_streamer->ResumeProcessing();
ASSERT_TRUE(sync.try_acquire_for(AZStd::chrono::seconds(5)));
EXPECT_EQ(IStreamerTypes::RequestStatus::Completed, m_streamer->GetRequestStatus(cancel));
EXPECT_EQ(IStreamerTypes::RequestStatus::Canceled, m_streamer->GetRequestStatus(read));
}
TEST_F(Streamer_SchedulerTest, Reschedule_SetNewDeadlineAndPriority_ReadRequestInMockHasUpdatedTime)
{
using ::testing::_;
using ::testing::AtLeast;
using ::testing::Invoke;
using ::testing::Return;
EXPECT_CALL(*m_mock, UpdateStatus(_)).Times(AtLeast(1));
EXPECT_CALL(*m_mock, UpdateCompletionEstimates(_, _, _, _)).Times(AtLeast(1));
EXPECT_CALL(*m_mock, PrepareRequest(_)).Times(AtLeast(1));
EXPECT_CALL(*m_mock, ExecuteRequests()).Times(AtLeast(1));
EXPECT_CALL(*m_mock, QueueRequest(_)).Times(1)
.WillOnce(Invoke([this](FileRequest* request)
{
auto* read = request->GetCommandFromChain<FileRequest::ReadRequestData>();
ASSERT_NE(nullptr, read);
EXPECT_LT(read->m_deadline, FileRequest::s_noDeadlineTime);
EXPECT_EQ(read->m_priority, IStreamerTypes::s_priorityHighest);
m_mock->ForwardQueueRequest(request);
}));
AZStd::atomic_int counter = 2;
AZStd::binary_semaphore sync;
auto wait = [&sync, &counter](FileRequestHandle)
{
if (--counter == 0)
{
sync.release();
}
};
char fakeBuffer[8];
FileRequestPtr read = m_streamer->Read("TestPath", fakeBuffer, sizeof(fakeBuffer), 8,
IStreamerTypes::s_noDeadline, IStreamerTypes::s_priorityMedium);
FileRequestPtr reschedule = m_streamer->RescheduleRequest(read, IStreamerTypes::s_deadlineNow, IStreamerTypes::s_priorityHighest);
m_streamer->SetRequestCompleteCallback(read, wait);
m_streamer->SetRequestCompleteCallback(reschedule, wait);
m_streamer->SuspendProcessing();
m_streamer->QueueRequest(read);
m_streamer->QueueRequest(reschedule);
m_streamer->ResumeProcessing();
ASSERT_TRUE(sync.try_acquire_for(AZStd::chrono::seconds(5)));
EXPECT_EQ(IStreamerTypes::RequestStatus::Completed, m_streamer->GetRequestStatus(reschedule));
}
TEST_F(Streamer_SchedulerTest, ProcessTillIdle_ShutDownIsDelayedUntilIdle_SchedulerThreadDoesNotImmediatelyShutDown)
{
using::testing::_;
using ::testing::AnyNumber;
using ::testing::Invoke;
constexpr static size_t Iterations = 16;
AZStd::atomic_int counter{ 0 };
EXPECT_CALL(*m_mock, UpdateStatus(_)).Times(AnyNumber());
EXPECT_CALL(*m_mock, UpdateCompletionEstimates(_, _, _, _)).Times(AnyNumber());
// Pretend to be busy [Iterations] times, then set the status to idle so the Scheduler thread can exit.
EXPECT_CALL(*m_mock, ExecuteRequests())
.Times(Iterations + 1)
.WillRepeatedly(Invoke([this, &counter]()
{
if (counter++ >= Iterations)
{
m_isStackIdle = true;
return false;
}
else
{
AZStd::this_thread::sleep_for(AZStd::chrono::milliseconds(32));
return true;
}
}));
if (m_streamer)
{
Interface<IO::IStreamer>::Unregister(m_streamer);
delete m_streamer;
m_streamer = nullptr;
}
EXPECT_EQ(Iterations + 1, counter);
}
} // namespace AZ::IO
@@ -0,0 +1,340 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <limits>
#include <AzCore/IO/IStreamerTypes.h>
#include <AzCore/IO/Streamer/StreamerContext.h>
#include <AzCore/IO/Streamer/StreamStackEntry.h>
#include <AzCore/std/smart_ptr/unique_ptr.h>
#include <AzCore/std/smart_ptr/make_shared.h>
#include <AzCore/UnitTest/TestTypes.h>
#include <AzTest/AzTest.h>
#include <Tests/Streamer/StreamStackEntryMock.h>
namespace AZ::IO
{
using MaxConcurrentRequestsType = decltype(IStreamerTypes::Recommendations::m_maxConcurrentRequests);
template<typename T>
class StreamStackEntryConformityTestsDescriptor
{
public:
using Type = T;
virtual ~StreamStackEntryConformityTestsDescriptor() = default;
virtual void SetUp() {}
virtual void TearDown() {}
virtual T CreateInstance() = 0;
virtual bool UsesSlots() const
{
return true;
}
};
template<typename T>
class StreamStackEntryConformityTests
: public UnitTest::AllocatorsTestFixture
{
public:
using Descriptor = T;
using Type = typename T::Type;
void SetUp() override
{
UnitTest::AllocatorsFixture::SetUp();
m_description.SetUp();
m_context = AZStd::make_unique<StreamerContext>();
}
void TearDown() override
{
m_context.reset();
m_description.TearDown();
UnitTest::AllocatorsFixture::TearDown();
}
FileRequest* CreateUnknownRequest()
{
FileRequest* request = m_context->GetNewInternalRequest();
AZStd::any command{ AZStd::string("Unknown request") };
request->CreateCustom(AZStd::move(command));
return request;
}
T m_description{};
AZStd::unique_ptr<StreamerContext> m_context;
};
TYPED_TEST_CASE_P(StreamStackEntryConformityTests);
TYPED_TEST_P(StreamStackEntryConformityTests, GetName_RetrieveNameSetOnConstruction_NameIsNotEmpty)
{
auto entry = this->m_description.CreateInstance();
const AZStd::string& name = entry.GetName();
EXPECT_FALSE(name.empty());
}
TYPED_TEST_P(StreamStackEntryConformityTests, Next_SetAndGetNext_NextIsSetAndCanBeRetrieved)
{
auto next = AZStd::make_shared<StreamStackEntry>("Next");
auto entry = this->m_description.CreateInstance();
entry.SetNext(next);
auto storedNext = entry.GetNext();
EXPECT_EQ(next.get(), storedNext.get());
}
TYPED_TEST_P(StreamStackEntryConformityTests, SetContext_ContextIsForwardedToNext_SetContextOnMockIsCalled)
{
using ::testing::_;
auto mock = AZStd::make_shared<StreamStackEntryMock>();
auto entry = this->m_description.CreateInstance();
entry.SetNext(mock);
EXPECT_CALL(*mock, SetContext(_))
.Times(1)
.WillOnce([this](StreamerContext& context) { EXPECT_EQ(this->m_context.get(), &context); });
entry.SetContext(*(this->m_context));
}
TYPED_TEST_P(StreamStackEntryConformityTests, PrepareRequest_UnsupportedRequestIsForwarded_RequestIsForwarded)
{
using ::testing::_;
using ::testing::Invoke;
auto mock = AZStd::make_shared<StreamStackEntryMock>();
auto entry = this->m_description.CreateInstance();
entry.SetNext(mock);
FileRequest* request = this->CreateUnknownRequest();
EXPECT_CALL(*mock, PrepareRequest(_))
.Times(1)
.WillOnce(Invoke([request](FileRequest* forwarded) { EXPECT_EQ(request, forwarded); }));
entry.PrepareRequest(request);
this->m_context->RecycleRequest(request);
}
TYPED_TEST_P(StreamStackEntryConformityTests, QueueRequest_UnsupportedRequestIsForwarded_RequestIsForwarded)
{
using ::testing::_;
using ::testing::Invoke;
auto mock = AZStd::make_shared<StreamStackEntryMock>();
auto entry = this->m_description.CreateInstance();
entry.SetNext(mock);
FileRequest* request = this->CreateUnknownRequest();
EXPECT_CALL(*mock, QueueRequest(_))
.Times(1)
.WillOnce(Invoke([request](FileRequest* forwarded) { EXPECT_EQ(request, forwarded); }));
entry.QueueRequest(request);
this->m_context->RecycleRequest(request);
}
TYPED_TEST_P(StreamStackEntryConformityTests, ExecuteRequests_CallIsForwardedToNext_MockReceivedCall)
{
using ::testing::_;
using ::testing::Return;
auto mock = AZStd::make_shared<StreamStackEntryMock>();
auto entry = this->m_description.CreateInstance();
entry.SetNext(mock);
EXPECT_CALL(*mock, ExecuteRequests())
.Times(1)
.WillOnce(Return(false));
entry.ExecuteRequests();
}
TYPED_TEST_P(StreamStackEntryConformityTests, ExecuteRequests_ForwardsFalseResultFromNext_ReturnsFalse)
{
using ::testing::_;
using ::testing::Return;
auto mock = AZStd::make_shared<StreamStackEntryMock>();
auto entry = this->m_description.CreateInstance();
entry.SetNext(mock);
EXPECT_CALL(*mock, ExecuteRequests()).WillRepeatedly(Return(false));
EXPECT_FALSE(entry.ExecuteRequests());
}
TYPED_TEST_P(StreamStackEntryConformityTests, ExecuteRequests_ForwardsTrueResultFromNext_ReturnsTrue)
{
using ::testing::_;
using ::testing::Return;
auto mock = AZStd::make_shared<StreamStackEntryMock>();
auto entry = this->m_description.CreateInstance();
entry.SetNext(mock);
EXPECT_CALL(*mock, ExecuteRequests()).WillRepeatedly(Return(true));
EXPECT_TRUE(entry.ExecuteRequests());
}
TYPED_TEST_P(StreamStackEntryConformityTests, UpdateStatus_ForwardsCallToNext_NextRecievedCall)
{
using ::testing::_;
auto mock = AZStd::make_shared<StreamStackEntryMock>();
auto entry = this->m_description.CreateInstance();
entry.SetNext(mock);
EXPECT_CALL(*mock, UpdateStatus(_))
.Times(1);
StreamStackEntry::Status status;
entry.UpdateStatus(status);
}
TYPED_TEST_P(StreamStackEntryConformityTests, UpdateStatus_IsIdleByDefault_StatusReturnsIsIdleWithTrue)
{
auto entry = this->m_description.CreateInstance();
StreamStackEntry::Status status;
entry.UpdateStatus(status);
EXPECT_TRUE(status.m_isIdle);
}
TYPED_TEST_P(StreamStackEntryConformityTests, UpdateStatus_HasAtLeastOneSlot_ReturnsSlotCountLargerThanOne)
{
if (this->m_description.UsesSlots())
{
auto entry = this->m_description.CreateInstance();
StreamStackEntry::Status status;
entry.UpdateStatus(status);
EXPECT_GE(status.m_numAvailableSlots, 1);
}
}
TYPED_TEST_P(StreamStackEntryConformityTests, UpdateStatus_InitialValueFitsInRecommendations_NumberOfSlotsSmallEnough)
{
if (this->m_description.UsesSlots())
{
auto entry = this->m_description.CreateInstance();
StreamStackEntry::Status status;
entry.UpdateStatus(status);
static constexpr size_t MaxRecommendationsRequestsSize = std::numeric_limits<AZ::IO::MaxConcurrentRequestsType>::max();
EXPECT_LE(status.m_numAvailableSlots, MaxRecommendationsRequestsSize);
}
}
TYPED_TEST_P(StreamStackEntryConformityTests, UpdateStatus_NextHasSmallerNumSlots_ReturnsSmallestNumSlots)
{
using ::testing::_;
if (this->m_description.UsesSlots())
{
auto mock = AZStd::make_shared<StreamStackEntryMock>();
auto entry = this->m_description.CreateInstance();
entry.SetNext(mock);
constexpr s32 minValue = std::numeric_limits<s32>::min();
EXPECT_CALL(*mock, UpdateStatus(_))
.WillOnce([minValue](StreamStackEntry::Status& status)
{
status.m_numAvailableSlots = minValue;
});
StreamStackEntry::Status status;
entry.UpdateStatus(status);
EXPECT_EQ(minValue, status.m_numAvailableSlots);
}
}
TYPED_TEST_P(StreamStackEntryConformityTests, UpdateStatus_NextHasLargerNumSlots_ReturnsSmallestNumSlots)
{
using ::testing::_;
if (this->m_description.UsesSlots())
{
auto mock = AZStd::make_shared<StreamStackEntryMock>();
auto entry = this->m_description.CreateInstance();
entry.SetNext(mock);
s32 maxValue = std::numeric_limits<s32>::max();
EXPECT_CALL(*mock, UpdateStatus(_))
.WillOnce([maxValue](StreamStackEntry::Status& status)
{
status.m_numAvailableSlots = maxValue;
});
StreamStackEntry::Status status;
entry.UpdateStatus(status);
EXPECT_NE(maxValue, status.m_numAvailableSlots);
// When nothing is happening there should always be at least one slot available.
EXPECT_GT(status.m_numAvailableSlots, 0);
}
}
TYPED_TEST_P(StreamStackEntryConformityTests, UpdateCompletionEstimates_ForwardsCallToNext_NextRecievedCall)
{
using ::testing::_;
auto mock = AZStd::make_shared<StreamStackEntryMock>();
auto entry = this->m_description.CreateInstance();
entry.SetNext(mock);
EXPECT_CALL(*mock, UpdateCompletionEstimates(_, _, _, _)).Times(1);
auto now = AZStd::chrono::system_clock::now();
AZStd::vector<FileRequest*> internalRequests;
StreamerContext::PreparedQueue pendingRequests;
entry.UpdateCompletionEstimates(now, internalRequests, pendingRequests.begin(), pendingRequests.end());
}
TYPED_TEST_P(StreamStackEntryConformityTests, CollectStatistics_ForwardsCallToNext_NextRecievedCall)
{
using ::testing::_;
auto mock = AZStd::make_shared<StreamStackEntryMock>();
auto entry = this->m_description.CreateInstance();
entry.SetNext(mock);
EXPECT_CALL(*mock, CollectStatistics(_)).Times(1);
AZStd::vector<Statistic> statistics;
entry.CollectStatistics(statistics);
}
REGISTER_TYPED_TEST_CASE_P(StreamStackEntryConformityTests,
GetName_RetrieveNameSetOnConstruction_NameIsNotEmpty,
Next_SetAndGetNext_NextIsSetAndCanBeRetrieved,
SetContext_ContextIsForwardedToNext_SetContextOnMockIsCalled,
PrepareRequest_UnsupportedRequestIsForwarded_RequestIsForwarded,
QueueRequest_UnsupportedRequestIsForwarded_RequestIsForwarded,
ExecuteRequests_CallIsForwardedToNext_MockReceivedCall,
ExecuteRequests_ForwardsFalseResultFromNext_ReturnsFalse,
ExecuteRequests_ForwardsTrueResultFromNext_ReturnsTrue,
UpdateStatus_ForwardsCallToNext_NextRecievedCall,
UpdateStatus_IsIdleByDefault_StatusReturnsIsIdleWithTrue,
UpdateStatus_HasAtLeastOneSlot_ReturnsSlotCountLargerThanOne,
UpdateStatus_InitialValueFitsInRecommendations_NumberOfSlotsSmallEnough,
UpdateStatus_NextHasSmallerNumSlots_ReturnsSmallestNumSlots,
UpdateStatus_NextHasLargerNumSlots_ReturnsSmallestNumSlots,
UpdateCompletionEstimates_ForwardsCallToNext_NextRecievedCall,
CollectStatistics_ForwardsCallToNext_NextRecievedCall);
} // namespace AZ::IO
@@ -0,0 +1,48 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <AzTest/AzTest.h>
#include <AzCore/IO/Streamer/StreamStackEntry.h>
namespace AZ::IO
{
class StreamStackEntryMock
: public StreamStackEntry
{
public:
StreamStackEntryMock() : StreamStackEntry("StreamStackEntryMock") {}
~StreamStackEntryMock() override = default;
MOCK_CONST_METHOD0(GetName, const AZStd::string&());
MOCK_METHOD1(SetNext, void(AZStd::shared_ptr<StreamStackEntry>));
MOCK_CONST_METHOD0(GetNext, AZStd::shared_ptr<StreamStackEntry>());
MOCK_METHOD1(SetContext, void(StreamerContext&));
MOCK_METHOD1(PrepareRequest, void(FileRequest*));
MOCK_METHOD1(QueueRequest, void(FileRequest*));
MOCK_METHOD0(ExecuteRequests, bool());
MOCK_CONST_METHOD1(UpdateStatus, void(Status& status));
MOCK_METHOD4(UpdateCompletionEstimates, void(AZStd::chrono::system_clock::time_point,
AZStd::vector<FileRequest*>&, StreamerContext::PreparedQueue::iterator, StreamerContext::PreparedQueue::iterator));
MOCK_CONST_METHOD1(CollectStatistics, void(AZStd::vector<Statistic>&));
inline void ForwardSetContext(StreamerContext& context) { StreamStackEntry::SetContext(context); }
inline void ForwardPrepareRequest(FileRequest* request) { StreamStackEntry::PrepareRequest(request); }
inline void ForwardQueueRequest(FileRequest* request) { StreamStackEntry::QueueRequest(request); }
};
} // namespace AZ::IO
@@ -0,0 +1,33 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <Tests/Streamer/StreamStackEntryConformityTests.h>
namespace AZ::IO
{
class StreamStackEntryTestDescription :
public StreamStackEntryConformityTestsDescriptor<StreamStackEntry>
{
public:
StreamStackEntry CreateInstance() override
{
return StreamStackEntry("Name");
}
bool UsesSlots() const override
{
return false;
}
};
INSTANTIATE_TYPED_TEST_CASE_P(Streamer_StreamStackEntryConformityTests, StreamStackEntryConformityTests, StreamStackEntryTestDescription);
} // namespace AZ::IO