birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1452 from aws-lumberyard-dev/Streamer/ConfigUpdates

Browse Source 
Streamer config updates
This commit is contained in:
AMZN-koppersr
2021-06-22 14:57:15 -07:00
committed by GitHub
parent ea9822acb2 c78fa200ef
commit 5dbfa2d497
14 changed files with 429 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Code/Framework/AzCore/Platform/Windows/AzCore/IO/Streamer/StorageDrive_Windows.h
+1 -2
View File
@@ -44,8 +44,7 @@ namespace AZ::IO
//! make adjustments. For the most optimal performance align read buffers to the physicalSectorSize.
u8 m_enableUnbufferedReads : 1;
//! Globally enable file sharing. This allows files to used outside AZ::IO::Streamer, including other applications
//! while in use by AZ::IO::Streamer. File sharing can negatively impact performance and is recommended for
//! development only.
//! while in use by AZ::IO::Streamer.
u8 m_enableSharing : 1;
//! If true, only information that's explicitly requested or issues are reported. If false, status information
//! such as when drives are created and destroyed is reported as well.
Code/Framework/AzCore/Platform/Windows/AzCore/IO/Streamer/StreamerConfiguration_Windows.cpp
+1 -1
View File
@@ -312,7 +312,7 @@ namespace AZ::IO
{
if (reportHardware)
{
AZ_Printf("Streamer", "Skipping drive '%s' because to no paths make use of it.\n", driveIt);
AZ_Printf("Streamer", "Skipping drive '%s' because no paths make use of it.\n", driveIt);
}
while (*driveIt++);
continue;
Code/Framework/AzCore/Tests/Platform/Windows/Tests/IO/Streamer/StorageDriveTests_Windows.cpp
+141
View File
@@ -46,6 +46,7 @@ namespace AZ::IO
options.m_hasSeekPenalty = HasSeekPenalty;
options.m_enableUnbufferedReads = TestEnableUnbufferReads;
options.m_enableSharing = TestEnableSharedReads;
options.m_minimalReporting = true;
return StorageDriveWin({ "c:/" }, TestMaxFileHandles, TestMaxMetaDataEntries, TestPhysicalSectorSize,
TestLogicalSectorSize, TestMaxIOChannels, TestOverCommit, options);
@@ -151,6 +152,7 @@ namespace AZ::IO
m_configurationOptions.m_hasSeekPenalty = HasSeekPenalty;
m_configurationOptions.m_enableUnbufferedReads = TestEnableUnbufferReads;
m_configurationOptions.m_enableSharing = TestEnableSharedReads;
m_configurationOptions.m_minimalReporting = true;
m_storageDriveWin = AZStd::make_shared<AZ::IO::StorageDriveWin>(AZStd::vector<AZStd::string_view>{drive}, TestMaxFileHandles,
TestMaxMetaDataEntries, TestPhysicalSectorSize, TestLogicalSectorSize, TestMaxIOChannels, overCommit, m_configurationOptions);
@@ -1148,3 +1150,142 @@ namespace AZ::IO
azfree(buffers[numRequests - 1]);
}
} // namespace AZ::IO
#if defined(HAVE_BENCHMARK)
#include <benchmark/benchmark.h>
namespace Benchmark
{
class StorageDriveWindowsFixture : public benchmark::Fixture
{
public:
constexpr static char* TestFileName = "StreamerBenchmark.bin";
constexpr static size_t FileSize = 64_mib;
void SetupStreamer(bool enableFileSharing)
{
using namespace AZ::IO;
m_fileIO = new UnitTest::TestFileIOBase();
m_previousFileIO = AZ::IO::FileIOBase::GetInstance();
AZ::IO::FileIOBase::SetInstance(nullptr);
AZ::IO::FileIOBase::SetInstance(m_fileIO);
SystemFile file;
file.Open(TestFileName, SystemFile::OpenMode::SF_OPEN_CREATE | SystemFile::OpenMode::SF_OPEN_READ_WRITE);
AZStd::unique_ptr<char[]> buffer(new char[FileSize]);
::memset(buffer.get(), 'c', FileSize);
file.Write(buffer.get(), FileSize);
file.Close();
AZStd::optional<AZ::IO::FixedMaxPathString> absolutePath = AZ::Utils::ConvertToAbsolutePath(TestFileName);
if (absolutePath.has_value())
{
AZStd::string drive;
AZ::StringFunc::Path::GetDrive(absolutePath->c_str(), drive);
m_absolutePath = *absolutePath;
StorageDriveWin::ConstructionOptions options;
options.m_hasSeekPenalty = false;
options.m_enableUnbufferedReads = true; // Leave this on otherwise repeated loads will be using the Windows cache instead.
options.m_enableSharing = enableFileSharing;
options.m_minimalReporting = true;
AZStd::shared_ptr<StreamStackEntry> storageDriveWin =
AZStd::make_shared<StorageDriveWin>(AZStd::vector<AZStd::string_view>{ drive }, 32, 32, 4_kib, 512, 8, 0, options);
AZStd::unique_ptr<Scheduler> stack = AZStd::make_unique<Scheduler>(AZStd::move(storageDriveWin));
m_streamer = aznew Streamer(AZStd::thread_desc{}, AZStd::move(stack));
}
}
void TearDown([[maybe_unused]] const ::benchmark::State& state) override
{
using namespace AZ::IO;
AZStd::string temp;
m_absolutePath.swap(temp);
delete m_streamer;
SystemFile::Delete(TestFileName);
AZ::IO::FileIOBase::SetInstance(nullptr);
AZ::IO::FileIOBase::SetInstance(m_previousFileIO);
delete m_fileIO;
}
void RepeatedlyReadFile(benchmark::State& state)
{
using namespace AZ::IO;
using namespace AZStd::chrono;
AZStd::unique_ptr<char[]> buffer(new char[FileSize]);
for (auto _ : state)
{
AZStd::binary_semaphore waitForReads;
AZStd::atomic<system_clock::time_point> end;
auto callback = [&end, &waitForReads]([[maybe_unused]] FileRequestHandle request)
{
benchmark::DoNotOptimize(end = high_resolution_clock::now());
waitForReads.release();
};
FileRequestPtr request = m_streamer->Read(m_absolutePath, buffer.get(), state.range(0), state.range(0));
m_streamer->SetRequestCompleteCallback(request, callback);
system_clock::time_point start;
benchmark::DoNotOptimize(start = high_resolution_clock::now());
m_streamer->QueueRequest(request);
waitForReads.try_acquire_for(AZStd::chrono::seconds(5));
auto durationInSeconds = duration_cast<duration<double>>(end.load() - start);
state.SetIterationTime(durationInSeconds.count());
m_streamer->QueueRequest(m_streamer->FlushCaches());
}
}
AZStd::string m_absolutePath;
AZ::IO::Streamer* m_streamer{};
AZ::IO::FileIOBase* m_previousFileIO{};
UnitTest::TestFileIOBase* m_fileIO{};
};
BENCHMARK_DEFINE_F(StorageDriveWindowsFixture, ReadsBaseline)(benchmark::State& state)
{
constexpr bool EnableFileSharing = false;
SetupStreamer(EnableFileSharing);
RepeatedlyReadFile(state);
}
BENCHMARK_DEFINE_F(StorageDriveWindowsFixture, ReadsWithFileReadSharingEnabled)(benchmark::State& state)
{
using namespace AZ::IO;
constexpr bool EnableFileSharing = true;
SetupStreamer(EnableFileSharing);
RepeatedlyReadFile(state);
}
// For these benchmarks the CPU stat doesn't provide useful information because it uses GetThreadTimes on Window but since the main
// thread is mostly sleeping while waiting for the read on the Streamer thread to complete this will report values (close to) zero.
BENCHMARK_REGISTER_F(StorageDriveWindowsFixture, ReadsBaseline)
->RangeMultiplier(8)
->Range(1024, 64_mib)
->UseManualTime()
->Unit(benchmark::kMillisecond);
BENCHMARK_REGISTER_F(StorageDriveWindowsFixture, ReadsWithFileReadSharingEnabled)
->RangeMultiplier(8)
->Range(1024, 64_mib)
->UseManualTime()
->Unit(benchmark::kMillisecond);
} // namespace Benchmark
#endif // HAVE_BENCHMARK
Registry/Platform/Windows/streamer.editor.setreg
+1 -15
View File
@@ -13,33 +13,19 @@
[
{
"$type": "AZ::IO::StorageDriveConfig",
// The maximum number of file handles that the drive will cache.
"MaxFileHandles": 1024
},
{
"$type": "AZ::IO::WindowsStorageDriveConfig",
// The maximum number of file handles that the drive will cache.
"MaxFileHandles": 1024,
// The maximum number of files to keep the meta data such as the size around for.
"MaxMetaDataCache": 1024,
// Number of requests the drive keeps after its queue is full.
// Overcommitting allows for requests to be immediately available after a request completes without needing
// any scheduling, but this also doesn't allow these requests to be rescheduled or updated.
"Overcommit": 8,
// Allows files to be shared. This can be needed if the file needs to be opened in multiple locations, such
// as the editor and the Asset Processor. Turning this feature on comes at a small performance cost.
"EnableFileSharing": true,
// Unbuffered reads bypass the OS file cache for faster file reads. This helps speed up initial file loads
// and is best for applications that only read a file once such as the game. For applications that frequently
// re-read files such as the editor it's better to turn this feature off.
"EnableUnbufferedReads": false,
// If true, only information that's explicitly requested or issues are reported. If false, status information
// such as when drives are created and destroyed is reported as well.
"MinimalReporting": false
},
{
"$type": "AzFramework::RemoteStorageDriveConfig",
// The maximum number of file handles that the drive will cache.
"MaxFileHandles": 1024
}
]
@@ -48,4 +34,4 @@
}
}
}
}
}
Registry/Platform/Windows/streamer.game.debug.setreg
+74
View File
@@ -0,0 +1,74 @@
{
"Amazon":
{
"AzCore":
{
"Streamer":
{
"Profiles":
{
"Generic":
{
"Stack":
[
{
"$type": "AZ::IO::WindowsStorageDriveConfig",
"MaxFileHandles": 32,
"MaxMetaDataCache": 32,
"Overcommit": 8,
"EnableFileSharing": true,
"EnableUnbufferedReads": true,
"MinimalReporting": false
},
{
"$type": "AZ::IO::ReadSplitterConfig",
"BufferSizeMib": 6,
"SplitSize": "MaxTransfer",
"AdjustOffset": true,
"SplitAlignedRequests": false
},
{
"$type": "AzFramework::RemoteStorageDriveConfig",
"MaxFileHandles": 1024
},
{
"$type": "AZ::IO::BlockCacheConfig",
"CacheSizeMib": 10,
"BlockSize": "MaxTransfer"
},
{
"$type": "AZ::IO::DedicatedCacheConfig",
"CacheSizeMib": 2,
"BlockSize": "MemoryAlignment",
"WriteOnlyEpilog": true
},
{
"$type": "AZ::IO::FullFileDecompressorConfig",
"MaxNumReads": 2,
"MaxNumJobs": 2
}
]
},
"DevMode":
{
"Stack":
[
{
"$type": "AZ::IO::WindowsStorageDriveConfig",
"MaxFileHandles": 1024,
"MaxMetaDataCache": 1024,
"Overcommit": 8,
"EnableFileSharing": true,
"EnableUnbufferedReads": false
},
{
"$type": "AzFramework::RemoteStorageDriveConfig",
"MaxFileHandles": 1024
}
]
}
}
}
}
}
}
Registry/Platform/Windows/streamer.game.profile.setreg
+74
View File
@@ -0,0 +1,74 @@
{
"Amazon":
{
"AzCore":
{
"Streamer":
{
"Profiles":
{
"Generic":
{
"Stack":
[
{
"$type": "AZ::IO::WindowsStorageDriveConfig",
"MaxFileHandles": 32,
"MaxMetaDataCache": 32,
"Overcommit": 8,
"EnableFileSharing": true,
"EnableUnbufferedReads": true,
"MinimalReporting": false
},
{
"$type": "AZ::IO::ReadSplitterConfig",
"BufferSizeMib": 6,
"SplitSize": "MaxTransfer",
"AdjustOffset": true,
"SplitAlignedRequests": false
},
{
"$type": "AzFramework::RemoteStorageDriveConfig",
"MaxFileHandles": 1024
},
{
"$type": "AZ::IO::BlockCacheConfig",
"CacheSizeMib": 10,
"BlockSize": "MaxTransfer"
},
{
"$type": "AZ::IO::DedicatedCacheConfig",
"CacheSizeMib": 2,
"BlockSize": "MemoryAlignment",
"WriteOnlyEpilog": true
},
{
"$type": "AZ::IO::FullFileDecompressorConfig",
"MaxNumReads": 2,
"MaxNumJobs": 2
}
]
},
"DevMode":
{
"Stack":
[
{
"$type": "AZ::IO::WindowsStorageDriveConfig",
"MaxFileHandles": 1024,
"MaxMetaDataCache": 1024,
"Overcommit": 8,
"EnableFileSharing": true,
"EnableUnbufferedReads": false
},
{
"$type": "AzFramework::RemoteStorageDriveConfig",
"MaxFileHandles": 1024
}
]
}
}
}
}
}
}
Registry/Platform/Windows/streamer.game.setreg
+2 -21
View File
@@ -27,8 +27,7 @@
// will avoid saturating the IO controller which can be needed if the drive is used by other applications.
"Overcommit": 8,
// Globally enable file sharing. This allows files to used outside AZ::IO::Streamer, including other
// applications while in use by AZ::IO::Streamer. File sharing can negatively impact performance and is
// recommended for development only.
// applications while in use by AZ::IO::Streamer.
"EnableFileSharing": false,
// Use unbuffered reads for the fastest possible read speeds by bypassing the Windows file cache. This
// results in a faster read the first time a file is read, but subsequent reads will possibly be slower as
@@ -63,27 +62,9 @@
"MaxNumJobs": 2
}
]
},
"DevMode":
{
"Stack":
[
{
"$type": "AZ::IO::WindowsStorageDriveConfig",
"MaxFileHandles": 1024,
"MaxMetaDataCache": 1024,
"Overcommit": 8,
"EnableFileSharing": true,
"EnableUnbufferedReads": false
},
{
"$type": "AzFramework::RemoteStorageDriveConfig",
"MaxFileHandles": 1024
}
]
}
}
}
}
}
}
}
Registry/Platform/Windows/streamer.test.setreg
+1 -1
View File
@@ -54,4 +54,4 @@
}
}
}
}
}
Registry/streamer.editor.setreg
+1 -1
View File
@@ -27,4 +27,4 @@
}
}
}
}
}
Registry/streamer.game.debug.setreg
+65
View File
@@ -0,0 +1,65 @@
{
"Amazon":
{
"AzCore":
{
"Streamer":
{
"Profiles":
{
"Generic":
{
"Stack":
[
{
"$type": "AZ::IO::StorageDriveConfig",
"MaxFileHandles": 32
},
{
"$type": "AZ::IO::ReadSplitterConfig",
"BufferSizeMib": 6,
"SplitSize": "MaxTransfer",
"AdjustOffset": true,
"SplitAlignedRequests": false
},
{
"$type": "AzFramework::RemoteStorageDriveConfig",
"MaxFileHandles": 1024
},
{
"$type": "AZ::IO::BlockCacheConfig",
"CacheSizeMib": 10,
"BlockSize": "MaxTransfer"
},
{
"$type": "AZ::IO::DedicatedCacheConfig",
"CacheSizeMib": 2,
"BlockSize": "MemoryAlignment",
"WriteOnlyEpilog": true
},
{
"$type": "AZ::IO::FullFileDecompressorConfig",
"MaxNumReads": 2,
"MaxNumJobs": 2
}
]
},
"DevMode":
{
"Stack":
[
{
"$type": "AZ::IO::StorageDriveConfig",
"MaxFileHandles": 1024
},
{
"$type": "AzFramework::RemoteStorageDriveConfig",
"MaxFileHandles": 1024
}
]
}
}
}
}
}
}
Registry/streamer.game.profile.setreg
+65
View File
@@ -0,0 +1,65 @@
{
"Amazon":
{
"AzCore":
{
"Streamer":
{
"Profiles":
{
"Generic":
{
"Stack":
[
{
"$type": "AZ::IO::StorageDriveConfig",
"MaxFileHandles": 32
},
{
"$type": "AZ::IO::ReadSplitterConfig",
"BufferSizeMib": 6,
"SplitSize": "MaxTransfer",
"AdjustOffset": true,
"SplitAlignedRequests": false
},
{
"$type": "AzFramework::RemoteStorageDriveConfig",
"MaxFileHandles": 1024
},
{
"$type": "AZ::IO::BlockCacheConfig",
"CacheSizeMib": 10,
"BlockSize": "MaxTransfer"
},
{
"$type": "AZ::IO::DedicatedCacheConfig",
"CacheSizeMib": 2,
"BlockSize": "MemoryAlignment",
"WriteOnlyEpilog": true
},
{
"$type": "AZ::IO::FullFileDecompressorConfig",
"MaxNumReads": 2,
"MaxNumJobs": 2
}
]
},
"DevMode":
{
"Stack":
[
{
"$type": "AZ::IO::StorageDriveConfig",
"MaxFileHandles": 1024
},
{
"$type": "AzFramework::RemoteStorageDriveConfig",
"MaxFileHandles": 1024
}
]
}
}
}
}
}
}
Registry/streamer.game.setreg
+1 -15
View File
@@ -57,23 +57,9 @@
"MaxNumJobs": 2
}
]
},
"DevMode":
{
"Stack":
[
{
"$type": "AZ::IO::StorageDriveConfig",
"MaxFileHandles": 1024
},
{
"$type": "AzFramework::RemoteStorageDriveConfig",
"MaxFileHandles": 1024
}
]
}
}
}
}
}
}
}
Registry/streamer.setreg
+1 -1
View File
@@ -26,4 +26,4 @@
}
}
}
}
}
Registry/streamer.test.setreg
+1 -1
View File
@@ -45,4 +45,4 @@
}
}
}
}
}