diff --git a/Code/Framework/AzCore/Platform/Windows/AzCore/IO/Streamer/StorageDrive_Windows.h b/Code/Framework/AzCore/Platform/Windows/AzCore/IO/Streamer/StorageDrive_Windows.h index 479828c9d3..d51fdc36f3 100644 --- a/Code/Framework/AzCore/Platform/Windows/AzCore/IO/Streamer/StorageDrive_Windows.h +++ b/Code/Framework/AzCore/Platform/Windows/AzCore/IO/Streamer/StorageDrive_Windows.h @@ -44,8 +44,7 @@ namespace AZ::IO //! make adjustments. For the most optimal performance align read buffers to the physicalSectorSize. u8 m_enableUnbufferedReads : 1; //! Globally enable file sharing. This allows files to used outside AZ::IO::Streamer, including other applications - //! while in use by AZ::IO::Streamer. File sharing can negatively impact performance and is recommended for - //! development only. + //! while in use by AZ::IO::Streamer. u8 m_enableSharing : 1; //! If true, only information that's explicitly requested or issues are reported. If false, status information //! such as when drives are created and destroyed is reported as well. diff --git a/Code/Framework/AzCore/Platform/Windows/AzCore/IO/Streamer/StreamerConfiguration_Windows.cpp b/Code/Framework/AzCore/Platform/Windows/AzCore/IO/Streamer/StreamerConfiguration_Windows.cpp index 5a19bc4f54..28831a00b0 100644 --- a/Code/Framework/AzCore/Platform/Windows/AzCore/IO/Streamer/StreamerConfiguration_Windows.cpp +++ b/Code/Framework/AzCore/Platform/Windows/AzCore/IO/Streamer/StreamerConfiguration_Windows.cpp @@ -312,7 +312,7 @@ namespace AZ::IO { if (reportHardware) { - AZ_Printf("Streamer", "Skipping drive '%s' because to no paths make use of it.\n", driveIt); + AZ_Printf("Streamer", "Skipping drive '%s' because no paths make use of it.\n", driveIt); } while (*driveIt++); continue; diff --git a/Code/Framework/AzCore/Tests/Platform/Windows/Tests/IO/Streamer/StorageDriveTests_Windows.cpp b/Code/Framework/AzCore/Tests/Platform/Windows/Tests/IO/Streamer/StorageDriveTests_Windows.cpp index bab531deb6..ecf6873052 100644 --- a/Code/Framework/AzCore/Tests/Platform/Windows/Tests/IO/Streamer/StorageDriveTests_Windows.cpp +++ b/Code/Framework/AzCore/Tests/Platform/Windows/Tests/IO/Streamer/StorageDriveTests_Windows.cpp @@ -46,6 +46,7 @@ namespace AZ::IO options.m_hasSeekPenalty = HasSeekPenalty; options.m_enableUnbufferedReads = TestEnableUnbufferReads; options.m_enableSharing = TestEnableSharedReads; + options.m_minimalReporting = true; return StorageDriveWin({ "c:/" }, TestMaxFileHandles, TestMaxMetaDataEntries, TestPhysicalSectorSize, TestLogicalSectorSize, TestMaxIOChannels, TestOverCommit, options); @@ -151,6 +152,7 @@ namespace AZ::IO m_configurationOptions.m_hasSeekPenalty = HasSeekPenalty; m_configurationOptions.m_enableUnbufferedReads = TestEnableUnbufferReads; m_configurationOptions.m_enableSharing = TestEnableSharedReads; + m_configurationOptions.m_minimalReporting = true; m_storageDriveWin = AZStd::make_shared(AZStd::vector{drive}, TestMaxFileHandles, TestMaxMetaDataEntries, TestPhysicalSectorSize, TestLogicalSectorSize, TestMaxIOChannels, overCommit, m_configurationOptions); @@ -1148,3 +1150,142 @@ namespace AZ::IO azfree(buffers[numRequests - 1]); } } // namespace AZ::IO + +#if defined(HAVE_BENCHMARK) + +#include + +namespace Benchmark +{ + class StorageDriveWindowsFixture : public benchmark::Fixture + { + public: + constexpr static char* TestFileName = "StreamerBenchmark.bin"; + constexpr static size_t FileSize = 64_mib; + + void SetupStreamer(bool enableFileSharing) + { + using namespace AZ::IO; + + m_fileIO = new UnitTest::TestFileIOBase(); + m_previousFileIO = AZ::IO::FileIOBase::GetInstance(); + AZ::IO::FileIOBase::SetInstance(nullptr); + AZ::IO::FileIOBase::SetInstance(m_fileIO); + + SystemFile file; + file.Open(TestFileName, SystemFile::OpenMode::SF_OPEN_CREATE | SystemFile::OpenMode::SF_OPEN_READ_WRITE); + AZStd::unique_ptr buffer(new char[FileSize]); + ::memset(buffer.get(), 'c', FileSize); + + file.Write(buffer.get(), FileSize); + file.Close(); + + AZStd::optional absolutePath = AZ::Utils::ConvertToAbsolutePath(TestFileName); + if (absolutePath.has_value()) + { + AZStd::string drive; + AZ::StringFunc::Path::GetDrive(absolutePath->c_str(), drive); + + m_absolutePath = *absolutePath; + + StorageDriveWin::ConstructionOptions options; + options.m_hasSeekPenalty = false; + options.m_enableUnbufferedReads = true; // Leave this on otherwise repeated loads will be using the Windows cache instead. + options.m_enableSharing = enableFileSharing; + options.m_minimalReporting = true; + AZStd::shared_ptr storageDriveWin = + AZStd::make_shared(AZStd::vector{ drive }, 32, 32, 4_kib, 512, 8, 0, options); + + AZStd::unique_ptr stack = AZStd::make_unique(AZStd::move(storageDriveWin)); + m_streamer = aznew Streamer(AZStd::thread_desc{}, AZStd::move(stack)); + } + } + + void TearDown([[maybe_unused]] const ::benchmark::State& state) override + { + using namespace AZ::IO; + + AZStd::string temp; + m_absolutePath.swap(temp); + + delete m_streamer; + + SystemFile::Delete(TestFileName); + + AZ::IO::FileIOBase::SetInstance(nullptr); + AZ::IO::FileIOBase::SetInstance(m_previousFileIO); + delete m_fileIO; + } + + void RepeatedlyReadFile(benchmark::State& state) + { + using namespace AZ::IO; + using namespace AZStd::chrono; + + AZStd::unique_ptr buffer(new char[FileSize]); + + for (auto _ : state) + { + AZStd::binary_semaphore waitForReads; + AZStd::atomic end; + auto callback = [&end, &waitForReads]([[maybe_unused]] FileRequestHandle request) + { + benchmark::DoNotOptimize(end = high_resolution_clock::now()); + waitForReads.release(); + }; + + FileRequestPtr request = m_streamer->Read(m_absolutePath, buffer.get(), state.range(0), state.range(0)); + m_streamer->SetRequestCompleteCallback(request, callback); + + system_clock::time_point start; + benchmark::DoNotOptimize(start = high_resolution_clock::now()); + m_streamer->QueueRequest(request); + + waitForReads.try_acquire_for(AZStd::chrono::seconds(5)); + auto durationInSeconds = duration_cast>(end.load() - start); + + state.SetIterationTime(durationInSeconds.count()); + + m_streamer->QueueRequest(m_streamer->FlushCaches()); + } + } + + AZStd::string m_absolutePath; + AZ::IO::Streamer* m_streamer{}; + AZ::IO::FileIOBase* m_previousFileIO{}; + UnitTest::TestFileIOBase* m_fileIO{}; + }; + + BENCHMARK_DEFINE_F(StorageDriveWindowsFixture, ReadsBaseline)(benchmark::State& state) + { + constexpr bool EnableFileSharing = false; + SetupStreamer(EnableFileSharing); + RepeatedlyReadFile(state); + } + + BENCHMARK_DEFINE_F(StorageDriveWindowsFixture, ReadsWithFileReadSharingEnabled)(benchmark::State& state) + { + using namespace AZ::IO; + + constexpr bool EnableFileSharing = true; + SetupStreamer(EnableFileSharing); + RepeatedlyReadFile(state); + } + + // For these benchmarks the CPU stat doesn't provide useful information because it uses GetThreadTimes on Window but since the main + // thread is mostly sleeping while waiting for the read on the Streamer thread to complete this will report values (close to) zero. + + BENCHMARK_REGISTER_F(StorageDriveWindowsFixture, ReadsBaseline) + ->RangeMultiplier(8) + ->Range(1024, 64_mib) + ->UseManualTime() + ->Unit(benchmark::kMillisecond); + + BENCHMARK_REGISTER_F(StorageDriveWindowsFixture, ReadsWithFileReadSharingEnabled) + ->RangeMultiplier(8) + ->Range(1024, 64_mib) + ->UseManualTime() + ->Unit(benchmark::kMillisecond); + +} // namespace Benchmark +#endif // HAVE_BENCHMARK diff --git a/Registry/Platform/Windows/streamer.editor.setreg b/Registry/Platform/Windows/streamer.editor.setreg index 0f7eb3e63d..42f3fc9b7f 100644 --- a/Registry/Platform/Windows/streamer.editor.setreg +++ b/Registry/Platform/Windows/streamer.editor.setreg @@ -13,33 +13,19 @@ [ { "$type": "AZ::IO::StorageDriveConfig", - // The maximum number of file handles that the drive will cache. "MaxFileHandles": 1024 }, { "$type": "AZ::IO::WindowsStorageDriveConfig", - // The maximum number of file handles that the drive will cache. "MaxFileHandles": 1024, - // The maximum number of files to keep the meta data such as the size around for. "MaxMetaDataCache": 1024, - // Number of requests the drive keeps after its queue is full. - // Overcommitting allows for requests to be immediately available after a request completes without needing - // any scheduling, but this also doesn't allow these requests to be rescheduled or updated. "Overcommit": 8, - // Allows files to be shared. This can be needed if the file needs to be opened in multiple locations, such - // as the editor and the Asset Processor. Turning this feature on comes at a small performance cost. "EnableFileSharing": true, - // Unbuffered reads bypass the OS file cache for faster file reads. This helps speed up initial file loads - // and is best for applications that only read a file once such as the game. For applications that frequently - // re-read files such as the editor it's better to turn this feature off. "EnableUnbufferedReads": false, - // If true, only information that's explicitly requested or issues are reported. If false, status information - // such as when drives are created and destroyed is reported as well. "MinimalReporting": false }, { "$type": "AzFramework::RemoteStorageDriveConfig", - // The maximum number of file handles that the drive will cache. "MaxFileHandles": 1024 } ] @@ -48,4 +34,4 @@ } } } -} \ No newline at end of file +} diff --git a/Registry/Platform/Windows/streamer.game.debug.setreg b/Registry/Platform/Windows/streamer.game.debug.setreg new file mode 100644 index 0000000000..9165829b1b --- /dev/null +++ b/Registry/Platform/Windows/streamer.game.debug.setreg @@ -0,0 +1,74 @@ +{ + "Amazon": + { + "AzCore": + { + "Streamer": + { + "Profiles": + { + "Generic": + { + "Stack": + [ + { + "$type": "AZ::IO::WindowsStorageDriveConfig", + "MaxFileHandles": 32, + "MaxMetaDataCache": 32, + "Overcommit": 8, + "EnableFileSharing": true, + "EnableUnbufferedReads": true, + "MinimalReporting": false + }, + { + "$type": "AZ::IO::ReadSplitterConfig", + "BufferSizeMib": 6, + "SplitSize": "MaxTransfer", + "AdjustOffset": true, + "SplitAlignedRequests": false + }, + { + "$type": "AzFramework::RemoteStorageDriveConfig", + "MaxFileHandles": 1024 + }, + { + "$type": "AZ::IO::BlockCacheConfig", + "CacheSizeMib": 10, + "BlockSize": "MaxTransfer" + }, + { + "$type": "AZ::IO::DedicatedCacheConfig", + "CacheSizeMib": 2, + "BlockSize": "MemoryAlignment", + "WriteOnlyEpilog": true + }, + { + "$type": "AZ::IO::FullFileDecompressorConfig", + "MaxNumReads": 2, + "MaxNumJobs": 2 + } + ] + }, + "DevMode": + { + "Stack": + [ + { + "$type": "AZ::IO::WindowsStorageDriveConfig", + "MaxFileHandles": 1024, + "MaxMetaDataCache": 1024, + "Overcommit": 8, + "EnableFileSharing": true, + "EnableUnbufferedReads": false + }, + { + "$type": "AzFramework::RemoteStorageDriveConfig", + "MaxFileHandles": 1024 + } + ] + } + } + } + } + } +} diff --git a/Registry/Platform/Windows/streamer.game.profile.setreg b/Registry/Platform/Windows/streamer.game.profile.setreg new file mode 100644 index 0000000000..9165829b1b --- /dev/null +++ b/Registry/Platform/Windows/streamer.game.profile.setreg @@ -0,0 +1,74 @@ +{ + "Amazon": + { + "AzCore": + { + "Streamer": + { + "Profiles": + { + "Generic": + { + "Stack": + [ + { + "$type": "AZ::IO::WindowsStorageDriveConfig", + "MaxFileHandles": 32, + "MaxMetaDataCache": 32, + "Overcommit": 8, + "EnableFileSharing": true, + "EnableUnbufferedReads": true, + "MinimalReporting": false + }, + { + "$type": "AZ::IO::ReadSplitterConfig", + "BufferSizeMib": 6, + "SplitSize": "MaxTransfer", + "AdjustOffset": true, + "SplitAlignedRequests": false + }, + { + "$type": "AzFramework::RemoteStorageDriveConfig", + "MaxFileHandles": 1024 + }, + { + "$type": "AZ::IO::BlockCacheConfig", + "CacheSizeMib": 10, + "BlockSize": "MaxTransfer" + }, + { + "$type": "AZ::IO::DedicatedCacheConfig", + "CacheSizeMib": 2, + "BlockSize": "MemoryAlignment", + "WriteOnlyEpilog": true + }, + { + "$type": "AZ::IO::FullFileDecompressorConfig", + "MaxNumReads": 2, + "MaxNumJobs": 2 + } + ] + }, + "DevMode": + { + "Stack": + [ + { + "$type": "AZ::IO::WindowsStorageDriveConfig", + "MaxFileHandles": 1024, + "MaxMetaDataCache": 1024, + "Overcommit": 8, + "EnableFileSharing": true, + "EnableUnbufferedReads": false + }, + { + "$type": "AzFramework::RemoteStorageDriveConfig", + "MaxFileHandles": 1024 + } + ] + } + } + } + } + } +} diff --git a/Registry/Platform/Windows/streamer.game.setreg b/Registry/Platform/Windows/streamer.game.setreg index 7788227fb5..29f85d1727 100644 --- a/Registry/Platform/Windows/streamer.game.setreg +++ b/Registry/Platform/Windows/streamer.game.setreg @@ -27,8 +27,7 @@ // will avoid saturating the IO controller which can be needed if the drive is used by other applications. "Overcommit": 8, // Globally enable file sharing. This allows files to used outside AZ::IO::Streamer, including other - // applications while in use by AZ::IO::Streamer. File sharing can negatively impact performance and is - // recommended for development only. + // applications while in use by AZ::IO::Streamer. "EnableFileSharing": false, // Use unbuffered reads for the fastest possible read speeds by bypassing the Windows file cache. This // results in a faster read the first time a file is read, but subsequent reads will possibly be slower as @@ -63,27 +62,9 @@ "MaxNumJobs": 2 } ] - }, - "DevMode": - { - "Stack": - [ - { - "$type": "AZ::IO::WindowsStorageDriveConfig", - "MaxFileHandles": 1024, - "MaxMetaDataCache": 1024, - "Overcommit": 8, - "EnableFileSharing": true, - "EnableUnbufferedReads": false - }, - { - "$type": "AzFramework::RemoteStorageDriveConfig", - "MaxFileHandles": 1024 - } - ] } } } } } -} \ No newline at end of file +} diff --git a/Registry/Platform/Windows/streamer.test.setreg b/Registry/Platform/Windows/streamer.test.setreg index 13e4fa2d57..c9a225a0b0 100644 --- a/Registry/Platform/Windows/streamer.test.setreg +++ b/Registry/Platform/Windows/streamer.test.setreg @@ -54,4 +54,4 @@ } } } -} \ No newline at end of file +} diff --git a/Registry/streamer.editor.setreg b/Registry/streamer.editor.setreg index 33eff0e5b6..7acc3f5486 100644 --- a/Registry/streamer.editor.setreg +++ b/Registry/streamer.editor.setreg @@ -27,4 +27,4 @@ } } } -} \ No newline at end of file +} diff --git a/Registry/streamer.game.debug.setreg b/Registry/streamer.game.debug.setreg new file mode 100644 index 0000000000..be1be3c073 --- /dev/null +++ b/Registry/streamer.game.debug.setreg @@ -0,0 +1,65 @@ +{ + "Amazon": + { + "AzCore": + { + "Streamer": + { + "Profiles": + { + "Generic": + { + "Stack": + [ + { + "$type": "AZ::IO::StorageDriveConfig", + "MaxFileHandles": 32 + }, + { + "$type": "AZ::IO::ReadSplitterConfig", + "BufferSizeMib": 6, + "SplitSize": "MaxTransfer", + "AdjustOffset": true, + "SplitAlignedRequests": false + }, + { + "$type": "AzFramework::RemoteStorageDriveConfig", + "MaxFileHandles": 1024 + }, + { + "$type": "AZ::IO::BlockCacheConfig", + "CacheSizeMib": 10, + "BlockSize": "MaxTransfer" + }, + { + "$type": "AZ::IO::DedicatedCacheConfig", + "CacheSizeMib": 2, + "BlockSize": "MemoryAlignment", + "WriteOnlyEpilog": true + }, + { + "$type": "AZ::IO::FullFileDecompressorConfig", + "MaxNumReads": 2, + "MaxNumJobs": 2 + } + ] + }, + "DevMode": + { + "Stack": + [ + { + "$type": "AZ::IO::StorageDriveConfig", + "MaxFileHandles": 1024 + }, + { + "$type": "AzFramework::RemoteStorageDriveConfig", + "MaxFileHandles": 1024 + } + ] + } + } + } + } + } +} diff --git a/Registry/streamer.game.profile.setreg b/Registry/streamer.game.profile.setreg new file mode 100644 index 0000000000..be1be3c073 --- /dev/null +++ b/Registry/streamer.game.profile.setreg @@ -0,0 +1,65 @@ +{ + "Amazon": + { + "AzCore": + { + "Streamer": + { + "Profiles": + { + "Generic": + { + "Stack": + [ + { + "$type": "AZ::IO::StorageDriveConfig", + "MaxFileHandles": 32 + }, + { + "$type": "AZ::IO::ReadSplitterConfig", + "BufferSizeMib": 6, + "SplitSize": "MaxTransfer", + "AdjustOffset": true, + "SplitAlignedRequests": false + }, + { + "$type": "AzFramework::RemoteStorageDriveConfig", + "MaxFileHandles": 1024 + }, + { + "$type": "AZ::IO::BlockCacheConfig", + "CacheSizeMib": 10, + "BlockSize": "MaxTransfer" + }, + { + "$type": "AZ::IO::DedicatedCacheConfig", + "CacheSizeMib": 2, + "BlockSize": "MemoryAlignment", + "WriteOnlyEpilog": true + }, + { + "$type": "AZ::IO::FullFileDecompressorConfig", + "MaxNumReads": 2, + "MaxNumJobs": 2 + } + ] + }, + "DevMode": + { + "Stack": + [ + { + "$type": "AZ::IO::StorageDriveConfig", + "MaxFileHandles": 1024 + }, + { + "$type": "AzFramework::RemoteStorageDriveConfig", + "MaxFileHandles": 1024 + } + ] + } + } + } + } + } +} diff --git a/Registry/streamer.game.setreg b/Registry/streamer.game.setreg index 972fb80fb2..69f3189d46 100644 --- a/Registry/streamer.game.setreg +++ b/Registry/streamer.game.setreg @@ -57,23 +57,9 @@ "MaxNumJobs": 2 } ] - }, - "DevMode": - { - "Stack": - [ - { - "$type": "AZ::IO::StorageDriveConfig", - "MaxFileHandles": 1024 - }, - { - "$type": "AzFramework::RemoteStorageDriveConfig", - "MaxFileHandles": 1024 - } - ] } } } } } -} \ No newline at end of file +} diff --git a/Registry/streamer.setreg b/Registry/streamer.setreg index 79e80bbf5c..48177628c2 100644 --- a/Registry/streamer.setreg +++ b/Registry/streamer.setreg @@ -26,4 +26,4 @@ } } } -} \ No newline at end of file +} diff --git a/Registry/streamer.test.setreg b/Registry/streamer.test.setreg index 21504f659a..0adb01fca8 100644 --- a/Registry/streamer.test.setreg +++ b/Registry/streamer.test.setreg @@ -45,4 +45,4 @@ } } } -} \ No newline at end of file +}