birkeh
/
o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1452 from aws-lumberyard-dev/Streamer/ConfigUpdates

Browse Source 
Streamer config updates
main
AMZN-koppersr 5 years ago committed by GitHub
parent ea9822acb2 c78fa200ef
commit 5dbfa2d497
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 429 additions and 58 deletions
Show Stats Download Patch File Download Diff File

3
Code/Framework/AzCore/Platform/Windows/AzCore/IO/Streamer/StorageDrive_Windows.h
Unescape Escape View File

@ -44,8 +44,7 @@ namespace AZ::IO
//! make adjustments. For the most optimal performance align read buffers to the physicalSectorSize. //! make adjustments. For the most optimal performance align read buffers to the physicalSectorSize.
u8 m_enableUnbufferedReads : 1; u8 m_enableUnbufferedReads : 1;
//! Globally enable file sharing. This allows files to used outside AZ::IO::Streamer, including other applications //! Globally enable file sharing. This allows files to used outside AZ::IO::Streamer, including other applications
//! while in use by AZ::IO::Streamer. File sharing can negatively impact performance and is recommended for //! while in use by AZ::IO::Streamer.
//! development only.
u8 m_enableSharing : 1; u8 m_enableSharing : 1;
//! If true, only information that's explicitly requested or issues are reported. If false, status information //! If true, only information that's explicitly requested or issues are reported. If false, status information
//! such as when drives are created and destroyed is reported as well. //! such as when drives are created and destroyed is reported as well.

2
Code/Framework/AzCore/Platform/Windows/AzCore/IO/Streamer/StreamerConfiguration_Windows.cpp
Unescape Escape View File

@ -312,7 +312,7 @@ namespace AZ::IO
{ {
if (reportHardware) if (reportHardware)
{ {
AZ_Printf("Streamer", "Skipping drive '%s' because to no paths make use of it.\n", driveIt); AZ_Printf("Streamer", "Skipping drive '%s' because no paths make use of it.\n", driveIt);
} }
while (*driveIt++); while (*driveIt++);
continue; continue;

141
Code/Framework/AzCore/Tests/Platform/Windows/Tests/IO/Streamer/StorageDriveTests_Windows.cpp
Unescape Escape View File

@ -46,6 +46,7 @@ namespace AZ::IO
options.m_hasSeekPenalty = HasSeekPenalty; options.m_hasSeekPenalty = HasSeekPenalty;
options.m_enableUnbufferedReads = TestEnableUnbufferReads; options.m_enableUnbufferedReads = TestEnableUnbufferReads;
options.m_enableSharing = TestEnableSharedReads; options.m_enableSharing = TestEnableSharedReads;
options.m_minimalReporting = true;
return StorageDriveWin({ "c:/" }, TestMaxFileHandles, TestMaxMetaDataEntries, TestPhysicalSectorSize, return StorageDriveWin({ "c:/" }, TestMaxFileHandles, TestMaxMetaDataEntries, TestPhysicalSectorSize,
TestLogicalSectorSize, TestMaxIOChannels, TestOverCommit, options); TestLogicalSectorSize, TestMaxIOChannels, TestOverCommit, options);
@ -151,6 +152,7 @@ namespace AZ::IO
m_configurationOptions.m_hasSeekPenalty = HasSeekPenalty; m_configurationOptions.m_hasSeekPenalty = HasSeekPenalty;
m_configurationOptions.m_enableUnbufferedReads = TestEnableUnbufferReads; m_configurationOptions.m_enableUnbufferedReads = TestEnableUnbufferReads;
m_configurationOptions.m_enableSharing = TestEnableSharedReads; m_configurationOptions.m_enableSharing = TestEnableSharedReads;
m_configurationOptions.m_minimalReporting = true;
m_storageDriveWin = AZStd::make_shared<AZ::IO::StorageDriveWin>(AZStd::vector<AZStd::string_view>{drive}, TestMaxFileHandles, m_storageDriveWin = AZStd::make_shared<AZ::IO::StorageDriveWin>(AZStd::vector<AZStd::string_view>{drive}, TestMaxFileHandles,
TestMaxMetaDataEntries, TestPhysicalSectorSize, TestLogicalSectorSize, TestMaxIOChannels, overCommit, m_configurationOptions); TestMaxMetaDataEntries, TestPhysicalSectorSize, TestLogicalSectorSize, TestMaxIOChannels, overCommit, m_configurationOptions);
@ -1148,3 +1150,142 @@ namespace AZ::IO
azfree(buffers[numRequests - 1]); azfree(buffers[numRequests - 1]);
} }
} // namespace AZ::IO } // namespace AZ::IO
#if defined(HAVE_BENCHMARK)
#include <benchmark/benchmark.h>
namespace Benchmark
{
class StorageDriveWindowsFixture : public benchmark::Fixture
{
public:
constexpr static char* TestFileName = "StreamerBenchmark.bin";
constexpr static size_t FileSize = 64_mib;
void SetupStreamer(bool enableFileSharing)
{
using namespace AZ::IO;
m_fileIO = new UnitTest::TestFileIOBase();
m_previousFileIO = AZ::IO::FileIOBase::GetInstance();
AZ::IO::FileIOBase::SetInstance(nullptr);
AZ::IO::FileIOBase::SetInstance(m_fileIO);
SystemFile file;
file.Open(TestFileName, SystemFile::OpenMode::SF_OPEN_CREATE | SystemFile::OpenMode::SF_OPEN_READ_WRITE);
AZStd::unique_ptr<char[]> buffer(new char[FileSize]);
::memset(buffer.get(), 'c', FileSize);
file.Write(buffer.get(), FileSize);
file.Close();
AZStd::optional<AZ::IO::FixedMaxPathString> absolutePath = AZ::Utils::ConvertToAbsolutePath(TestFileName);
if (absolutePath.has_value())
{
AZStd::string drive;
AZ::StringFunc::Path::GetDrive(absolutePath->c_str(), drive);
m_absolutePath = *absolutePath;
StorageDriveWin::ConstructionOptions options;
options.m_hasSeekPenalty = false;
options.m_enableUnbufferedReads = true; // Leave this on otherwise repeated loads will be using the Windows cache instead.
options.m_enableSharing = enableFileSharing;
options.m_minimalReporting = true;
AZStd::shared_ptr<StreamStackEntry> storageDriveWin =
AZStd::make_shared<StorageDriveWin>(AZStd::vector<AZStd::string_view>{ drive }, 32, 32, 4_kib, 512, 8, 0, options);
AZStd::unique_ptr<Scheduler> stack = AZStd::make_unique<Scheduler>(AZStd::move(storageDriveWin));
m_streamer = aznew Streamer(AZStd::thread_desc{}, AZStd::move(stack));
}
}
void TearDown([[maybe_unused]] const ::benchmark::State& state) override
{
using namespace AZ::IO;
AZStd::string temp;
m_absolutePath.swap(temp);
delete m_streamer;
SystemFile::Delete(TestFileName);
AZ::IO::FileIOBase::SetInstance(nullptr);
AZ::IO::FileIOBase::SetInstance(m_previousFileIO);
delete m_fileIO;
}
void RepeatedlyReadFile(benchmark::State& state)
{
using namespace AZ::IO;
using namespace AZStd::chrono;
AZStd::unique_ptr<char[]> buffer(new char[FileSize]);
for (auto _ : state)
{
AZStd::binary_semaphore waitForReads;
AZStd::atomic<system_clock::time_point> end;
auto callback = [&end, &waitForReads]([[maybe_unused]] FileRequestHandle request)
{
benchmark::DoNotOptimize(end = high_resolution_clock::now());
waitForReads.release();
};
FileRequestPtr request = m_streamer->Read(m_absolutePath, buffer.get(), state.range(0), state.range(0));
m_streamer->SetRequestCompleteCallback(request, callback);
system_clock::time_point start;
benchmark::DoNotOptimize(start = high_resolution_clock::now());
m_streamer->QueueRequest(request);
waitForReads.try_acquire_for(AZStd::chrono::seconds(5));
auto durationInSeconds = duration_cast<duration<double>>(end.load() - start);
state.SetIterationTime(durationInSeconds.count());
m_streamer->QueueRequest(m_streamer->FlushCaches());
}
}
AZStd::string m_absolutePath;
AZ::IO::Streamer* m_streamer{};
AZ::IO::FileIOBase* m_previousFileIO{};
UnitTest::TestFileIOBase* m_fileIO{};
};
BENCHMARK_DEFINE_F(StorageDriveWindowsFixture, ReadsBaseline)(benchmark::State& state)
{
constexpr bool EnableFileSharing = false;
SetupStreamer(EnableFileSharing);
RepeatedlyReadFile(state);
}
BENCHMARK_DEFINE_F(StorageDriveWindowsFixture, ReadsWithFileReadSharingEnabled)(benchmark::State& state)
{
using namespace AZ::IO;
constexpr bool EnableFileSharing = true;
SetupStreamer(EnableFileSharing);
RepeatedlyReadFile(state);
}
// For these benchmarks the CPU stat doesn't provide useful information because it uses GetThreadTimes on Window but since the main
// thread is mostly sleeping while waiting for the read on the Streamer thread to complete this will report values (close to) zero.
BENCHMARK_REGISTER_F(StorageDriveWindowsFixture, ReadsBaseline)
->RangeMultiplier(8)
->Range(1024, 64_mib)
->UseManualTime()
->Unit(benchmark::kMillisecond);
BENCHMARK_REGISTER_F(StorageDriveWindowsFixture, ReadsWithFileReadSharingEnabled)
->RangeMultiplier(8)
->Range(1024, 64_mib)
->UseManualTime()
->Unit(benchmark::kMillisecond);
} // namespace Benchmark
#endif // HAVE_BENCHMARK

14
Registry/Platform/Windows/streamer.editor.setreg
Unescape Escape View File

@ -13,33 +13,19 @@
[ [
{ {
"$type": "AZ::IO::StorageDriveConfig", "$type": "AZ::IO::StorageDriveConfig",
// The maximum number of file handles that the drive will cache.
"MaxFileHandles": 1024 "MaxFileHandles": 1024
}, },
{ {
"$type": "AZ::IO::WindowsStorageDriveConfig", "$type": "AZ::IO::WindowsStorageDriveConfig",
// The maximum number of file handles that the drive will cache.
"MaxFileHandles": 1024, "MaxFileHandles": 1024,
// The maximum number of files to keep the meta data such as the size around for.
"MaxMetaDataCache": 1024, "MaxMetaDataCache": 1024,
// Number of requests the drive keeps after its queue is full.
// Overcommitting allows for requests to be immediately available after a request completes without needing
// any scheduling, but this also doesn't allow these requests to be rescheduled or updated.
"Overcommit": 8, "Overcommit": 8,
// Allows files to be shared. This can be needed if the file needs to be opened in multiple locations, such
// as the editor and the Asset Processor. Turning this feature on comes at a small performance cost.
"EnableFileSharing": true, "EnableFileSharing": true,
// Unbuffered reads bypass the OS file cache for faster file reads. This helps speed up initial file loads
// and is best for applications that only read a file once such as the game. For applications that frequently
// re-read files such as the editor it's better to turn this feature off.
"EnableUnbufferedReads": false, "EnableUnbufferedReads": false,
// If true, only information that's explicitly requested or issues are reported. If false, status information
// such as when drives are created and destroyed is reported as well.
"MinimalReporting": false "MinimalReporting": false
}, },
{ {
"$type": "AzFramework::RemoteStorageDriveConfig", "$type": "AzFramework::RemoteStorageDriveConfig",
// The maximum number of file handles that the drive will cache.
"MaxFileHandles": 1024 "MaxFileHandles": 1024
} }
] ]

74
Registry/Platform/Windows/streamer.game.debug.setreg
Unescape Escape View File

@ -0,0 +1,74 @@
{
"Amazon":
{
"AzCore":
{
"Streamer":
{
"Profiles":
{
"Generic":
{
"Stack":
[
{
"$type": "AZ::IO::WindowsStorageDriveConfig",
"MaxFileHandles": 32,
"MaxMetaDataCache": 32,
"Overcommit": 8,
"EnableFileSharing": true,
"EnableUnbufferedReads": true,
"MinimalReporting": false
},
{
"$type": "AZ::IO::ReadSplitterConfig",
"BufferSizeMib": 6,
"SplitSize": "MaxTransfer",
"AdjustOffset": true,
"SplitAlignedRequests": false
},
{
"$type": "AzFramework::RemoteStorageDriveConfig",
"MaxFileHandles": 1024
},
{
"$type": "AZ::IO::BlockCacheConfig",
"CacheSizeMib": 10,
"BlockSize": "MaxTransfer"
},
{
"$type": "AZ::IO::DedicatedCacheConfig",
"CacheSizeMib": 2,
"BlockSize": "MemoryAlignment",
"WriteOnlyEpilog": true
},
{
"$type": "AZ::IO::FullFileDecompressorConfig",
"MaxNumReads": 2,
"MaxNumJobs": 2
}
]
},
"DevMode":
{
"Stack":
[
{
"$type": "AZ::IO::WindowsStorageDriveConfig",
"MaxFileHandles": 1024,
"MaxMetaDataCache": 1024,
"Overcommit": 8,
"EnableFileSharing": true,
"EnableUnbufferedReads": false
},
{
"$type": "AzFramework::RemoteStorageDriveConfig",
"MaxFileHandles": 1024
}
]
}
}
}
}
}
}

74
Registry/Platform/Windows/streamer.game.profile.setreg
Unescape Escape View File

@ -0,0 +1,74 @@
{
"Amazon":
{
"AzCore":
{
"Streamer":
{
"Profiles":
{
"Generic":
{
"Stack":
[
{
"$type": "AZ::IO::WindowsStorageDriveConfig",
"MaxFileHandles": 32,
"MaxMetaDataCache": 32,
"Overcommit": 8,
"EnableFileSharing": true,
"EnableUnbufferedReads": true,
"MinimalReporting": false
},
{
"$type": "AZ::IO::ReadSplitterConfig",
"BufferSizeMib": 6,
"SplitSize": "MaxTransfer",
"AdjustOffset": true,
"SplitAlignedRequests": false
},
{
"$type": "AzFramework::RemoteStorageDriveConfig",
"MaxFileHandles": 1024
},
{
"$type": "AZ::IO::BlockCacheConfig",
"CacheSizeMib": 10,
"BlockSize": "MaxTransfer"
},
{
"$type": "AZ::IO::DedicatedCacheConfig",
"CacheSizeMib": 2,
"BlockSize": "MemoryAlignment",
"WriteOnlyEpilog": true
},
{
"$type": "AZ::IO::FullFileDecompressorConfig",
"MaxNumReads": 2,
"MaxNumJobs": 2
}
]
},
"DevMode":
{
"Stack":
[
{
"$type": "AZ::IO::WindowsStorageDriveConfig",
"MaxFileHandles": 1024,
"MaxMetaDataCache": 1024,
"Overcommit": 8,
"EnableFileSharing": true,
"EnableUnbufferedReads": false
},
{
"$type": "AzFramework::RemoteStorageDriveConfig",
"MaxFileHandles": 1024
}
]
}
}
}
}
}
}

21
Registry/Platform/Windows/streamer.game.setreg
Unescape Escape View File

@ -27,8 +27,7 @@
// will avoid saturating the IO controller which can be needed if the drive is used by other applications. // will avoid saturating the IO controller which can be needed if the drive is used by other applications.
"Overcommit": 8, "Overcommit": 8,
// Globally enable file sharing. This allows files to used outside AZ::IO::Streamer, including other // Globally enable file sharing. This allows files to used outside AZ::IO::Streamer, including other
// applications while in use by AZ::IO::Streamer. File sharing can negatively impact performance and is // applications while in use by AZ::IO::Streamer.
// recommended for development only.
"EnableFileSharing": false, "EnableFileSharing": false,
// Use unbuffered reads for the fastest possible read speeds by bypassing the Windows file cache. This // Use unbuffered reads for the fastest possible read speeds by bypassing the Windows file cache. This
// results in a faster read the first time a file is read, but subsequent reads will possibly be slower as // results in a faster read the first time a file is read, but subsequent reads will possibly be slower as
@ -63,24 +62,6 @@
"MaxNumJobs": 2 "MaxNumJobs": 2
} }
] ]
},
"DevMode":
{
"Stack":
[
{
"$type": "AZ::IO::WindowsStorageDriveConfig",
"MaxFileHandles": 1024,
"MaxMetaDataCache": 1024,
"Overcommit": 8,
"EnableFileSharing": true,
"EnableUnbufferedReads": false
},
{
"$type": "AzFramework::RemoteStorageDriveConfig",
"MaxFileHandles": 1024
}
]
} }
} }
} }

65
Registry/streamer.game.debug.setreg
Unescape Escape View File

@ -0,0 +1,65 @@
{
"Amazon":
{
"AzCore":
{
"Streamer":
{
"Profiles":
{
"Generic":
{
"Stack":
[
{
"$type": "AZ::IO::StorageDriveConfig",
"MaxFileHandles": 32
},
{
"$type": "AZ::IO::ReadSplitterConfig",
"BufferSizeMib": 6,
"SplitSize": "MaxTransfer",
"AdjustOffset": true,
"SplitAlignedRequests": false
},
{
"$type": "AzFramework::RemoteStorageDriveConfig",
"MaxFileHandles": 1024
},
{
"$type": "AZ::IO::BlockCacheConfig",
"CacheSizeMib": 10,
"BlockSize": "MaxTransfer"
},
{
"$type": "AZ::IO::DedicatedCacheConfig",
"CacheSizeMib": 2,
"BlockSize": "MemoryAlignment",
"WriteOnlyEpilog": true
},
{
"$type": "AZ::IO::FullFileDecompressorConfig",
"MaxNumReads": 2,
"MaxNumJobs": 2
}
]
},
"DevMode":
{
"Stack":
[
{
"$type": "AZ::IO::StorageDriveConfig",
"MaxFileHandles": 1024
},
{
"$type": "AzFramework::RemoteStorageDriveConfig",
"MaxFileHandles": 1024
}
]
}
}
}
}
}
}

65
Registry/streamer.game.profile.setreg
Unescape Escape View File

@ -0,0 +1,65 @@
{
"Amazon":
{
"AzCore":
{
"Streamer":
{
"Profiles":
{
"Generic":
{
"Stack":
[
{
"$type": "AZ::IO::StorageDriveConfig",
"MaxFileHandles": 32
},
{
"$type": "AZ::IO::ReadSplitterConfig",
"BufferSizeMib": 6,
"SplitSize": "MaxTransfer",
"AdjustOffset": true,
"SplitAlignedRequests": false
},
{
"$type": "AzFramework::RemoteStorageDriveConfig",
"MaxFileHandles": 1024
},
{
"$type": "AZ::IO::BlockCacheConfig",
"CacheSizeMib": 10,
"BlockSize": "MaxTransfer"
},
{
"$type": "AZ::IO::DedicatedCacheConfig",
"CacheSizeMib": 2,
"BlockSize": "MemoryAlignment",
"WriteOnlyEpilog": true
},
{
"$type": "AZ::IO::FullFileDecompressorConfig",
"MaxNumReads": 2,
"MaxNumJobs": 2
}
]
},
"DevMode":
{
"Stack":
[
{
"$type": "AZ::IO::StorageDriveConfig",
"MaxFileHandles": 1024
},
{
"$type": "AzFramework::RemoteStorageDriveConfig",
"MaxFileHandles": 1024
}
]
}
}
}
}
}
}

14
Registry/streamer.game.setreg
Unescape Escape View File

@ -57,20 +57,6 @@
"MaxNumJobs": 2 "MaxNumJobs": 2
} }
] ]
},
"DevMode":
{
"Stack":
[
{
"$type": "AZ::IO::StorageDriveConfig",
"MaxFileHandles": 1024
},
{
"$type": "AzFramework::RemoteStorageDriveConfig",
"MaxFileHandles": 1024
}
]
} }
} }
} }

Write Preview
Loading…
Cancel
Save