From f062b563c89e88b1facf32c6660bdca6cfac9057 Mon Sep 17 00:00:00 2001 From: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> Date: Tue, 30 Nov 2021 17:30:55 -0800 Subject: [PATCH] Improving runtime and making the whole duration manageable Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> --- .../Tests/Memory/AllocatorBenchmarks.cpp | 44 +++++++--- .../AzCore/Tests/Memory/HphaSchema.cpp | 88 ------------------- 2 files changed, 31 insertions(+), 101 deletions(-) diff --git a/Code/Framework/AzCore/Tests/Memory/AllocatorBenchmarks.cpp b/Code/Framework/AzCore/Tests/Memory/AllocatorBenchmarks.cpp index 7cd7e7ce0a..c1fdd4ca32 100644 --- a/Code/Framework/AzCore/Tests/Memory/AllocatorBenchmarks.cpp +++ b/Code/Framework/AzCore/Tests/Memory/AllocatorBenchmarks.cpp @@ -364,21 +364,27 @@ namespace Benchmark AZStd::vector& perThreadAllocations = base::GetPerThreadAllocations(state.thread_index); const size_t numberOfAllocations = perThreadAllocations.size(); + size_t totalAllocationSize = 0; for (size_t allocationIndex = 0; allocationIndex < numberOfAllocations; ++allocationIndex) { const AllocationSizeArray& allocationArray = s_allocationSizes[TAllocationSize]; const size_t allocationSize = allocationArray[allocationIndex % allocationArray.size()]; + totalAllocationSize += allocationSize; state.ResumeTiming(); perThreadAllocations[allocationIndex] = TestAllocatorType::Allocate(allocationSize, 0); state.PauseTiming(); } - state.PauseTiming(); - state.counters[s_counterAllocatorMemoryRatio] = - benchmark::Counter(static_cast(TestAllocatorType::NumAllocatedBytes()), benchmark::Counter::kDefaults); + // In allocation cases, s_counterAllocatorMemoryRatio is measuring how much over-allocation our allocators + // are doing to keep track of the memory and because of fragmentation/under-use of blocks. A ratio over 1 means + // that we are using more memory than requested. Ideally we would approximate to a ratio of 1. + state.counters[s_counterAllocatorMemoryRatio] = benchmark::Counter( + static_cast(TestAllocatorType::NumAllocatedBytes()) / static_cast(totalAllocationSize), + benchmark::Counter::kDefaults); + // s_counterProcessMemoryRatio is measuring the same ratio but using the OS to measure the used process memory state.counters[s_counterProcessMemoryRatio] = benchmark::Counter( - static_cast(Platform::GetProcessMemoryUsageBytes() - processMemoryBaseline), + static_cast(Platform::GetProcessMemoryUsageBytes() - processMemoryBaseline) / static_cast(totalAllocationSize), benchmark::Counter::kDefaults); for (size_t allocationIndex = 0; allocationIndex < numberOfAllocations; ++allocationIndex) @@ -391,7 +397,6 @@ namespace Benchmark TestAllocatorType::GarbageCollect(); state.SetItemsProcessed(numberOfAllocations); - state.ResumeTiming(); } } }; @@ -422,10 +427,12 @@ namespace Benchmark const size_t processMemoryBaseline = Platform::GetProcessMemoryUsageBytes(); const size_t numberOfAllocations = perThreadAllocations.size(); + size_t totalAllocationSize = 0; for (size_t allocationIndex = 0; allocationIndex < numberOfAllocations; ++allocationIndex) { const AllocationSizeArray& allocationArray = s_allocationSizes[TAllocationSize]; const size_t allocationSize = allocationArray[allocationIndex % allocationArray.size()]; + totalAllocationSize += allocationSize; perThreadAllocations[allocationIndex] = TestAllocatorType::Allocate(allocationSize, 0); } @@ -439,29 +446,40 @@ namespace Benchmark perThreadAllocations[allocationIndex] = nullptr; } - state.counters[s_counterAllocatorMemoryRatio] = - benchmark::Counter(static_cast(TestAllocatorType::NumAllocatedBytes()), benchmark::Counter::kDefaults); + // In deallocation cases, s_counterAllocatorMemoryRatio is measuring how much "left-over" memory our allocators + // have after deallocations happen. This is memory that is not returned to the operative system. A ratio of 1 means + // that no memory was returned to the OS. A ratio over 1 means that we are holding more memory than requested. A ratio + // lower than 1 means that we have returned some memory. + state.counters[s_counterAllocatorMemoryRatio] = benchmark::Counter( + static_cast(TestAllocatorType::NumAllocatedBytes()) / static_cast(totalAllocationSize), + benchmark::Counter::kDefaults); + // s_counterProcessMemoryRatio is measuring the same ratio but using the OS to measure the used process memory state.counters[s_counterProcessMemoryRatio] = benchmark::Counter( - static_cast(Platform::GetProcessMemoryUsageBytes() - processMemoryBaseline), + static_cast(Platform::GetProcessMemoryUsageBytes() - processMemoryBaseline) / static_cast(totalAllocationSize), benchmark::Counter::kDefaults); state.SetItemsProcessed(numberOfAllocations); TestAllocatorType::GarbageCollect(); - - state.ResumeTiming(); } } }; + // For non-threaded ranges, run 100, 400, 1600 amounts static void RunRanges(benchmark::internal::Benchmark* b) { - for (int i = 0; i < 6; ++i) + for (int i = 0; i < 6; i += 2) { - b->Arg((1 << i) * 1000); + b->Arg((1 << i) * 100); } } + // For threaded ranges, run just 200, multi-threaded will already multiply by thread + static void ThreadedRunRanges(benchmark::internal::Benchmark* b) + { + b->Arg(100); + } + // Test under and over-subscription of threads vs the amount of CPUs available static const unsigned int MaxThreadRange = 2 * AZStd::thread::hardware_concurrency(); @@ -475,7 +493,7 @@ namespace Benchmark BM_REGISTER_TEMPLATE(FIXTURE, TESTNAME##_SMALL, ALLOCATORTYPE, SMALL)->Apply(RunRanges); \ BM_REGISTER_TEMPLATE(FIXTURE, TESTNAME##_BIG, ALLOCATORTYPE, BIG)->Apply(RunRanges); \ BM_REGISTER_TEMPLATE(FIXTURE, TESTNAME##_MIXED, ALLOCATORTYPE, MIXED)->Apply(RunRanges); \ - BM_REGISTER_TEMPLATE(FIXTURE, TESTNAME##_MIXED_THREADED, ALLOCATORTYPE, MIXED)->ThreadRange(1, MaxThreadRange)->Apply(RunRanges); + BM_REGISTER_TEMPLATE(FIXTURE, TESTNAME##_MIXED_THREADED, ALLOCATORTYPE, MIXED)->ThreadRange(2, MaxThreadRange)->Apply(ThreadedRunRanges); #define BM_REGISTER_ALLOCATOR(TESTNAME, ALLOCATORTYPE) \ namespace TESTNAME \ diff --git a/Code/Framework/AzCore/Tests/Memory/HphaSchema.cpp b/Code/Framework/AzCore/Tests/Memory/HphaSchema.cpp index 85dd79931d..08b84416e6 100644 --- a/Code/Framework/AzCore/Tests/Memory/HphaSchema.cpp +++ b/Code/Framework/AzCore/Tests/Memory/HphaSchema.cpp @@ -10,10 +10,6 @@ #include #include -#if defined(HAVE_BENCHMARK) -#include -#endif // HAVE_BENCHMARK - class HphaSchema_TestAllocator : public AZ::SimpleSchemaAllocator { @@ -112,87 +108,3 @@ namespace UnitTest HphaSchemaTestFixture, ::testing::ValuesIn(s_mixedInstancesParameters)); } - - -#if defined(HAVE_BENCHMARK) -namespace Benchmark -{ - class HphaSchemaBenchmarkFixture - : public ::benchmark::Fixture - { - void internalSetUp() - { - AZ::AllocatorInstance::Create(); - } - - void internalTearDown() - { - AZ::AllocatorInstance::Destroy(); - } - - public: - void SetUp(const benchmark::State&) override - { - internalSetUp(); - } - void SetUp(benchmark::State&) override - { - internalSetUp(); - } - void TearDown(const benchmark::State&) override - { - internalTearDown(); - } - void TearDown(benchmark::State&) override - { - internalTearDown(); - } - - static void BM_Allocations(benchmark::State& state, const AllocationSizeArray& allocationArray) - { - AZStd::vector allocations; - while (state.KeepRunning()) - { - state.PauseTiming(); - const size_t allocationIndex = allocations.size(); - const size_t allocationSize = allocationArray[allocationIndex % allocationArray.size()]; - - state.ResumeTiming(); - void* allocation = AZ::AllocatorInstance::Get().Allocate(allocationSize, 0); - - state.PauseTiming(); - allocations.emplace_back(allocation); - - state.ResumeTiming(); - } - - const size_t numberOfAllocations = allocations.size(); - state.SetItemsProcessed(numberOfAllocations); - - for (size_t allocationIndex = 0; allocationIndex < numberOfAllocations; ++allocationIndex) - { - AZ::AllocatorInstance::Get().DeAllocate(allocations[allocationIndex], allocationArray[allocationIndex % allocationArray.size()]); - } - AZ::AllocatorInstance::Get().GarbageCollect(); - } - }; - - // Small allocations, these are allocations that are going to end up in buckets in the HphaSchema - BENCHMARK_F(HphaSchemaBenchmarkFixture, SmallAllocations)(benchmark::State& state) - { - BM_Allocations(state, s_smallAllocationSizes); - } - - BENCHMARK_F(HphaSchemaBenchmarkFixture, BigAllocations)(benchmark::State& state) - { - BM_Allocations(state, s_bigAllocationSizes); - } - - BENCHMARK_F(HphaSchemaBenchmarkFixture, MixedAllocations)(benchmark::State& state) - { - BM_Allocations(state, s_mixedAllocationSizes); - } - - -} // Benchmark -#endif // HAVE_BENCHMARK