From 5b76d47e17b3ab7d80cfecc5fbfb85a03558e302 Mon Sep 17 00:00:00 2001 From: Jeremy Ong Date: Thu, 23 Dec 2021 17:15:49 -0700 Subject: [PATCH] Remove time tests that relied on main-thread sleeps Sleeping in tests and attempting to rely on fine-grained measurements to check code validity is intrinsically brittle. Wall-clock time is unreliable in an environment where tests are run under a hypervisor that may choose to suspend your VM at any point, or in situations where the OS cannot schedule your thread in time. The correct way to reintroduce these tests in the future is provide an override for the timestamp queries that can be injected in the test environment to control the wall time deterministically. Signed-off-by: Jeremy Ong --- .../Framework/AzCore/Tests/Time/TimeTests.cpp | 188 ------------------ 1 file changed, 188 deletions(-) diff --git a/Code/Framework/AzCore/Tests/Time/TimeTests.cpp b/Code/Framework/AzCore/Tests/Time/TimeTests.cpp index 446368a8ca..164e72343b 100644 --- a/Code/Framework/AzCore/Tests/Time/TimeTests.cpp +++ b/Code/Framework/AzCore/Tests/Time/TimeTests.cpp @@ -76,192 +76,4 @@ namespace UnitTest int64_t delta = static_cast(timeMs) - static_cast(timeUsToMs); EXPECT_LT(abs(delta), 1); } - - class TimeSystemTests : public AllocatorsTestFixture - { - public: - void SetUp() override - { - SetupAllocator(); - m_controlTime = static_cast(AZStd::GetTimeNowMicroSecond()); - m_timeSystem = AZStd::make_unique(); - } - - void TearDown() override - { - m_controlTime = AZ::Time::ZeroTimeUs; - m_timeSystem.reset(); - TeardownAllocator(); - } - - AZ::TimeUs GetDiff(AZ::TimeUs time1, AZ::TimeUs time2) const - { - // AZ::TimeUs is unsigned so make sure to not underflow. - return time1 > time2 ? time1 - time2 : time2 - time1; - } - - AZ::TimeUs m_controlTime; - AZStd::unique_ptr m_timeSystem; - }; - - TEST_F(TimeSystemTests, GetRealElapsedTimeUs) - { - // sleep for a bit to advance time. - AZStd::this_thread::sleep_for(AZStd::chrono::milliseconds(2)); - - // find the delta for the control and from GetRealElapsedTimeUs - const AZ::TimeUs baseline = static_cast(AZStd::GetTimeNowMicroSecond()) - m_controlTime; - const AZ::TimeUs elapsedTime = m_timeSystem->GetRealElapsedTimeUs(); - - const AZ::TimeUs diff = GetDiff(baseline, elapsedTime); - - // elapsedTime should be within 10 microseconds from baseline. - EXPECT_LT(diff, AZ::TimeUs{ 10 }); - } - - TEST_F(TimeSystemTests, GetElapsedTimeUs) - { - // sleep for a bit to advance time. - AZStd::this_thread::sleep_for(AZStd::chrono::milliseconds(2)); - - // find the delta for the control and from GetElapsedTimeUs - const AZ::TimeUs baseline = static_cast(AZStd::GetTimeNowMicroSecond()) - m_controlTime; - const AZ::TimeUs elapsedTime = m_timeSystem->GetElapsedTimeUs(); - - const AZ::TimeUs diff = GetDiff(baseline, elapsedTime); - - // elapsedTime should be within 10 microseconds from baseline. - EXPECT_LT(diff, AZ::TimeUs{ 10 }); - } - - TEST_F(TimeSystemTests, ElapsedTimeScales) - { - // slow down 'time' - m_timeSystem->SetSimulationTickScale(0.5f); - - // sleep for a bit to advance time. - AZStd::this_thread::sleep_for(AZStd::chrono::milliseconds(2)); - - // find the delta for the control and from GetElapsedTimeUs - const AZ::TimeUs baseline = static_cast(AZStd::GetTimeNowMicroSecond()) - m_controlTime; - const AZ::TimeUs elapsedTime = m_timeSystem->GetElapsedTimeUs(); - const AZ::TimeUs halfBaseline = (baseline / AZ::TimeUs{ 2 }); - - // elapsedTime should be about half of the control. - const AZ::TimeUs diff = GetDiff(halfBaseline, elapsedTime); - - // elapsedTime should be within 10 microseconds from baseline. - EXPECT_LT(diff, AZ::TimeUs{ 10 }); - - // reset time scale - m_timeSystem->SetSimulationTickScale(1.0f); - } - - TEST_F(TimeSystemTests, AdvanceTickDeltaTimes) - { - // advance the tick delta to get a clean base. - m_timeSystem->AdvanceTickDeltaTimes(); - const AZ::TimeUs baselineStart = static_cast(AZStd::GetTimeNowMicroSecond()); - - // sleep for a bit to advance time. - AZStd::this_thread::sleep_for(AZStd::chrono::milliseconds(2)); - - // advance the tick delta. - const AZ::TimeUs delta = m_timeSystem->AdvanceTickDeltaTimes(); - const AZ::TimeUs baselineDelta = static_cast(AZStd::GetTimeNowMicroSecond()) - baselineStart; - - // the delta should be close to the baselineDelta. - const AZ::TimeUs diff = GetDiff(delta, baselineDelta); - EXPECT_LT(diff, AZ::TimeUs{ 10 }); - } - - TEST_F(TimeSystemTests, SimulationAndRealTickDeltaTimesWithNoTimeScale) - { - // advance the tick delta to get a clean base. - m_timeSystem->AdvanceTickDeltaTimes(); - const AZ::TimeUs baselineStart = static_cast(AZStd::GetTimeNowMicroSecond()); - - // sleep for a bit to advance time. - AZStd::this_thread::sleep_for(AZStd::chrono::milliseconds(2)); - - // advance the tick delta. - const AZ::TimeUs delta = m_timeSystem->AdvanceTickDeltaTimes(); - const AZ::TimeUs baselineDelta = static_cast(AZStd::GetTimeNowMicroSecond()) - baselineStart; - - // the delta should be close to the baselineDelta. - AZ::TimeUs diff = GetDiff(delta, baselineDelta); - EXPECT_LT(diff, AZ::TimeUs{ 10 }); - - // the delta should be the same as GetSimulationTickDeltaTimeUs and near GetRealTickDeltaTimeUs - const AZ::TimeUs simDeltaTime = m_timeSystem->GetSimulationTickDeltaTimeUs(); - EXPECT_EQ(delta, simDeltaTime); - - const AZ::TimeUs realDeltaTime = m_timeSystem->GetRealTickDeltaTimeUs(); - diff = GetDiff(delta, realDeltaTime); - EXPECT_LT(diff, AZ::TimeUs{ 10 }); - } - - TEST_F(TimeSystemTests, SimulationAndRealTickDeltaTimesWithTimeScale) - { - // advance the tick delta to get a clean base. - m_timeSystem->AdvanceTickDeltaTimes(); - const AZ::TimeUs baselineStart = static_cast(AZStd::GetTimeNowMicroSecond()); - - // slow down 'time'; - m_timeSystem->SetSimulationTickScale(0.5f); - - // sleep for a bit to advance time. - AZStd::this_thread::sleep_for(AZStd::chrono::milliseconds(2)); - - // advance the tick delta. - const AZ::TimeUs delta = m_timeSystem->AdvanceTickDeltaTimes(); - const AZ::TimeUs baselineDelta = static_cast(AZStd::GetTimeNowMicroSecond()) - baselineStart; - const AZ::TimeUs halfBaselineDelta = (baselineDelta / AZ::TimeUs{ 2 }); - - // the delta should be half the baselineDelta - AZ::TimeUs diff = GetDiff(delta, halfBaselineDelta); - EXPECT_LT(diff, AZ::TimeUs{ 10 }); - - // the delta should be the same as GetSimulationTickDeltaTimeUs - const AZ::TimeUs simDeltaTime = m_timeSystem->GetSimulationTickDeltaTimeUs(); - EXPECT_EQ(delta, simDeltaTime); - - // the delta should be near half the GetRealTickDeltaTimeUs - const AZ::TimeUs realDeltaTime = m_timeSystem->GetRealTickDeltaTimeUs(); - const AZ::TimeUs halfRealDeltaTime = (realDeltaTime / AZ::TimeUs{ 2 }); - diff = GetDiff(delta, halfRealDeltaTime); - EXPECT_LT(diff, AZ::TimeUs{ 10 }); - - // reset time scale - m_timeSystem->SetSimulationTickScale(1.0f); - } - - TEST_F(TimeSystemTests, SimulationTickDeltaOverride) - { - // advance the tick delta to get a clean base. - m_timeSystem->AdvanceTickDeltaTimes(); - const AZ::TimeUs baselineStart = static_cast(AZStd::GetTimeNowMicroSecond()); - - // set the tick delta override - const AZ::TimeMs tickOverride = AZ::TimeMs{ 3462 }; - m_timeSystem->SetSimulationTickDeltaOverride(tickOverride); - - // sleep for a bit to advance time. - AZStd::this_thread::sleep_for(AZStd::chrono::milliseconds(2)); - - // advance the tick delta. - const AZ::TimeUs delta = m_timeSystem->AdvanceTickDeltaTimes(); - const AZ::TimeUs baselineDelta = static_cast(AZStd::GetTimeNowMicroSecond()) - baselineStart; - - // the delta should be equal to the tickOverride - EXPECT_EQ(delta, AZ::TimeMsToUs(tickOverride)); - - // real tick delta should be near the baselineDelta - const AZ::TimeUs realDeltaTime = m_timeSystem->GetRealTickDeltaTimeUs(); - const AZ::TimeUs diff = GetDiff(realDeltaTime, baselineDelta); - EXPECT_LT(diff, AZ::TimeUs{ 10 }); - - // reset the tick delta override - m_timeSystem->SetSimulationTickDeltaOverride(AZ::Time::ZeroTimeMs); - } } // namespace UnitTest