From c2a6addf1ff60dfed88c19d3ceb5f00937bcc904 Mon Sep 17 00:00:00 2001 From: alexpete Date: Tue, 20 Apr 2021 10:42:39 -0700 Subject: [PATCH 001/102] Integrating latest from github/TIF/Runtime --- .../Console/Code/Source/TestImpactConsole.cpp | 20 +- .../Runtime/Code/CMakeLists.txt | 64 +- .../TestImpactFramework/TestImpactBitwise.h | 35 + .../TestImpactFramework/TestImpactCallback.h | 23 + .../TestImpactFramework/TestImpactException.h | 51 + .../TestImpactFrameworkPath.h | 41 + .../Artifact/Dynamic/TestImpactChangeList.h | 27 + .../Artifact/Dynamic/TestImpactCoverage.h | 41 + .../Dynamic/TestImpactTestEnumerationSuite.h | 21 + .../Artifact/Dynamic/TestImpactTestRunSuite.h | 51 + .../Artifact/Dynamic/TestImpactTestSuite.h | 35 + ...TestImpactBuildTargetDescriptorFactory.cpp | 172 ++ .../TestImpactBuildTargetDescriptorFactory.h | 33 + .../Factory/TestImpactChangeListFactory.cpp | 175 ++ .../Factory/TestImpactChangeListFactory.h | 29 + .../TestImpactModuleCoverageFactory.cpp | 154 ++ .../Factory/TestImpactModuleCoverageFactory.h | 26 + .../TestImpactTestEnumerationSuiteFactory.cpp | 94 + .../TestImpactTestEnumerationSuiteFactory.h | 26 + .../Factory/TestImpactTestRunSuiteFactory.cpp | 143 ++ .../Factory/TestImpactTestRunSuiteFactory.h | 26 + .../TestImpactTestTargetMetaArtifactFactory.h | 25 + .../TestImpactTestTargetMetaMapFactory.cpp | 89 + .../TestImpactTestTargetMetaMapFactory.h | 25 + .../TestImpactBuildTargetDescriptor.cpp | 22 + .../Static/TestImpactBuildTargetDescriptor.h | 55 + .../TestImpactProductionTargetDescriptor.cpp | 21 + .../TestImpactProductionTargetDescriptor.h | 25 + .../TestImpactTargetDescriptorCompiler.cpp | 45 + .../TestImpactTargetDescriptorCompiler.h | 31 + .../Static/TestImpactTestTargetDescriptor.cpp | 22 + .../Static/TestImpactTestTargetDescriptor.h | 28 + .../Static/TestImpactTestTargetMeta.h | 36 + .../Artifact/TestImpactArtifactException.h | 26 + .../Runtime/Code/Source/Dummy.cpp | 11 - .../Clang/testimpactframework_clang.cmake | 12 + .../MSVC/testimpactframework_msvc.cmake | 12 + .../Source/Platform/Windows/Dummy_Windows.cpp | 11 - .../Windows/Process/TestImpactWin32_Handle.h | 92 + .../Windows/Process/TestImpactWin32_Pipe.cpp | 67 + .../Windows/Process/TestImpactWin32_Pipe.h | 52 + .../Process/TestImpactWin32_Process.cpp | 259 +++ .../Windows/Process/TestImpactWin32_Process.h | 101 + .../TestImpactWin32_ProcessLauncher.cpp | 24 + .../Windows/platform_windows_files.cmake | 7 +- .../Process/JobRunner/TestImpactProcessJob.h | 139 ++ .../JobRunner/TestImpactProcessJobInfo.h | 73 + .../JobRunner/TestImpactProcessJobRunner.h | 190 ++ .../Scheduler/TestImpactProcessScheduler.cpp | 270 +++ .../Scheduler/TestImpactProcessScheduler.h | 110 + .../Code/Source/Process/TestImpactProcess.cpp | 32 + .../Code/Source/Process/TestImpactProcess.h | 61 + .../Process/TestImpactProcessException.h | 26 + .../Source/Process/TestImpactProcessInfo.cpp | 67 + .../Source/Process/TestImpactProcessInfo.h | 93 + .../Process/TestImpactProcessLauncher.h | 27 + .../Source/Target/TestImpactBuildTarget.cpp | 48 + .../Source/Target/TestImpactBuildTarget.h | 55 + .../Source/Target/TestImpactBuildTargetList.h | 125 ++ .../Target/TestImpactProductionTarget.cpp | 21 + .../Target/TestImpactProductionTarget.h | 31 + .../Target/TestImpactProductionTargetList.h | 22 + .../Source/Target/TestImpactTargetException.h | 25 + .../Source/Target/TestImpactTestTarget.cpp | 32 + .../Code/Source/Target/TestImpactTestTarget.h | 41 + .../Source/Target/TestImpactTestTargetList.h | 22 + .../Enumeration/TestImpactTestEnumeration.h | 22 + .../TestImpactTestEnumerationException.h | 26 + .../TestImpactTestEnumerationSerializer.cpp | 100 + .../TestImpactTestEnumerationSerializer.h | 26 + .../Enumeration/TestImpactTestEnumerator.cpp | 190 ++ .../Enumeration/TestImpactTestEnumerator.h | 94 + .../Source/Test/Job/TestImpactTestJobCommon.h | 36 + .../Test/Job/TestImpactTestJobException.h | 26 + .../Source/Test/Job/TestImpactTestJobRunner.h | 141 ++ .../Run/TestImpactInstrumentedTestRunner.cpp | 89 + .../Run/TestImpactInstrumentedTestRunner.h | 71 + .../Test/Run/TestImpactTestCoverage.cpp | 68 + .../Source/Test/Run/TestImpactTestCoverage.h | 54 + .../Source/Test/Run/TestImpactTestRun.cpp | 70 + .../Code/Source/Test/Run/TestImpactTestRun.h | 51 + .../Test/Run/TestImpactTestRunException.h | 25 + .../Test/Run/TestImpactTestRunJobData.cpp | 26 + .../Test/Run/TestImpactTestRunJobData.h | 31 + .../Test/Run/TestImpactTestRunSerializer.cpp | 186 ++ .../Test/Run/TestImpactTestRunSerializer.h | 26 + .../Source/Test/Run/TestImpactTestRunner.cpp | 58 + .../Source/Test/Run/TestImpactTestRunner.h | 46 + .../Test/TestImpactTestSuiteContainer.h | 101 + .../Code/Source/TestImpactException.cpp | 31 + .../Code/Source/TestImpactFrameworkPath.cpp | 38 + ...ImpactBuildTargetDescriptorFactoryTest.cpp | 370 ++++ .../TestImpactChangeListFactoryTest.cpp | 288 +++ .../TestImpactModuleCoverageFactoryTest.cpp | 522 +++++ ...TestImpactTargetDescriptorCompilerTest.cpp | 150 ++ ...tImpactTestEnumerationSuiteFactoryTest.cpp | 589 ++++++ .../TestImpactTestRunSuiteFactoryTest.cpp | 592 ++++++ ...TestImpactTestTargetMetaMapFactoryTest.cpp | 239 +++ .../TestImpactProcessSchedulerTest.cpp | 718 +++++++ .../Tests/Process/TestImpactProcessTest.cpp | 491 +++++ .../Target/TestImpactBuildTargetTest.cpp | 352 ++++ .../TestImpactInstrumentedTestRunnerTest.cpp | 822 ++++++++ .../Tests/Test/TestImpactTestCoverageTest.cpp | 208 ++ .../Test/TestImpactTestEnumeratorTest.cpp | 890 ++++++++ ...TestImpactTestEumerationSerializerTest.cpp | 99 + .../Test/TestImpactTestRunSerializerTest.cpp | 99 + .../Tests/Test/TestImpactTestRunnerTest.cpp | 516 +++++ .../Code/Tests/TestImpactExceptionTest.cpp | 150 ++ .../Tests/TestImpactFrameworkPathTest.cpp | 137 ++ .../Tests/TestImpactTestJobRunnerCommon.h | 159 ++ .../Runtime/Code/Tests/TestImpactTestMain.cpp | 33 + .../Code/Tests/TestImpactTestUtils.cpp | 1843 +++++++++++++++++ .../Runtime/Code/Tests/TestImpactTestUtils.h | 220 ++ .../Code/Tests/TestProcess/CMakeLists.txt | 12 + .../Tests/TestProcess/Code/CMakeLists.txt | 24 + .../Code/Source/TestImpactTestProcess.cpp | 93 + .../Code/Source/TestImpactTestProcess.h | 38 + .../Source/TestImpactTestProcessLargeText.cpp | 531 +++++ .../Source/TestImpactTestProcessLargeText.h | 19 + .../Code/Source/TestImpactTestProcessMain.cpp | 19 + ...estimpactframework_testprocess_files.cmake | 18 + .../Code/Tests/TestTargetA/CMakeLists.txt | 12 + .../Tests/TestTargetA/Code/CMakeLists.txt | 28 + .../Code/Tests/TestImpactTestTargetA.cpp | 73 + ...actframework_testtargeta_tests_files.cmake | 14 + .../Code/Tests/TestTargetB/CMakeLists.txt | 12 + .../Tests/TestTargetB/Code/CMakeLists.txt | 29 + .../Code/Tests/TestImpactTestTargetB.cpp | 78 + ...actframework_testtargetb_tests_files.cmake | 14 + .../Code/Tests/TestTargetC/CMakeLists.txt | 12 + .../Tests/TestTargetC/Code/CMakeLists.txt | 29 + .../Code/Tests/TestImpactTestTargetC.cpp | 63 + ...actframework_testtargetc_tests_files.cmake | 14 + .../Code/Tests/TestTargetD/CMakeLists.txt | 12 + .../Tests/TestTargetD/Code/CMakeLists.txt | 29 + .../Code/Tests/TestImpactTestTargetD.cpp | 188 ++ ...actframework_testtargetd_tests_files.cmake | 14 + .../testimpactframework_runtime_files.cmake | 17 +- ...timpactframework_runtime_tests_files.cmake | 22 + 139 files changed, 16391 insertions(+), 37 deletions(-) create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactBitwise.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactCallback.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactException.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactFrameworkPath.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Dynamic/TestImpactChangeList.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Dynamic/TestImpactCoverage.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Dynamic/TestImpactTestEnumerationSuite.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Dynamic/TestImpactTestRunSuite.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Dynamic/TestImpactTestSuite.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactBuildTargetDescriptorFactory.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactBuildTargetDescriptorFactory.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactChangeListFactory.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactChangeListFactory.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactModuleCoverageFactory.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactModuleCoverageFactory.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestEnumerationSuiteFactory.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestEnumerationSuiteFactory.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestRunSuiteFactory.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestRunSuiteFactory.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestTargetMetaArtifactFactory.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestTargetMetaMapFactory.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestTargetMetaMapFactory.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactBuildTargetDescriptor.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactBuildTargetDescriptor.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactProductionTargetDescriptor.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactProductionTargetDescriptor.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactTargetDescriptorCompiler.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactTargetDescriptorCompiler.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactTestTargetDescriptor.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactTestTargetDescriptor.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactTestTargetMeta.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/TestImpactArtifactException.h delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Dummy.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Common/Clang/testimpactframework_clang.cmake create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Common/MSVC/testimpactframework_msvc.cmake delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/Dummy_Windows.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/Process/TestImpactWin32_Handle.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/Process/TestImpactWin32_Pipe.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/Process/TestImpactWin32_Pipe.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/Process/TestImpactWin32_Process.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/Process/TestImpactWin32_Process.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/Process/TestImpactWin32_ProcessLauncher.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJob.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJobInfo.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJobRunner.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/Scheduler/TestImpactProcessScheduler.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/Scheduler/TestImpactProcessScheduler.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/TestImpactProcess.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/TestImpactProcess.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/TestImpactProcessException.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/TestImpactProcessInfo.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/TestImpactProcessInfo.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/TestImpactProcessLauncher.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactBuildTarget.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactBuildTarget.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactBuildTargetList.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactProductionTarget.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactProductionTarget.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactProductionTargetList.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactTargetException.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactTestTarget.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactTestTarget.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactTestTargetList.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumeration.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerationException.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerationSerializer.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerationSerializer.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerator.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerator.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Job/TestImpactTestJobCommon.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Job/TestImpactTestJobException.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Job/TestImpactTestJobRunner.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactInstrumentedTestRunner.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactInstrumentedTestRunner.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestCoverage.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestCoverage.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRun.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRun.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunException.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunJobData.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunJobData.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunSerializer.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunSerializer.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunner.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunner.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/TestImpactTestSuiteContainer.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactException.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactFrameworkPath.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactBuildTargetDescriptorFactoryTest.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactChangeListFactoryTest.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactModuleCoverageFactoryTest.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactTargetDescriptorCompilerTest.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactTestEnumerationSuiteFactoryTest.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactTestRunSuiteFactoryTest.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactTestTargetMetaMapFactoryTest.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/Process/TestImpactProcessSchedulerTest.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/Process/TestImpactProcessTest.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/Target/TestImpactBuildTargetTest.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactInstrumentedTestRunnerTest.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestCoverageTest.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestEnumeratorTest.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestEumerationSerializerTest.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestRunSerializerTest.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestRunnerTest.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactExceptionTest.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactFrameworkPathTest.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactTestJobRunnerCommon.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactTestMain.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactTestUtils.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactTestUtils.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/CMakeLists.txt create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/CMakeLists.txt create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/Source/TestImpactTestProcess.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/Source/TestImpactTestProcess.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/Source/TestImpactTestProcessLargeText.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/Source/TestImpactTestProcessLargeText.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/Source/TestImpactTestProcessMain.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/testimpactframework_testprocess_files.cmake create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetA/CMakeLists.txt create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetA/Code/CMakeLists.txt create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetA/Code/Tests/TestImpactTestTargetA.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetA/Code/testimpactframework_testtargeta_tests_files.cmake create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetB/CMakeLists.txt create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetB/Code/CMakeLists.txt create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetB/Code/Tests/TestImpactTestTargetB.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetB/Code/testimpactframework_testtargetb_tests_files.cmake create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetC/CMakeLists.txt create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetC/Code/CMakeLists.txt create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetC/Code/Tests/TestImpactTestTargetC.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetC/Code/testimpactframework_testtargetc_tests_files.cmake create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetD/CMakeLists.txt create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetD/Code/CMakeLists.txt create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetD/Code/Tests/TestImpactTestTargetD.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetD/Code/testimpactframework_testtargetd_tests_files.cmake create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_tests_files.cmake diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactConsole.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactConsole.cpp index c20b3c60fe..12743f4b38 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactConsole.cpp +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactConsole.cpp @@ -1,14 +1,14 @@ /* -* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or -* its licensors. -* -* For complete copyright and license terms please see the LICENSE at the root of this -* distribution (the "License"). All use of this software is governed by the License, -* or, if provided, by the license below or the license accompanying this file. Do not -* remove or modify any license notices. This file is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* -*/ + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ int main([[maybe_unused]] int argc, [[maybe_unused]] char** argv) { diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/CMakeLists.txt b/Code/Tools/TestImpactFramework/Runtime/Code/CMakeLists.txt index d48acd73c5..4c021a12b4 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/CMakeLists.txt +++ b/Code/Tools/TestImpactFramework/Runtime/Code/CMakeLists.txt @@ -9,8 +9,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # - ly_get_list_relative_pal_filename(pal_dir ${CMAKE_CURRENT_LIST_DIR}/Source/Platform/${PAL_PLATFORM_NAME}) +ly_get_list_relative_pal_filename(common_dir ${CMAKE_CURRENT_LIST_DIR}/Source/Platform/Common) + +add_subdirectory(Tests/TestProcess) +add_subdirectory(Tests/TestTargetA) +add_subdirectory(Tests/TestTargetB) +add_subdirectory(Tests/TestTargetC) +add_subdirectory(Tests/TestTargetD) ly_add_target( NAME TestImpact.Runtime.Static STATIC @@ -18,12 +24,64 @@ ly_add_target( FILES_CMAKE testimpactframework_runtime_files.cmake ${pal_dir}/platform_${PAL_PLATFORM_NAME_LOWERCASE}_files.cmake + PLATFORM_INCLUDE_FILES + ${common_dir}/${PAL_TRAIT_COMPILER_ID}/testimpactframework_${PAL_TRAIT_COMPILER_ID_LOWERCASE}.cmake INCLUDE_DIRECTORIES PRIVATE Source PUBLIC Include BUILD_DEPENDENCIES - Public + PUBLIC AZ::AzCore -) \ No newline at end of file +) + +################################################################################ +# Tests +################################################################################ + +ly_add_target( + NAME TestImpact.Runtime.Tests ${PAL_TRAIT_TEST_TARGET_TYPE} + NAMESPACE AZ + FILES_CMAKE + testimpactframework_runtime_tests_files.cmake + INCLUDE_DIRECTORIES + PRIVATE + Include + Source + Tests + BUILD_DEPENDENCIES + PRIVATE + AZ::AzTestShared + AZ::AzTest + AZ::TestImpact.Runtime.Static + RUNTIME_DEPENDENCIES + AZ::AzTestRunner + AZ::TestImpact.TestProcess.Console + AZ::TestImpact.TestTargetA.Tests + AZ::TestImpact.TestTargetB.Tests + AZ::TestImpact.TestTargetC.Tests + AZ::TestImpact.TestTargetD.Tests + COMPILE_DEFINITIONS + PRIVATE + LY_TEST_IMPACT_AZ_TESTRUNNER_BIN="$" + LY_TEST_IMPACT_TEST_PROCESS_BIN="$" + LY_TEST_IMPACT_TEST_TARGET_A_BIN="$" + LY_TEST_IMPACT_TEST_TARGET_B_BIN="$" + LY_TEST_IMPACT_TEST_TARGET_C_BIN="$" + LY_TEST_IMPACT_TEST_TARGET_D_BIN="$" + LY_TEST_IMPACT_TEST_TARGET_A_BASE_NAME="$" + LY_TEST_IMPACT_TEST_TARGET_B_BASE_NAME="$" + LY_TEST_IMPACT_TEST_TARGET_C_BASE_NAME="$" + LY_TEST_IMPACT_TEST_TARGET_D_BASE_NAME="$" + LY_TEST_IMPACT_TEST_TARGET_ENUMERATION_DIR="${GTEST_XML_OUTPUT_DIR}/TestImpact/Temp/Exclusive/Enum" + LY_TEST_IMPACT_TEST_TARGET_RESULTS_DIR="${GTEST_XML_OUTPUT_DIR}/TestImpact/Temp/Exclusive/Result" + LY_TEST_IMPACT_TEST_TARGET_COVERAGE_DIR="${GTEST_XML_OUTPUT_DIR}/TestImpact/Temp/Exclusive/Coverage" + LY_TEST_IMPACT_INSTRUMENTATION_BIN="${LY_TEST_IMPACT_INSTRUMENTATION_BIN}" + LY_TEST_IMPACT_MODULES_DIR="${CMAKE_BINARY_DIR}" + LY_TEST_IMPACT_COVERAGE_SOURCES_DIR="${CMAKE_CURRENT_SOURCE_DIR}" +) + +ly_add_googletest( + NAME AZ::TestImpact.Runtime.Tests +) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactBitwise.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactBitwise.h new file mode 100644 index 0000000000..73278f6be6 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactBitwise.h @@ -0,0 +1,35 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +namespace TestImpact +{ + //! Convinience namespace for allowing bitwise operations on enum classes. + //! @note Any types declared in this namespace will have the bitwise operator overloads declared within. + namespace Bitwise + { + template + Flags operator|(Flags lhs, Flags rhs) + { + return static_cast( + static_cast::type>(lhs) | static_cast::type>(rhs)); + } + + template + bool IsFlagSet(Flags flags, Flags flag) + { + return static_cast( + static_cast::type>(flags) & static_cast::type>(flag)); + } + } // namespace Bitwise +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactCallback.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactCallback.h new file mode 100644 index 0000000000..a8c9542b47 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactCallback.h @@ -0,0 +1,23 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +namespace TestImpact +{ + //! Generic callback result used by test impact systems. + enum class CallbackResult : bool + { + Continue, + Abort + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactException.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactException.h new file mode 100644 index 0000000000..83ba48aa46 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactException.h @@ -0,0 +1,51 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +#include + +//! Evaluates the specified condition and throws the specified exception with the specified +// !message upon failure. +#define AZ_TestImpact_Eval(CONDITION, EXCEPTION_TYPE, MSG) \ + do \ + { \ + static_assert( \ + AZStd::is_base_of_v, \ + "TestImpact Eval macro must only be used with TestImpact exceptions"); \ + if(!(CONDITION)) \ + { \ + throw(EXCEPTION_TYPE(MSG)); \ + } \ + } \ + while (0) + +namespace TestImpact +{ + //! Base class for test impact framework exceptions. + //! @note The message passed in to the constructor is copied and thus safe with dynamic strings. + class Exception + : public std::exception + { + public: + explicit Exception() = default; + explicit Exception(const AZStd::string& msg); + explicit Exception(const char* msg); + const char* what() const noexcept override; + + private: + //! Error message detailing the reason for the exception. + AZStd::string m_msg; + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactFrameworkPath.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactFrameworkPath.h new file mode 100644 index 0000000000..affab6daa7 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactFrameworkPath.h @@ -0,0 +1,41 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +namespace TestImpact +{ + //! Wrapper for OS paths relative to a specified parent path. + //! @note Mimics path semantics only, makes no guarantees about validity. + class FrameworkPath + { + public: + FrameworkPath() = default; + //! Creates a path with no parent. + explicit FrameworkPath(const AZ::IO::Path& absolutePath); + //! Creates a path with an absolute path and path relative to the specified parent path. + explicit FrameworkPath(const AZ::IO::Path& absolutePath, const FrameworkPath& relativeTo); + + //! Retrieves the absolute path. + const AZ::IO::Path& Absolute() const; + //! Retrieves the path relative to the specified parent path. + const AZ::IO::Path& Relative() const; + + private: + //! The absolute path value. + AZ::IO::Path m_absolutePath; + //! The path value relative to the specified parent path. + AZ::IO::Path m_relativePath; + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Dynamic/TestImpactChangeList.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Dynamic/TestImpactChangeList.h new file mode 100644 index 0000000000..6d966385ac --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Dynamic/TestImpactChangeList.h @@ -0,0 +1,27 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include +#include + +namespace TestImpact +{ + //! Artifact produced by the unified diff parsing process representing the file CRUD operations of a given diff. + struct ChangeList + { + AZStd::vector m_createdFiles; + AZStd::vector m_updatedFiles; + AZStd::vector m_deletedFiles; + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Dynamic/TestImpactCoverage.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Dynamic/TestImpactCoverage.h new file mode 100644 index 0000000000..3c5b5353a7 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Dynamic/TestImpactCoverage.h @@ -0,0 +1,41 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include +#include +#include + +namespace TestImpact +{ + //! Coverage information about a particular line. + struct LineCoverage + { + size_t m_lineNumber = 0; //!< The source line number this covers. + size_t m_hitCount = 0; //!< Number of times this line was covered (zero if not covered). + }; + + //! Coverage information about a particular source file. + struct SourceCoverage + { + AZ::IO::Path m_path; //!< Source file path. + AZStd::optional> m_coverage; //!< Source file line coverage (empty if source level coverage only). + }; + + //! Coverage information about a particular module (executable, shared library). + struct ModuleCoverage + { + AZ::IO::Path m_path; //!< Module path. + AZStd::vector m_sources; //!< Sources of this module that are covered. + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Dynamic/TestImpactTestEnumerationSuite.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Dynamic/TestImpactTestEnumerationSuite.h new file mode 100644 index 0000000000..e4c22a8787 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Dynamic/TestImpactTestEnumerationSuite.h @@ -0,0 +1,21 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +namespace TestImpact +{ + using TestEnumerationCase = TestCase; //!< Test case for test enumeration artifacts. + using TestEnumerationSuite = TestSuite; //!< Test suite for test enumeration artifacts. +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Dynamic/TestImpactTestRunSuite.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Dynamic/TestImpactTestRunSuite.h new file mode 100644 index 0000000000..22254db38e --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Dynamic/TestImpactTestRunSuite.h @@ -0,0 +1,51 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +#include +#include + +namespace TestImpact +{ + //! Result of a test that was ran. + enum class TestRunResult : bool + { + Failed, //! The given test failed. + Passed //! The given test passed. + }; + + //! Status of test as to whether or not it was ran. + enum class TestRunStatus : bool + { + NotRun, //!< The test was not run (typically because the test run was aborted by the client or runner before the test could run). + Run //!< The test was run (see TestRunResult for the result of this test). + }; + + //! Test case for test run artifacts. + struct TestRunCase + : public TestCase + { + AZStd::optional m_result; + AZStd::chrono::milliseconds m_duration = AZStd::chrono::milliseconds{0}; //! Duration this test took to run. + TestRunStatus m_status = TestRunStatus::NotRun; + }; + + //! Test suite for test run artifacts. + struct TestRunSuite + : public TestSuite + { + AZStd::chrono::milliseconds m_duration = AZStd::chrono::milliseconds{0}; //!< Duration this test suite took to run all of its tests. + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Dynamic/TestImpactTestSuite.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Dynamic/TestImpactTestSuite.h new file mode 100644 index 0000000000..0646dd384a --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Dynamic/TestImpactTestSuite.h @@ -0,0 +1,35 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include +#include + +namespace TestImpact +{ + //! Artifact describing basic information about a test case. + struct TestCase + { + AZStd::string m_name; + bool m_enabled = false; + }; + + //! Artifact describing basic information about a test suite. + template + struct TestSuite + { + AZStd::string m_name; + bool m_enabled = false; + AZStd::vector m_tests; + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactBuildTargetDescriptorFactory.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactBuildTargetDescriptorFactory.cpp new file mode 100644 index 0000000000..9e82170dff --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactBuildTargetDescriptorFactory.cpp @@ -0,0 +1,172 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include + +#include +#include +#include + +namespace TestImpact +{ + namespace + { + // Keys for pertinent JSON node and attribute names + constexpr const char* Keys[] = + { + "target", + "name", + "output_name", + "path", + "sources", + "static", + "input", + "output" + }; + + enum + { + TargetKey, + NameKey, + OutputNameKey, + PathKey, + SourcesKey, + StaticKey, + InputKey, + OutputKey + }; + + } // namespace + + AutogenSources PairAutogenSources( + const AZStd::vector& inputSources, + const AZStd::vector& outputSources, + const AZStd::string& autogenMatcher) + { + AutogenSources autogenSources; + const auto matcherPattern = AZStd::regex(autogenMatcher); + AZStd::smatch inputMatches, outputMatches; + + // This has the potential to be optimized to O(n(n-1)/2) time complexity but to be perfectly honest it's not a serious + // bottleneck right now and easier gains would be achieved by constructing build target artifacts in parallel rather than + // trying to squeeze any more juice here as each build target is independent of one and other with no shared memory + for (const auto& input : inputSources) + { + AutogenPairs autogenPairs; + autogenPairs.m_input = input.String(); + const AZStd::string inputString = input.Stem().Native(); + if (AZStd::regex_search(inputString, inputMatches, matcherPattern)) + { + for (const auto& output : outputSources) + { + const AZStd::string outputString = output.Stem().Native(); + if (AZStd::regex_search(outputString, outputMatches, matcherPattern)) + { + // Note: [0] contains the whole match, [1] contains the first capture group + const auto& inputMatch = inputMatches[1]; + const auto& outputMatch = outputMatches[1]; + if (inputMatch == outputMatch) + { + autogenPairs.m_outputs.emplace_back(output); + } + } + } + } + + if (!autogenPairs.m_outputs.empty()) + { + autogenSources.emplace_back(AZStd::move(autogenPairs)); + } + } + + return autogenSources; + } + + BuildTargetDescriptor BuildTargetDescriptorFactory( + const AZStd::string& buildTargetData, + const AZStd::vector& staticSourceExtensionExcludes, + const AZStd::vector& autogenInputExtensionExcludes, + const AZStd::string& autogenMatcher) + { + AZ_TestImpact_Eval(!autogenMatcher.empty(), ArtifactException, "Autogen matcher cannot be empty"); + + BuildTargetDescriptor buildTargetDescriptor; + rapidjson::Document buildTarget; + + if (buildTarget.Parse(buildTargetData.c_str()).HasParseError()) + { + throw TestImpact::ArtifactException("Could not parse build target data"); + } + + const auto& target = buildTarget[Keys[TargetKey]]; + buildTargetDescriptor.m_buildMetaData.m_name = target[Keys[NameKey]].GetString(); + buildTargetDescriptor.m_buildMetaData.m_outputName = target[Keys[OutputNameKey]].GetString(); + buildTargetDescriptor.m_buildMetaData.m_path = target["path"].GetString(); + + AZ_TestImpact_Eval(!buildTargetDescriptor.m_buildMetaData.m_name.empty(), ArtifactException, "Target name cannot be empty"); + AZ_TestImpact_Eval( + !buildTargetDescriptor.m_buildMetaData.m_outputName.empty(), ArtifactException, "Target output name cannot be empty"); + AZ_TestImpact_Eval(!buildTargetDescriptor.m_buildMetaData.m_path.empty(), ArtifactException, "Target path cannot be empty"); + + const auto& sources = buildTarget[Keys[SourcesKey]]; + const auto& staticSources = sources[Keys[StaticKey]].GetArray(); + if (!staticSources.Empty()) + { + buildTargetDescriptor.m_sources.m_staticSources = AZStd::vector(); + + for (const auto& source : staticSources) + { + const AZ::IO::Path sourcePath = AZ::IO::Path(source.GetString()); + if (AZStd::find( + staticSourceExtensionExcludes.begin(), staticSourceExtensionExcludes.end(), sourcePath.Extension().Native()) == + staticSourceExtensionExcludes.end()) + { + buildTargetDescriptor.m_sources.m_staticSources.emplace_back(AZStd::move(sourcePath)); + } + } + } + + const auto& inputSources = buildTarget[Keys[SourcesKey]][Keys[InputKey]].GetArray(); + const auto& outputSources = buildTarget[Keys[SourcesKey]][Keys[OutputKey]].GetArray(); + if (!inputSources.Empty() || !outputSources.Empty()) + { + AZ_TestImpact_Eval( + !inputSources.Empty() && !outputSources.Empty(), ArtifactException, "Autogen malformed, input or output sources are empty"); + + AZStd::vector inputPaths; + AZStd::vector outputPaths; + inputPaths.reserve(inputSources.Size()); + outputPaths.reserve(outputSources.Size()); + + for (const auto& source : inputSources) + { + const AZ::IO::Path sourcePath = AZ::IO::Path(source.GetString()); + if (AZStd::find( + autogenInputExtensionExcludes.begin(), autogenInputExtensionExcludes.end(), sourcePath.Extension().Native()) == + autogenInputExtensionExcludes.end()) + { + inputPaths.emplace_back(AZStd::move(sourcePath)); + } + } + + for (const auto& source : outputSources) + { + outputPaths.emplace_back(AZStd::move(AZ::IO::Path(source.GetString()))); + } + + buildTargetDescriptor.m_sources.m_autogenSources = PairAutogenSources(inputPaths, outputPaths, autogenMatcher); + } + + return buildTargetDescriptor; + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactBuildTargetDescriptorFactory.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactBuildTargetDescriptorFactory.h new file mode 100644 index 0000000000..b23e00478a --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactBuildTargetDescriptorFactory.h @@ -0,0 +1,33 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +#include +#include + +namespace TestImpact +{ + //! Constructs a build target artifact from the specified build target data. + //! @param buildTargetData The raw build target data in JSON format. + //! @param staticSourceExcludes The list of file extensions to exclude for static sources. + //! @param autogenInputExtentsionExcludes The list of file extensions to exclude for autogen input sources. + //! @param autogenMatcher The regex pattern used to match autogen input filenames with output filenames. + //! @return The constructed build target artifact. + BuildTargetDescriptor BuildTargetDescriptorFactory( + const AZStd::string& buildTargetData, + const AZStd::vector& staticSourceExtentsionExcludes, + const AZStd::vector& autogenInputExtentsionExcludes, + const AZStd::string& autogenMatcher); +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactChangeListFactory.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactChangeListFactory.cpp new file mode 100644 index 0000000000..4b2a33355b --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactChangeListFactory.cpp @@ -0,0 +1,175 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include + +#include + +namespace TestImpact +{ + namespace Utils + { + template + auto split(C&& str, const AZStd::string& delimiter) + { + AZStd::vector strings; + + for (auto p = str.data(), end = p + str.length(); p != end; p += ((p == end) ? 0 : delimiter.length())) + { + const auto pre = p; + p = AZStd::search(pre, end, delimiter.cbegin(), delimiter.cend()); + + if (p != pre) + { + strings.emplace_back(pre, p - pre); + } + } + + return strings; + } + } // namespace Utils + + namespace UnifiedDiff + { + class UnifiedDiffParser + { + public: + ChangeList Parse(const AZStd::string& unifiedDiff); + + private: + AZStd::optional GetTargetFile(const AZStd::string_view& targetFile); + ChangeList GenerateChangelist( + const AZStd::vector>& src, + const AZStd::vector>& dst); + + const AZStd::string m_srcFilePrefix = "--- "; + const AZStd::string m_dstFilePrefix = "+++ "; + const AZStd::string m_gitTargetPrefix = "b/"; + const AZStd::string m_perforceTargetPrefix = "/b/"; + const AZStd::string m_renameFromPrefix = "rename from "; + const AZStd::string m_renameToPrefix = "rename to "; + const AZStd::string m_nullFile = "/dev/null"; + bool m_hasGitHeader = false; + }; + + AZStd::optional UnifiedDiffParser::GetTargetFile(const AZStd::string_view& targetFile) + { + size_t startIndex = 0; + + if (targetFile.starts_with(m_renameFromPrefix)) + { + startIndex = m_renameFromPrefix.length(); + } + else if (targetFile.starts_with(m_renameToPrefix)) + { + startIndex = m_renameToPrefix.length(); + } + else if (targetFile.find(m_nullFile) != AZStd::string::npos) + { + return AZStd::nullopt; + } + else + { + startIndex = m_hasGitHeader ? m_gitTargetPrefix.size() + m_dstFilePrefix.size() + : m_perforceTargetPrefix.size() + m_dstFilePrefix.size(); + } + + const auto endIndex = targetFile.find('\t'); + + if (endIndex != AZStd::string::npos) + { + return targetFile.substr(startIndex, endIndex - startIndex); + } + + return targetFile.substr(startIndex); + } + + ChangeList UnifiedDiffParser::GenerateChangelist( + const AZStd::vector>& src, const AZStd::vector>& dst) + { + AZ_TestImpact_Eval(src.size() == dst.size(), ArtifactException, "Change list source and destination file count mismatch"); + ChangeList changelist; + + for (size_t i = 0; i < src.size(); i++) + { + if (!src[i].has_value()) + { + changelist.m_createdFiles.emplace_back(dst[i].value()); + } + else if (!dst[i].has_value()) + { + changelist.m_deletedFiles.emplace_back(src[i].value()); + } + else if (src[i] != dst[i]) + { + changelist.m_deletedFiles.emplace_back(src[i].value()); + changelist.m_createdFiles.emplace_back(dst[i].value()); + } + else + { + changelist.m_updatedFiles.emplace_back(src[i].value()); + } + } + + return changelist; + } + + ChangeList UnifiedDiffParser::Parse(const AZStd::string& unifiedDiff) + { + const AZStd::string GitHeader = "diff --git"; + const auto lines = Utils::split(unifiedDiff, "\n"); + + AZStd::vector> src; + AZStd::vector> dst; + + for (const auto& line : lines) + { + if (line.starts_with(GitHeader)) + { + m_hasGitHeader = true; + } + else if (line.starts_with(m_srcFilePrefix)) + { + src.emplace_back(GetTargetFile(line)); + } + else if (line.starts_with(m_dstFilePrefix)) + { + dst.emplace_back(GetTargetFile(line)); + } + else if (line.starts_with(m_renameFromPrefix)) + { + src.emplace_back(GetTargetFile(line)); + } + else if (line.starts_with(m_renameToPrefix)) + { + dst.emplace_back(GetTargetFile(line)); + } + } + + return GenerateChangelist(src, dst); + } + + ChangeList ChangeListFactory(const AZStd::string& unifiedDiff) + { + AZ_TestImpact_Eval(!unifiedDiff.empty(), ArtifactException, "Unified diff is empty"); + UnifiedDiffParser diff; + ChangeList changeList = diff.Parse(unifiedDiff); + AZ_TestImpact_Eval( + !changeList.m_createdFiles.empty() || + !changeList.m_updatedFiles.empty() || + !changeList.m_deletedFiles.empty(), + ArtifactException, "The unified diff contained no changes"); + return changeList; + } + } // namespace UnifiedDiff +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactChangeListFactory.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactChangeListFactory.h new file mode 100644 index 0000000000..82074a66ef --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactChangeListFactory.h @@ -0,0 +1,29 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +#include +#include + +namespace TestImpact +{ + namespace UnifiedDiff + { + //! Constructs a change list artifact from the specified unified diff data. + //! @param unifiedDiffData The raw change list data in unified diff format. + //! @return The constructed change list artifact. + ChangeList ChangeListFactory(const AZStd::string& unifiedDiffData); + } // namespace UnifiedDiff +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactModuleCoverageFactory.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactModuleCoverageFactory.cpp new file mode 100644 index 0000000000..54ba65c762 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactModuleCoverageFactory.cpp @@ -0,0 +1,154 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include + +#include +#include + +namespace TestImpact +{ + namespace + { + // Keys for pertinent XML node and attribute names + constexpr const char* Keys[] = + { + "packages", + "name", + "filename", + "coverage", + "classes", + "lines", + "line", + "number", + "hits", + "sources", + "source" + }; + + enum + { + PackagesKey, + NameKey, + FileNameKey, + CoverageKey, + ClassesKey, + LinesKey, + LineKey, + NumberKey, + HitsKey, + SourcesKey, + SourceKey + }; + } // namespace + + namespace Cobertura + { + // Note: OpenCppCoverage appears to have a very liberal interpretation of the Cobertura coverage file format so consider + // this implementation to be provisional and coupled to the Windows platform and OpenCppCoverage tool + AZStd::vector ModuleCoveragesFactory(const AZStd::string& coverageData) + { + AZ_TestImpact_Eval(!coverageData.empty(), ArtifactException, "Cannot parse coverage, string is empty"); + AZStd::vector modules; + AZStd::vector rawData(coverageData.begin(), coverageData.end()); + + try + { + AZ::rapidxml::xml_document<> doc; + // Parse the XML doc with default flags + doc.parse<0>(rawData.data()); + + // Coverage + const auto coverage_node = doc.first_node(Keys[CoverageKey]); + AZ_TestImpact_Eval(coverage_node, ArtifactException, "Could not parse coverage node"); + + // Sources + const auto sources_node = coverage_node->first_node(Keys[SourcesKey]); + if (!sources_node) + { + return {}; + } + + // Source + const auto source_node = sources_node->first_node(Keys[SourceKey]); + if (!source_node) + { + return {}; + } + + // Root drive (this seems to be an unconventional use of the sources section by OpenCppCoverage) + const AZStd::string pathRoot = AZStd::string(source_node->value(), source_node->value() + source_node->value_size()) + "\\"; + + const auto packages_node = coverage_node->first_node(Keys[PackagesKey]); + if (packages_node) + { + // Modules + for (auto package_node = packages_node->first_node(); package_node; package_node = package_node->next_sibling()) + { + // Module + ModuleCoverage moduleCoverage; + moduleCoverage.m_path = AZ::IO::Path(package_node->first_attribute(Keys[NameKey])->value()); + + const auto classes_node = package_node->first_node(Keys[ClassesKey]); + if (classes_node) + { + // Sources + for (auto class_node = classes_node->first_node(); class_node; class_node = class_node->next_sibling()) + { + // Source + SourceCoverage sourceCoverage; + sourceCoverage.m_path = AZ::IO::Path(pathRoot + class_node->first_attribute(Keys[FileNameKey])->value()); + + const auto lines_node = class_node->first_node(Keys[LinesKey]); + if (lines_node) + { + AZStd::vector lineCoverage; + + // Lines + for (auto line_node = lines_node->first_node(); line_node; line_node = line_node->next_sibling()) + { + // Line + const size_t number = + AZStd::stol(AZStd::string(line_node->first_attribute(Keys[NumberKey])->value())); + const size_t hits = AZStd::stol(AZStd::string(line_node->first_attribute(Keys[HitsKey])->value())); + lineCoverage.emplace_back(LineCoverage{number, hits}); + } + + if (!lineCoverage.empty()) + { + sourceCoverage.m_coverage.emplace(AZStd::move(lineCoverage)); + } + } + + moduleCoverage.m_sources.emplace_back(AZStd::move(sourceCoverage)); + } + } + + modules.emplace_back(AZStd::move(moduleCoverage)); + } + } + } + catch (const std::exception& e) + { + AZ_Error("ModuleCoveragesFactory", false, e.what()); + throw ArtifactException(e.what()); + } + catch (...) + { + throw ArtifactException("An unknown error occurred parsing the XML data"); + } + + return modules; + } + } // namespace Cobertura +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactModuleCoverageFactory.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactModuleCoverageFactory.h new file mode 100644 index 0000000000..c238901017 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactModuleCoverageFactory.h @@ -0,0 +1,26 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +namespace TestImpact +{ + namespace Cobertura + { + //! Constructs a list of module coverage artifacts from the specified coverage data. + //! @param coverageData The raw coverage data in XML format. + //! @return The constructed list of module coverage artifacts. + AZStd::vector ModuleCoveragesFactory(const AZStd::string& coverageData); + } // namespace Cobertura +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestEnumerationSuiteFactory.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestEnumerationSuiteFactory.cpp new file mode 100644 index 0000000000..bf8a19c812 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestEnumerationSuiteFactory.cpp @@ -0,0 +1,94 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include + +#include +#include + +namespace TestImpact +{ + namespace + { + // Keys for pertinent XML node and attribute names + constexpr const char* Keys[] = + { + "testsuites", + "testsuite", + "name", + "testcase" + }; + + enum + { + TestSuitesKey, + TestSuiteKey, + NameKey, + TestCaseKey + }; + } // namespace + + namespace GTest + { + AZStd::vector TestEnumerationSuitesFactory(const AZStd::string& testEnumerationData) + { + AZ_TestImpact_Eval(!testEnumerationData.empty(), ArtifactException, "Cannot parse enumeration, string is empty"); + AZStd::vector testSuites; + AZStd::vector rawData(testEnumerationData.begin(), testEnumerationData.end()); + + try + { + AZ::rapidxml::xml_document<> doc; + // Parse the XML doc with default flags + doc.parse<0>(rawData.data()); + + const auto testsuites_node = doc.first_node(Keys[TestSuitesKey]); + AZ_TestImpact_Eval(testsuites_node, ArtifactException, "Could not parse enumeration, XML is invalid"); + for (auto testsuite_node = testsuites_node->first_node(Keys[TestSuiteKey]); testsuite_node; + testsuite_node = testsuite_node->next_sibling()) + { + const auto isEnabled = [](const AZStd::string& name) + { + return !name.starts_with("DISABLED_") && name.find("/DISABLED_") == AZStd::string::npos; + }; + + TestEnumerationSuite testSuite; + testSuite.m_name = testsuite_node->first_attribute(Keys[NameKey])->value(); + testSuite.m_enabled = isEnabled(testSuite.m_name); + + for (auto testcase_node = testsuite_node->first_node(Keys[TestCaseKey]); testcase_node; + testcase_node = testcase_node->next_sibling()) + { + TestEnumerationCase testCase; + testCase.m_name = testcase_node->first_attribute(Keys[NameKey])->value(); + testCase.m_enabled = isEnabled(testCase.m_name); + testSuite.m_tests.emplace_back(AZStd::move(testCase)); + } + + testSuites.emplace_back(AZStd::move(testSuite)); + } + } + catch (const std::exception& e) + { + AZ_Error("TestEnumerationSuitesFactory", false, e.what()); + throw ArtifactException(e.what()); + } + catch (...) + { + throw ArtifactException("An unknown error occured parsing the XML data"); + } + + return testSuites; + } + } // namespace GTest +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestEnumerationSuiteFactory.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestEnumerationSuiteFactory.h new file mode 100644 index 0000000000..4be58b2116 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestEnumerationSuiteFactory.h @@ -0,0 +1,26 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +namespace TestImpact +{ + namespace GTest + { + //! Constructs a list of test enumeration suite artifacts from the specified test enumeraion data. + //! @param testEnumerationData The raw test enumeration data in XML format. + //! @return The constructed list of test enumeration suite artifacts. + AZStd::vector TestEnumerationSuitesFactory(const AZStd::string& testEnumerationData); + } // namespace GTest +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestRunSuiteFactory.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestRunSuiteFactory.cpp new file mode 100644 index 0000000000..506e058824 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestRunSuiteFactory.cpp @@ -0,0 +1,143 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include + +#include +#include +#include + +namespace TestImpact +{ + namespace + { + // Keys for pertinent XML node and attribute names + constexpr const char* Keys[] = + { + "testsuites", + "testsuite", + "name", + "testcase", + "status", + "run", + "notrun", + "time" + }; + + enum + { + TestSuitesKey, + TestSuiteKey, + NameKey, + TestCaseKey, + StatusKey, + RunKey, + NotRunKey, + DurationKey + }; + } // namespace + + namespace GTest + { + AZStd::vector TestRunSuitesFactory(const AZStd::string& testEnumerationData) + { + AZ_TestImpact_Eval(!testEnumerationData.empty(), ArtifactException, "Cannot parse test run, string is empty"); + AZStd::vector testSuites; + AZStd::vector rawData(testEnumerationData.begin(), testEnumerationData.end()); + + try + { + AZ::rapidxml::xml_document<> doc; + // Parse the XML doc with default flags + doc.parse<0>(rawData.data()); + + const auto testsuites_node = doc.first_node(Keys[TestSuitesKey]); + AZ_TestImpact_Eval(testsuites_node, ArtifactException, "Could not parse enumeration, XML is invalid"); + for (auto testsuite_node = testsuites_node->first_node(Keys[TestSuiteKey]); testsuite_node; + testsuite_node = testsuite_node->next_sibling()) + { + const auto isEnabled = [](const AZStd::string& name) + { + return !name.starts_with("DISABLED_") && name.find("/DISABLED_") == AZStd::string::npos; + }; + + const auto getDuration = [](const AZ::rapidxml::xml_node<>* node) + { + const AZStd::string duration = node->first_attribute(Keys[DurationKey])->value(); + return AZStd::chrono::milliseconds(AZStd::stof(duration) * 1000.f); + }; + + TestRunSuite testSuite; + testSuite.m_name = testsuite_node->first_attribute(Keys[NameKey])->value(); + testSuite.m_enabled = isEnabled(testSuite.m_name); + testSuite.m_duration = getDuration(testsuite_node); + + for (auto testcase_node = testsuite_node->first_node(Keys[TestCaseKey]); testcase_node; + testcase_node = testcase_node->next_sibling()) + { + const auto getStatus = [](const AZ::rapidxml::xml_node<>* node) + { + const AZStd::string status = node->first_attribute(Keys[StatusKey])->value(); + if (status == Keys[RunKey]) + { + return TestRunStatus::Run; + } + else if (status == Keys[NotRunKey]) + { + return TestRunStatus::NotRun; + } + + throw ArtifactException(AZStd::string::format("Unexpected run status: %s", status.c_str())); + }; + + const auto getResult = [](const AZ::rapidxml::xml_node<>* node) + { + for (auto child_node = node->first_node("failure"); child_node; child_node = child_node->next_sibling()) + { + return TestRunResult::Failed; + } + + return TestRunResult::Passed; + }; + + TestRunCase testCase; + testCase.m_name = testcase_node->first_attribute(Keys[NameKey])->value(); + testCase.m_enabled = isEnabled(testCase.m_name); + testCase.m_duration = getDuration(testcase_node); + testCase.m_status = getStatus(testcase_node); + + if (testCase.m_status == TestRunStatus::Run) + { + testCase.m_result = getResult(testcase_node); + } + + testSuite.m_tests.emplace_back(AZStd::move(testCase)); + } + + testSuites.emplace_back(AZStd::move(testSuite)); + } + } + catch (const std::exception& e) + { + AZ_Error("TestRunSuitesFactory", false, e.what()); + throw ArtifactException(e.what()); + } + catch (...) + { + throw ArtifactException("An unknown error occurred parsing the XML data"); + } + + return testSuites; + } + } // namespace GTest +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestRunSuiteFactory.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestRunSuiteFactory.h new file mode 100644 index 0000000000..fb16c837b2 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestRunSuiteFactory.h @@ -0,0 +1,26 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +namespace TestImpact +{ + namespace GTest + { + //! Constructs a list of test run suite artifacts from the specified test run data. + //! @param testRunData The raw test run data in XML format. + //! @return The constructed list of test run suite artifacts. + AZStd::vector TestRunSuitesFactory(const AZStd::string& testRunData); + } // namespace GTest +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestTargetMetaArtifactFactory.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestTargetMetaArtifactFactory.h new file mode 100644 index 0000000000..f25af960b9 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestTargetMetaArtifactFactory.h @@ -0,0 +1,25 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +#include + +namespace TestImpact +{ + //! Constructs a list of test target meta-data artifacts from the specified master test list data. + //! @param masterTestListData The raw master test list data in JSON format. + //! @return The constructed list of test target meta-data artifacts. + TestTargetMetas TestTargetMetaMapFactory(const AZStd::string& masterTestListData); +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestTargetMetaMapFactory.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestTargetMetaMapFactory.cpp new file mode 100644 index 0000000000..9135a3c683 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestTargetMetaMapFactory.cpp @@ -0,0 +1,89 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include + +#include + +#include + +namespace TestImpact +{ + namespace + { + // Keys for pertinent JSON node and attribute names + constexpr const char* Keys[] = + { + "google", + "test", + "tests", + "suite", + "launch_method", + "test_runner", + "stand_alone", + "name" + }; + + enum + { + GoogleKey, + TestKey, + TestsKey, + SuiteKey, + LaunchMethodKey, + TestRunnerKey, + StandAloneKey, + NameKey + }; + } // namespace + + TestTargetMetaMap TestTargetMetaMapFactory(const AZStd::string& masterTestListData) + { + TestTargetMetaMap testMetas; + rapidjson::Document masterTestList; + + if (masterTestList.Parse(masterTestListData.c_str()).HasParseError()) + { + throw TestImpact::ArtifactException("Could not parse test meta-data file"); + } + + const auto tests = masterTestList[Keys[GoogleKey]][Keys[TestKey]][Keys[TestsKey]].GetArray(); + for (const auto& test : tests) + { + TestTargetMeta testMeta; + testMeta.m_suite = test[Keys[SuiteKey]].GetString(); + + if (const auto buildTypeString = test[Keys[LaunchMethodKey]].GetString(); strcmp(buildTypeString, Keys[TestRunnerKey]) == 0) + { + testMeta.m_launchMethod = LaunchMethod::TestRunner; + } + else if (strcmp(buildTypeString, Keys[StandAloneKey]) == 0) + { + testMeta.m_launchMethod = LaunchMethod::StandAlone; + } + else + { + throw(ArtifactException("Unexpected test build type")); + } + + AZStd::string name = test[Keys[NameKey]].GetString(); + AZ_TestImpact_Eval(!name.empty(), ArtifactException, "Test name field cannot be empty"); + testMetas.emplace(AZStd::move(name), AZStd::move(testMeta)); + } + + // If there's no tests in the repo then something is seriously wrong + AZ_TestImpact_Eval(!testMetas.empty(), ArtifactException, "No tests were found in the repository"); + + return testMetas; + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestTargetMetaMapFactory.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestTargetMetaMapFactory.h new file mode 100644 index 0000000000..039a9e89e2 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestTargetMetaMapFactory.h @@ -0,0 +1,25 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +#include + +namespace TestImpact +{ + //! Constructs a list of test target meta-data artifacts from the specified master test list data. + //! @param masterTestListData The raw master test list data in JSON format. + //! @return The constructed list of test target meta-data artifacts. + TestTargetMetaMap TestTargetMetaMapFactory(const AZStd::string& masterTestListData); +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactBuildTargetDescriptor.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactBuildTargetDescriptor.cpp new file mode 100644 index 0000000000..935ae14452 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactBuildTargetDescriptor.cpp @@ -0,0 +1,22 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include + +namespace TestImpact +{ + BuildTargetDescriptor::BuildTargetDescriptor(BuildMetaData&& buildMetaData, TargetSources&& sources) + : m_buildMetaData(AZStd::move(buildMetaData)) + , m_sources(AZStd::move(sources)) + { + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactBuildTargetDescriptor.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactBuildTargetDescriptor.h new file mode 100644 index 0000000000..e247b663f2 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactBuildTargetDescriptor.h @@ -0,0 +1,55 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include +#include +#include +#include + +namespace TestImpact +{ + //! Pairing between a given autogen input source and the generated output source(s). + struct AutogenPairs + { + AZStd::string m_input; + AZStd::vector m_outputs; + }; + + using AutogenSources = AZStd::vector; + + //! Representation of a given built target's source list. + struct TargetSources + { + AZStd::vector m_staticSources; //!< Source files used to build this target (if any). + AutogenSources m_autogenSources; //!< Autogen source files (if any). + }; + + //! Representation of a given build target's basic build infotmation. + struct BuildMetaData + { + AZStd::string m_name; //!< Build target name. + AZStd::string m_outputName; //!< Output name (sans extension) of build target binary. + AZ::IO::Path m_path; //!< Path to build target location in source tree (relative to repository root). + }; + + //! Artifact produced by the build system for each build target. Contains source and output information about said targets. + struct BuildTargetDescriptor + { + BuildTargetDescriptor() = default; + BuildTargetDescriptor(BuildMetaData&& buildMetaData, TargetSources&& sources); + + BuildMetaData m_buildMetaData; + TargetSources m_sources; + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactProductionTargetDescriptor.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactProductionTargetDescriptor.cpp new file mode 100644 index 0000000000..1b9cb70dab --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactProductionTargetDescriptor.cpp @@ -0,0 +1,21 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include + +namespace TestImpact +{ + ProductionTargetDescriptor::ProductionTargetDescriptor(BuildTargetDescriptor&& buildTargetDescriptor) + : BuildTargetDescriptor(AZStd::move(buildTargetDescriptor)) + { + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactProductionTargetDescriptor.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactProductionTargetDescriptor.h new file mode 100644 index 0000000000..e9fccbc51a --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactProductionTargetDescriptor.h @@ -0,0 +1,25 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +namespace TestImpact +{ + //! Artifact produced by the target artifact compiler that represents a production build target in the repository. + struct ProductionTargetDescriptor + : public BuildTargetDescriptor + { + ProductionTargetDescriptor(BuildTargetDescriptor&& buildTarget); + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactTargetDescriptorCompiler.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactTargetDescriptorCompiler.cpp new file mode 100644 index 0000000000..e13b2083ae --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactTargetDescriptorCompiler.cpp @@ -0,0 +1,45 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include + +#include + +namespace TestImpact +{ + AZStd::tuple, AZStd::vector> CompileTargetDescriptors( + AZStd::vector&& buildTargets, TestTargetMetaMap&& testTargetMetaMap) + { + AZ_TestImpact_Eval(!buildTargets.empty(), ArtifactException, "Build target descriptor list cannot be null"); + AZ_TestImpact_Eval(!testTargetMetaMap.empty(), ArtifactException, "Test target meta map cannot be null"); + + AZStd::tuple, AZStd::vector> outputTargets; + auto& [productionTargets, testTargets] = outputTargets; + + for (auto&& buildTarget : buildTargets) + { + // If this build target has an associated test artifact then it is a test target, otherwise it is a production target + if (auto&& testTargetMeta = testTargetMetaMap.find(buildTarget.m_buildMetaData.m_name); + testTargetMeta != testTargetMetaMap.end()) + { + testTargets.emplace_back(TestTargetDescriptor(AZStd::move(buildTarget), AZStd::move(testTargetMeta->second))); + } + else + { + productionTargets.emplace_back(ProductionTargetDescriptor(AZStd::move(buildTarget))); + } + } + + return outputTargets; + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactTargetDescriptorCompiler.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactTargetDescriptorCompiler.h new file mode 100644 index 0000000000..608fa9d072 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactTargetDescriptorCompiler.h @@ -0,0 +1,31 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include +#include +#include + +#include +#include + +namespace TestImpact +{ + //! Compiles the production target artifacts and test target artifactss from the supplied build target artifacts and test target meta + //! map artifact. + //! @param buildTargets The list of build target artifacts to be sorted into production and test artifact types. + //! @param testTargetMetaMap The map of test target meta artifacts containing the additional meta-data about each test target. + //! @return A tuple containing the production artifacts and test artifacts. + AZStd::tuple, AZStd::vector> CompileTargetDescriptors( + AZStd::vector&& buildTargets, TestTargetMetaMap&& testTargetMetaMap); +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactTestTargetDescriptor.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactTestTargetDescriptor.cpp new file mode 100644 index 0000000000..f45e61c554 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactTestTargetDescriptor.cpp @@ -0,0 +1,22 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include + +namespace TestImpact +{ + TestTargetDescriptor::TestTargetDescriptor(BuildTargetDescriptor&& buildTarget, TestTargetMeta&& testTargetMeta) + : BuildTargetDescriptor(AZStd::move(buildTarget)) + , m_testMetaData(AZStd::move(testTargetMeta)) + { + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactTestTargetDescriptor.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactTestTargetDescriptor.h new file mode 100644 index 0000000000..7044c963e8 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactTestTargetDescriptor.h @@ -0,0 +1,28 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include +#include + +namespace TestImpact +{ + //! Artifact produced by the target artifact compiler that represents a test build target in the repository. + struct TestTargetDescriptor + : public BuildTargetDescriptor + { + TestTargetDescriptor(BuildTargetDescriptor&& buildTarget, TestTargetMeta&& testTargetMeta); + + TestTargetMeta m_testMetaData; + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactTestTargetMeta.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactTestTargetMeta.h new file mode 100644 index 0000000000..57ae2fe894 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactTestTargetMeta.h @@ -0,0 +1,36 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include +#include + +namespace TestImpact +{ + //! Method used to launch the test target. + enum class LaunchMethod : bool + { + TestRunner, //!< Target is launched through a separate test runner binary. + StandAlone //!< Target is launched directly by itself. + }; + + //! Artifact produced by the build system for each test target containing the additional meta-data about the test. + struct TestTargetMeta + { + AZStd::string m_suite; + LaunchMethod m_launchMethod = LaunchMethod::TestRunner; + }; + + //! Map between test target name and test target meta-data. + using TestTargetMetaMap = AZStd::unordered_map; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/TestImpactArtifactException.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/TestImpactArtifactException.h new file mode 100644 index 0000000000..5d2a4fabd2 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/TestImpactArtifactException.h @@ -0,0 +1,26 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +namespace TestImpact +{ + //! Exception for artifacts and artifact parsing operations. + class ArtifactException + : public Exception + { + public: + using Exception::Exception; + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dummy.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dummy.cpp deleted file mode 100644 index 1ac4c2487f..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dummy.cpp +++ /dev/null @@ -1,11 +0,0 @@ -/* -* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or -* its licensors. -* -* For complete copyright and license terms please see the LICENSE at the root of this -* distribution (the "License"). All use of this software is governed by the License, -* or, if provided, by the license below or the license accompanying this file. Do not -* remove or modify any license notices. This file is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* -*/ diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Common/Clang/testimpactframework_clang.cmake b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Common/Clang/testimpactframework_clang.cmake new file mode 100644 index 0000000000..af1177e359 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Common/Clang/testimpactframework_clang.cmake @@ -0,0 +1,12 @@ +# +# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +# its licensors. +# +# For complete copyright and license terms please see the LICENSE at the root of this +# distribution (the "License"). All use of this software is governed by the License, +# or, if provided, by the license below or the license accompanying this file. Do not +# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# + +set(LY_COMPILE_OPTIONS PUBLIC -fexceptions) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Common/MSVC/testimpactframework_msvc.cmake b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Common/MSVC/testimpactframework_msvc.cmake new file mode 100644 index 0000000000..79d4b190a2 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Common/MSVC/testimpactframework_msvc.cmake @@ -0,0 +1,12 @@ +# +# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +# its licensors. +# +# For complete copyright and license terms please see the LICENSE at the root of this +# distribution (the "License"). All use of this software is governed by the License, +# or, if provided, by the license below or the license accompanying this file. Do not +# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# + +set(LY_COMPILE_OPTIONS PUBLIC /EHsc) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/Dummy_Windows.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/Dummy_Windows.cpp deleted file mode 100644 index 1ac4c2487f..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/Dummy_Windows.cpp +++ /dev/null @@ -1,11 +0,0 @@ -/* -* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or -* its licensors. -* -* For complete copyright and license terms please see the LICENSE at the root of this -* distribution (the "License"). All use of this software is governed by the License, -* or, if provided, by the license below or the license accompanying this file. Do not -* remove or modify any license notices. This file is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* -*/ diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/Process/TestImpactWin32_Handle.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/Process/TestImpactWin32_Handle.h new file mode 100644 index 0000000000..3930732bd0 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/Process/TestImpactWin32_Handle.h @@ -0,0 +1,92 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +namespace TestImpact +{ + //! OS function to cleanup handle + using CleanupFunc = BOOL (*)(HANDLE); + + //! RAII wrapper around OS handles. + template + class Handle + { + public: + Handle() = default; + explicit Handle(HANDLE handle); + ~Handle(); + + operator HANDLE&(); + PHANDLE operator&(); + HANDLE& operator=(HANDLE handle); + void Close(); + + private: + HANDLE m_handle = INVALID_HANDLE_VALUE; + }; + + template + Handle::Handle(HANDLE handle) + : m_handle(handle) + { + } + + template + Handle::~Handle() + { + Close(); + } + + template + Handle::operator HANDLE&() + { + return m_handle; + } + + template + PHANDLE Handle::operator&() + { + return &m_handle; + } + + template + HANDLE& Handle::operator=(HANDLE handle) + { + // Setting the handle to INVALID_HANDLE_VALUE will close the handle + if (handle == INVALID_HANDLE_VALUE && m_handle != INVALID_HANDLE_VALUE) + { + Close(); + } + else + { + m_handle = handle; + } + + return m_handle; + } + + template + void Handle::Close() + { + if (m_handle != INVALID_HANDLE_VALUE) + { + CleanupFuncT(m_handle); + m_handle = INVALID_HANDLE_VALUE; + } + } + + using ObjectHandle = Handle; + using WaitHandle = Handle; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/Process/TestImpactWin32_Pipe.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/Process/TestImpactWin32_Pipe.cpp new file mode 100644 index 0000000000..b872487942 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/Process/TestImpactWin32_Pipe.cpp @@ -0,0 +1,67 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include "TestImpactWin32_Pipe.h" + +#include + +#include +#include + +namespace TestImpact +{ + Pipe::Pipe(SECURITY_ATTRIBUTES& sa, HANDLE& stdChannel) + { + if (!CreatePipe(&m_parent, &m_child, &sa, 0)) + { + throw ProcessException("Couldn't create pipe"); + } + + SetHandleInformation(m_parent, HANDLE_FLAG_INHERIT, 0); + stdChannel = m_child; + } + + void Pipe::ReleaseChild() + { + m_child.Close(); + } + + void Pipe::EmptyPipe() + { + DWORD bytesAvailable = 0; + while (PeekNamedPipe(m_parent, NULL, 0, NULL, &bytesAvailable, NULL) && bytesAvailable > 0) + { + // Grow the buffer by the number of bytes available in the pipe and append the new data + DWORD bytesRead; + const size_t currentSize = m_buffer.size(); + m_buffer.resize(m_buffer.size() + bytesAvailable); + if (!ReadFile(m_parent, m_buffer.data() + currentSize, bytesAvailable, &bytesRead, NULL) || bytesRead == 0) + { + throw ProcessException("Couldn't read child output from pipe"); + } + } + } + + AZStd::string Pipe::GetContentsAndClearInternalBuffer() + { + EmptyPipe(); + AZStd::string contents; + + if (m_buffer.size() > 0) + { + contents = AZStd::string(m_buffer.begin(), m_buffer.end()); + m_buffer.clear(); + } + + return contents; + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/Process/TestImpactWin32_Pipe.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/Process/TestImpactWin32_Pipe.h new file mode 100644 index 0000000000..d7b0d616f7 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/Process/TestImpactWin32_Pipe.h @@ -0,0 +1,52 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include "TestImpactWin32_Handle.h" + +#include +#include +#include + +namespace TestImpact +{ + //! RAII wrapper around OS pipes. + //! Used to connect the standard output and standard error of the child process to a sink accessible to the + //! parent process to allow the parent process to read the output(s) of the child process. + class Pipe + { + public: + Pipe(SECURITY_ATTRIBUTES& sa, HANDLE& stdChannel); + Pipe(Pipe&& other) = delete; + Pipe(Pipe& other) = delete; + Pipe& operator=(Pipe& other) = delete; + Pipe& operator=(Pipe&& other) = delete; + + //! Releases the child end of the pipe (not needed once parent has their end). + void ReleaseChild(); + + //! Empties the contents of the pipe into the internal buffer. + void EmptyPipe(); + + //! Empties the contents of the pipe into a string, clearing the internal buffer. + AZStd::string GetContentsAndClearInternalBuffer(); + + private: + // Parent and child process ends of pipe + ObjectHandle m_parent; + ObjectHandle m_child; + + // Buffer for emptying pipe upon child processes exit + AZStd::vector m_buffer; + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/Process/TestImpactWin32_Process.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/Process/TestImpactWin32_Process.cpp new file mode 100644 index 0000000000..0a8a8643f3 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/Process/TestImpactWin32_Process.cpp @@ -0,0 +1,259 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include "TestImpactWin32_Process.h" + +#include + +#include + +namespace TestImpact +{ + // Note: this is called from an OS thread + VOID ProcessWin32::ProcessExitCallback(PVOID processPtr, [[maybe_unused]] BOOLEAN EventSignalled) + { + // Lock the process destructor from being entered from the client thread + AZStd::lock_guard lifeLock(m_lifeCycleMutex); + + ProcessId id = reinterpret_cast(processPtr); + auto process = m_masterProcessList[id]; + + // Check that the process hasn't already been destructed from the client thread + if (process && process->m_isRunning) + { + // Lock state access and/or mutation from the client thread + AZStd::lock_guard stateLock(process->m_stateMutex); + process->RetrieveOSReturnCodeAndCleanUpProcess(); + } + } + + ProcessWin32::ProcessWin32(const ProcessInfo& processInfo) + : Process(processInfo) + { + AZStd::string args(m_processInfo.GetProcessPath().String()); + + if (m_processInfo.GetStartupArgs().length()) + { + args = AZStd::string::format("%s %s", args.c_str(), m_processInfo.GetStartupArgs().c_str()); + } + + SECURITY_ATTRIBUTES sa; + sa.nLength = sizeof(sa); + sa.lpSecurityDescriptor = nullptr; + sa.bInheritHandle = IsPiping(); + + STARTUPINFO si; + ZeroMemory(&si, sizeof(STARTUPINFO)); + si.cb = sizeof(STARTUPINFO); + PROCESS_INFORMATION pi; + ZeroMemory(&pi, sizeof(PROCESS_INFORMATION)); + + CreatePipes(sa, si); + + if (!CreateProcess( + NULL, + &args[0], + NULL, + NULL, + IsPiping(), + CREATE_NEW_PROCESS_GROUP | CREATE_NO_WINDOW, + NULL, NULL, + &si, &pi)) + { + throw ProcessException("Couldn't create process"); + } + + ReleaseChildPipes(); + + m_process = pi.hProcess; + m_thread = pi.hThread; + m_isRunning = true; + + { + // Lock reading of the master process list from the OS thread + AZStd::lock_guard lock(m_lifeCycleMutex); + + // Register this process with a unique id in the master process list + m_uniqueId = m_uniqueIdCounter++; + m_masterProcessList[m_uniqueId] = this; + } + + // Register the process exit signal callback + if (!RegisterWaitForSingleObject( + &m_waitCallback, + pi.hProcess, + ProcessExitCallback, + reinterpret_cast(m_uniqueId), + INFINITE, + WT_EXECUTEONLYONCE)) + { + throw ProcessException("Couldn't register wait object for process exit event"); + } + } + + bool ProcessWin32::IsPiping() const + { + return m_processInfo.ParentHasStdOutput() || m_processInfo.ParentHasStdError(); + } + + void ProcessWin32::CreatePipes(SECURITY_ATTRIBUTES& sa, STARTUPINFO& si) + { + if (IsPiping()) + { + si.dwFlags = STARTF_USESTDHANDLES; + + if (m_processInfo.ParentHasStdOutput()) + { + m_stdOutPipe.emplace(sa, si.hStdOutput); + } + + if (m_processInfo.ParentHasStdError()) + { + m_stdErrPipe.emplace(sa, si.hStdError); + } + } + } + + void ProcessWin32::ReleaseChildPipes() + { + if (m_stdOutPipe) + { + m_stdOutPipe->ReleaseChild(); + } + + if (m_stdErrPipe) + { + m_stdErrPipe->ReleaseChild(); + } + } + + void ProcessWin32::EmptyPipes() + { + if (m_stdOutPipe) + { + m_stdOutPipe->EmptyPipe(); + } + + if (m_stdErrPipe) + { + m_stdErrPipe->EmptyPipe(); + } + } + + AZStd::optional ProcessWin32::ConsumeStdOut() + { + if (m_stdOutPipe) + { + AZStd::string contents = m_stdOutPipe->GetContentsAndClearInternalBuffer(); + if (!contents.empty()) + { + return contents; + } + } + + return AZStd::nullopt; + } + + AZStd::optional ProcessWin32::ConsumeStdErr() + { + if (m_stdErrPipe) + { + AZStd::string contents = m_stdErrPipe->GetContentsAndClearInternalBuffer(); + if (!contents.empty()) + { + return contents; + } + } + + return AZStd::nullopt; + } + + void ProcessWin32::Terminate(ReturnCode returnCode) + { + // Lock process cleanup from the OS thread + AZStd::lock_guard lock(m_stateMutex); + + if (m_isRunning) + { + // Cancel the callback so we can wait for the signal ourselves + // Note: we keep the state mutex locked as closing the callback is not guaranteed to be instantaneous + m_waitCallback.Close(); + + // Terminate the process and set the error code to the terminate code + TerminateProcess(m_process, returnCode); + SetReturnCodeAndCleanUpProcesses(returnCode); + } + } + + bool ProcessWin32::IsRunning() const + { + return m_isRunning; + } + + void ProcessWin32::BlockUntilExit() + { + // Lock process cleanup from the OS thread + AZStd::lock_guard lock(m_stateMutex); + + if (m_isRunning) + { + // Cancel the callback so we can wait for the signal ourselves + // Note: we keep the state mutex locked as closing the callback is not guaranteed to be instantaneous + m_waitCallback.Close(); + + if (IsPiping()) + { + // This process will be blocked from exiting if pipe not emptied so will deadlock if we wait + // indefintely whilst there is still output in the pipes so instead keep waiting and checking + // if the pipes need emptying until the process exits + while (WAIT_OBJECT_0 != WaitForSingleObject(m_process, 1)) + { + EmptyPipes(); + } + } + else + { + // No possibility of pipe deadlocking, safe to wait indefinitely for process exit + WaitForSingleObject(m_process, INFINITE); + } + + // Now that the this process has definately exited we are safe to clean up + RetrieveOSReturnCodeAndCleanUpProcess(); + } + } + + void ProcessWin32::RetrieveOSReturnCodeAndCleanUpProcess() + { + DWORD returnCode; + GetExitCodeProcess(m_process, &returnCode); + SetReturnCodeAndCleanUpProcesses(returnCode); + } + + void ProcessWin32::SetReturnCodeAndCleanUpProcesses(ReturnCode returnCode) + { + m_returnCode = returnCode; + m_process.Close(); + m_thread.Close(); + m_waitCallback.Close(); + m_isRunning = false; + } + + ProcessWin32::~ProcessWin32() + { + // Lock the process exit signal callback from being entered OS thread + AZStd::lock_guard lock(m_lifeCycleMutex); + + // Remove this process from the master list so the process exit signal doesn't attempt to cleanup + // this process if it is deleted client side + m_masterProcessList[m_uniqueId] = nullptr; + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/Process/TestImpactWin32_Process.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/Process/TestImpactWin32_Process.h new file mode 100644 index 0000000000..a29fecada1 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/Process/TestImpactWin32_Process.h @@ -0,0 +1,101 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include "TestImpactWin32_Handle.h" +#include "TestImpactWin32_Pipe.h" + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace TestImpact +{ + //! Platform-specific implementation of Process. + class ProcessWin32 + : public Process + { + public: + explicit ProcessWin32(const ProcessInfo& processInfo); + ProcessWin32(ProcessWin32&& other) = delete; + ProcessWin32(ProcessWin32& other) = delete; + ProcessWin32& operator=(ProcessWin32& other) = delete; + ProcessWin32& operator=(ProcessWin32&& other) = delete; + ~ProcessWin32(); + + // Process overrides... + void Terminate(ReturnCode returnCode) override; + void BlockUntilExit() override; + bool IsRunning() const override; + AZStd::optional ConsumeStdOut() override; + AZStd::optional ConsumeStdErr() override; + + private: + //! Callback for process exit signal. + static VOID ProcessExitCallback(PVOID processPtr, BOOLEAN EventSignalled); + + //! Retrieves the return code and cleans up the OS handles. + void RetrieveOSReturnCodeAndCleanUpProcess(); + + //! Sets the return code and cleans up the OS handles + void SetReturnCodeAndCleanUpProcesses(ReturnCode returnCode); + + //! Returns true if either stdout or stderr is beign redirected. + bool IsPiping() const; + + //! Creates the parent and child pipes for stdout and/or stderr. + void CreatePipes(SECURITY_ATTRIBUTES& sa, STARTUPINFO& si); + + //! Empties all pipes so the process can exit without deadlocking. + void EmptyPipes(); + + //! Releases the child end of the stdout and/or stderr pipes/ + void ReleaseChildPipes(); + + // Flag to determine whether or not the process is in flight + AZStd::atomic_bool m_isRunning = false; + + // Unique id assigned to this process (not the same as the id assigned by the client in the ProcessInfo class) + // as used in the master process list + size_t m_uniqueId = 0; + + // Handles to OS process + ObjectHandle m_process; + ObjectHandle m_thread; + + // Handle to process exit signal callback + WaitHandle m_waitCallback; + + // Process to parent standard output piping + AZStd::optional m_stdOutPipe; + AZStd::optional m_stdErrPipe; + + // Mutex protecting process state access/mutation from the OS thread and client thread + mutable AZStd::mutex m_stateMutex; + + // Mutex keeping the process life cycles in sync between the OS thread and client thread + inline static AZStd::mutex m_lifeCycleMutex; + + // Unique counter to give each launched process a unique id + inline static size_t m_uniqueIdCounter = 1; + + // Master process list used to ensure consistency of process lifecycles between OS thread and client thread + inline static std::unordered_map m_masterProcessList; + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/Process/TestImpactWin32_ProcessLauncher.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/Process/TestImpactWin32_ProcessLauncher.cpp new file mode 100644 index 0000000000..d798e4ff7b --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/Process/TestImpactWin32_ProcessLauncher.cpp @@ -0,0 +1,24 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include "TestImpactWin32_Process.h" + +#include +#include + +namespace TestImpact +{ + AZStd::unique_ptr LaunchProcess(const ProcessInfo& processInfo) + { + return AZStd::make_unique(processInfo); + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/platform_windows_files.cmake b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/platform_windows_files.cmake index 9b83a02475..32f996cdf8 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/platform_windows_files.cmake +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/platform_windows_files.cmake @@ -10,5 +10,10 @@ # set(FILES - Dummy_Windows.cpp + Process/TestImpactWin32_ProcessLauncher.cpp + Process/TestImpactWin32_Process.cpp + Process/TestImpactWin32_Process.h + Process/TestImpactWin32_Handle.h + Process/TestImpactWin32_Pipe.cpp + Process/TestImpactWin32_Pipe.h ) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJob.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJob.h new file mode 100644 index 0000000000..d9dd4870dd --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJob.h @@ -0,0 +1,139 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include +#include + +#include +#include + +namespace TestImpact +{ + //! Result of a job that was run. + enum class JobResult + { + NotExecuted, //!< The job was not executed (e.g. the job runner terminated before the job could be executed). + FailedToExecute, //!< The job failed to execute (e.g. due to the arguments used to execute the job being invalid). + Terminated, //!< The job was terminated by the job runner (e.g. job or runner timeout exceeded while job was in-flight). + ExecutedWithFailure, //!< The job was executed but exited in an erroneous state (the underlying process returned non-zero). + ExecutedWithSuccess //!< The job was executed and exited in a successful state (the underlying processes returned zero). + }; + + //! The meta-data for a given job. + struct JobMeta + { + JobResult m_result = JobResult::NotExecuted; + AZStd::optional + m_startTime; //!< The time, relative to the job runner start, that this job started. + AZStd::optional m_duration; //!< The duration that this job took to complete. + AZStd::optional m_returnCode; //!< The return code of the underlying processes of this job. + }; + + //! Representation of a unit of work to be performed by a process. + //! @tparam JobInfoT The JobInfo structure containing the information required to run this job. + //! @tparam JobPayloadT The resulting output of the processed artifact produced by this job. + template + class Job + { + public: + using Info = JobInfoT; + using Payload = JobPayloadT; + + //! Constructor with r-values for the specific use case of the job runner. + Job(Info jobInfo, JobMeta&& jobMeta, AZStd::optional&& payload); + + //! Returns the job info associated with this job. + const Info& GetJobInfo() const; + + //! Returns the result of this job. + JobResult GetResult() const; + + //! Returns the start time, relative to the job runner start, that this job started. + AZStd::chrono::high_resolution_clock::time_point GetStartTime() const; + + //! Returns the end time, relative to the job runner start, that this job ended. + AZStd::chrono::high_resolution_clock::time_point GetEndTime() const; + + //! Returns the duration that this job took to complete. + AZStd::chrono::milliseconds GetDuration() const; + + //! Returns the return code of the underlying processes of this job. + AZStd::optional GetReturnCode() const; + + //! Returns the payload produced by this job. + const AZStd::optional& GetPayload() const; + + private: + Info m_jobInfo; + JobMeta m_meta; + AZStd::optional m_payload; + }; + + template + Job::Job(Info jobInfo, JobMeta&& jobMeta, AZStd::optional&& payload) + : m_jobInfo(jobInfo) + , m_meta(AZStd::move(jobMeta)) + , m_payload(AZStd::move(payload)) + { + } + + template + const JobInfoT& Job::GetJobInfo() const + { + return m_jobInfo; + } + + template + JobResult Job::GetResult() const + { + return m_meta.m_result; + } + + template + AZStd::optional Job::GetReturnCode() const + { + return m_meta.m_returnCode; + } + + template + AZStd::chrono::high_resolution_clock::time_point Job::GetStartTime() const + { + return m_meta.m_startTime.value_or(AZStd::chrono::high_resolution_clock::time_point()); + } + + template + AZStd::chrono::high_resolution_clock::time_point Job::GetEndTime() const + { + if (m_meta.m_startTime.has_value() && m_meta.m_duration.has_value()) + { + return m_meta.m_startTime.value() + m_meta.m_duration.value(); + } + else + { + return AZStd::chrono::high_resolution_clock::time_point(); + } + } + + template + AZStd::chrono::milliseconds Job::GetDuration() const + { + return m_meta.m_duration.value_or(AZStd::chrono::milliseconds{0}); + } + + template + const AZStd::optional& Job::GetPayload() const + { + return m_payload; + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJobInfo.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJobInfo.h new file mode 100644 index 0000000000..fd9d76652a --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJobInfo.h @@ -0,0 +1,73 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +namespace TestImpact +{ + //! Per-job information to configure and run jobs and process the resulting artifacts. + //! @tparam AdditionalInfo Additional information to be provided to each job to be consumed by client. + template + class JobInfo + : public AdditionalInfo + { + public: + using IdType = size_t; + + //! Client-provided identifier to distinguish between different jobs. + //! @note Ids of different job types are not interchangeable. + struct Id + { + IdType m_value; + }; + + //! Constructs the job information with any additional information required by the job. + //! @param jobId The client-provided unique identifier for the job. + //! @param args The arguments used to launch the process running the job. + //! @param additionalInfo The arguments to be provided to the additional information data structure. + template + JobInfo(Id jobId, const AZStd::string& args, AdditionalInfoArgs&&... additionalInfo); + + //! Returns the id of this job. + Id GetId() const; + + //! Returns the command arguments used to execute this job. + const AZStd::string& GetArgs() const; + + private: + Id m_id; + AZStd::string m_args; + }; + + template + template + JobInfo::JobInfo(Id jobId, const AZStd::string& args, AdditionalInfoArgs&&... additionalInfo) + : AdditionalInfo{std::forward(additionalInfo)...} + , m_id(jobId) + , m_args(args) + { + } + + template + typename JobInfo::Id JobInfo::GetId() const + { + return m_id; + } + + template + const AZStd::string& JobInfo::GetArgs() const + { + return m_args; + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJobRunner.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJobRunner.h new file mode 100644 index 0000000000..8b9e4fa69e --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJobRunner.h @@ -0,0 +1,190 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include +#include + +#include +#include +#include +#include + +namespace TestImpact +{ + //! Callback for job completion/failure. + //! @param jobInfo The job information associated with this job. + //! @param meta The meta-data about the job run. + //! @param std The standard output and standard error of the process running the job. + template + using JobCallback = AZStd::function; + + //! The payloads produced by the job-specific payload producer in the form of a map associating each job id with the job's payload. + template + using PayloadMap = AZStd::unordered_map>; + + //! The map used by the client to associate the job information and meta-data with the job ids. + template + using JobDataMap = AZStd::unordered_map>; + + //! The callback for producing the payloads for the jobs after all jobs have finished executing. + //! @param jobInfos The information for each job run. + //! @param jobDataMap The job data (in the form of job info and meta-data) for each job run. + template + using PayloadMapProducer = AZStd::function(const JobDataMap& jobDataMap)>; + + //! Generic job runner that launches a process for each job, records metrics about each job run and hands the payload artifacts + //! produced by each job to the client before compositing the metrics and payload artifacts for each job into a single interface + //! to be consumed by the client. + template + class JobRunner + { + public: + //! Constructs the job runner with the specified parameters to constrain job runs. + //! @param stdOutRouting The standard output routing to be specified for all jobs. + //! @param stdErrRouting The standard error routing to be specified for all jobs. + //! @param maxConcurrentProcesses he maximum number of concurrent jobs in-flight. + //! @param processTimeout The maximum duration a job may be in-flight before being forcefully terminated (nullopt if no timeout). + //! @param scheduleTimeout The maximum duration the scheduler may run before forcefully terminating all in-flight jobs (nullopt if + //! no timeout). + JobRunner( + StdOutputRouting stdOutRouting, + StdErrorRouting stdErrRouting, + size_t maxConcurrentProcesses, + AZStd::optional processTimeout, + AZStd::optional scheduleTimeout); + + //! Executes the specified jobs and returns the products of their labor. + //! @note: the job and payload callbacks are specified here rather than in the constructor to allow clients to use capturing lambdas + //! should they desire to. + //! @param jobs The arguments (and other pertinent information) required for each job to be run. + //! @param jobCallback The client callback to be called when each job changes state. + //! @param payloadMapProducer The client callback to be called when all jobs have finished to transform the work produced by each + //! job into the desired output. + AZStd::vector Execute( + const AZStd::vector& jobs, JobCallback jobCallback, + PayloadMapProducer payloadMapProducer); + + private: + size_t m_maxConcurrentProcesses = 0; //!< Maximum number of concurrent jobs being executed at a given time. + StdOutputRouting m_stdOutRouting; //!< Standard output routing from each job process to job runner. + StdErrorRouting m_stdErrRouting; //!< Standard error routing from each job process to job runner + AZStd::optional m_jobTimeout; //!< Maximum time a job can run for before being forcefully terminated. + AZStd::optional m_runnerTimeout; //!< Maximum time the job runner can run before forcefully terminating all in-flight jobs and shutting down. + }; + + template + JobRunner::JobRunner( + StdOutputRouting stdOutRouting, + StdErrorRouting stdErrRouting, + size_t maxConcurrentProcesses, + AZStd::optional jobTimeout, + AZStd::optional runnerTimeout) + : m_maxConcurrentProcesses(maxConcurrentProcesses) + , m_stdOutRouting(stdOutRouting) + , m_stdErrRouting(stdErrRouting) + , m_jobTimeout(jobTimeout) + , m_runnerTimeout(runnerTimeout) + { + } + + template + AZStd::vector JobRunner::Execute( + const AZStd::vector& jobInfos, + JobCallback jobCallback, + PayloadMapProducer payloadMapProducer) + { + AZStd::vector processes; + AZStd::unordered_map> metas; + AZStd::vector jobs; + jobs.reserve(jobInfos.size()); + processes.reserve(jobInfos.size()); + + // Transform the job infos into the underlying process infos required for each job + for (size_t jobIndex = 0; jobIndex < jobInfos.size(); jobIndex++) + { + const auto* jobInfo = &jobInfos[jobIndex]; + const auto jobId = jobInfo->GetId().m_value; + metas.emplace(jobId, AZStd::pair{JobMeta{}, jobInfo}); + processes.emplace_back(jobId, m_stdOutRouting, m_stdErrRouting, jobInfo->GetArgs()); + } + + // Wrapper around low-level process launch callback to gather job meta-data and present a simplified callback interface to the client + const auto processLaunchCallback = [&jobCallback, &jobInfos, &metas]( + TestImpact::ProcessId pid, + TestImpact::LaunchResult launchResult, + AZStd::chrono::high_resolution_clock::time_point createTime) + { + auto& [meta, jobInfo] = metas.at(pid); + if (launchResult == LaunchResult::Failure) + { + meta.m_result = JobResult::FailedToExecute; + return jobCallback(*jobInfo, meta, {}); + } + else + { + meta.m_startTime = createTime; + return CallbackResult::Continue; + } + }; + + // Wrapper around low-level process exit callback to gather job meta-data and present a simplified callback interface to the client + const auto processExitCallback = [&jobCallback, &jobInfos, &metas]( + TestImpact::ProcessId pid, + TestImpact::ExitCondition exitCondition, + TestImpact::ReturnCode returnCode, + TestImpact::StdContent&& std, + AZStd::chrono::high_resolution_clock::time_point exitTime) + { + auto& [meta, jobInfo] = metas.at(pid); + meta.m_returnCode = returnCode; + meta.m_duration = AZStd::chrono::duration_cast(exitTime - *meta.m_startTime); + if (exitCondition == ExitCondition::Gracefull && returnCode == 0) + { + meta.m_result = JobResult::ExecutedWithSuccess; + } + else if (exitCondition == ExitCondition::Terminated || exitCondition == ExitCondition::Timeout) + { + meta.m_result = JobResult::Terminated; + } + else + { + meta.m_result = JobResult::ExecutedWithFailure; + } + + return jobCallback(*jobInfo, meta, AZStd::move(std)); + }; + + // Schedule all jobs for execution + ProcessScheduler scheduler( + processes, + processLaunchCallback, + processExitCallback, + m_maxConcurrentProcesses, + m_jobTimeout, + m_runnerTimeout + ); + + // Hand off the jobs to the client for payload generation + auto payloadMap = payloadMapProducer(metas); + + // Unpack the payload map produced by the client into a vector of jobs containing the job data and payload for each job + for (const auto& jobInfo : jobInfos) + { + const auto jobId = jobInfo.GetId().m_value; + jobs.emplace_back(JobT(jobInfo, AZStd::move(metas.at(jobId).first), AZStd::move(payloadMap[jobId]))); + } + + return jobs; + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/Scheduler/TestImpactProcessScheduler.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/Scheduler/TestImpactProcessScheduler.cpp new file mode 100644 index 0000000000..15230b1a46 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/Scheduler/TestImpactProcessScheduler.cpp @@ -0,0 +1,270 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include +#include +#include +#include + +namespace TestImpact +{ + struct ProcessScheduler::ProcessInFlight + { + AZStd::unique_ptr m_process; + AZStd::optional m_startTime; + AZStd::string m_stdOutput; + AZStd::string m_stdError; + }; + + ProcessScheduler::ProcessScheduler( + const AZStd::vector& processes, + const ProcessLaunchCallback& processLaunchCallback, + const ProcessExitCallback& processExitCallback, + size_t maxConcurrentProcesses, + AZStd::optional processTimeout, + AZStd::optional scheduleTimeout) + : m_processCreateCallback(processLaunchCallback) + , m_processExitCallback(processExitCallback) + , m_processTimeout(processTimeout) + , m_scheduleTimeout(scheduleTimeout) + , m_startTime(AZStd::chrono::high_resolution_clock::now()) + { + AZ_TestImpact_Eval(maxConcurrentProcesses != 0, ProcessException, "Max Number of concurrent processes in flight cannot be 0"); + AZ_TestImpact_Eval(!processes.empty(), ProcessException, "Number of processes to launch cannot be 0"); + AZ_TestImpact_Eval( + !m_processTimeout.has_value() || m_processTimeout->count() > 0, ProcessException, + "Process timeout must be empty or non-zero value"); + AZ_TestImpact_Eval( + !m_scheduleTimeout.has_value() || m_scheduleTimeout->count() > 0, ProcessException, + "Scheduler timeout must be empty or non-zero value"); + + const size_t numConcurrentProcesses = AZStd::min(processes.size(), maxConcurrentProcesses); + m_processPool.resize(numConcurrentProcesses); + + for (const auto& process : processes) + { + m_processQueue.emplace(process); + } + + for (auto& process : m_processPool) + { + if (PopAndLaunch(process) == CallbackResult::Abort) + { + TerminateAllProcesses(ExitCondition::Terminated); + return; + } + } + + MonitorProcesses(); + } + + ProcessScheduler::~ProcessScheduler() + { + TerminateAllProcesses(ExitCondition::Terminated); + } + + void ProcessScheduler::MonitorProcesses() + { + while (true) + { + // Check to see whether or not the scheduling has exceeded its specified runtime + if (m_scheduleTimeout.has_value()) + { + const auto shedulerRunTime = AZStd::chrono::milliseconds(AZStd::chrono::high_resolution_clock::now() - m_startTime); + + if (shedulerRunTime > m_scheduleTimeout) + { + // Runtime exceeded, terminate all proccesses and schedule no further + TerminateAllProcesses(ExitCondition::Timeout); + return; + } + } + + // Flag to determine whether or not there are currently any processes in-flight + bool processesInFlight = false; + + // Loop round the process pool and visit round robin queued up processes for launch + for (auto& processInFlight : m_processPool) + { + if (processInFlight.m_process) + { + // Process is alive (note: not necessarilly currently running) + AccumulateProcessStdContent(processInFlight); + const ProcessId processId = processInFlight.m_process->GetProcessInfo().GetId(); + + if (!processInFlight.m_process->IsRunning()) + { + // Process has exited of its own accord + const ReturnCode returnCode = processInFlight.m_process->GetReturnCode().value(); + processInFlight.m_process.reset(); + const auto exitTime = AZStd::chrono::high_resolution_clock::now(); + + // Inform the client that the processes has exited + if (CallbackResult::Abort == m_processExitCallback( + processId, + ExitCondition::Gracefull, + returnCode, + ConsumeProcessStdContent(processInFlight), + exitTime)) + { + // Client chose to abort the scheduler + TerminateAllProcesses(ExitCondition::Terminated); + return; + } + else if (!m_processQueue.empty()) + { + // This slot in the pool is free so launch one of the processes waiting in the queue + if (PopAndLaunch(processInFlight) == CallbackResult::Abort) + { + // Client chose to abort the scheduler + TerminateAllProcesses(ExitCondition::Terminated); + return; + } + else + { + // We know from the above PopAndLaunch there is at least one process in-flight this iteration + processesInFlight = true; + } + } + } + else + { + // Process is still in-flight + const auto exitTime = AZStd::chrono::high_resolution_clock::now(); + const auto runTime = AZStd::chrono::milliseconds(exitTime - processInFlight.m_startTime.value()); + + // Check to see whether or not the processes has exceeded its specified flight time + if (m_processTimeout.has_value() && runTime > m_processTimeout) + { + processInFlight.m_process->Terminate(ProcessTimeoutErrorCode); + const ReturnCode returnCode = processInFlight.m_process->GetReturnCode().value(); + processInFlight.m_process.reset(); + + if (CallbackResult::Abort == m_processExitCallback( + processId, + ExitCondition::Timeout, + returnCode, + ConsumeProcessStdContent(processInFlight), + exitTime)) + { + // Flight time exceeded, terminate this process + TerminateAllProcesses(ExitCondition::Terminated); + return; + } + } + + // We know that at least this process is in-flight this iteration + processesInFlight = true; + } + } + else + { + // Queue is empty, no more processes to launch + if (!m_processQueue.empty()) + { + if (PopAndLaunch(processInFlight) == CallbackResult::Abort) + { + // Client chose to abort the scheduler + TerminateAllProcesses(ExitCondition::Terminated); + return; + } + else + { + // We know from the above PopAndLaunch there is at least one process in-flight this iteration + processesInFlight = true; + } + } + } + } + + if (!processesInFlight) + { + break; + } + } + } + + CallbackResult ProcessScheduler::PopAndLaunch(ProcessInFlight& processInFlight) + { + auto processInfo = m_processQueue.front(); + m_processQueue.pop(); + const auto createTime = AZStd::chrono::high_resolution_clock::now(); + LaunchResult createResult = LaunchResult::Success; + + try + { + processInFlight.m_process = LaunchProcess(AZStd::move(processInfo)); + processInFlight.m_startTime = createTime; + } + catch (ProcessException& e) + { + AZ_Warning("ProcessScheduler", false, e.what()); + createResult = LaunchResult::Failure; + } + + return m_processCreateCallback(processInfo.GetId(), createResult, createTime); + } + + void ProcessScheduler::AccumulateProcessStdContent(ProcessInFlight& processInFlight) + { + // Accumulate the stdout/stderr so we don't deadlock with the process waiting for the pipe to empty before finishing + processInFlight.m_stdOutput += processInFlight.m_process->ConsumeStdOut().value_or(""); + processInFlight.m_stdError += processInFlight.m_process->ConsumeStdErr().value_or(""); + } + + StdContent ProcessScheduler::ConsumeProcessStdContent(ProcessInFlight& processInFlight) + { + return + { + !processInFlight.m_stdOutput.empty() + ? AZStd::optional{AZStd::move(processInFlight.m_stdOutput)} + : AZStd::nullopt, + !processInFlight.m_stdError.empty() + ? AZStd::optional{AZStd::move(processInFlight.m_stdError)} + : AZStd::nullopt + }; + } + + void ProcessScheduler::TerminateAllProcesses(ExitCondition exitStatus) + { + bool isCallingBackToClient = true; + const ReturnCode returnCode = static_cast(exitStatus); + + for (auto& processInFlight : m_processPool) + { + if (processInFlight.m_process) + { + processInFlight.m_process->Terminate(ProcessTerminateErrorCode); + AccumulateProcessStdContent(processInFlight); + const ProcessId processId = processInFlight.m_process->GetProcessInfo().GetId(); + + if (isCallingBackToClient) + { + const auto exitTime = AZStd::chrono::high_resolution_clock::now(); + if (CallbackResult::Abort == m_processExitCallback( + processInFlight.m_process->GetProcessInfo().GetId(), + exitStatus, + returnCode, + ConsumeProcessStdContent(processInFlight), + exitTime)) + { + // Client chose to abort the scheduler, do not make any further callbacks + isCallingBackToClient = false; + } + } + + processInFlight.m_process.reset(); + } + } + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/Scheduler/TestImpactProcessScheduler.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/Scheduler/TestImpactProcessScheduler.h new file mode 100644 index 0000000000..5531c78397 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/Scheduler/TestImpactProcessScheduler.h @@ -0,0 +1,110 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace TestImpact +{ + //! Result of the attempt to launch a process. + enum class LaunchResult : bool + { + Failure, + Success + }; + + //! The condition under which the processes exited. + //! @note For convinience, the terminate and timeout condition values are set to the corresponding return value sent to the + //! process. + enum class ExitCondition : ReturnCode + { + Gracefull, //!< Process has exited of its own accord. + Terminated = ProcessTerminateErrorCode, //!< The process was terminated by the client/scheduler. + Timeout = ProcessTimeoutErrorCode //!< The process was terminated by the scheduler due to exceeding runtime limit. + }; + + //! Callback for process launch attempt. + //! @param processId The id of the process that attempted to launch. + //! @param launchResult The result of the process launch attempt. + //! @param createTime The timestamp of the process launch attempt. + using ProcessLaunchCallback = + AZStd::function; + + //! Callback for process exit of successfully launched process. + //! @param processId The id of the process that attempted to launch. + //! @param exitStatus The circumstances upon which the processes exited. + //! @param returnCode The return code of the exited process. + //! @param std The standard output and standard error of the process. + //! @param createTime The timestamp of the process exit. + using ProcessExitCallback = + AZStd::function; + + //! Schedules a batch of processes for launch using a round robin approach to distribute the in-flight processes over + //! the specified number of concurrent process slots. + class ProcessScheduler + { + public: + //! Constructs the scheduler with the specified batch of processes. + //! @param processes The batch of processes to schedule. + //! @param processLaunchCallback The process launch callback function. + //! @param processExitCallback The process exit callback function. + //! @param maxConcurrentProcesses The maximum number of concurrent processes in-flight. + //! @param processTimeout The maximum duration a process may be in-flight for before being forcefully terminated. + //! @param scheduleTimeout The maximum duration the scheduler may run before forcefully terminating all in-flight processes. + //! processes and abandoning any queued processes. + ProcessScheduler( + const AZStd::vector& processes, + const ProcessLaunchCallback& processLaunchCallback, + const ProcessExitCallback& processExitCallback, + size_t maxConcurrentProcesses, + AZStd::optional processTimeout, + AZStd::optional scheduleTimeout); + + ~ProcessScheduler(); + + private: + struct ProcessInFlight; + + void MonitorProcesses(); + CallbackResult PopAndLaunch(ProcessInFlight& processInFlight); + void TerminateAllProcesses(ExitCondition exitStatus); + StdContent ConsumeProcessStdContent(ProcessInFlight& processInFlight); + void AccumulateProcessStdContent(ProcessInFlight& processInFlight); + + const ProcessLaunchCallback m_processCreateCallback; + const ProcessExitCallback m_processExitCallback; + const AZStd::optional m_processTimeout; + const AZStd::optional m_scheduleTimeout; + const AZStd::chrono::high_resolution_clock::time_point m_startTime; + AZStd::vector m_processPool; + AZStd::queue m_processQueue; + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/TestImpactProcess.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/TestImpactProcess.cpp new file mode 100644 index 0000000000..3edc5af28f --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/TestImpactProcess.cpp @@ -0,0 +1,32 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include + +namespace TestImpact +{ + Process::Process(const ProcessInfo& processInfo) + : m_processInfo(processInfo) + { + } + + const ProcessInfo& Process::GetProcessInfo() const + { + return m_processInfo; + } + + AZStd::optional Process::GetReturnCode() const + { + return m_returnCode; + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/TestImpactProcess.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/TestImpactProcess.h new file mode 100644 index 0000000000..763290d787 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/TestImpactProcess.h @@ -0,0 +1,61 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +#include + +namespace TestImpact +{ + //! Abstraction of platform-specific process. + class Process + { + public: + explicit Process(const ProcessInfo& processInfo); + virtual ~Process() = default; + + //! Terminates the process with the specified return code. + virtual void Terminate(ReturnCode returnCode) = 0; + + //! Block the calling thread until the process exits. + virtual void BlockUntilExit() = 0; + + //! Returns whether or not the process is still running. + virtual bool IsRunning() const = 0; + + //! Returns the process info associated with this process. + const ProcessInfo& GetProcessInfo() const; + + //! Returns the return code of the exited process. + //! Will be empty if the process is still running or was not successfully launched. + AZStd::optional GetReturnCode() const; + + //! Flushes the internal buffer and returns the process's buffered standard output. + //! Subsequent calls will keep returning data so long as the process is producing output. + //! Will return nullopt if no output routing or no output produced. + virtual AZStd::optional ConsumeStdOut() = 0; + + //! Flushes the internal buffer and returns the process's buffered standard error. + //! Subsequent calls will keep returning data so long as the process is producing errors. + //! Will return nullopt if no error routing or no errors produced. + virtual AZStd::optional ConsumeStdErr() = 0; + + protected: + //! The information used to launch the process. + ProcessInfo m_processInfo; + + //! The return code of a successfully launched process (otherwise is empty) + AZStd::optional m_returnCode; + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/TestImpactProcessException.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/TestImpactProcessException.h new file mode 100644 index 0000000000..0c53800287 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/TestImpactProcessException.h @@ -0,0 +1,26 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +namespace TestImpact +{ + //! Exception for processes and process-related operations. + class ProcessException + : public Exception + { + public: + using Exception::Exception; + }; +} diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/TestImpactProcessInfo.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/TestImpactProcessInfo.cpp new file mode 100644 index 0000000000..8dd81e4a21 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/TestImpactProcessInfo.cpp @@ -0,0 +1,67 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include + +namespace TestImpact +{ + ProcessInfo::ProcessInfo(ProcessId id, const AZ::IO::Path& processPath, const AZStd::string& startupArgs) + : m_id(id) + , m_parentHasStdOutput(false) + , m_parentHasStdErr(false) + , m_processPath(processPath) + , m_startupArgs(startupArgs) + { + AZ_TestImpact_Eval(processPath.String().length() > 0, ProcessException, "Process path cannot be empty"); + } + + ProcessInfo::ProcessInfo( + ProcessId id, + StdOutputRouting stdOut, + StdErrorRouting stdErr, + const AZ::IO::Path& processPath, + const AZStd::string& startupArgs) + : m_id(id) + , m_processPath(processPath) + , m_startupArgs(startupArgs) + , m_parentHasStdOutput(stdOut == StdOutputRouting::ToParent ? true : false) + , m_parentHasStdErr(stdErr == StdErrorRouting::ToParent ? true : false) + { + AZ_TestImpact_Eval(processPath.String().length() > 0, ProcessException, "Process path cannot be empty"); + } + + ProcessId ProcessInfo::GetId() const + { + return m_id; + } + + const AZ::IO::Path& ProcessInfo::GetProcessPath() const + { + return m_processPath; + } + + const AZStd::string& ProcessInfo::GetStartupArgs() const + { + return m_startupArgs; + } + + bool ProcessInfo::ParentHasStdOutput() const + { + return m_parentHasStdOutput; + } + + bool ProcessInfo::ParentHasStdError() const + { + return m_parentHasStdErr; + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/TestImpactProcessInfo.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/TestImpactProcessInfo.h new file mode 100644 index 0000000000..b73613db0d --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/TestImpactProcessInfo.h @@ -0,0 +1,93 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include +#include +#include + +namespace TestImpact +{ + //! Identifier to distinguish between processes. + using ProcessId = size_t; + + //! Return code of successfully launched process. + using ReturnCode = int; + + //! Error code for processes that are forcefully terminated whilst in-flight by the client. + inline constexpr const ReturnCode ProcessTerminateErrorCode = 0xF10BAD; + + //! Error code for processes that are forcefully terminated whilst in-flight by the scheduler due to timing out. + inline constexpr const ReturnCode ProcessTimeoutErrorCode = 0xBADF10; + + //! Specifier for how the process's standard out willt be routed + enum class StdOutputRouting + { + ToParent, + None + }; + + enum class StdErrorRouting + { + ToParent, + None + }; + + //! Container for process standard output and standard error. + struct StdContent + { + AZStd::optional m_out; + AZStd::optional m_err; + }; + + //! Information about a process the arguments used to launch it. + class ProcessInfo + { + public: + //! Provides the information required to launch a process. + //! @param processId Client-supplied id to diffrentiate between processes. + //! @param stdOut Routing of process standard output. + //! @param stdErr Routing of process standard error. + //! @param processPath Path to executable binary to launch. + //! @param startupArgs Arguments to launch the process with. + ProcessInfo( + ProcessId processId, + StdOutputRouting stdOut, + StdErrorRouting stdErr, + const AZ::IO::Path& processPath, + const AZStd::string& startupArgs = ""); + ProcessInfo(ProcessId processId, const AZ::IO::Path& processPath, const AZStd::string& startupArgs = ""); + + //! Returns the identifier of this process. + ProcessId GetId() const; + + //! Returns whether or not stdoutput is routed to the parent process. + bool ParentHasStdOutput() const; + + //! Returns whether or not stderror is routed to the parent process. + bool ParentHasStdError() const; + + // Returns the path to the process binary. + const AZ::IO::Path& GetProcessPath() const; + + //! Returns the command line arguments used to launch the process. + const AZStd::string& GetStartupArgs() const; + + private: + const ProcessId m_id; + const bool m_parentHasStdOutput; + const bool m_parentHasStdErr; + const AZ::IO::Path m_processPath; + const AZStd::string m_startupArgs; + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/TestImpactProcessLauncher.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/TestImpactProcessLauncher.h new file mode 100644 index 0000000000..cefd16f003 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/TestImpactProcessLauncher.h @@ -0,0 +1,27 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +#include + +namespace TestImpact +{ + class Process; + class ProcessInfo; + + //! Attempts to launch a process with the provided command line arguments. + //! @param processInfo The path and command line arguments to launch the process with. + AZStd::unique_ptr LaunchProcess(const ProcessInfo& processInfo); +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactBuildTarget.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactBuildTarget.cpp new file mode 100644 index 0000000000..bce6fa25d6 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactBuildTarget.cpp @@ -0,0 +1,48 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include "TestImpactBuildTarget.h" + +namespace TestImpact +{ + BuildTarget::BuildTarget(BuildTargetDescriptor&& descriptor, TargetType type) + : m_buildMetaData(AZStd::move(descriptor.m_buildMetaData)) + , m_sources(AZStd::move(descriptor.m_sources)) + , m_type(type) + { + } + + const AZStd::string& BuildTarget::GetName() const + { + return m_buildMetaData.m_name; + } + + const AZStd::string& BuildTarget::GetOutputName() const + { + return m_buildMetaData.m_outputName; + } + + const AZ::IO::Path& BuildTarget::GetPath() const + { + return m_buildMetaData.m_path; + } + + const TargetSources& BuildTarget::GetSources() const + { + return m_sources; + } + + TargetType BuildTarget::GetType() const + { + return m_type; + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactBuildTarget.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactBuildTarget.h new file mode 100644 index 0000000000..d4346c15ef --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactBuildTarget.h @@ -0,0 +1,55 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +#include + +namespace TestImpact +{ + //! Type id for querying specialized derived target types from base pointer/reference. + enum class TargetType : bool + { + Production, //!< Production build target. + Test //!< Test build target. + }; + + //! Representation of a generic build target in the repository. + class BuildTarget + { + public: + BuildTarget(BuildTargetDescriptor&& descriptor, TargetType type); + virtual ~BuildTarget() = default; + + //! Returns the build target name. + const AZStd::string& GetName() const; + + //! Returns the build target's compiled binary name. + const AZStd::string& GetOutputName() const; + + //! Returns the path in the source tree to the build target location. + const AZ::IO::Path& GetPath() const; + + //! Returns the build target's sources. + const TargetSources& GetSources() const; + + //! Returns the build target type. + TargetType GetType() const; + + private: + BuildMetaData m_buildMetaData; + TargetSources m_sources; + TargetType m_type; + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactBuildTargetList.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactBuildTargetList.h new file mode 100644 index 0000000000..dfd102b55f --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactBuildTargetList.h @@ -0,0 +1,125 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +#include +#include +#include +#include + +#include + +namespace TestImpact +{ + //! Container for unique set of sorted build target types. + //! @tparam Target The specialized build target type. + template + class BuildTargetList + { + public: + using TargetType = Target; + + BuildTargetList(AZStd::vector&& descriptors); + + //! Returns the targets in the collection. + const AZStd::vector& GetTargets() const; + + //! Returns the target with the specified name. + const Target* GetTarget(const AZStd::string& name) const; + + //! Returns the target with the specified name or throws if target not found. + const Target* GetTargetOrThrow(const AZStd::string& name) const; + + // Returns the number of targets in the list. + size_t GetNumTargets() const; + + private: + AZStd::vector m_targets; + }; + + template + BuildTargetList::BuildTargetList(AZStd::vector&& descriptors) + { + AZ_TestImpact_Eval(!descriptors.empty(), TargetException, "Target list is empty"); + + AZStd::sort( + descriptors.begin(), descriptors.end(), [](const typename Target::Descriptor& lhs, const typename Target::Descriptor& rhs) + { + return lhs.m_buildMetaData.m_name < rhs.m_buildMetaData.m_name; + }); + + const auto duplicateElement = AZStd::adjacent_find( + descriptors.begin(), descriptors.end(), [](const typename Target::Descriptor& lhs, const typename Target::Descriptor& rhs) + { + return lhs.m_buildMetaData.m_name == rhs.m_buildMetaData.m_name; + }); + + AZ_TestImpact_Eval(duplicateElement == descriptors.end(), TargetException, "Target list contains duplicate targets"); + + m_targets.reserve(descriptors.size()); + for (auto&& descriptor : descriptors) + { + m_targets.emplace_back(Target(AZStd::move(descriptor))); + } + } + + template + const AZStd::vector& BuildTargetList::GetTargets() const + { + return m_targets; + } + + template + size_t BuildTargetList::GetNumTargets() const + { + return m_targets.size(); + } + + template + const Target* BuildTargetList::GetTarget(const AZStd::string& name) const + { + struct TargetComparator + { + bool operator()(const Target& target, const AZStd::string& name) const + { + return target.GetName() < name; + } + + bool operator()(const AZStd::string& name, const Target& target) const + { + return name < target.GetName(); + } + }; + + const auto targetRange = std::equal_range(m_targets.begin(), m_targets.end(), name, TargetComparator{}); + + if (targetRange.first != targetRange.second) + { + return targetRange.first; + } + else + { + return nullptr; + } + } + + template + const Target* BuildTargetList::GetTargetOrThrow(const AZStd::string& name) const + { + const Target* target = GetTarget(name); + AZ_TestImpact_Eval(target, TargetException, AZStd::string::format("Couldn't find target %s", name.c_str()).c_str()); + return target; + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactProductionTarget.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactProductionTarget.cpp new file mode 100644 index 0000000000..88c3cf932a --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactProductionTarget.cpp @@ -0,0 +1,21 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include "TestImpactProductionTarget.h" + +namespace TestImpact +{ + ProductionTarget::ProductionTarget(Descriptor&& descriptor) + : BuildTarget(AZStd::move(descriptor), TargetType::Production) + { + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactProductionTarget.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactProductionTarget.h new file mode 100644 index 0000000000..d4ddb6dc50 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactProductionTarget.h @@ -0,0 +1,31 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include +#include + +namespace TestImpact +{ + //! Build target specialization for production targets (build targets containing production code and no test code). + class ProductionTarget + : public BuildTarget + { + public: + using Descriptor = ProductionTargetDescriptor; + ProductionTarget(Descriptor&& descriptor); + }; + + template + inline constexpr bool IsProductionTarget = AZStd::is_same_v>>>; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactProductionTargetList.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactProductionTargetList.h new file mode 100644 index 0000000000..d2f23abe2e --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactProductionTargetList.h @@ -0,0 +1,22 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include +#include + +namespace TestImpact +{ + //! Container for set of sorted production targets containing no duplicates. + using ProductionTargetList = BuildTargetList; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactTargetException.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactTargetException.h new file mode 100644 index 0000000000..d3f2ec25ee --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactTargetException.h @@ -0,0 +1,25 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +namespace TestImpact +{ + //! Exception for target and target-related operations. + class TargetException : public Exception + { + public: + using Exception::Exception; + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactTestTarget.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactTestTarget.cpp new file mode 100644 index 0000000000..0189bc9e78 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactTestTarget.cpp @@ -0,0 +1,32 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include "TestImpactTestTarget.h" + +namespace TestImpact +{ + TestTarget::TestTarget(Descriptor&& descriptor) + : BuildTarget(AZStd::move(descriptor), TargetType::Test) + , m_testMetaData(AZStd::move(descriptor.m_testMetaData)) + { + } + + const AZStd::string& TestTarget::GetSuite() const + { + return m_testMetaData.m_suite; + } + + LaunchMethod TestTarget::GetLaunchMethod() const + { + return m_testMetaData.m_launchMethod; + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactTestTarget.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactTestTarget.h new file mode 100644 index 0000000000..b6f44e7cc1 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactTestTarget.h @@ -0,0 +1,41 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include +#include + +namespace TestImpact +{ + //! Build target specialization for test targets (build targets containing test code and no production code). + class TestTarget + : public BuildTarget + { + public: + using Descriptor = TestTargetDescriptor; + + TestTarget(Descriptor&& descriptor); + + //! Returns the test target suite. + const AZStd::string& GetSuite() const; + + //! Returns the test target launch method. + LaunchMethod GetLaunchMethod() const; + + private: + const TestTargetMeta m_testMetaData; + }; + + template + inline constexpr bool IsTestTarget = AZStd::is_same_v>>>; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactTestTargetList.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactTestTargetList.h new file mode 100644 index 0000000000..f8cafcb735 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactTestTargetList.h @@ -0,0 +1,22 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include +#include + +namespace TestImpact +{ + //! Container for set of sorted test targets containing no duplicates. + using TestTargetList = BuildTargetList; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumeration.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumeration.h new file mode 100644 index 0000000000..c12abec006 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumeration.h @@ -0,0 +1,22 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include +#include + +namespace TestImpact +{ + //! Representation of a given test target's enumerated tests. + using TestEnumeration = TestSuiteContainer; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerationException.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerationException.h new file mode 100644 index 0000000000..0c9d27df7a --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerationException.h @@ -0,0 +1,26 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +namespace TestImpact +{ + //! Exception for test enumerations and test enumeration related operations. + class TestEnumerationException + : public Exception + { + public: + using Exception::Exception; + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerationSerializer.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerationSerializer.cpp new file mode 100644 index 0000000000..0b171ba404 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerationSerializer.cpp @@ -0,0 +1,100 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include + +#include +#include +#include +#include + +namespace TestImpact +{ + namespace + { + // Keys for pertinent JSON node and attribute names + constexpr const char* Keys[] = + { + "suites", + "name", + "enabled", + "tests" + }; + + enum + { + SuitesKey, + NameKey, + EnabledKey, + TestsKey + }; + } // namespace + + AZStd::string SerializeTestEnumeration(const TestEnumeration& testEnum) + { + rapidjson::StringBuffer stringBuffer; + rapidjson::PrettyWriter writer(stringBuffer); + + writer.StartObject(); + writer.Key(Keys[SuitesKey]); + writer.StartArray(); + for (const auto& suite : testEnum.GetTestSuites()) + { + writer.StartObject(); + writer.Key(Keys[NameKey]); + writer.String(suite.m_name.c_str()); + writer.Key(Keys[EnabledKey]); + writer.Bool(suite.m_enabled); + writer.Key(Keys[TestsKey]); + writer.StartArray(); + for (const auto& test : suite.m_tests) + { + writer.StartObject(); + writer.Key(Keys[NameKey]); + writer.String(test.m_name.c_str()); + writer.Key(Keys[EnabledKey]); + writer.Bool(test.m_enabled); + writer.EndObject(); + } + writer.EndArray(); + writer.EndObject(); + } + writer.EndArray(); + writer.EndObject(); + + return stringBuffer.GetString(); + } + + TestEnumeration DeserializeTestEnumeration(const AZStd::string& testEnumString) + { + AZStd::vector testSuites; + rapidjson::Document doc; + + if (doc.Parse<0>(testEnumString.c_str()).HasParseError()) + { + throw TestEnumerationException("Could not parse enumeration data"); + } + + for (const auto& suite : doc[Keys[SuitesKey]].GetArray()) + { + testSuites.emplace_back(TestEnumerationSuite{suite[Keys[NameKey]].GetString(), suite[Keys[EnabledKey]].GetBool(), {}}); + for (const auto& test : suite[Keys[TestsKey]].GetArray()) + { + testSuites.back().m_tests.emplace_back( + TestEnumerationCase{test[Keys[NameKey]].GetString(), test[Keys[EnabledKey]].GetBool()}); + } + } + + return TestEnumeration(std::move(testSuites)); + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerationSerializer.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerationSerializer.h new file mode 100644 index 0000000000..422308d862 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerationSerializer.h @@ -0,0 +1,26 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +#include + +namespace TestImpact +{ + //! Serializes the specified test enumeration to JSON format. + AZStd::string SerializeTestEnumeration(const TestEnumeration& testEnumeration); + + //! Deserializes a test enumeration from the specified test enumeration data in JSON format. + TestEnumeration DeserializeTestEnumeration(const AZStd::string& testEnumerationString); +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerator.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerator.cpp new file mode 100644 index 0000000000..1964f8876e --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerator.cpp @@ -0,0 +1,190 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include +#include +#include +#include + +#include + +namespace TestImpact +{ + namespace + { + void WriteCacheFile( + const TestEnumeration& enumeration, const AZ::IO::Path& path, Bitwise::CacheExceptionPolicy cacheExceptionPolicy) + { + const AZStd::string cacheJSON = SerializeTestEnumeration(enumeration); + const AZStd::vector cacheBytes(cacheJSON.begin(), cacheJSON.end()); + AZ::IO::SystemFile cacheFile; + + if (!cacheFile.Open( + path.c_str(), + AZ::IO::SystemFile::SF_OPEN_CREATE | AZ::IO::SystemFile::SF_OPEN_CREATE_PATH | AZ::IO::SystemFile::SF_OPEN_WRITE_ONLY)) + { + AZ_TestImpact_Eval( + !IsFlagSet(cacheExceptionPolicy, Bitwise::CacheExceptionPolicy::OnCacheWriteFailure), TestEnumerationException, + "Couldn't open cache file for writing"); + return; + } + + if (cacheFile.Write(cacheBytes.data(), cacheBytes.size()) == 0) + { + AZ_TestImpact_Eval( + !IsFlagSet(cacheExceptionPolicy, Bitwise::CacheExceptionPolicy::OnCacheWriteFailure), TestEnumerationException, + "Couldn't write cache file data"); + return; + } + } + + AZStd::optional ReadCacheFile(const AZ::IO::Path& path, Bitwise::CacheExceptionPolicy cacheExceptionPolicy) + { + AZ::IO::SystemFile cacheFile; + AZStd::string cacheJSON; + if (!cacheFile.Open(path.c_str(), AZ::IO::SystemFile::SF_OPEN_READ_ONLY)) + { + AZ_TestImpact_Eval( + !IsFlagSet(cacheExceptionPolicy, Bitwise::CacheExceptionPolicy::OnCacheNotExist), TestEnumerationException, + "Couldn't locate cache file"); + return AZStd::nullopt; + } + + const AZ::IO::SystemFile::SizeType length = cacheFile.Length(); + if (length == 0) + { + AZ_TestImpact_Eval( + !IsFlagSet(cacheExceptionPolicy, Bitwise::CacheExceptionPolicy::OnCacheReadFailure), TestEnumerationException, + "Cache file is empty"); + return AZStd::nullopt; + } + + cacheFile.Seek(0, AZ::IO::SystemFile::SF_SEEK_BEGIN); + cacheJSON.resize(length); + if (cacheFile.Read(length, cacheJSON.data()) != length) + { + AZ_TestImpact_Eval( + !IsFlagSet(cacheExceptionPolicy, Bitwise::CacheExceptionPolicy::OnCacheReadFailure), TestEnumerationException, + "Couldn't read cache file"); + return AZStd::nullopt; + } + + return DeserializeTestEnumeration(cacheJSON); + } + } // namespace + + TestEnumeration ParseTestEnumerationFile(const AZ::IO::Path& enumerationFile) + { + return TestEnumeration(GTest::TestEnumerationSuitesFactory(ReadFileContents(enumerationFile))); + } + + TestEnumerationJobData::TestEnumerationJobData(const AZ::IO::Path& enumerationArtifact, AZStd::optional&& cache) + : m_enumerationArtifact(enumerationArtifact) + , m_cache(AZStd::move(cache)) + { + } + + const AZ::IO::Path& TestEnumerationJobData::GetEnumerationArtifactPath() const + { + return m_enumerationArtifact; + } + + const AZStd::optional& TestEnumerationJobData::GetCache() const + { + return m_cache; + } + + TestEnumerator::TestEnumerator( + AZStd::optional clientCallback, + size_t maxConcurrentEnumerations, + AZStd::optional enumerationTimeout, + AZStd::optional enumeratorTimeout) + : JobRunner(clientCallback, AZStd::nullopt, StdOutputRouting::None, StdErrorRouting::None, maxConcurrentEnumerations, enumerationTimeout, enumeratorTimeout) + { + } + + AZStd::vector TestEnumerator::Enumerate( + const AZStd::vector& jobInfos, + CacheExceptionPolicy cacheExceptionPolicy, + JobExceptionPolicy jobExceptionPolicy) + { + AZStd::vector cachedJobs; + AZStd::vector jobQueue; + + for (const auto& jobInfo : jobInfos) + { + // If this job has a cache read policy attempt to read the cache + if (jobInfo.GetCache().has_value() && jobInfo.GetCache()->m_policy == JobData::CachePolicy::Read) + { + JobMeta meta; + auto enumeration = ReadCacheFile(jobInfo.GetCache()->m_file, cacheExceptionPolicy); + + // Even though cached jobs don't get executed we still give the client the opportunity to handle the job state + // change in order to make the caching process transparent to the client + if (enumeration.has_value()) + { + if (m_clientJobCallback.has_value()) + { + (*m_clientJobCallback)(jobInfo, meta); + } + + // Cache read successfully, this job will not be placed in the job queue + cachedJobs.emplace_back(Job(jobInfo, AZStd::move(meta), AZStd::move(enumeration))); + } + else + { + // The cache read failed and exception policy for cache read failures is not to throw so instead place this job in the + // job queue + jobQueue.emplace_back(jobInfo); + } + } + else + { + // This job has no cache read policy so place in job queue + jobQueue.emplace_back(jobInfo); + } + } + + const auto payloadGenerator = [this, cacheExceptionPolicy](const JobDataMap& jobDataMap) + { + PayloadMap enumerations; + for (const auto& [jobId, jobData] : jobDataMap) + { + const auto& [meta, jobInfo] = jobData; + if (meta.m_result == JobResult::ExecutedWithSuccess) + { + const auto& enumeration = enumerations[jobId] = ParseTestEnumerationFile(jobInfo->GetEnumerationArtifactPath()); + + // Write out the enumeration to a cache file if we have a cache write policy for this job + if (jobInfo->GetCache().has_value() && jobInfo->GetCache()->m_policy == JobData::CachePolicy::Write) + { + WriteCacheFile(enumeration.value(), jobInfo->GetCache()->m_file, cacheExceptionPolicy); + } + } + } + + return enumerations; + }; + + // Generate the enumeration results for the jobs that weren't cached + auto jobs = ExecuteJobs(jobQueue, payloadGenerator, jobExceptionPolicy); + + // We need to add the cached jobs to the completed job list even though they technically weren't executed + for (auto&& job : cachedJobs) + { + jobs.emplace_back(AZStd::move(job)); + } + + return jobs; + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerator.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerator.h new file mode 100644 index 0000000000..a005a32b0d --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerator.h @@ -0,0 +1,94 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include +#include +#include + +#include + +namespace TestImpact +{ + //! Per-job data for test enumerations. + class TestEnumerationJobData + { + public: + //! Policy for how a test enumeration will be written/read from the a previous cache instead of enumerated from the test target. + enum class CachePolicy + { + Read, //!< Do read from a cache file but do not overwrite any existing cache file. + Write //!< Do not read from a cache file but instead overwrite any existing cache file. + }; + + //! Cache configuration for a given test enumeration command. + struct Cache + { + CachePolicy m_policy; + AZ::IO::Path m_file; + }; + + TestEnumerationJobData(const AZ::IO::Path& enumerationArtifact, AZStd::optional&& cache); + + //! Returns the path to the enumeration artifact produced by the test target. + const AZ::IO::Path& GetEnumerationArtifactPath() const; + + //! Returns the cache details for this job. + const AZStd::optional& GetCache() const; + + private: + AZ::IO::Path m_enumerationArtifact; //!< Path to enumeration artifact to be processed. + AZStd::optional m_cache = AZStd::nullopt; //!< No caching takes place if cache is empty. + }; + + namespace Bitwise + { + //! Exception policy for test enumeration cache reads/writes. + enum class CacheExceptionPolicy + { + Never = 0, //! Never throw. + OnCacheNotExist = 1, //! Throw when a cache read policy was in place but a cache file for this job doesn't exist. + OnCacheReadFailure = 1 << 1, //! Throw when a cache read policy is in place but the cache file could not be read. + OnCacheWriteFailure = 1 << 2 //! Throw when a cache write policy is in place but the cache file could not be written. + }; + } // namespace Bitwise + + //! Enumerate a batch of test targets to determine the test suites and fixtures they contain, caching the results where applicable. + class TestEnumerator + : public TestJobRunner + { + using JobRunner = TestJobRunner; + + public: + using CacheExceptionPolicy = Bitwise::CacheExceptionPolicy; + + //! Constructs a test enumerator with the specified parameters common to all enumeration job runs of this enumerator. + //! @param clientCallback The optional client callback to be called whenever an enumeration job changes state. + //! @param maxConcurrentEnumerations The maximum number of enumerations to be in flight at any given time. + //! @param enumerationTimeout The maximum duration an enumeration may be in-flight for before being forcefully terminated. + //! @param enumeratorTimeout The maximum duration the enumerator may run before forcefully terminating all in-flight enumerations. + TestEnumerator( + AZStd::optional clientCallback, + size_t maxConcurrentEnumerations, + AZStd::optional enumerationTimeout, + AZStd::optional enumeratorTimeout); + + //! Executes the specified test enumeration jobs according to the specified cache and job exception policies. + //! @param jobInfos The enumeration jobs to execute. + //! @param cacheExceptionPolicy The cache exception policy to be used for this run. + //! @param jobExceptionPolicy The enumeration job exception policy to be used for this run. + //! @return the test enumeration jobs with their associated test enumeration payloads. + AZStd::vector Enumerate( + const AZStd::vector& jobInfos, CacheExceptionPolicy cacheExceptionPolicy, JobExceptionPolicy jobExceptionPolicy); + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Job/TestImpactTestJobCommon.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Job/TestImpactTestJobCommon.h new file mode 100644 index 0000000000..ee245f1ff3 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Job/TestImpactTestJobCommon.h @@ -0,0 +1,36 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include + +#include +#include +#include +#include + +namespace TestImpact +{ + template + AZStd::string ReadFileContents(const AZ::IO::Path& file) + { + static_assert(AZStd::is_base_of::value, "Exception must be a TestImpact exception or derived type"); + + const auto fileSize = AZ::IO::SystemFile::Length(file.c_str()); + AZ_TestImpact_Eval(fileSize > 0, ExceptionType, AZStd::string::format("File %s does not exist", file.c_str())); + + AZStd::vector buffer(fileSize + 1); + buffer[fileSize] = '\0'; + AZ_TestImpact_Eval(AZ::IO::SystemFile::Read(file.c_str(), buffer.data()), ExceptionType, "Could not read file contents"); + + return AZStd::string(buffer.begin(), buffer.end()); + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Job/TestImpactTestJobException.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Job/TestImpactTestJobException.h new file mode 100644 index 0000000000..263d70322e --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Job/TestImpactTestJobException.h @@ -0,0 +1,26 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +namespace TestImpact +{ + //! Exception for test job related operations. + class TestJobException + : public Exception + { + public: + using Exception::Exception; + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Job/TestImpactTestJobRunner.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Job/TestImpactTestJobRunner.h new file mode 100644 index 0000000000..a4a0b19845 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Job/TestImpactTestJobRunner.h @@ -0,0 +1,141 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +#include +#include +#include + +#include +#include +#include + +namespace TestImpact +{ + namespace Bitwise + { + //! Exception policy for test jobs (and derived jobs). + enum class TestJobExceptionPolicy + { + Never = 0, //!< Never throw. + OnFailedToExecute = 1, //!< Throw when a job fails to execute. + OnExecutedWithFailure = 1 << 1 //!< Throw when a job returns with an error code. + }; + } // namespace Bitwise + + //! Base class for test related job runners. + //! @tparam AdditionalInfo The data structure containing the information additional to the command arguments necessary to execute and + //! complete a job. + //! @tparam Payload The output produced by a job. + template + class TestJobRunner + { + public: + using JobData = AdditionalInfo; + using JobInfo = JobInfo; + using JobPayload = Payload; + using Job = Job; + using ClientJobCallback = AZStd::function; + using DerivedJobCallback = JobCallback; + using JobExceptionPolicy = Bitwise::TestJobExceptionPolicy; + using JobDataMap = JobDataMap; + + //! Constructs the job runner with the specified parameters common to all job runs of this runner. + //! @param clientCallback The optional callback function provided by the client to be called upon job state change. + //! @param clientCallback The optional callback function provided by the derived job runner to be called upon job state change. + //! @param stdOutRouting The standard output routing from the underlying job processes to the derived runner. + //! @param stdErrorRouting The standard error routing from the underlying job processes to the derived runner. + //! @param maxConcurrentJobs The maximum number of jobs to be in flight at any given time. + //! @param jobTimeout The maximum duration a job may be in-flight for before being forcefully terminated (nullopt if no timeout). + //! @param runnerTimeout The maximum duration the runner may run before forcefully terminating all in-flight jobs (nullopt if no + //! timeout). + TestJobRunner( + AZStd::optional clientCallback, + AZStd::optional derivedJobCallback, + StdOutputRouting stdOutRouting, + StdErrorRouting stdErrRouting, + size_t maxConcurrentJobs, + AZStd::optional jobTimeout, + AZStd::optional runnerTimeout); + + protected: + //! Runs the specified jobs and returns the completed payloads produced by each job. + //! @param jobInfos The batch of jobs to execute. + //! @param payloadMapProducer The client callback for producing the payload map based on the completed job data. + //! @param jobExceptionPolicy The job execution policy for this job run. + AZStd::vector ExecuteJobs( + const AZStd::vector& jobInfos, + PayloadMapProducer payloadMapProducer, + JobExceptionPolicy jobExceptionPolicy); + + const AZStd::optional m_clientJobCallback; + + private: + JobRunner m_jobRunner; + const AZStd::optional m_derivedJobCallback; + }; + + template + TestJobRunner::TestJobRunner( + AZStd::optional clientCallback, + AZStd::optional derivedJobCallback, + StdOutputRouting stdOutRouting, + StdErrorRouting stdErrRouting, + size_t maxConcurrentJobs, + AZStd::optional jobTimeout, + AZStd::optional runnerTimeout) + : m_jobRunner(stdOutRouting, stdErrRouting, maxConcurrentJobs, jobTimeout, runnerTimeout) + , m_clientJobCallback(clientCallback) + , m_derivedJobCallback(derivedJobCallback) + { + } + + template + AZStd::vector::Job> TestJobRunner::ExecuteJobs( + const AZStd::vector& jobInfos, + PayloadMapProducer payloadMapProducer, + JobExceptionPolicy jobExceptionPolicy) + { + // Callback to handle job exception policies and client/derived callbacks + const auto jobCallback = [this, &jobExceptionPolicy](const JobInfo& jobInfo, const JobMeta& meta, StdContent&& std) + { + if (meta.m_result == JobResult::FailedToExecute && IsFlagSet(jobExceptionPolicy, JobExceptionPolicy::OnFailedToExecute)) + { + throw TestJobException("Job failed to execute"); + } + else if (meta.m_result == JobResult::ExecutedWithFailure && IsFlagSet(jobExceptionPolicy, JobExceptionPolicy::OnExecutedWithFailure)) + { + throw TestJobException("Job executed with failure"); + } + + if (m_derivedJobCallback.has_value()) + { + if (const auto result = (*m_derivedJobCallback)(jobInfo, meta, AZStd::move(std)); result != CallbackResult::Continue) + { + return result; + } + } + + if (m_clientJobCallback.has_value()) + { + (*m_clientJobCallback)(jobInfo, meta); + } + + return CallbackResult::Continue; + }; + + return m_jobRunner.Execute(jobInfos, jobCallback, payloadMapProducer); + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactInstrumentedTestRunner.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactInstrumentedTestRunner.cpp new file mode 100644 index 0000000000..3b7a088fb4 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactInstrumentedTestRunner.cpp @@ -0,0 +1,89 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include +#include +#include +#include +#include + +#include + +namespace TestImpact +{ + InstrumentedTestRunJobData::InstrumentedTestRunJobData(const AZ::IO::Path& resultsArtifact, const AZ::IO::Path& coverageArtifact) + : TestRunJobData(resultsArtifact) + , m_coverageArtifact(coverageArtifact) + { + } + + const AZ::IO::Path& InstrumentedTestRunJobData::GetCoverageArtifactPath() const + { + return m_coverageArtifact; + } + + InstrumentedTestRunner::JobPayload ParseTestRunAndCoverageFiles( + const AZ::IO::Path& runFile, + const AZ::IO::Path& coverageFile, + AZStd::chrono::milliseconds duration, + InstrumentedTestRunner::CoverageExceptionPolicy coverageExceptionPolicy) + { + TestRun run(GTest::TestRunSuitesFactory(ReadFileContents(runFile)), duration); + AZStd::vector moduleCoverages = Cobertura::ModuleCoveragesFactory(ReadFileContents(coverageFile)); + if (moduleCoverages.empty()) + { + AZ_TestImpact_Eval( + !IsFlagSet(coverageExceptionPolicy, Bitwise::CoverageExceptionPolicy::OnEmptyCoverage), TestRunException, + "No coverage data generated"); + } + + TestCoverage coverage(AZStd::move(moduleCoverages)); + return {AZStd::move(run), AZStd::move(coverage)}; + } + + InstrumentedTestRunner::InstrumentedTestRunner( + AZStd::optional clientCallback, + size_t maxConcurrentRuns, + AZStd::optional runTimeout, + AZStd::optional runnerTimeout) + : JobRunner(clientCallback, AZStd::nullopt, StdOutputRouting::None, StdErrorRouting::None, maxConcurrentRuns, runTimeout, runnerTimeout) + { + } + + AZStd::vector InstrumentedTestRunner::RunInstrumentedTests( + const AZStd::vector& jobInfos, + CoverageExceptionPolicy coverageExceptionPolicy, + JobExceptionPolicy jobExceptionPolicy) + { + const auto payloadGenerator = [this, coverageExceptionPolicy](const JobDataMap& jobDataMap) + { + PayloadMap runs; + for (const auto& [jobId, jobData] : jobDataMap) + { + const auto& [meta, jobInfo] = jobData; + if (meta.m_result == JobResult::ExecutedWithSuccess || meta.m_result == JobResult::ExecutedWithFailure) + { + runs[jobId] = ParseTestRunAndCoverageFiles( + jobInfo->GetRunArtifactPath(), + jobInfo->GetCoverageArtifactPath(), + meta.m_duration.value(), + coverageExceptionPolicy); + } + } + + return runs; + }; + + return ExecuteJobs(jobInfos, payloadGenerator, jobExceptionPolicy); + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactInstrumentedTestRunner.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactInstrumentedTestRunner.h new file mode 100644 index 0000000000..e6f059b5d3 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactInstrumentedTestRunner.h @@ -0,0 +1,71 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include +#include +#include +#include + +namespace TestImpact +{ + //! Per-job data for instrumented test runs. + class InstrumentedTestRunJobData : public TestRunJobData + { + public: + InstrumentedTestRunJobData(const AZ::IO::Path& resultsArtifact, const AZ::IO::Path& coverageArtifact); + + //! Returns the path to the coverage artifact produced by the test target. + const AZ::IO::Path& GetCoverageArtifactPath() const; + + private: + AZ::IO::Path m_coverageArtifact; //!< Path to coverage data. + }; + + namespace Bitwise + { + //! Exception policy for test coverage artifacts. + enum class CoverageExceptionPolicy + { + Never = 0, //! Never throw. + OnEmptyCoverage = 1 //! Throw when no coverage data was produced. + }; + } // namespace Bitwise + + //! Runs a batch of test targets to determine the test coverage and passes/failures. + class InstrumentedTestRunner : public TestJobRunner> + { + using JobRunner = TestJobRunner>; + + public: + using CoverageExceptionPolicy = Bitwise::CoverageExceptionPolicy; + + //! Constructs an instrumented test runner with the specified parameters common to all job runs of this runner. + //! @param clientCallback The optional client callback to be called whenever a run job changes state. + //! @param maxConcurrentRuns The maximum number of runs to be in flight at any given time. + //! @param runTimeout The maximum duration a run may be in-flight for before being forcefully terminated. + //! @param runnerTimeout The maximum duration the runner may run before forcefully terminating all in-flight runs. + InstrumentedTestRunner( + AZStd::optional clientCallback, size_t maxConcurrentRuns, + AZStd::optional runTimeout, AZStd::optional runnerTimeout); + + //! Executes the specified instrumented test run jobs according to the specified job exception policies. + //! @param jobInfos The test run jobs to execute. + //! @param CoverageExceptionPolicy The coverage exception policy to be used for this run. + //! @param jobExceptionPolicy The test run job exception policy to be used for this run (use + //! TestJobExceptionPolicy::OnFailedToExecute to throw on test failures). + //! @return the instrumented test run jobs with their associated test run and test coverage payloads. + AZStd::vector RunInstrumentedTests( + const AZStd::vector& jobInfos, CoverageExceptionPolicy coverageExceptionPolicy, JobExceptionPolicy jobExceptionPolicy); + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestCoverage.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestCoverage.cpp new file mode 100644 index 0000000000..3a6f5d0b79 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestCoverage.cpp @@ -0,0 +1,68 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include + +#include +#include + +namespace TestImpact +{ + TestCoverage::TestCoverage(AZStd::vector&& moduleCoverages) + : m_modules(AZStd::move(moduleCoverages)) + { + for (const auto& moduleCovered : m_modules) + { + for (const auto& sourceCovered : moduleCovered.m_sources) + { + m_sourcesCovered.emplace_back(sourceCovered.m_path); + if (sourceCovered.m_coverage.has_value()) + { + m_coverageLevel = CoverageLevel::Line; + } + } + } + + AZStd::sort(m_sourcesCovered.begin(), m_sourcesCovered.end()); + m_sourcesCovered.erase(AZStd::unique(m_sourcesCovered.begin(), m_sourcesCovered.end()), m_sourcesCovered.end()); + + if (!m_coverageLevel.has_value() && !m_sourcesCovered.empty()) + { + m_coverageLevel = CoverageLevel::Source; + } + } + + size_t TestCoverage::GetNumSourcesCovered() const + { + return m_sourcesCovered.size(); + } + + size_t TestCoverage::GetNumModulesCovered() const + { + return m_modules.size(); + } + + const AZStd::vector& TestCoverage::GetSourcesCovered() const + { + return m_sourcesCovered; + } + + const AZStd::vector& TestCoverage::GetModuleCoverages() const + { + return m_modules; + } + + AZStd::optional TestCoverage::GetCoverageLevel() const + { + return m_coverageLevel; + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestCoverage.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestCoverage.h new file mode 100644 index 0000000000..ab92b1bd80 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestCoverage.h @@ -0,0 +1,54 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +#include + +namespace TestImpact +{ + //! Scope of coverage data. + enum class CoverageLevel : bool + { + Source, //!< Line-level coverage data. + Line //!< Source-level coverage data. + }; + + //! Representation of a given test target's test coverage results. + class TestCoverage + { + public: + TestCoverage(AZStd::vector&& moduleCoverages); + + //! Returns the number of unique sources covered. + size_t GetNumSourcesCovered() const; + + //! Returns the number of modules (dynamic libraries, child processes, etc.) covered. + size_t GetNumModulesCovered() const; + + //! Returns the sorted set of unique sources covered (empty if no coverage). + const AZStd::vector& GetSourcesCovered() const; + + //! Returns the modules covered (empty if no coverage). + const AZStd::vector& GetModuleCoverages() const; + + //! Returns the coverage level (empty if no coverage). + AZStd::optional GetCoverageLevel() const; + + private: + AZStd::vector m_modules; + AZStd::vector m_sourcesCovered; + AZStd::optional m_coverageLevel; + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRun.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRun.cpp new file mode 100644 index 0000000000..1ba2a3e849 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRun.cpp @@ -0,0 +1,70 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include "TestImpactTestRun.h" + +namespace TestImpact +{ + TestRun::TestRun(AZStd::vector&& testSuites, AZStd::chrono::milliseconds duration) + : TestSuiteContainer(AZStd::move(testSuites)) + , m_duration(duration) + { + for (const auto& suite : m_testSuites) + { + for (const auto& test : suite.m_tests) + { + if (test.m_status == TestRunStatus::Run) + { + m_numRuns++; + + if (test.m_result.value() == TestRunResult::Passed) + { + m_numPasses++; + } + else + { + m_numFailures++; + } + } + else + { + m_numNotRuns++; + } + } + } + } + + size_t TestRun::GetNumRuns() const + { + return m_numRuns; + } + + size_t TestRun::GetNumNotRuns() const + { + return m_numNotRuns; + } + + size_t TestRun::GetNumPasses() const + { + return m_numPasses; + } + + size_t TestRun::GetNumFailures() const + { + return m_numFailures; + } + + AZStd::chrono::milliseconds TestRun::GetDuration() const + { + return m_duration; + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRun.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRun.h new file mode 100644 index 0000000000..75d2140a68 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRun.h @@ -0,0 +1,51 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include +#include + +namespace TestImpact +{ + //! Representation of a given test target's test run results. + class TestRun + : public TestSuiteContainer + { + using TestSuiteContainer = TestSuiteContainer; + + public: + TestRun(AZStd::vector&& testSuites, AZStd::chrono::milliseconds duration); + + //! Returns the total number of tests that were run. + size_t GetNumRuns() const; + + //! Returns the total number of tests that were not run. + size_t GetNumNotRuns() const; + + //! Returns the total number of tests that were run and passed. + size_t GetNumPasses() const; + + //! Returns the total number of tests that were run and failed. + size_t GetNumFailures() const; + + //! Returns the duration of the job that was executed to yield this run data. + AZStd::chrono::milliseconds GetDuration() const; + + private: + size_t m_numRuns = 0; + size_t m_numNotRuns = 0; + size_t m_numPasses = 0; + size_t m_numFailures = 0; + AZStd::chrono::milliseconds m_duration = AZStd::chrono::milliseconds{0}; + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunException.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunException.h new file mode 100644 index 0000000000..ec13a17f57 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunException.h @@ -0,0 +1,25 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +namespace TestImpact +{ + //! Exception for test runs and test run related operations. + class TestRunException : public Exception + { + public: + using Exception::Exception; + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunJobData.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunJobData.cpp new file mode 100644 index 0000000000..fa40b30263 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunJobData.cpp @@ -0,0 +1,26 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include + +namespace TestImpact +{ + TestRunJobData::TestRunJobData(const AZ::IO::Path& resultsArtifact) + : m_runArtifact(resultsArtifact) + { + } + + const AZ::IO::Path& TestRunJobData::GetRunArtifactPath() const + { + return m_runArtifact; + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunJobData.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunJobData.h new file mode 100644 index 0000000000..802aa0d30f --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunJobData.h @@ -0,0 +1,31 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +namespace TestImpact +{ + //! Per-job data for test runs. + class TestRunJobData + { + public: + TestRunJobData(const AZ::IO::Path& resultsArtifact); + + //! Returns the path to the test run artifact produced by the test target. + const AZ::IO::Path& GetRunArtifactPath() const; + + private: + AZ::IO::Path m_runArtifact; //!< Path to results data. + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunSerializer.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunSerializer.cpp new file mode 100644 index 0000000000..15a135ad25 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunSerializer.cpp @@ -0,0 +1,186 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include + +#include +#include +#include +#include + +namespace TestImpact +{ + namespace + { + // Keys for pertinent JSON node and attribute names + constexpr const char* Keys[] = + { + "suites", + "name", + "enabled", + "tests", + "duration", + "status", + "result" + }; + + enum + { + SuitesKey, + NameKey, + EnabledKey, + TestsKey, + DurationKey, + StatusKey, + ResultKey + }; + } // namespace + + AZStd::string SerializeTestRun(const TestRun& testRun) + { + rapidjson::StringBuffer stringBuffer; + rapidjson::PrettyWriter writer(stringBuffer); + + // Run + writer.StartObject(); + + // Run duration + writer.Key(Keys[DurationKey]); + writer.Uint(testRun.GetDuration().count()); + + // Suites + writer.Key(Keys[SuitesKey]); + writer.StartArray(); + + for (const auto& suite : testRun.GetTestSuites()) + { + // Suite + writer.StartObject(); + + // Suite name + writer.Key(Keys[NameKey]); + writer.String(suite.m_name.c_str()); + + // Suite duration + writer.Key(Keys[DurationKey]); + writer.Uint(suite.m_duration.count()); + + // Suite enabled + writer.Key(Keys[EnabledKey]); + writer.Bool(suite.m_enabled); + + // Suite tests + writer.Key(Keys[TestsKey]); + writer.StartArray(); + for (const auto& test : suite.m_tests) + { + // Test + writer.StartObject(); + + // Test name + writer.Key(Keys[NameKey]); + writer.String(test.m_name.c_str()); + + // Test enabled + writer.Key(Keys[EnabledKey]); + writer.Bool(test.m_enabled); + + // Test duration + writer.Key(Keys[DurationKey]); + writer.Uint(test.m_duration.count()); + + // Test status + writer.Key(Keys[StatusKey]); + writer.Bool(static_cast(test.m_status)); + + // Test result + if (test.m_status == TestRunStatus::Run) + { + writer.Key(Keys[ResultKey]); + writer.Bool(static_cast(test.m_result.value())); + } + else + { + writer.Key(Keys[ResultKey]); + writer.Null(); + } + + // End test + writer.EndObject(); + } + + // End tests + writer.EndArray(); + + // End suite + writer.EndObject(); + } + + // End suites + writer.EndArray(); + + // End run + writer.EndObject(); + + return stringBuffer.GetString(); + } + + TestRun DeserializeTestRun(const AZStd::string& testEnumString) + { + AZStd::vector testSuites; + rapidjson::Document doc; + + if (doc.Parse<0>(testEnumString.c_str()).HasParseError()) + { + throw TestRunException("Could not parse enumeration data"); + } + + // Run duration + const AZStd::chrono::milliseconds runDuration = AZStd::chrono::milliseconds{doc[Keys[DurationKey]].GetUint()}; + + // Suites + for (const auto& suite : doc[Keys[SuitesKey]].GetArray()) + { + // Suite name + const AZStd::string name = suite[Keys[NameKey]].GetString(); + + // Suite duration + const AZStd::chrono::milliseconds suiteDuration = AZStd::chrono::milliseconds{suite[Keys[DurationKey]].GetUint()}; + + // Suite enabled + const bool enabled = suite[Keys[EnabledKey]].GetBool(); + + testSuites.emplace_back(TestRunSuite{ + suite[Keys[NameKey]].GetString(), + suite[Keys[EnabledKey]].GetBool(), + {}, + AZStd::chrono::milliseconds{suite[Keys[DurationKey]].GetUint()}}); + + // Suite tests + for (const auto& test : suite[Keys[TestsKey]].GetArray()) + { + AZStd::optional result; + TestRunStatus status = static_cast(test[Keys[StatusKey]].GetBool()); + if (status == TestRunStatus::Run) + { + result = static_cast(test[Keys[ResultKey]].GetBool()); + } + const AZStd::chrono::milliseconds testDuration = AZStd::chrono::milliseconds{test[Keys[DurationKey]].GetUint()}; + testSuites.back().m_tests.emplace_back( + TestRunCase{test[Keys[NameKey]].GetString(), test[Keys[EnabledKey]].GetBool(), result, testDuration, status}); + } + } + + return TestRun(std::move(testSuites), runDuration); + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunSerializer.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunSerializer.h new file mode 100644 index 0000000000..5978b27105 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunSerializer.h @@ -0,0 +1,26 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +#include + +namespace TestImpact +{ + //! Serializes the specified test run to JSON format. + AZStd::string SerializeTestRun(const TestRun& testRun); + + //! Deserializes a test run from the specified test run data in JSON format. + TestRun DeserializeTestRun(const AZStd::string& testRunString); +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunner.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunner.cpp new file mode 100644 index 0000000000..dde7dd7d47 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunner.cpp @@ -0,0 +1,58 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include +#include +#include +#include + +#include + +namespace TestImpact +{ + TestRun ParseTestRunFile(const AZ::IO::Path& runFile, AZStd::chrono::milliseconds duration) + { + return TestRun(GTest::TestRunSuitesFactory(ReadFileContents(runFile)), duration); + } + + TestRunner::TestRunner( + AZStd::optional clientCallback, + size_t maxConcurrentRuns, + AZStd::optional runTimeout, + AZStd::optional runnerTimeout) + : JobRunner(clientCallback, AZStd::nullopt, StdOutputRouting::None, StdErrorRouting::None, maxConcurrentRuns, runTimeout, runnerTimeout) + { + } + + AZStd::vector TestRunner::RunTests( + const AZStd::vector& jobInfos, + JobExceptionPolicy jobExceptionPolicy) + { + const auto payloadGenerator = [this](const JobDataMap& jobDataMap) + { + PayloadMap runs; + for (const auto& [jobId, jobData] : jobDataMap) + { + const auto& [meta, jobInfo] = jobData; + if (meta.m_result == JobResult::ExecutedWithSuccess || meta.m_result == JobResult::ExecutedWithFailure) + { + runs[jobId] = ParseTestRunFile(jobInfo->GetRunArtifactPath(), meta.m_duration.value()); + } + } + + return runs; + }; + + return ExecuteJobs(jobInfos, payloadGenerator, jobExceptionPolicy); + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunner.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunner.h new file mode 100644 index 0000000000..0bf66601fe --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunner.h @@ -0,0 +1,46 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include +#include +#include + +namespace TestImpact +{ + //! Runs a batch of test targets to determine the test passes/failures. + class TestRunner + : public TestJobRunner + { + using JobRunner = TestJobRunner; + + public: + //! Constructs a test runner with the specified parameters common to all job runs of this runner. + //! @param clientCallback The optional client callback to be called whenever a run job changes state. + //! @param maxConcurrentRuns The maximum number of runs to be in flight at any given time. + //! @param runTimeout The maximum duration a run may be in-flight for before being forcefully terminated. + //! @param runnerTimeout The maximum duration the runner may run before forcefully terminating all in-flight runs. + TestRunner( + AZStd::optional clientCallback, + size_t maxConcurrentRuns, + AZStd::optional runTimeout, + AZStd::optional runnerTimeout); + + //! Executes the specified test run jobs according to the specified job exception policies. + //! @param jobInfos The test run jobs to execute. + //! @param jobExceptionPolicy The test run job exception policy to be used for this run (use + //! TestJobExceptionPolicy::OnFailedToExecute to throw on test failures). + //! @return the test run jobs with their associated test run payloads. + AZStd::vector RunTests(const AZStd::vector& jobInfos, JobExceptionPolicy jobExceptionPolicy); + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/TestImpactTestSuiteContainer.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/TestImpactTestSuiteContainer.h new file mode 100644 index 0000000000..c41d76bd04 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/TestImpactTestSuiteContainer.h @@ -0,0 +1,101 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +namespace TestImpact +{ + //! Encapsulation of test suites into a class with meta-data about each the suites. + //! @tparam TestSuite The test suite data structure to encapsulate. + template + class TestSuiteContainer + { + public: + TestSuiteContainer(AZStd::vector&& testSuites); + + //! Returns the test suites in this container. + const AZStd::vector& GetTestSuites() const; + + //! Returns the number of test suites in this container. + size_t GetNumTestSuites() const; + + //! Returns the total number of tests across all test suites. + size_t GetNumTests() const; + + //! Returns the total number of enabled tests across all test suites. + size_t GetNumEnabledTests() const; + + //! Returns the total number of disabled tests across all test suites. + size_t GetNumDisabledTests() const; + + protected: + AZStd::vector m_testSuites; + size_t m_numDisabledTests = 0; + size_t m_numEnabledTests = 0; + }; + + template + TestSuiteContainer::TestSuiteContainer(AZStd::vector&& testSuites) + : m_testSuites(std::move(testSuites)) + { + for (const auto& suite : m_testSuites) + { + if (suite.m_enabled) + { + const auto enabled = std::count_if(suite.m_tests.begin(), suite.m_tests.end(), [](const auto& test) + { + return test.m_enabled; + }); + + m_numEnabledTests += enabled; + m_numDisabledTests += suite.m_tests.size() - enabled; + } + else + { + // Disabled status of suites propagates down to all tests regardless of whether or not each individual test is disabled + m_numDisabledTests += suite.m_tests.size(); + } + } + } + + template + const AZStd::vector& TestSuiteContainer::GetTestSuites() const + { + return m_testSuites; + } + + template + size_t TestSuiteContainer::GetNumTests() const + { + return m_numEnabledTests + m_numDisabledTests; + } + + template + size_t TestSuiteContainer::GetNumEnabledTests() const + { + return m_numEnabledTests; + } + + template + size_t TestSuiteContainer::GetNumDisabledTests() const + { + return m_numDisabledTests; + } + + template + size_t TestSuiteContainer::GetNumTestSuites() const + { + return m_testSuites.size(); + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactException.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactException.cpp new file mode 100644 index 0000000000..804ab26e4d --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactException.cpp @@ -0,0 +1,31 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include + +namespace TestImpact +{ + Exception::Exception(const AZStd::string& msg) + : m_msg(msg) + { + } + + Exception::Exception(const char* msg) + : m_msg(msg) + { + } + + const char* Exception::what() const noexcept + { + return m_msg.c_str(); + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactFrameworkPath.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactFrameworkPath.cpp new file mode 100644 index 0000000000..6b5d671776 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactFrameworkPath.cpp @@ -0,0 +1,38 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include + +namespace TestImpact +{ + FrameworkPath::FrameworkPath(const AZ::IO::Path& absolutePath) + { + m_absolutePath = AZ::IO::Path(absolutePath).MakePreferred(); + m_relativePath = m_absolutePath.LexicallyRelative(m_absolutePath); + } + + FrameworkPath::FrameworkPath(const AZ::IO::Path& absolutePath, const FrameworkPath& relativeTo) + { + m_absolutePath = AZ::IO::Path(absolutePath).MakePreferred(); + m_relativePath = m_absolutePath.LexicallyRelative(relativeTo.Absolute()); + } + + const AZ::IO::Path& FrameworkPath::Absolute() const + { + return m_absolutePath; + } + + const AZ::IO::Path& FrameworkPath::Relative() const + { + return m_relativePath; + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactBuildTargetDescriptorFactoryTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactBuildTargetDescriptorFactoryTest.cpp new file mode 100644 index 0000000000..a0a4a7309e --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactBuildTargetDescriptorFactoryTest.cpp @@ -0,0 +1,370 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include + +#include +#include + +#include +#include + +namespace UnitTest +{ + class BuildTargetDescriptorFactoryTestFixture + : public AllocatorsTestFixture + { + protected: + const AZStd::vector m_staticExclude = {".cmake"}; + const AZStd::vector m_inputExclude = {".jinja"}; + const AZStd::string m_autogenMatcher = {"(.*)\\..*"}; + + const AZStd::vector m_autogenInputs = + { + "Gems/ScriptCanvasDiagnosticLibrary/Code/Source/Log.ScriptCanvasNode.xml", + "Gems/ScriptCanvasDiagnosticLibrary/Code/Source/DrawText.ScriptCanvasNode.xml", + "Gems/ScriptCanvas/Code/Include/ScriptCanvas/AutoGen/ScriptCanvasNode_Header.jinja", + "Gems/ScriptCanvas/Code/Include/ScriptCanvas/AutoGen/ScriptCanvasNode_Source.jinja" + }; + + const AZStd::vector m_autogenOutputs = + { + "windows_vs2019/Gems/ScriptCanvasDiagnosticLibrary/Code/Azcg/Generated/Source/Log.generated.h", + "windows_vs2019/Gems/ScriptCanvasDiagnosticLibrary/Code/Azcg/Generated/Source/DrawText.generated.h", + "windows_vs2019/Gems/ScriptCanvasDiagnosticLibrary/Code/Azcg/Generated/Source/Log.generated.cpp", + "windows_vs2019/Gems/ScriptCanvasDiagnosticLibrary/Code/Azcg/Generated/Source/DrawText.generated.cpp" + }; + + const AZStd::vector m_staticSources = + { + "Gems/ScriptCanvasDiagnosticLibrary/Code/Source/precompiled.cpp", + "Gems/ScriptCanvasDiagnosticLibrary/Code/Source/precompiled.h", + "Gems/ScriptCanvasDiagnosticLibrary/Code/scriptcanvasdiagnosticlibrary_autogen_files.cmake", + "windows_vs2019/Gems/ScriptCanvasDiagnosticLibrary/Code/Azcg/Generated/Source/Log.generated.h", + "windows_vs2019/Gems/ScriptCanvasDiagnosticLibrary/Code/Azcg/Generated/Source/DrawText.generated.h", + "windows_vs2019/Gems/ScriptCanvasDiagnosticLibrary/Code/Azcg/Generated/Source/Log.generated.cpp", + "windows_vs2019/Gems/ScriptCanvasDiagnosticLibrary/Code/Azcg/Generated/Source/DrawText.generated.cpp" + }; + + const AZStd::vector m_expectedStaticSources = + { + "Gems/ScriptCanvasDiagnosticLibrary/Code/Source/precompiled.cpp", + "Gems/ScriptCanvasDiagnosticLibrary/Code/Source/precompiled.h", + "windows_vs2019/Gems/ScriptCanvasDiagnosticLibrary/Code/Azcg/Generated/Source/Log.generated.h", + "windows_vs2019/Gems/ScriptCanvasDiagnosticLibrary/Code/Azcg/Generated/Source/DrawText.generated.h", + "windows_vs2019/Gems/ScriptCanvasDiagnosticLibrary/Code/Azcg/Generated/Source/Log.generated.cpp", + "windows_vs2019/Gems/ScriptCanvasDiagnosticLibrary/Code/Azcg/Generated/Source/DrawText.generated.cpp" + }; + + const AZStd::string m_name = "ScriptCanvasDiagnosticLibrary.Static"; + const AZStd::string m_outputName = "ScriptCanvasDiagnosticLibrary"; + const AZStd::string m_path = "Gems/ScriptCanvasDiagnosticLibrary/Code"; + + const TestImpact::AutogenSources m_expectedAutogenSources = + { + { + "Gems/ScriptCanvasDiagnosticLibrary/Code/Source/Log.ScriptCanvasNode.xml", + { + "windows_vs2019/Gems/ScriptCanvasDiagnosticLibrary/Code/Azcg/Generated/Source/Log.generated.h", + "windows_vs2019/Gems/ScriptCanvasDiagnosticLibrary/Code/Azcg/Generated/Source/Log.generated.cpp" + } + }, + { + "Gems/ScriptCanvasDiagnosticLibrary/Code/Source/DrawText.ScriptCanvasNode.xml", + { + "windows_vs2019/Gems/ScriptCanvasDiagnosticLibrary/Code/Azcg/Generated/Source/DrawText.generated.h", + "windows_vs2019/Gems/ScriptCanvasDiagnosticLibrary/Code/Azcg/Generated/Source/DrawText.generated.cpp" + } + } + }; + }; + + TEST_F(BuildTargetDescriptorFactoryTestFixture, NoRawData_ExpectArtifactException) + { + // Given an empty raw descriptor string + const AZStd::string rawTargetDescriptor; + + try + { + // When attempting to construct the build target descriptor + const TestImpact::BuildTargetDescriptor buildTarget = + TestImpact::BuildTargetDescriptorFactory(rawTargetDescriptor, m_staticExclude, m_inputExclude, m_autogenMatcher); + + // Do not expect this statement to be reachable + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::ArtifactException& e) + { + // Expect an artifact exception + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(BuildTargetDescriptorFactoryTestFixture, InvalidRawData_ExpectArtifactException) + { + // Given a raw descriptor string of invalid data + const AZStd::string rawTargetDescriptor = "abcde"; + + try + { + // When attempting to construct the build target descriptor + const TestImpact::BuildTargetDescriptor buildTarget = + TestImpact::BuildTargetDescriptorFactory(rawTargetDescriptor, m_staticExclude, m_inputExclude, m_autogenMatcher); + + // Do not expect this statement to be reachable + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::ArtifactException& e) + { + // Expect an artifact exception + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(BuildTargetDescriptorFactoryTestFixture, NoAutogenMatcher_ExpectArtifactException) + { + // Given a valid raw descriptor string + const AZStd::string rawTargetDescriptor = + GenerateBuildTargetDescriptorString(m_name, m_outputName, m_path, m_staticSources, m_autogenInputs, m_autogenOutputs); + + try + { + // When attempting to construct the build target descriptor with an empty autogen matcher + const TestImpact::BuildTargetDescriptor buildTarget = + TestImpact::BuildTargetDescriptorFactory(rawTargetDescriptor, m_staticExclude, m_inputExclude, ""); + + // Do not expect this statement to be reachable + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::ArtifactException& e) + { + // Expect an artifact exception + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(BuildTargetDescriptorFactoryTestFixture, EmptyName_ExpectArtifactException) + { + // Given a invalid raw descriptor string lacking build meta-data name + const AZStd::string rawTargetDescriptor = + GenerateBuildTargetDescriptorString("", m_outputName, m_path, m_staticSources, m_autogenInputs, m_autogenOutputs); + + try + { + // When attempting to construct the build target descriptor + const TestImpact::BuildTargetDescriptor buildTarget = + TestImpact::BuildTargetDescriptorFactory(rawTargetDescriptor, m_staticExclude, m_inputExclude, m_autogenMatcher); + + // Do not expect this statement to be reachable + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::ArtifactException& e) + { + // Expect an artifact exception + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(BuildTargetDescriptorFactoryTestFixture, EmptyOutputName_ExpectArtifactException) + { + // Given a invalid raw descriptor string lacking build meta-data output name + const AZStd::string rawTargetDescriptor = + GenerateBuildTargetDescriptorString(m_name, "", m_path, m_staticSources, m_autogenInputs, m_autogenOutputs); + + try + { + // When attempting to construct the build target descriptor + const TestImpact::BuildTargetDescriptor buildTarget = + TestImpact::BuildTargetDescriptorFactory(rawTargetDescriptor, m_staticExclude, m_inputExclude, m_autogenMatcher); + + // Do not expect this statement to be reachable + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::ArtifactException& e) + { + // Expect an artifact exception + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(BuildTargetDescriptorFactoryTestFixture, EmptyPath_ExpectArtifactException) + { + // Given a invalid raw descriptor string lacking build meta-data path + const AZStd::string rawTargetDescriptor = + GenerateBuildTargetDescriptorString(m_name, m_outputName, "", m_staticSources, m_autogenInputs, m_autogenOutputs); + + try + { + // When attempting to construct the build target descriptor + const TestImpact::BuildTargetDescriptor buildTarget = + TestImpact::BuildTargetDescriptorFactory(rawTargetDescriptor, m_staticExclude, m_inputExclude, m_autogenMatcher); + + // Do not expect this statement to be reachable + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::ArtifactException& e) + { + // Expect an artifact exception + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(BuildTargetDescriptorFactoryTestFixture, NoStaticSources_ExpectValidDescriptor) + { + const TestImpact::BuildTargetDescriptor expectedBuiltTarget = + GenerateBuildTargetDescriptor(m_name, m_outputName, m_path, {}, m_expectedAutogenSources); + + // Given a valid raw descriptor string with no static sources + const AZStd::string rawTargetDescriptor = + GenerateBuildTargetDescriptorString(m_name, m_outputName, m_path, {}, m_autogenInputs, m_autogenOutputs); + + // When attempting to construct the build target descriptor + const TestImpact::BuildTargetDescriptor buildTarget = + TestImpact::BuildTargetDescriptorFactory(rawTargetDescriptor, {}, m_inputExclude, m_autogenMatcher); + + // Expect the constructed build target descriptor to match the specified descriptor + EXPECT_TRUE(buildTarget == expectedBuiltTarget); + } + + TEST_F(BuildTargetDescriptorFactoryTestFixture, NoAutogenSources_ExpectValidDescriptor) + { + const TestImpact::BuildTargetDescriptor expectedBuiltTarget = + GenerateBuildTargetDescriptor(m_name, m_outputName, m_path, m_expectedStaticSources, {}); + + // Given a valid raw descriptor string with no autogen sources + const AZStd::string rawTargetDescriptor = + GenerateBuildTargetDescriptorString(m_name, m_outputName, m_path, m_staticSources, {}, {}); + + // When attempting to construct the build target descriptor + const TestImpact::BuildTargetDescriptor buildTarget = + TestImpact::BuildTargetDescriptorFactory(rawTargetDescriptor, m_staticExclude, m_inputExclude, m_autogenMatcher); + + // Expect the constructed build target descriptor to match the specified descriptor + EXPECT_TRUE(buildTarget == expectedBuiltTarget); + } + + TEST_F(BuildTargetDescriptorFactoryTestFixture, NoStaticOrAutogenSources_ExpectValidDescriptor) + { + const TestImpact::BuildTargetDescriptor expectedBuiltTarget = GenerateBuildTargetDescriptor(m_name, m_outputName, m_path, {}, {}); + + // Given a valid raw descriptor string with no static or autogen sources + const AZStd::string rawTargetDescriptor = GenerateBuildTargetDescriptorString(m_name, m_outputName, m_path, {}, {}, {}); + + // When attempting to construct the build target descriptor + const TestImpact::BuildTargetDescriptor buildTarget = + TestImpact::BuildTargetDescriptorFactory(rawTargetDescriptor, m_staticExclude, m_inputExclude, m_autogenMatcher); + + // Expect the constructed build target descriptor to match the specified descriptor + EXPECT_TRUE(buildTarget == expectedBuiltTarget); + } + + TEST_F(BuildTargetDescriptorFactoryTestFixture, AutogenOutputSourcesButNoAutogenInputSources_ExpectArtifactException) + { + // Given a valid raw descriptor string with autogen output sources but no autogen input sources + const AZStd::string rawTargetDescriptor = + GenerateBuildTargetDescriptorString(m_name, m_outputName, m_path, m_staticSources, {}, m_autogenOutputs); + + try + { + // When attempting to construct the build target descriptor + const TestImpact::BuildTargetDescriptor buildTarget = + TestImpact::BuildTargetDescriptorFactory(rawTargetDescriptor, m_staticExclude, m_inputExclude, m_autogenMatcher); + + // Do not expect this statement to be reachable + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::ArtifactException& e) + { + // Expect an artifact exception + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(BuildTargetDescriptorFactoryTestFixture, AutogenInputSourcesButNoAutogenOutputSources_ExpectArtifactException) + { + // Given a valid raw descriptor string with autogen inout sources but no autogen output sources + const AZStd::string rawTargetDescriptor = + GenerateBuildTargetDescriptorString(m_name, m_outputName, m_path, m_staticSources, m_autogenInputs, {}); + + try + { + // When attempting to construct the build target descriptor + const TestImpact::BuildTargetDescriptor buildTarget = + TestImpact::BuildTargetDescriptorFactory(rawTargetDescriptor, m_staticExclude, m_inputExclude, m_autogenMatcher); + + // Do not expect this statement to be reachable + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::ArtifactException& e) + { + // Expect an artifact exception + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(BuildTargetDescriptorFactoryTestFixture, StaticAndAutogenSources_ExpectValidDescriptor) + { + const TestImpact::BuildTargetDescriptor expectedBuiltTarget = + GenerateBuildTargetDescriptor(m_name, m_outputName, m_path, m_expectedStaticSources, m_expectedAutogenSources); + + // Given a valid raw descriptor string with static and autogen sources + const AZStd::string rawTargetDescriptor = + GenerateBuildTargetDescriptorString(m_name, m_outputName, m_path, m_staticSources, m_autogenInputs, m_autogenOutputs); + + // When attempting to construct the build target descriptor + const TestImpact::BuildTargetDescriptor buildTarget = + TestImpact::BuildTargetDescriptorFactory(rawTargetDescriptor, m_staticExclude, m_inputExclude, m_autogenMatcher); + + // Expect the constructed build target descriptor to match the specified descriptor + EXPECT_TRUE(buildTarget == expectedBuiltTarget); + } +} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactChangeListFactoryTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactChangeListFactoryTest.cpp new file mode 100644 index 0000000000..aeb8f118a4 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactChangeListFactoryTest.cpp @@ -0,0 +1,288 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include + +#include +#include + +namespace UnitTest +{ + TEST(ChangeListFactoryTest, NoRawData_ExpectArtifactException) + { + // Given an empty unified diff string + const AZStd::string unifiedDiff; + + try + { + // When attempting to construct the change list + const TestImpact::ChangeList changeList = TestImpact::UnifiedDiff::ChangeListFactory(unifiedDiff); + + // Do not expect this statement to be reachable + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::ArtifactException& e) + { + // Expect an artifact exception + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST(ChangeListFactoryTest, NoChanges_ExpectArtifactException) + { + // Given a unified diff string with no changes + const AZStd::string unifiedDiff = "On this day in 1738 absolutely nothing happened"; + + try + { + // When attempting to construct the change list + const TestImpact::ChangeList changeList = TestImpact::UnifiedDiff::ChangeListFactory(unifiedDiff); + + // Do not expect this statement to be reachable + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::ArtifactException& e) + { + // Expect an artifact exception + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST(ChangeListFactoryTest, CreateOnly_ExpectValidChangeListWithFileCreateOperations) + { + // Given a unified diff with only one file creation and no file updates or deletions + const AZStd::string unifiedDiff = + "From f642a2f698452fc18484758b0046132415f09467 Mon Sep 17 00:00:00 2001\n" + "From: user \n" + "Date: Sat, 13 Mar 2021 22:58:07 +0000\n" + "Subject: Test\n" + "\n" + "---\n" + " New.txt | 1 +\n" + " create mode 100644 New.txt\n" + "diff --git a/New.txt b/New.txt\n" + "new file mode 100644\n" + "index 0000000..30d74d2\n" + "--- /dev/null\n" + "+++ b/New.txt\n" + "@@ -0,0 +1 @@\n" + "+test\n" + "\\ No newline at end of file\n" + "-- \n" + "2.30.0.windows.2\n" + "\n" + "\n"; + + // When attempting to construct the change list + const TestImpact::ChangeList changeList = TestImpact::UnifiedDiff::ChangeListFactory(unifiedDiff); + + // Expect the change list to contain the 1 created file + EXPECT_EQ(changeList.m_createdFiles.size(), 1); + EXPECT_TRUE( + AZStd::find(changeList.m_createdFiles.begin(), changeList.m_createdFiles.end(), "New.txt") != changeList.m_createdFiles.end()); + + // Expect the change list to contain no updated files + EXPECT_TRUE(changeList.m_updatedFiles.empty()); + + // Expect the change list to contain no deleted files + EXPECT_TRUE(changeList.m_deletedFiles.empty()); + } + + TEST(ChangeListFactoryTest, UpdateOnly_ExpectValidChangeListWithFileUpdateOperations) + { + // Given a unified diff with only one file update and no file creations or deletions + const AZStd::string unifiedDiff = + "From f642a2f698452fc18484758b0046132415f09467 Mon Sep 17 00:00:00 2001\n" + "From: user \n" + "Date: Sat, 13 Mar 2021 22:58:07 +0000\n" + "Subject: Test\n" + "\n" + "---\n" + " A.txt | 2 +-\n" + "diff --git a/A.txt b/A.txt\n" + "index 7c4a013..e132db2 100644\n" + "--- a/A.txt\n" + "+++ b/A.txt\n" + "@@ -1 +1 @@\n" + "-aaa\n" + "\\ No newline at end of file\n" + "+zzz\n" + "\\ No newline at end of file\n" + "-- \n" + "2.30.0.windows.2\n" + "\n" + "\n"; + + // When attempting to construct the change list + const TestImpact::ChangeList changeList = TestImpact::UnifiedDiff::ChangeListFactory(unifiedDiff); + + // Expect the change list to contain no created files + EXPECT_TRUE(changeList.m_createdFiles.empty()); + + // Expect the change list to contain one updated file + EXPECT_EQ(changeList.m_updatedFiles.size(), 1); + EXPECT_TRUE( + AZStd::find(changeList.m_updatedFiles.begin(), changeList.m_updatedFiles.end(), "A.txt") != changeList.m_updatedFiles.end()); + + // Expect the change list to contain no deleted files + EXPECT_TRUE(changeList.m_deletedFiles.empty()); + } + + TEST(ChangeListFactoryTest, DeleteOnly_ExpectValidChangeListWithFileDeleteOperations) + { + // Given a unified diff with only one file deletion and no file creations or updates + const AZStd::string unifiedDiff = + "From f642a2f698452fc18484758b0046132415f09467 Mon Sep 17 00:00:00 2001\n" + "From: user \n" + "Date: Sat, 13 Mar 2021 22:58:07 +0000\n" + "Subject: Test\n" + "\n" + "---\n" + " B.txt | 1 -\n" + " delete mode 100644 B.txt\n" + "diff --git a/B.txt b/B.txt\n" + "deleted file mode 100644\n" + "index 01f02e3..0000000\n" + "--- a/B.txt\n" + "+++ /dev/null\n" + "@@ -1 +0,0 @@\n" + "-bbb\n" + "\\ No newline at end of file\n" + "-- \n" + "2.30.0.windows.2\n" + "\n" + "\n"; + + // When attempting to construct the change list + const TestImpact::ChangeList changeList = TestImpact::UnifiedDiff::ChangeListFactory(unifiedDiff); + + // Expect the change list to contain no created files + EXPECT_TRUE(changeList.m_createdFiles.empty()); + + // Expect the change list to contain no updated files + EXPECT_TRUE(changeList.m_updatedFiles.empty()); + + // Expect the change list to contain one deleted file + EXPECT_EQ(changeList.m_deletedFiles.size(), 1); + EXPECT_TRUE( + AZStd::find(changeList.m_deletedFiles.begin(), changeList.m_deletedFiles.end(), "B.txt") != changeList.m_deletedFiles.end()); + } + + TEST(ChangeListFactoryTest, ParseUnifiedDiffWithAllPossibleOperations_ExpectChangeListMatchingOperations) + { + // Given a unified diff with created files, updated files, deleted files, renamed files and moved files + const AZStd::string unifiedDiff = + "From f642a2f698452fc18484758b0046132415f09467 Mon Sep 17 00:00:00 2001\n" + "From: user \n" + "Date: Sat, 13 Mar 2021 22:58:07 +0000\n" + "Subject: Test\n" + "\n" + "---\n" + " A.txt | 2 +-\n" + " B.txt | 1 -\n" + " D.txt => Foo/D.txt | 0\n" + " E.txt => Foo/Y.txt | 0\n" + " New.txt | 1 +\n" + " C.txt => X.txt | 0\n" + " 6 files changed, 2 insertions(+), 2 deletions(-)\n" + " delete mode 100644 B.txt\n" + " rename D.txt => Foo/D.txt (100%)\n" + " rename E.txt => Foo/Y.txt (100%)\n" + " create mode 100644 New.txt\n" + " rename C.txt => X.txt (100%)\n" + "\n" + "diff --git a/A.txt b/A.txt\n" + "index 7c4a013..e132db2 100644\n" + "--- a/A.txt\n" + "+++ b/A.txt\n" + "@@ -1 +1 @@\n" + "-aaa\n" + "\\ No newline at end of file\n" + "+zzz\n" + "\\ No newline at end of file\n" + "diff --git a/B.txt b/B.txt\n" + "deleted file mode 100644\n" + "index 01f02e3..0000000\n" + "--- a/B.txt\n" + "+++ /dev/null\n" + "@@ -1 +0,0 @@\n" + "-bbb\n" + "\\ No newline at end of file\n" + "diff --git a/D.txt b/Foo/D.txt\n" + "similarity index 100%\n" + "rename from D.txt\n" + "rename to Foo/D.txt\n" + "diff --git a/E.txt b/Foo/Y.txt\n" + "similarity index 100%\n" + "rename from E.txt\n" + "rename to Foo/Y.txt\n" + "diff --git a/New.txt b/New.txt\n" + "new file mode 100644\n" + "index 0000000..30d74d2\n" + "--- /dev/null\n" + "+++ b/New.txt\n" + "@@ -0,0 +1 @@\n" + "+test\n" + "\\ No newline at end of file\n" + "diff --git a/C.txt b/X.txt\n" + "similarity index 100%\n" + "rename from C.txt\n" + "rename to X.txt\n" + "-- \n" + "2.30.0.windows.2\n" + "\n" + "\n"; + + // When attempting to construct the change list + const TestImpact::ChangeList changeList = TestImpact::UnifiedDiff::ChangeListFactory(unifiedDiff); + + // Expect the change list to contain the 4 created files + EXPECT_EQ(changeList.m_createdFiles.size(), 4); + EXPECT_TRUE( + AZStd::find(changeList.m_createdFiles.begin(), changeList.m_createdFiles.end(), "Foo/D.txt") != + changeList.m_createdFiles.end()); + EXPECT_TRUE( + AZStd::find(changeList.m_createdFiles.begin(), changeList.m_createdFiles.end(), "Foo/Y.txt") != + changeList.m_createdFiles.end()); + EXPECT_TRUE( + AZStd::find(changeList.m_createdFiles.begin(), changeList.m_createdFiles.end(), "X.txt") != changeList.m_createdFiles.end()); + EXPECT_TRUE( + AZStd::find(changeList.m_createdFiles.begin(), changeList.m_createdFiles.end(), "New.txt") != changeList.m_createdFiles.end()); + + // Expect the change list to contain the 1 updated file + EXPECT_EQ(changeList.m_updatedFiles.size(), 1); + EXPECT_TRUE( + AZStd::find(changeList.m_updatedFiles.begin(), changeList.m_updatedFiles.end(), "A.txt") != changeList.m_updatedFiles.end()); + + // Expect the change list to contain the 4 deleted files + EXPECT_EQ(changeList.m_deletedFiles.size(), 4); + EXPECT_TRUE( + AZStd::find(changeList.m_deletedFiles.begin(), changeList.m_deletedFiles.end(), "B.txt") != changeList.m_deletedFiles.end()); + EXPECT_TRUE( + AZStd::find(changeList.m_deletedFiles.begin(), changeList.m_deletedFiles.end(), "D.txt") != changeList.m_deletedFiles.end()); + EXPECT_TRUE( + AZStd::find(changeList.m_deletedFiles.begin(), changeList.m_deletedFiles.end(), "E.txt") != changeList.m_deletedFiles.end()); + EXPECT_TRUE( + AZStd::find(changeList.m_deletedFiles.begin(), changeList.m_deletedFiles.end(), "C.txt") != changeList.m_deletedFiles.end()); + } +} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactModuleCoverageFactoryTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactModuleCoverageFactoryTest.cpp new file mode 100644 index 0000000000..137addc360 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactModuleCoverageFactoryTest.cpp @@ -0,0 +1,522 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include + +#include +#include + +#include +#include + +namespace UnitTest +{ + TEST(CoberturaModuleCoveragesFactory, ParseEmptyString_ThrowsArtifactException) + { + // Given an empty string + const AZStd::string rawCoverage; + + try + { + // When attempting to parse the empty coverage + const AZStd::vector coverage = TestImpact::Cobertura::ModuleCoveragesFactory(rawCoverage); + + // Do not expect the parsing to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::ArtifactException& e) + { + // Expect an artifact exception + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST(CoberturaModuleCoveragesFactory, ParseInvalidString_ThrowsArtifactException) + { + // Given an invalid string + const AZStd::string rawCoverage = "!@?"; + + try + { + // When attempting to parse the invalid coverage + const AZStd::vector coverage = TestImpact::Cobertura::ModuleCoveragesFactory(rawCoverage); + + // Do not expect the parsing to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::ArtifactException& e) + { + // Expect an artifact exception + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST(CoberturaModuleCoveragesFactory, ParseEmptyCoverage_ExpectEmptyModuleCoverages) + { + // Given an empty coverage string + const AZStd::string rawCoverage = + "" + "" + " " + " " + "" + ""; + + // When attempting to parse the empty coverage + const AZStd::vector coverage = TestImpact::Cobertura::ModuleCoveragesFactory(rawCoverage); + + // Expect an empty module coverages + EXPECT_TRUE(coverage.empty()); + } + + TEST(CoberturaModuleCoveragesFactory, ParseTestTargetLineCoverageA_ReturnsValidLineCoverage) + { + const AZStd::vector expectedCoverage = GetTestTargetALineModuleCoverages(); + + // Given the raw line coverage output of TestTargetA + const AZStd::string rawCoverage = + "" + "" + " " + " C:" + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + "" + ""; + + // When the raw line coverage text is parsed + const AZStd::vector coverage = TestImpact::Cobertura::ModuleCoveragesFactory(rawCoverage); + + // Expect the generated suite data to match that of the raw coverage text + EXPECT_TRUE(coverage == expectedCoverage); + } + + TEST(CoberturaModuleCoveragesFactory, ParseTestTargetSourceCoverageA_ReturnsValidSourceCoverage) + { + const AZStd::vector expectedCoverage = GetTestTargetASourceModuleCoverages(); + + // Given the raw source coverage output of TestTargetA + const AZStd::string rawCoverage = + "" + "" + " " + " C:" + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + "" + ""; + + // When the raw source coverage text is parsed + const AZStd::vector coverage = TestImpact::Cobertura::ModuleCoveragesFactory(rawCoverage); + + // Expect the generated suite data to match that of the raw coverage text + EXPECT_TRUE(coverage == expectedCoverage); + } + + TEST(CoberturaModuleCoveragesFactory, ParseTestTargetLineCoverageB_ReturnsValidLineCoverage) + { + const AZStd::vector expectedCoverage = GetTestTargetBLineModuleCoverages(); + + // Given the raw line coverage output of TestTargetB + const AZStd::string rawCoverage = + "" + "" + " " + " C:" + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + "" + ""; + + // When the raw line coverage text is parsed + const AZStd::vector coverage = TestImpact::Cobertura::ModuleCoveragesFactory(rawCoverage); + + // Expect the generated suite data to match that of the raw coverage text + EXPECT_TRUE(coverage == expectedCoverage); + } + + TEST(CoberturaModuleCoveragesFactory, ParseTestTargetSourceCoverageB_ReturnsValidSourceCoverage) + { + const AZStd::vector expectedCoverage = GetTestTargetBSourceModuleCoverages(); + + // Given the raw source coverage output of TestTargetB + const AZStd::string rawCoverage = + "" + "" + " " + " C:" + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + "" + ""; + + // When the raw source coverage text is parsed + const AZStd::vector coverage = TestImpact::Cobertura::ModuleCoveragesFactory(rawCoverage); + + // Expect the generated suite data to match that of the raw coverage text + EXPECT_TRUE(coverage == expectedCoverage); + } + + TEST(CoberturaModuleCoveragesFactory, ParseTestTargetLineCoverageC_ReturnsValidLineCoverage) + { + const AZStd::vector expectedCoverage = GetTestTargetCLineModuleCoverages(); + + // Given the raw line coverage output of TestTargetC + const AZStd::string rawCoverage = + "" + "" + " " + " C:" + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + "" + ""; + + // When the raw line coverage text is parsed + const AZStd::vector coverage = TestImpact::Cobertura::ModuleCoveragesFactory(rawCoverage); + + // Expect the generated suite data to match that of the raw coverage text + EXPECT_TRUE(coverage == expectedCoverage); + } + + TEST(CoberturaModuleCoveragesFactory, ParseTestTargetSourceCoverageC_ReturnsValidSourceCoverage) + { + const AZStd::vector expectedCoverage = GetTestTargetCSourceModuleCoverages(); + + // Given the raw source coverage output of TestTargetC + const AZStd::string rawCoverage = + "" + "" + " " + " C:" + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + "" + ""; + + // When the raw source coverage text is parsed + const AZStd::vector coverage = TestImpact::Cobertura::ModuleCoveragesFactory(rawCoverage); + + // Expect the generated suite data to match that of the raw coverage text + EXPECT_TRUE(coverage == expectedCoverage); + } + + TEST(CoberturaModuleCoveragesFactory, ParseTestTargetLineCoverageD_ReturnsValidLineCoverage) + { + const AZStd::vector expectedCoverage = GetTestTargetDLineModuleCoverages(); + + // Given the raw line coverage output of TestTargetD + const AZStd::string rawCoverage = + "" + "" + " " + " C:" + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + "" + ""; + + // When the raw line coverage text is parsed + const AZStd::vector coverage = TestImpact::Cobertura::ModuleCoveragesFactory(rawCoverage); + + // Expect the generated suite data to match that of the raw coverage text + EXPECT_TRUE(coverage == expectedCoverage); + } + + TEST(CoberturaModuleCoveragesFactory, ParseTestTargetSourceCoverageD_ReturnsValidSourceCoverage) + { + const AZStd::vector expectedCoverage = GetTestTargetDSourceModuleCoverages(); + + // Given the raw source coverage output of TestTargetD + const AZStd::string rawCoverage = + "" + "" + " " + " C:" + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + "" + ""; + + // When the raw source coverage text is parsed + const AZStd::vector coverage = TestImpact::Cobertura::ModuleCoveragesFactory(rawCoverage); + + // Expect the generated suite data to match that of the raw coverage text + EXPECT_TRUE(coverage == expectedCoverage); + } +} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactTargetDescriptorCompilerTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactTargetDescriptorCompilerTest.cpp new file mode 100644 index 0000000000..1e72902b4e --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactTargetDescriptorCompilerTest.cpp @@ -0,0 +1,150 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include + +#include +#include + +#include +#include + +namespace UnitTest +{ + class TargetDescriptorCompilerTestFixture + : public AllocatorsTestFixture + { + public: + void SetUp() override + { + AllocatorsTestFixture::SetUp(); + + m_buildTargetDescriptors.emplace_back(TestImpact::BuildTargetDescriptor{{"TestTargetA", "", ""}, {}}); + m_buildTargetDescriptors.emplace_back(TestImpact::BuildTargetDescriptor{{"TestTargetB", "", ""}, {}}); + m_buildTargetDescriptors.emplace_back(TestImpact::BuildTargetDescriptor{{"ProductionTargetA", "", ""}, {}}); + m_buildTargetDescriptors.emplace_back(TestImpact::BuildTargetDescriptor{{"ProductionTargetB", "", ""}, {}}); + m_buildTargetDescriptors.emplace_back(TestImpact::BuildTargetDescriptor{{"ProductionTargetC", "", ""}, {}}); + + m_testTargetMetaMap.emplace("TestTargetA", TestImpact::TestTargetMeta{"", TestImpact::LaunchMethod::TestRunner}); + m_testTargetMetaMap.emplace("TestTargetB", TestImpact::TestTargetMeta{"", TestImpact::LaunchMethod::StandAlone}); + } + + protected: + AZStd::vector m_buildTargetDescriptors; + TestImpact::TestTargetMetaMap m_testTargetMetaMap; + }; + + TestImpact::ProductionTargetDescriptor ConstructProductionTargetDescriptor(const AZStd::string& name) + { + return TestImpact::ProductionTargetDescriptor{TestImpact::BuildTargetDescriptor{{name, "", ""}, {{}, {}}}}; + } + + TestImpact::TestTargetDescriptor ConstructTestTargetDescriptor(const AZStd::string& name, TestImpact::LaunchMethod launchMethod) + { + return TestImpact::TestTargetDescriptor{ + TestImpact::BuildTargetDescriptor{{name, "", ""}, {{}, {}}}, TestImpact::TestTargetMeta{"", launchMethod}}; + } + + TEST_F(TargetDescriptorCompilerTestFixture, EmptyBuildTargetDescriptorList_ExpectArtifactException) + { + try + { + // Given an empty build target descriptor list but valid test target meta map + // When attempting to construct the test target + const auto& [productionTargetDescriptors, testTargetDescriptors] = + TestImpact::CompileTargetDescriptors({}, AZStd::move(m_testTargetMetaMap)); + + // Do not expect this statement to be reachable + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::ArtifactException& e) + { + // Expect an artifact exception + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(TargetDescriptorCompilerTestFixture, EmptyTestTargetMetaMap_ExpectArtifactException) + { + try + { + // Given a valid build target descriptor list but empty test target meta map + // When attempting to construct the test target + const auto& [productionTargetDescriptors, testTargetDescriptors] = + TestImpact::CompileTargetDescriptors(AZStd::move(m_buildTargetDescriptors), {}); + + // Do not expect this statement to be reachable + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::ArtifactException& e) + { + // Expect an artifact exception + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(TargetDescriptorCompilerTestFixture, TestTargetWithNoMatchingMeta_ExpectArtifactException) + { + try + { + // Given a valid build target descriptor list but a test target meta map with an orphan entry + m_testTargetMetaMap.emplace("Orphan", TestImpact::TestTargetMeta{"", TestImpact::LaunchMethod::TestRunner}); + + // When attempting to construct the test target + const auto& [productionTargetDescriptors, testTargetDescriptors] = + TestImpact::CompileTargetDescriptors(AZStd::move(m_buildTargetDescriptors), {}); + + // Do not expect this statement to be reachable + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::ArtifactException& e) + { + // Expect an artifact exception + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(TargetDescriptorCompilerTestFixture, ValidProductionTargetsAndTestTargetMetas_ExpectValidProductionAndTestTargets) + { + // Given a valid build target descriptor list and a valid test target meta map + // When attempting to construct the test target + const auto& [productionTargetDescriptors, testTargetDescriptors] = + TestImpact::CompileTargetDescriptors(AZStd::move(m_buildTargetDescriptors), AZStd::move(m_testTargetMetaMap)); + + // Expect the production targets to match the expected targets + EXPECT_TRUE(productionTargetDescriptors.size() == 3); + EXPECT_TRUE(productionTargetDescriptors[0] == ConstructProductionTargetDescriptor("ProductionTargetA")); + EXPECT_TRUE(productionTargetDescriptors[1] == ConstructProductionTargetDescriptor("ProductionTargetB")); + EXPECT_TRUE(productionTargetDescriptors[2] == ConstructProductionTargetDescriptor("ProductionTargetC")); + + // Expect the test targets to match the expected targets + EXPECT_TRUE(testTargetDescriptors.size() == 2); + EXPECT_TRUE(testTargetDescriptors[0] == ConstructTestTargetDescriptor("TestTargetA", TestImpact::LaunchMethod::TestRunner)); + EXPECT_TRUE(testTargetDescriptors[1] == ConstructTestTargetDescriptor("TestTargetB", TestImpact::LaunchMethod::StandAlone)); + } + +} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactTestEnumerationSuiteFactoryTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactTestEnumerationSuiteFactoryTest.cpp new file mode 100644 index 0000000000..d6e1dc5920 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactTestEnumerationSuiteFactoryTest.cpp @@ -0,0 +1,589 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include + +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace UnitTest +{ + TEST(GTestEnumerationSuiteFactory, ParseEmptyString_ThrowsArtifactException) + { + // Given an empty string + const AZStd::string rawEnum; + + try + { + // When attempting to parse the empty suite + const AZStd::vector suites = TestImpact::GTest::TestEnumerationSuitesFactory(rawEnum); + + // Do not expect the parsing to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::ArtifactException& e) + { + // Expect an artifact exception + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST(GTestEnumerationSuiteFactory, ParseInvalidString_ThrowsArtifactException) + { + // Given an invalid string + const AZStd::string rawEnum = "!@?"; + + try + { + // When attempting to parse the empty suite + const AZStd::vector suites = TestImpact::GTest::TestEnumerationSuitesFactory(rawEnum); + + // Do not expect the parsing to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::ArtifactException& e) + { + // Expect an artifact exception + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST(GTestEnumerationSuiteFactory, ParseTestTargetA_ReturnsValidSuitesAndTests) + { + const AZStd::vector expectedSuites = GetTestTargetATestEnumerationSuites(); + + // Given the raw enum output of TestTargetA + const AZStd::string rawEnum = + "\n" + "\n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + "\n" + "\n"; + + // When the raw enumeration text is parsed + const AZStd::vector suites = TestImpact::GTest::TestEnumerationSuitesFactory(rawEnum); + + // Expect the generated suite data to match that of the raw enum text + EXPECT_TRUE(suites == expectedSuites); + } + + TEST(GTestEnumerationSuiteFactory, ParseTestTargetB_ReturnsValidSuitesAndTests) + { + const AZStd::vector expectedSuites = GetTestTargetBTestEnumerationSuites(); + + // Given the raw enum output of TestTargetB + const AZStd::string rawEnum = + "\n" + "\n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + "\n" + "\n"; + + // When the raw enumeration text is parsed + const AZStd::vector suites = TestImpact::GTest::TestEnumerationSuitesFactory(rawEnum); + + // Expect the generated suite data to match that of the raw enum text + EXPECT_TRUE(suites == expectedSuites); + } + + TEST(GTestEnumerationSuiteFactory, ParseTestTargetC_ReturnsValidSuitesAndTests) + { + const AZStd::vector expectedSuites = GetTestTargetCTestEnumerationSuites(); + + // Given the raw enum output of TestTargetC + const AZStd::string rawEnum = + "\n" + "\n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + "\n" + "\n"; + + // When the raw enumeration text is parsed + const AZStd::vector suites = TestImpact::GTest::TestEnumerationSuitesFactory(rawEnum); + + // Expect the generated suite data to match that of the raw enum text + EXPECT_TRUE(suites == expectedSuites); + } + + TEST(GTestEnumerationSuiteFactory, ParseTestTargetD_ReturnsValidSuitesAndTests) + { + const AZStd::vector expectedSuites = GetTestTargetDTestEnumerationSuites(); + + // Given the raw enum output of TestTargetD + const AZStd::string rawEnum = + "\n" + "\n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + "\n" + "\n"; + + // When the raw enumeration text is parsed + const AZStd::vector suites = TestImpact::GTest::TestEnumerationSuitesFactory(rawEnum); + + // Expect the generated suite data to match that of the raw enum text + EXPECT_TRUE(suites == expectedSuites); + } +} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactTestRunSuiteFactoryTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactTestRunSuiteFactoryTest.cpp new file mode 100644 index 0000000000..0f0a7dfdd8 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactTestRunSuiteFactoryTest.cpp @@ -0,0 +1,592 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include + +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace UnitTest +{ + TEST(GTestRunSuiteFactory, ParseEmptyString_ThrowsArtifactException) + { + // Given an empty string + const AZStd::string rawRun; + + try + { + // When attempting to parse the empty suite + const AZStd::vector suites = TestImpact::GTest::TestRunSuitesFactory(rawRun); + + // Do not expect the parsing to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::ArtifactException& e) + { + // Expect a artifact exception + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST(GTestRunSuiteFactory, ParseInvalidString_ThrowsArtifactException) + { + // Given an invalid string + const AZStd::string rawRun = "!@?"; + + try + { + // When attempting to parse the invalid suite + const AZStd::vector suites = TestImpact::GTest::TestRunSuitesFactory(rawRun); + + // Do not expect the parsing to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::ArtifactException& e) + { + // Expect a artifact exception + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST(GTestRunSuiteFactory, ParseTestTargetA_ReturnsValidSuitesAndTests) + { + const AZStd::vector expectedSuites = GetTestTargetATestRunSuites(); + + // Given the raw run output of TestTargetA + const AZStd::string rawRun = + "" + "" + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + "" + ""; + + // When the raw run text is parsed + const AZStd::vector suites = TestImpact::GTest::TestRunSuitesFactory(rawRun); + + // Expect the generated suite data to match that of the raw run text + EXPECT_TRUE(suites == expectedSuites); + } + + TEST(GTestRunSuiteFactory, ParseTestTargetB_ReturnsValidSuitesAndTests) + { + const AZStd::vector expectedSuites = GetTestTargetBTestRunSuites(); + + // Given the raw run output of TestTargetB + const AZStd::string rawRun = + "" + "" + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + "" + ""; + + // When the raw run text is parsed + const AZStd::vector suites = TestImpact::GTest::TestRunSuitesFactory(rawRun); + + // Expect the generated suite data to match that of the raw run text + EXPECT_TRUE(suites == expectedSuites); + } + + TEST(GTestRunSuiteFactory, ParseTestTargetC_ReturnsValidSuitesAndTests) + { + const AZStd::vector expectedSuites = GetTestTargetCTestRunSuites(); + + // Given the raw run output of TestTargetC + const AZStd::string rawRun = + "" + "" + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + "" + ""; + + // When the raw run text is parsed + const AZStd::vector suites = TestImpact::GTest::TestRunSuitesFactory(rawRun); + + // Expect the generated suite data to match that of the raw run text + EXPECT_TRUE(suites == expectedSuites); + } + + TEST(GTestRunSuiteFactory, ParseTestTargetD_ReturnsValidSuitesAndTests) + { + const AZStd::vector expectedSuites = GetTestTargetDTestRunSuites(); + + // Given the raw run output of TestTargetD + const AZStd::string rawRun = + "" + "" + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + " " + "" + ""; + + // When the raw run text is parsed + const AZStd::vector suites = TestImpact::GTest::TestRunSuitesFactory(rawRun); + + // Expect the generated suite data to match that of the raw run text + EXPECT_TRUE(suites == expectedSuites); + } +} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactTestTargetMetaMapFactoryTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactTestTargetMetaMapFactoryTest.cpp new file mode 100644 index 0000000000..588c46d1ce --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactTestTargetMetaMapFactoryTest.cpp @@ -0,0 +1,239 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include + +#include +#include + +#include +#include + +namespace UnitTest +{ + TEST(TestTargetMetaMapFactoryTest, NoRawData_ExpectArtifactException) + { + // Given an empty meta data string + const AZStd::string rawTestTargetMetaData; + + try + { + // When attempting to construct the test target + const TestImpact::TestTargetMetaMap testTargetMetaData = TestImpact::TestTargetMetaMapFactory(rawTestTargetMetaData); + + // Do not expect this statement to be reachable + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::ArtifactException& e) + { + // Expect an artifact exception + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST(TestTargetMetaMapFactoryTest, InvalidRawData_ExpectArtifactException) + { + // Given a raw meta data string of invalid data + const AZStd::string rawTestTargetMetaData = "abcde"; + + try + { + // When attempting to construct the test target + const TestImpact::TestTargetMetaMap testTargetMetaData = TestImpact::TestTargetMetaMapFactory(rawTestTargetMetaData); + + // Do not expect this statement to be reachable + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::ArtifactException& e) + { + // Expect an artifact exception + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST(TestTargetMetaMapFactoryTest, EmptyTestMetaArray_ExpectArtifactException) + { + // Given a raw meta data string of valid JSON that contains no tests + const AZStd::string rawTestTargetMetaData = + "{" + " \"google\": {" + " \"test\": {" + " \"tests\": [" + " ]" + " }" + " }" + "}"; + + try + { + // When attempting to construct the test target + const TestImpact::TestTargetMetaMap testTargetMetaData = TestImpact::TestTargetMetaMapFactory(rawTestTargetMetaData); + + // Do not expect this statement to be reachable + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::ArtifactException& e) + { + // Expect an artifact exception + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST(TestTargetMetaMapFactoryTest, EmptyName_ExpectArtifactException) + { + // Given a raw meta data string with a test that has no name value + const AZStd::string rawTestTargetMetaData = + "{" + " \"google\": {" + " \"test\": {" + " \"tests\": [" + " { \"name\": \"\", \"namespace\": \"Legacy\", \"suite\": \"main\", \"launch_method\": \"test_runner\" }" + " ]" + " }" + " }" + "}"; + + try + { + // When attempting to construct the test target + const TestImpact::TestTargetMetaMap testTargetMetaData = TestImpact::TestTargetMetaMapFactory(rawTestTargetMetaData); + + // Do not expect this statement to be reachable + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::ArtifactException& e) + { + // Expect an artifact exception + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST(TestTargetMetaMapFactoryTest, EmptyBuildType_ExpectArtifactException) + { + // Given a raw meta data string with a test that has no build type + const AZStd::string rawTestTargetMetaData = + "{" + " \"google\": {" + " \"test\": {" + " \"tests\": [" + " { \"name\": \"TestName\", \"namespace\": \"Legacy\", \"suite\": \"main\", \"launch_method\": \"\" }" + " ]" + " }" + " }" + "}"; + + try + { + // When attempting to construct the test target + const TestImpact::TestTargetMetaMap testTargetMetaData = TestImpact::TestTargetMetaMapFactory(rawTestTargetMetaData); + + // Do not expect this statement to be reachable + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::ArtifactException& e) + { + // Expect an artifact exception + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST(TestTargetMetaMapFactoryTest, InvalidBuildType_ExpectArtifactException) + { + // Given a raw meta data string with a test that has an invalid build type + const AZStd::string rawTestTargetMetaData = + "{" + " \"google\": {" + " \"test\": {" + " \"tests\": [" + " { \"name\": \"TestName\", \"namespace\": \"Legacy\", \"suite\": \"main\", \"launch_method\": \"Unknown\" }" + " ]" + " }" + " }" + "}"; + + try + { + // When attempting to construct the test target + const TestImpact::TestTargetMetaMap testTargetMetaData = TestImpact::TestTargetMetaMapFactory(rawTestTargetMetaData); + + // Do not expect this statement to be reachable + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::ArtifactException& e) + { + // Expect an artifact exception + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST(TestTargetMetaMapFactoryTest, ValidMetaData_ExpectValidTestMetaData) + { + // Given a raw meta data string valid test meta-data + const AZStd::string rawTestTargetMetaData = + "{" + " \"google\": {" + " \"test\": {" + " \"tests\": [" + " { \"name\": \"TestA\", \"namespace\": \"Legacy\", \"suite\": \"main\", \"launch_method\": \"test_runner\" }," + " { \"name\": \"TestB\", \"namespace\": \"\", \"suite\": \"main\", \"launch_method\": \"test_runner\" }," + " { \"name\": \"TestC\", \"namespace\": \"\", \"suite\": \"\", \"launch_method\": \"test_runner\" }," + " { \"name\": \"TestD\", \"namespace\": \"Legacy\", \"suite\": \"main\", \"launch_method\": \"stand_alone\" }" + " ]" + " }" + " }" + "}"; + + // When attempting to construct the test target + const auto testTargetMetaData = TestImpact::TestTargetMetaMapFactory(rawTestTargetMetaData); + + // Expect the constructed test meta-data to match that of the supplied raw data + EXPECT_EQ(testTargetMetaData.size(), 4); + EXPECT_TRUE(testTargetMetaData.find("TestA") != testTargetMetaData.end()); + EXPECT_TRUE((testTargetMetaData.at("TestA") == TestImpact::TestTargetMeta{"main", TestImpact::LaunchMethod::TestRunner})); + EXPECT_TRUE(testTargetMetaData.find("TestB") != testTargetMetaData.end()); + EXPECT_TRUE((testTargetMetaData.at("TestB") == TestImpact::TestTargetMeta{"main", TestImpact::LaunchMethod::TestRunner})); + EXPECT_TRUE(testTargetMetaData.find("TestC") != testTargetMetaData.end()); + EXPECT_TRUE((testTargetMetaData.at("TestC") == TestImpact::TestTargetMeta{"", TestImpact::LaunchMethod::TestRunner})); + EXPECT_TRUE(testTargetMetaData.find("TestD") != testTargetMetaData.end()); + EXPECT_TRUE((testTargetMetaData.at("TestD") == TestImpact::TestTargetMeta{"main", TestImpact::LaunchMethod::StandAlone})); + } +} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Process/TestImpactProcessSchedulerTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Process/TestImpactProcessSchedulerTest.cpp new file mode 100644 index 0000000000..c64b05e431 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Process/TestImpactProcessSchedulerTest.cpp @@ -0,0 +1,718 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace UnitTest +{ + using SchedulerPermutation = AZStd::tuple + < + size_t, // Number of concurrent processes + size_t // Number of processes to launch + >; + + class ProcessSchedulerTestFixtureWithParams + : public AllocatorsTestFixture + , private AZ::Debug::TraceMessageBus::Handler + , public ::testing::WithParamInterface + { + private: + bool OnOutput(const char*, const char*) override; + void SetUp() override; + void TearDown() override; + + protected: + void ConfigurePermutation(); + void ExpectSuccessfulLaunch(TestImpact::ProcessId pid); + void ExpectGracefulExit(TestImpact::ProcessId pid); + void ExpectUnsuccessfulLaunch(TestImpact::ProcessId pid); + void ExpectExitCondition( + TestImpact::ProcessId pid, AZStd::optional expectedExitCondition = AZStd::nullopt); + void DoNotExpectExitCondition(TestImpact::ProcessId pid); + void ExpectTerminatedProcess(TestImpact::ProcessId pid); + void ExpectTimeoutProcess(TestImpact::ProcessId pid); + void ExpectUnlaunchedProcess(TestImpact::ProcessId pid); + void ExpectLargeStdOutput(TestImpact::ProcessId pid); + void ExpectLargeStdError(TestImpact::ProcessId pid); + void ExpectSmallStdOutput(TestImpact::ProcessId pid); + void ExpectSmallStdError(TestImpact::ProcessId pid); + void DoNotExpectStdOutput(TestImpact::ProcessId pid); + void DoNotExpectStdError(TestImpact::ProcessId pid); + + struct ProcessResult + { + AZStd::optional m_launchResult; + AZStd::optional m_exitStatus; + AZStd::optional m_createTime; + AZStd::optional m_exitTime; + AZStd::chrono::milliseconds m_duration = AZStd::chrono::milliseconds(0); + AZStd::optional m_returnCode; + TestImpact::StdContent m_std; + }; + + size_t m_numMaxConcurrentProcesses = 0; + size_t m_numProcessesToLaunch = 0; + + AZStd::unique_ptr m_processScheduler; + TestImpact::ProcessLaunchCallback m_processLaunchCallback; + TestImpact::ProcessExitCallback m_processExitCallback; + AZStd::vector m_processInfos; + AZStd::vector m_processResults; + }; + + // Permutation values for small process batches + const std::vector SmallNumMaxConcurrentProcesses = {1, 4, 8}; + const std::vector SmallNumProcessesToLaunch = {1, 8, 32}; + + // Permutation values for large process batches + const std::vector LargeNumMaxConcurrentProcesses = {16, 32, 64}; + const std::vector LargeNumProcessesToLaunch = {128, 256, 512}; + + void ProcessSchedulerTestFixtureWithParams::ConfigurePermutation() + { + const auto& [numMaxConcurrentProcesses, numProcessesToLaunch] = GetParam(); + m_numMaxConcurrentProcesses = numMaxConcurrentProcesses; + m_numProcessesToLaunch = numProcessesToLaunch; + } + + // Consume AZ log spam + bool ProcessSchedulerTestFixtureWithParams::OnOutput([[maybe_unused]] const char*, [[maybe_unused]] const char*) + { + return true; + } + + void ProcessSchedulerTestFixtureWithParams::SetUp() + { + UnitTest::AllocatorsTestFixture::SetUp(); + AZ::Debug::TraceMessageBus::Handler::BusConnect(); + ConfigurePermutation(); + + m_processLaunchCallback = [this]( + TestImpact::ProcessId pid, + TestImpact::LaunchResult launchResult, + AZStd::chrono::high_resolution_clock::time_point createTime) + { + m_processResults[pid].m_launchResult = launchResult; + m_processResults[pid].m_createTime.emplace(createTime); + return TestImpact::CallbackResult::Continue; + }; + + m_processExitCallback = [this]( + TestImpact::ProcessId pid, + TestImpact::ExitCondition exitStatus, + TestImpact::ReturnCode returnCode, + TestImpact::StdContent&& std, + AZStd::chrono::high_resolution_clock::time_point exitTime) + { + m_processResults[pid].m_std = std::move(std); + m_processResults[pid].m_returnCode.emplace(returnCode); + m_processResults[pid].m_exitStatus = exitStatus; + m_processResults[pid].m_exitTime = exitTime; + m_processResults[pid].m_duration = + AZStd::chrono::duration_cast(exitTime - *m_processResults[pid].m_createTime); + return TestImpact::CallbackResult::Continue; + }; + + m_processResults.resize(m_numProcessesToLaunch); + } + + void ProcessSchedulerTestFixtureWithParams::TearDown() + { + AZ::Debug::TraceMessageBus::Handler::BusDisconnect(); + UnitTest::AllocatorsTestFixture::TearDown(); + } + + // Expects the process to have exited under the specified circumstances + void ProcessSchedulerTestFixtureWithParams::ExpectExitCondition( + TestImpact::ProcessId pid, AZStd::optional expectedExitCondition) + { + const auto& [launchResult, exitStatus, createTime, exitTime, duration, returnCode, std] = m_processResults[pid]; + + // Expect the processes to have exited + EXPECT_TRUE(exitStatus.has_value()); + + // Expect the return code to be valid + EXPECT_TRUE(returnCode.has_value()); + + if (expectedExitCondition.has_value()) + { + // Expect the process to have exited under the specified conditions + EXPECT_EQ(exitStatus, expectedExitCondition); + + // Expect the return code to be that of the process's is (for gracefull exists) or that of the error code + // associated with the abnormal exit condition + EXPECT_EQ(returnCode, + expectedExitCondition == TestImpact::ExitCondition::Gracefull + ? pid + : static_cast(*exitStatus)); + } + + // Expect the duration to be non zero and the create time and exit time to have values as the processes has been in-flight + EXPECT_GT(duration.count(), 0); + EXPECT_TRUE(createTime.has_value()); + EXPECT_TRUE(exitTime.has_value()); + + // Expect the exit time to be later than the start time + EXPECT_GT(exitTime, createTime); + } + + // Expects the process to not have exited for to lack of being in-flight + void ProcessSchedulerTestFixtureWithParams::DoNotExpectExitCondition(TestImpact::ProcessId pid) + { + const auto& [launchResult, exitStatus, createTime, exitTime, duration, returnCode, std] = m_processResults[pid]; + + // Expect the processes to not have exited as the process has not been launched successfully + EXPECT_FALSE(exitStatus.has_value()); + + // Expect the return code to be invalid + EXPECT_FALSE(returnCode.has_value()); + + // Expect the duration to be zero as the process has not been in-flight + EXPECT_EQ(duration.count(), 0); + + // Do not expect the exit time to have a value + EXPECT_FALSE(exitTime.has_value()); + } + + // Expects the process to have been launched successfully, makes no assumptions about how it exited + void ProcessSchedulerTestFixtureWithParams::ExpectSuccessfulLaunch(TestImpact::ProcessId pid) + { + const auto& [launchResult, exitStatus, createTime, exitTime, duration, returnCode, std] = m_processResults[pid]; + + // Expect a launch to have been attempted by this process (not still in queue) + EXPECT_TRUE(launchResult.has_value()); + + // Expect the process to have launched successfully + EXPECT_EQ(launchResult, TestImpact::LaunchResult::Success); + } + + // Expects the process to have failed to launch + void ProcessSchedulerTestFixtureWithParams::ExpectUnsuccessfulLaunch(TestImpact::ProcessId pid) + { + const auto& [launchResult, exitStatus, createTime, exitTime, duration, returnCode, std] = m_processResults[pid]; + + // Expect a launch to have been attempted by this process (not still in queue) + EXPECT_TRUE(launchResult.has_value()); + + // Expect the process to have launched unsuccessfully + EXPECT_EQ(launchResult, TestImpact::LaunchResult::Failure); + + // Expect the create time to have a value as the process was technically created + EXPECT_TRUE(createTime.has_value()); + + DoNotExpectExitCondition(pid); + } + + // Expects the process to have exited gracefully of its own accord (i.e. not terminated for any reason by the scheduler) + void ProcessSchedulerTestFixtureWithParams::ExpectGracefulExit(TestImpact::ProcessId pid) + { + const auto& [launchResult, exitStatus, createTime, exitTime, duration, returnCode, std] = m_processResults[pid]; + + ExpectSuccessfulLaunch(pid); + ExpectExitCondition(pid, TestImpact::ExitCondition::Gracefull); + } + + // Expects the process to have been terminated by the client or scheduler + void ProcessSchedulerTestFixtureWithParams::ExpectTerminatedProcess(TestImpact::ProcessId pid) + { + const auto& [launchResult, exitStatus, createTime, exitTime, duration, returnCode, std] = m_processResults[pid]; + + ExpectSuccessfulLaunch(pid); + ExpectExitCondition(pid, TestImpact::ExitCondition::Terminated); + } + + // Expects the process to have been terminated by the scheduler due to the process or scheduler timing out + void ProcessSchedulerTestFixtureWithParams::ExpectTimeoutProcess(TestImpact::ProcessId pid) + { + const auto& [launchResult, exitStatus, createTime, exitTime, duration, returnCode, std] = m_processResults[pid]; + + ExpectSuccessfulLaunch(pid); + ExpectExitCondition(pid, TestImpact::ExitCondition::Timeout); + } + + // Expects the process to have not been attempted to launch (was still in the queue) + void ProcessSchedulerTestFixtureWithParams::ExpectUnlaunchedProcess(TestImpact::ProcessId pid) + { + const auto& [launchResult, exitStatus, createTime, exitTime, duration, returnCode, std] = m_processResults[pid]; + + // Expect a launch to not have been attempted by this process (still in queue) + EXPECT_FALSE(launchResult.has_value()); + + // Expect the create time to have a value as the process was technically created + EXPECT_FALSE(createTime.has_value()); + + DoNotExpectExitCondition(pid); + } + + // Expects the std output to have been captured from the process with a large volume of text + void ProcessSchedulerTestFixtureWithParams::ExpectLargeStdOutput(TestImpact::ProcessId pid) + { + const auto& stdOut = m_processResults[pid].m_std.m_out; + + // Expect standard output to have a value + EXPECT_TRUE(stdOut.has_value()); + + // Expect the output length to be that of the large text output from the child process + EXPECT_EQ(stdOut.value().length(), LargeTextSize); + } + + // Expects the std error to have been captured from the process with a large volume of text + void ProcessSchedulerTestFixtureWithParams::ExpectLargeStdError(TestImpact::ProcessId pid) + { + const auto& stdErr = m_processResults[pid].m_std.m_err; + + // Expect standard error to have a value + EXPECT_TRUE(stdErr.has_value()); + + // Expect the error length to be that of the large text output from the child process + EXPECT_EQ(stdErr.value().length(), LargeTextSize); + } + + // Expects the std output to have been captured from the process with a small known text string + void ProcessSchedulerTestFixtureWithParams::ExpectSmallStdOutput(TestImpact::ProcessId pid) + { + const auto& stdOut = m_processResults[pid].m_std.m_out; + + // Expect standard output to have a value + EXPECT_TRUE(stdOut.has_value()); + + // Expect the output to match the known stdout of the child + EXPECT_EQ(stdOut.value(), KnownTestProcessOutputString(pid)); + } + + // Expects the std error to have been captured from the process with a small known text string + void ProcessSchedulerTestFixtureWithParams::ExpectSmallStdError(TestImpact::ProcessId pid) + { + const auto& stdErr = m_processResults[pid].m_std.m_err; + + // Expect standard error to have a value + EXPECT_TRUE(stdErr.has_value()); + + // Expect the output to match the known stderr of the child + EXPECT_EQ(stdErr.value(), KnownTestProcessErrorString(pid)); + } + + // Expects no standard output from the child process + void ProcessSchedulerTestFixtureWithParams::DoNotExpectStdOutput(TestImpact::ProcessId pid) + { + const auto& stdOut = m_processResults[pid].m_std.m_out; + + // Do not expect standard output to have a value + EXPECT_FALSE(stdOut.has_value()); + } + + // Expects no standard error from the child process + void ProcessSchedulerTestFixtureWithParams::DoNotExpectStdError(TestImpact::ProcessId pid) + { + const auto& stdErr = m_processResults[pid].m_std.m_err; + + // Do not expect standard error to have a value + EXPECT_FALSE(stdErr.has_value()); + } + + TEST_P(ProcessSchedulerTestFixtureWithParams, ValidProcesses_SuccessfulLaunches) + { + // Given a set of processes to launch with valid arguments + for (size_t pid = 0; pid < m_numProcessesToLaunch; pid++) + { + m_processInfos.emplace_back( + pid, + ValidProcessPath, + ConstructTestProcessArgs(pid, NoSleep)); + } + + // When the process scheduler launches the processes + m_processScheduler = AZStd::make_unique( + m_processInfos, + m_processLaunchCallback, + m_processExitCallback, + m_numMaxConcurrentProcesses, + AZStd::nullopt, + AZStd::nullopt + ); + + for (size_t pid = 0; pid < m_numProcessesToLaunch; pid++) + { + ExpectGracefulExit(pid); + } + } + + TEST_P(ProcessSchedulerTestFixtureWithParams, ValidAndInvalidProcesses_LaunchSuccessesAndFailures) + { + const size_t invalidProcessGroup = 4; + + // Given a mixture of processes to launch with valid and invalid arguments + for (size_t pid = 0; pid < m_numProcessesToLaunch; pid++) + { + if (pid % invalidProcessGroup == 0) + { + m_processInfos.emplace_back( + pid, + InvalidProcessPath, + ConstructTestProcessArgs(pid, NoSleep)); + } + else + { + m_processInfos.emplace_back( + pid, + ValidProcessPath, + ConstructTestProcessArgs(pid, NoSleep)); + } + } + + // When the process scheduler launches the processes + m_processScheduler = AZStd::make_unique( + m_processInfos, + m_processLaunchCallback, + m_processExitCallback, + m_numMaxConcurrentProcesses, + AZStd::nullopt, + AZStd::nullopt + ); + + for (size_t pid = 0; pid < m_numProcessesToLaunch; pid++) + { + if (pid % invalidProcessGroup == 0) + { + ExpectUnsuccessfulLaunch(pid); + } + else + { + ExpectGracefulExit(pid); + } + + DoNotExpectStdOutput(pid); + DoNotExpectStdError(pid); + } + } + + TEST_P(ProcessSchedulerTestFixtureWithParams, ProcessTimeouts_InFlightProcessesTimeout) + { + const size_t timeoutProcessGroup = 4; + + // Given a mixture of processes to launch with some processes sleeping indefinately + for (size_t pid = 0; pid < m_numProcessesToLaunch; pid++) + { + if (pid % timeoutProcessGroup == 0) + { + m_processInfos.emplace_back(pid, ValidProcessPath, ConstructTestProcessArgs(pid, LongSleep)); + } + else + { + m_processInfos.emplace_back(pid, ValidProcessPath, ConstructTestProcessArgs(pid, NoSleep)); + } + } + + // When the process scheduler launches the processes with a process timeout value + m_processScheduler = AZStd::make_unique( + m_processInfos, + m_processLaunchCallback, + m_processExitCallback, + m_numMaxConcurrentProcesses, + AZStd::chrono::milliseconds(100), + AZStd::nullopt + ); + + for (size_t pid = 0; pid < m_numProcessesToLaunch; pid++) + { + if (pid % timeoutProcessGroup == 0) + { + ExpectTimeoutProcess(pid); + } + else + { + ExpectExitCondition(pid); + } + + DoNotExpectStdOutput(pid); + DoNotExpectStdError(pid); + } + } + + TEST_P( + ProcessSchedulerTestFixtureWithParams, ProcessLaunchCallbackAbort_InFlightProcessesTerminatedAndQueuedProcessesUnlaunched) + { + const size_t processToAbort = 8; + + // Given a set of processes to launch + for (size_t pid = 0; pid < m_numProcessesToLaunch; pid++) + { + m_processInfos.emplace_back( + pid, + ValidProcessPath, + ConstructTestProcessArgs(pid, NoSleep)); + } + + // Given a launch callback that will return the abort value for the process to abort + const auto abortingLaunchCallback = [this, processToAbort]( + TestImpact::ProcessId pid, + TestImpact::LaunchResult launchResult, + AZStd::chrono::high_resolution_clock::time_point createTime) + { + m_processLaunchCallback(pid, launchResult, createTime); + + if (pid == processToAbort) + { + return TestImpact::CallbackResult::Abort; + } + else + { + return TestImpact::CallbackResult::Continue; + } + }; + + // When the process scheduler launches the processes + m_processScheduler = AZStd::make_unique( + m_processInfos, + abortingLaunchCallback, + m_processExitCallback, + m_numMaxConcurrentProcesses, + AZStd::nullopt, + AZStd::nullopt + ); + + for (size_t pid = 0; pid < m_numProcessesToLaunch; pid++) + { + if (pid < processToAbort) + { + ExpectSuccessfulLaunch(pid); + } + else if (pid == processToAbort) + { + ExpectTerminatedProcess(pid); + } + else + { + ExpectUnlaunchedProcess(pid); + } + + DoNotExpectStdOutput(pid); + DoNotExpectStdError(pid); + } + } + + TEST_P(ProcessSchedulerTestFixtureWithParams, ProcessExitCallbackAbort_InFlightProcessesTerminated) + { + const size_t processToAbort = 4; + + // Given a set of processes to launch + for (size_t pid = 0; pid < m_numProcessesToLaunch; pid++) + { + m_processInfos.emplace_back( + pid, + ValidProcessPath, + ConstructTestProcessArgs(pid, NoSleep)); + } + + // Given an exit callback that will return the abort value for the process to abort + const auto abortingExitCallback = [this, processToAbort]( + TestImpact::ProcessId pid, + TestImpact::ExitCondition exitStatus, + TestImpact::ReturnCode returnCode, + TestImpact::StdContent&& std, + AZStd::chrono::high_resolution_clock::time_point exitTime) + { + m_processExitCallback(pid, exitStatus, returnCode, std::move(std), exitTime); + + if (pid == processToAbort) + { + return TestImpact::CallbackResult::Abort; + } + else + { + return TestImpact::CallbackResult::Continue; + } + }; + + // When the process scheduler launches the processes + m_processScheduler = AZStd::make_unique( + m_processInfos, + m_processLaunchCallback, + abortingExitCallback, + m_numMaxConcurrentProcesses, + AZStd::nullopt, + AZStd::nullopt + ); + + for (size_t pid = 0; pid < m_numProcessesToLaunch; pid++) + { + if (pid < processToAbort) + { + ExpectSuccessfulLaunch(pid); + } + else if (pid == processToAbort) + { + ExpectGracefulExit(pid); + } + else + { + if (m_processResults[pid].m_launchResult.has_value()) + { + ExpectSuccessfulLaunch(pid); + } + else + { + ExpectUnlaunchedProcess(pid); + } + } + + DoNotExpectStdOutput(pid); + DoNotExpectStdError(pid); + } + } + + TEST_P(ProcessSchedulerTestFixtureWithParams, SchedulerTimeout_QueuedAndInFlightProcessesTerminated) + { + const size_t processToTimeout = 0; + + // Given a set of processes to launch where one process will sleep indefinately + for (size_t pid = 0; pid < m_numProcessesToLaunch; pid++) + { + if (pid == processToTimeout) + { + m_processInfos.emplace_back( + pid, + ValidProcessPath, + ConstructTestProcessArgs(pid, LongSleep)); + } + else + { + m_processInfos.emplace_back( + pid, + ValidProcessPath, + ConstructTestProcessArgs(pid, NoSleep)); + } + } + + // When the process scheduler launches the processes with a scheduler timeout value + m_processScheduler = AZStd::make_unique( + m_processInfos, + m_processLaunchCallback, + m_processExitCallback, + m_numMaxConcurrentProcesses, + AZStd::nullopt, + AZStd::chrono::milliseconds(100) + ); + + for (size_t pid = 0; pid < m_numProcessesToLaunch; pid++) + { + if (pid == processToTimeout) + { + ExpectTimeoutProcess(pid); + } + else + { + if (m_processResults[pid].m_launchResult.has_value()) + { + ExpectSuccessfulLaunch(pid); + } + else + { + ExpectUnlaunchedProcess(pid); + } + } + + DoNotExpectStdOutput(pid); + DoNotExpectStdError(pid); + } + } + + TEST_P(ProcessSchedulerTestFixtureWithParams, RedirectStdOut_StdOutputIsLargeTextString) + { + // Given a set of processes to launch + for (size_t pid = 0; pid < m_numProcessesToLaunch; pid++) + { + m_processInfos.emplace_back( + pid, + TestImpact::StdOutputRouting::ToParent, + TestImpact::StdErrorRouting::None, + ValidProcessPath, + ConstructTestProcessArgsLargeText(pid, NoSleep)); + } + + // When the process scheduler launches the processes + m_processScheduler = AZStd::make_unique( + m_processInfos, + m_processLaunchCallback, + m_processExitCallback, + m_numMaxConcurrentProcesses, + AZStd::nullopt, + AZStd::nullopt + ); + + for (size_t pid = 0; pid < m_numProcessesToLaunch; pid++) + { + ExpectGracefulExit(pid); + ExpectLargeStdOutput(pid); + DoNotExpectStdError(pid); + } + } + + TEST_P(ProcessSchedulerTestFixtureWithParams, RedirectStdError_StdErrorIsLargeTextString) + { + // Given a set of processes to launch + for (size_t pid = 0; pid < m_numProcessesToLaunch; pid++) + { + m_processInfos.emplace_back( + pid, + TestImpact::StdOutputRouting::None, + TestImpact::StdErrorRouting::ToParent, + ValidProcessPath, + ConstructTestProcessArgsLargeText(pid, NoSleep)); + } + + // When the process scheduler launches the processes + m_processScheduler = AZStd::make_unique( + m_processInfos, + m_processLaunchCallback, + m_processExitCallback, + m_numMaxConcurrentProcesses, + AZStd::nullopt, + AZStd::nullopt + ); + + for (size_t pid = 0; pid < m_numProcessesToLaunch; pid++) + { + ExpectGracefulExit(pid); + DoNotExpectStdOutput(pid); + ExpectLargeStdError(pid); + } + } + + INSTANTIATE_TEST_CASE_P( + SmallConcurrentProcesses, + ProcessSchedulerTestFixtureWithParams, + ::testing::Combine( + ::testing::ValuesIn(SmallNumMaxConcurrentProcesses), + ::testing::ValuesIn(SmallNumProcessesToLaunch) + )); + + INSTANTIATE_TEST_CASE_P( + LargeConcurrentProcesses, + ProcessSchedulerTestFixtureWithParams, + ::testing::Combine( + ::testing::ValuesIn(LargeNumMaxConcurrentProcesses), + ::testing::ValuesIn(LargeNumProcessesToLaunch) + )); +} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Process/TestImpactProcessTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Process/TestImpactProcessTest.cpp new file mode 100644 index 0000000000..df4ea5a967 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Process/TestImpactProcessTest.cpp @@ -0,0 +1,491 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include + +#include +#include + +#include +#include +#include + +namespace UnitTest +{ + class ProcessTestFixture + : public AllocatorsTestFixture + { + protected: + void SetUp() override + { + UnitTest::AllocatorsTestFixture::SetUp(); + } + + void TearDown() override + { + if (m_process && m_process->IsRunning()) + { + m_process->Terminate(0); + } + + UnitTest::AllocatorsTestFixture::TearDown(); + } + + AZStd::unique_ptr m_process; + static constexpr const TestImpact::ProcessId m_id = 1; + static constexpr const TestImpact::ReturnCode m_terminateErrorCode = 666; + }; + + TEST_F(ProcessTestFixture, LaunchValidProcess_ProcessReturnsSuccessfully) + { + // Given a process launched with a valid binary + TestImpact::ProcessInfo processInfo(m_id, ValidProcessPath); + m_process = TestImpact::LaunchProcess(processInfo); + + // When the process has exited + m_process->BlockUntilExit(); + + // Expect the process to no longer be running + EXPECT_FALSE(m_process->IsRunning()); + } + + TEST_F(ProcessTestFixture, LaunchInvalidProcessInfo_ThrowsProcessException) + { + try + { + // Given a process launched with an invalid binary + TestImpact::ProcessInfo processInfo(m_id, ""); + + // Do not expect this code to be reachable + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::ProcessException& e) + { + // Except a process exception to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(ProcessTestFixture, LaunchInvalidBinary_ThrowsProcessException) + { + try + { + // Given a process launched with an ivalid binary + TestImpact::ProcessInfo processInfo(m_id, "#!#zz:/z/z/z.exe.z@"); + + // When the process is attempted launch + m_process = TestImpact::LaunchProcess(processInfo); + + // Do not expect this code to be reachable + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::ProcessException& e) + { + // Except a process exception to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(ProcessTestFixture, GetProcessInfo_ReturnsProcessInfo) + { + // Given a process launched with a valid binary and args + const AZStd::string args = ConstructTestProcessArgs(m_id, NoSleep); + TestImpact::ProcessInfo processInfo(m_id, ValidProcessPath, args); + m_process = TestImpact::LaunchProcess(processInfo); + + // When the process has exited + m_process->BlockUntilExit(); + + // Expect the retrieved process info to match to process info used to launch the process + EXPECT_EQ(m_process->GetProcessInfo().GetId(), m_id); + EXPECT_EQ(m_process->GetProcessInfo().GetProcessPath(), ValidProcessPath); + EXPECT_EQ(m_process->GetProcessInfo().GetStartupArgs(), args); + } + + TEST_F(ProcessTestFixture, GetReturnCodeAfterExit_ReturnsArg) + { + // Given a process launched with a valid binary and args + const AZStd::string args = ConstructTestProcessArgs(m_id, NoSleep); + TestImpact::ProcessInfo processInfo(m_id, ValidProcessPath, args); + m_process = TestImpact::LaunchProcess(processInfo); + + // When the process has exited + m_process->BlockUntilExit(); + + // Expect the process to no longer be running + EXPECT_FALSE(m_process->IsRunning()); + + // Expect the return value to be the process id + EXPECT_EQ(m_process->GetReturnCode(), m_id); + } + + TEST_F(ProcessTestFixture, GetReturnCodeInFlight_ReturnsNullOpt) + { + // Given a process launched with a valid binary and args to sleep for 100 seconds + const AZStd::string args = ConstructTestProcessArgs(m_id, LongSleep); + TestImpact::ProcessInfo processInfo(m_id, ValidProcessPath, args); + + // When the process is running + m_process = TestImpact::LaunchProcess(processInfo); + + // Expect the return value to be empty + EXPECT_EQ(m_process->GetReturnCode(), AZStd::nullopt); + } + + TEST_F(ProcessTestFixture, TerminateWithErrorCode_ReturnsErrorCode) + { + // Given a process launched with a valid binary and args to sleep for 100 seconds + const AZStd::string args = ConstructTestProcessArgs(m_id, LongSleep); + TestImpact::ProcessInfo processInfo(m_id, ValidProcessPath, args); + m_process = TestImpact::LaunchProcess(processInfo); + + // When the process is terminated by the client with the specified error code + m_process->Terminate(m_terminateErrorCode); + + // Expect the return value to be the error code specified in the Terminate call + EXPECT_EQ(m_process->GetReturnCode(), m_terminateErrorCode); + + // Expect the process to no longer be running + EXPECT_FALSE(m_process->IsRunning()); + } + + TEST_F(ProcessTestFixture, CheckIsRunningWhilstRunning_ProcessIsRunning) + { + // Given a process launched with a valid binary and args to sleep for 100 seconds + const AZStd::string args = ConstructTestProcessArgs(m_id, LongSleep); + TestImpact::ProcessInfo processInfo(m_id, ValidProcessPath, args); + + // When the process is running + m_process = TestImpact::LaunchProcess(processInfo); + + // Expect the process to be running + EXPECT_TRUE(m_process->IsRunning()); + } + + TEST_F(ProcessTestFixture, CheckIsRunningWhilstNotRunning_ReturnsFalse) + { + // Given a process launched with a valid binary + TestImpact::ProcessInfo processInfo(m_id, ValidProcessPath); + m_process = TestImpact::LaunchProcess(processInfo); + + // When the process is terminated by the client + m_process->Terminate(0); + + // Expect the process to be running + EXPECT_FALSE(m_process->IsRunning()); + } + + TEST_F(ProcessTestFixture, RedirectStdOut_OutputIsKnownTestProcessOutputString) + { + // Given a process launched with a valid binary and standard output redirected to parent + const AZStd::string args = ConstructTestProcessArgs(m_id, NoSleep); + TestImpact::ProcessInfo processInfo( + m_id, + TestImpact::StdOutputRouting::ToParent, + TestImpact::StdErrorRouting::None, + ValidProcessPath, + args); + m_process = TestImpact::LaunchProcess(processInfo); + + // When the process exits + m_process->BlockUntilExit(); + + // Expect stdoutput to be rerouted to the parent + EXPECT_TRUE(m_process->GetProcessInfo().ParentHasStdOutput()); + + // Do not expect stdouterr to be rerouted to the parent + EXPECT_FALSE(m_process->GetProcessInfo().ParentHasStdError()); + + // Expect the output to match the known stdout of the child + EXPECT_EQ(m_process->ConsumeStdOut(), KnownTestProcessOutputString(m_id)); + } + + TEST_F(ProcessTestFixture, RedirectStdErr_OutputIsKnownTestProcessOutputString) + { + // Given a process launched with a valid binary and standard error redirected to parent + const AZStd::string args = ConstructTestProcessArgs(m_id, NoSleep); + TestImpact::ProcessInfo processInfo( + m_id, + TestImpact::StdOutputRouting::None, + TestImpact::StdErrorRouting::ToParent, + ValidProcessPath, + args); + m_process = TestImpact::LaunchProcess(processInfo); + + // When the process exits + m_process->BlockUntilExit(); + + // Do not expect stdoutput to be rerouted to the parent + EXPECT_FALSE(m_process->GetProcessInfo().ParentHasStdOutput()); + + // Expect stderror to be rerouted to the parent + EXPECT_TRUE(m_process->GetProcessInfo().ParentHasStdError()); + + // Expect the output to match the known stdout of the child + EXPECT_EQ(m_process->ConsumeStdErr(), KnownTestProcessErrorString(m_id)); + } + + TEST_F(ProcessTestFixture, RedirectStdOutAndTerminate_OutputIsEmpty) + { + // Given a process launched with a valid binary and standard output redirected to parent + const AZStd::string args = ConstructTestProcessArgs(m_id, NoSleep); + TestImpact::ProcessInfo processInfo( + m_id, + TestImpact::StdOutputRouting::ToParent, + TestImpact::StdErrorRouting::None, + ValidProcessPath, + args); + m_process = TestImpact::LaunchProcess(processInfo); + + // When the process is terminated + m_process->Terminate(0); + + // Expect stdoutput to be rerouted to the parent + EXPECT_TRUE(m_process->GetProcessInfo().ParentHasStdOutput()); + + // Do not expect stdouterr to be rerouted to the parent + EXPECT_FALSE(m_process->GetProcessInfo().ParentHasStdError()); + + // Expect the output to be empty + EXPECT_FALSE(m_process->ConsumeStdOut().has_value()); + } + + TEST_F(ProcessTestFixture, RedirectStdErrAndTerminate_OutputIsEmpty) + { + // Given a process launched with a valid binary and standard error redirected to parent + const AZStd::string args = ConstructTestProcessArgs(m_id, NoSleep); + TestImpact::ProcessInfo processInfo( + m_id, + TestImpact::StdOutputRouting::None, + TestImpact::StdErrorRouting::ToParent, + ValidProcessPath, + args); + m_process = TestImpact::LaunchProcess(processInfo); + + // When the process is terminated + m_process->Terminate(0); + + // Do not expect stdoutput to be rerouted to the parent + EXPECT_FALSE(m_process->GetProcessInfo().ParentHasStdOutput()); + + // Expect stderror to be rerouted to the parent + EXPECT_TRUE(m_process->GetProcessInfo().ParentHasStdError()); + + // Expect the output to be empty + EXPECT_FALSE(m_process->ConsumeStdErr().has_value()); + } + + TEST_F(ProcessTestFixture, RedirectStdOutAndStdErrorRouting_OutputsAreKnownTestProcessOutputStrings) + { + // Given a process launched with a valid binary and both standard output and standard error redirected to parent + const AZStd::string args = ConstructTestProcessArgs(m_id, NoSleep); + TestImpact::ProcessInfo processInfo( + m_id, + TestImpact::StdOutputRouting::ToParent, + TestImpact::StdErrorRouting::ToParent, + ValidProcessPath, + args); + m_process = TestImpact::LaunchProcess(processInfo); + + // When the process exits + m_process->BlockUntilExit(); + + // Expect stdoutput to be rerouted to the parent + EXPECT_TRUE(m_process->GetProcessInfo().ParentHasStdOutput()); + + // Expect stderror to be rerouted to the parent + EXPECT_TRUE(m_process->GetProcessInfo().ParentHasStdError()); + + // Expect the output to match the known stdout and stderr of the child + EXPECT_EQ(m_process->ConsumeStdOut(), KnownTestProcessOutputString(m_id)); + EXPECT_EQ(m_process->ConsumeStdErr(), KnownTestProcessErrorString(m_id)); + } + + TEST_F(ProcessTestFixture, NoStdOutOrStdErrRedirect_OutputIsEmpty) + { + // Given a process launched with a valid binary and both standard output and standard error redirected to parent + const AZStd::string args = ConstructTestProcessArgs(m_id, NoSleep); + TestImpact::ProcessInfo processInfo( + m_id, + TestImpact::StdOutputRouting::None, + TestImpact::StdErrorRouting::None, + ValidProcessPath, + args); + m_process = TestImpact::LaunchProcess(processInfo); + + // When the process exits + m_process->BlockUntilExit(); + + // Do not expect stdoutput to be rerouted to the parent + EXPECT_FALSE(m_process->GetProcessInfo().ParentHasStdOutput()); + + // Do not expect stdouterr to be rerouted to the parent + EXPECT_FALSE(m_process->GetProcessInfo().ParentHasStdError()); + + // Expect the output to to be empty + EXPECT_FALSE(m_process->ConsumeStdOut().has_value()); + EXPECT_FALSE(m_process->ConsumeStdErr().has_value()); + } + + TEST_F(ProcessTestFixture, LargePipeNoRedirects_OutputsAreEmpty) + { + // Given a process outputting large text with no standard output and no standard error redirected to parent + const AZStd::string args = ConstructTestProcessArgsLargeText(m_id, NoSleep); + TestImpact::ProcessInfo processInfo( + m_id, + TestImpact::StdOutputRouting::None, + TestImpact::StdErrorRouting::None, + ValidProcessPath, + args); + m_process = TestImpact::LaunchProcess(processInfo); + + // When the process exits + m_process->BlockUntilExit(); + + // Expect the output to to be empty + EXPECT_FALSE(m_process->ConsumeStdOut().has_value()); + EXPECT_FALSE(m_process->ConsumeStdErr().has_value()); + } + + TEST_F(ProcessTestFixture, LargePipeNoRedirectsAndTerminated_OutputsAreEmpty) + { + // Given a process outputting large text with no standard output and standard error redirected to parent + const AZStd::string args = ConstructTestProcessArgsLargeText(m_id, LongSleep); + TestImpact::ProcessInfo processInfo( + m_id, + TestImpact::StdOutputRouting::None, + TestImpact::StdErrorRouting::None, + ValidProcessPath, + args); + + // When the process is running + m_process = TestImpact::LaunchProcess(processInfo); + + // Expect the output to to be empty + EXPECT_FALSE(m_process->ConsumeStdOut().has_value()); + EXPECT_FALSE(m_process->ConsumeStdErr().has_value()); + + // When the process is terminated + m_process->Terminate(0); + + // Expect the output to to be empty + EXPECT_FALSE(m_process->ConsumeStdOut().has_value()); + EXPECT_FALSE(m_process->ConsumeStdErr().has_value()); + } + + TEST_F(ProcessTestFixture, LargePipeRedirectsAndReadWhilstRunning_OutputsAreEmpty) + { + // Given a process outputting large text with no standard output and standard error redirected to parent + const AZStd::string args = ConstructTestProcessArgsLargeText(m_id, ShortSleep); + TestImpact::ProcessInfo processInfo( + m_id, + TestImpact::StdOutputRouting::None, + TestImpact::StdErrorRouting::None, + ValidProcessPath, + args); + m_process = TestImpact::LaunchProcess(processInfo); + + // When the outputs are read as the process is running + while (m_process->IsRunning()) + { + // Expect the outputs to be empty + EXPECT_FALSE(m_process->ConsumeStdOut().has_value()); + EXPECT_FALSE(m_process->ConsumeStdErr().has_value()); + } + } + + TEST_F(ProcessTestFixture, LargePipeRedirectsAndTerminated_OutputsAreEmpty) + { + // Given a process outputting large text with both standard output and standard error redirected to parent + const AZStd::string args = ConstructTestProcessArgsLargeText(m_id, LongSleep); + TestImpact::ProcessInfo processInfo( + m_id, + TestImpact::StdOutputRouting::ToParent, + TestImpact::StdErrorRouting::ToParent, + ValidProcessPath, + args); + m_process = TestImpact::LaunchProcess(processInfo); + + // When the process is terminated + m_process->Terminate(0); + + // Expect the outputs to be empty + EXPECT_FALSE(m_process->ConsumeStdOut().has_value()); + EXPECT_FALSE(m_process->ConsumeStdErr().has_value()); + } + + TEST_F(ProcessTestFixture, LargePipeRedirectsAndBlockedUntilExit_OutputsAreLargeTextFileSize) + { + // Given a process outputting large text with both standard output and standard error redirected to parent + const AZStd::string args = ConstructTestProcessArgsLargeText(m_id, NoSleep); + TestImpact::ProcessInfo processInfo( + m_id, + TestImpact::StdOutputRouting::ToParent, + TestImpact::StdErrorRouting::ToParent, + ValidProcessPath, + args); + m_process = TestImpact::LaunchProcess(processInfo); + + // When the process exits + m_process->BlockUntilExit(); + + // Expect the output lengths to be that of the large text output from the child process + EXPECT_EQ(m_process->ConsumeStdOut().value().length(), LargeTextSize); + EXPECT_EQ(m_process->ConsumeStdErr().value().length(), LargeTextSize); + } + + TEST_F(ProcessTestFixture, LargePipeRedirectsAndReadWhilstRunning_TotalOutputsAreLargeTextFileSize) + { + // Given a process outputting large text with both standard output and standard error redirected to parent + const AZStd::string args = ConstructTestProcessArgsLargeText(m_id, NoSleep); + TestImpact::ProcessInfo processInfo( + m_id, + TestImpact::StdOutputRouting::ToParent, + TestImpact::StdErrorRouting::ToParent, + ValidProcessPath, args); + m_process = TestImpact::LaunchProcess(processInfo); + + size_t stdOutBytes = 0; + size_t stdErrBytes = 0; + while (m_process->IsRunning()) + { + // When the outputs are read as the process is running + if (auto output = m_process->ConsumeStdOut(); output.has_value()) + { + stdOutBytes += output.value().length(); + } + + if (auto output = m_process->ConsumeStdErr(); output.has_value()) + { + stdErrBytes += output.value().length(); + } + } + + // Expect the output lengths to be that of the large text output from the child process + EXPECT_EQ(stdOutBytes, LargeTextSize); + EXPECT_EQ(stdErrBytes, LargeTextSize); + + // Expect the outputs to be empty + EXPECT_FALSE(m_process->ConsumeStdOut().has_value()); + EXPECT_FALSE(m_process->ConsumeStdErr().has_value()); + } +} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Target/TestImpactBuildTargetTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Target/TestImpactBuildTargetTest.cpp new file mode 100644 index 0000000000..e416d3836e --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Target/TestImpactBuildTargetTest.cpp @@ -0,0 +1,352 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include + +#include +#include + +#include +#include + +namespace UnitTest +{ + namespace + { + AZStd::string GenerateBuildTargetName(size_t index) + { + return AZStd::string::format("Target%u", index); + } + + AZStd::string GenerateBuildTargetOutputName(size_t index) + { + return AZStd::string::format("Output%u", index); + } + + AZ::IO::Path GenerateBuildTargetPath(size_t index) + { + return AZStd::string::format("C:\\Repo\\Dir%u", index); + } + + AZStd::string GenerateTestTargetSuite(size_t index) + { + return AZStd::string::format("Suite%u", index); + } + + TestImpact::LaunchMethod GenerateLaunchMethod(size_t index) + { + return index % 2 == 0 ? TestImpact::LaunchMethod::StandAlone : TestImpact::LaunchMethod::TestRunner; + } + + AZStd::string GenerateStaticSourceFile(size_t index) + { + return AZStd::string::format("StaticSource%u", index); + } + + TestImpact::AutogenPairs GenerateAutogenSourceFiles(size_t index) + { + TestImpact::AutogenPairs autogen; + autogen.m_input = AZStd::string::format("InputSource%u", index); + autogen.m_outputs.push_back(AZStd::string::format("OutputSource%u", index)); + autogen.m_outputs.push_back(AZStd::string::format("OutputHeader%u", index)); + return autogen; + } + + TestImpact::TargetSources GenerateTargetSources(size_t index) + { + TestImpact::TargetSources sources; + sources.m_staticSources.resize(index + 1); + for (size_t i = 0; i < sources.m_staticSources.size(); i++) + { + sources.m_staticSources[i] = GenerateStaticSourceFile(i); + } + + // Only generate autogen sources for even-numbered indexes + if (index % 2 == 0) + { + sources.m_autogenSources.resize(index + 1); + for (size_t i = 0; i < sources.m_autogenSources.size(); i++) + { + sources.m_autogenSources[i] = GenerateAutogenSourceFiles(i); + } + } + + return sources; + } + + TestImpact::ProductionTargetDescriptor GenerateProductionTargetDescriptor(size_t index = 0) + { + return TestImpact::ProductionTargetDescriptor( + { + TestImpact::BuildMetaData + { + GenerateBuildTargetName(index), GenerateBuildTargetOutputName(index), GenerateBuildTargetPath(index) + }, + GenerateTargetSources(index) + }); + } + + TestImpact::TestTargetDescriptor GenerateTestTargetDescriptor(size_t index = 0) + { + return TestImpact::TestTargetDescriptor( + { + TestImpact::BuildMetaData + { + GenerateBuildTargetName(index), GenerateBuildTargetOutputName(index), GenerateBuildTargetPath(index) + }, + GenerateTargetSources(index) + }, + TestImpact::TestTargetMeta + { + GenerateTestTargetSuite(index), GenerateLaunchMethod(index) + }); + } + } // namespace + + template + typename Target::Descriptor GenerateTargetDescriptor(size_t index = 0) + { + static_assert( + AZStd::is_same_v || AZStd::is_same_v, + "Unexpected target type, wants TestImpact::ProductionTarget or TestImpact::TestTarget"); + + if constexpr (AZStd::is_same_v) + { + return GenerateProductionTargetDescriptor(index); + } + else + { + return GenerateTestTargetDescriptor(index); + } + } + + void ValidateSources(const TestImpact::TargetSources& sources, size_t index = 0) + { + EXPECT_EQ(sources.m_staticSources.size(), index + 1); + for (size_t i = 0; i < sources.m_staticSources.size(); i++) + { + EXPECT_EQ(sources.m_staticSources[i], GenerateStaticSourceFile(i)); + } + + // Even numbered indexes have autogen sources + if (index % 2 == 0) + { + EXPECT_EQ(sources.m_autogenSources.size(), index + 1); + for (size_t i = 0; i < sources.m_autogenSources.size(); i++) + { + enum + { + OutputSource = 0, + OutputHeader = 1 + }; + + const TestImpact::AutogenPairs expectedAutogenSources = GenerateAutogenSourceFiles(i); + EXPECT_EQ(sources.m_autogenSources[i].m_input, expectedAutogenSources.m_input); + EXPECT_EQ(sources.m_autogenSources[i].m_outputs[OutputSource], expectedAutogenSources.m_outputs[OutputSource]); + EXPECT_EQ(sources.m_autogenSources[i].m_outputs[OutputHeader], expectedAutogenSources.m_outputs[OutputHeader]); + } + } + else + { + EXPECT_TRUE(sources.m_autogenSources.empty()); + } + } + + void ValidateTarget(const TestImpact::ProductionTarget& target, size_t index = 0) + { + EXPECT_EQ(target.GetName(), GenerateBuildTargetName(index)); + EXPECT_EQ(target.GetOutputName(), GenerateBuildTargetOutputName(index)); + EXPECT_EQ(target.GetPath(), GenerateBuildTargetPath(index)); + EXPECT_EQ(target.GetType(), TestImpact::TargetType::Production); + ValidateSources(target.GetSources(), index); + } + + void ValidateTarget(const TestImpact::TestTarget& target, size_t index = 0) + { + EXPECT_EQ(target.GetName(), GenerateBuildTargetName(index)); + EXPECT_EQ(target.GetOutputName(), GenerateBuildTargetOutputName(index)); + EXPECT_EQ(target.GetPath(), GenerateBuildTargetPath(index)); + EXPECT_EQ(target.GetType(), TestImpact::TargetType::Test); + EXPECT_EQ(target.GetSuite(), GenerateTestTargetSuite(index)); + EXPECT_EQ(target.GetLaunchMethod(), GenerateLaunchMethod(index)); + ValidateSources(target.GetSources(), index); + } + + template + class TargetListFixture + : public AllocatorsTestFixture + { + public: + using TargetListType = TargetList; + using TargetType = typename TargetList::TargetType; + + static constexpr size_t m_numTargets = 10; + }; + + using TargetTypes = testing::Types; + TYPED_TEST_CASE(TargetListFixture, TargetTypes); + + TYPED_TEST(TargetListFixture, CreateTarget_ExpectValidTarget) + { + // Given a target of the specified type + TargetType target(GenerateTargetDescriptor()); + + // Expect the target to match the procedurally generated target descriptor + ValidateTarget(target); + } + + TYPED_TEST(TargetListFixture, CreateEmptyTargetList_ExpectTargetException) + { + try + { + // Given an empty target list + TargetListType targetList({}); + + // Do not expect the target list construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::TargetException& e) + { + // Expect a target exception to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TYPED_TEST(TargetListFixture, CreateTargetListWithDuplicateDescriptor_ExpectTargetException) + { + try + { + // Given a set of target descriptors containing a single duplicate + AZStd::vector descriptors; + descriptors.reserve(m_numTargets); + for (size_t i = 0; i < m_numTargets; i++) + { + // Wrap the last index round to repeat the first index + descriptors.push_back(GenerateTargetDescriptor(i % (m_numTargets - 1))); + } + + // When constructing the target list containing the duplicate target descriptor + TargetListType targetList(AZStd::move(descriptors)); + + // Do not expect the target list construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::TargetException& e) + { + // Expect a target exception to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TYPED_TEST(TargetListFixture, CreateTargetListWithValidDescriptors_ExpectValidTargetList) + { + // Given a valid set of target descriptor + AZStd::vector descriptors; + descriptors.reserve(m_numTargets); + for (size_t i = 0; i < m_numTargets; i++) + { + descriptors.push_back(GenerateTargetDescriptor(i)); + } + + // When constructing the target list containing the valid target descriptors + TargetListType targetList(AZStd::move(descriptors)); + + // Expect the number of targets in the list to match the number of target descriptors used to construct the list + EXPECT_EQ(targetList.GetNumTargets(), m_numTargets); + + for (size_t i = 0; i < targetList.GetNumTargets(); i++) + { + // Expect the target to match the procedurally generated target descriptor + ValidateTarget(targetList.GetTargets()[i], i); + + // Expect the target obtained by name to match the procedurally generated target descriptor + auto target = targetList.GetTarget(GenerateBuildTargetName(i)); + EXPECT_TRUE(target); + EXPECT_EQ(target->GetName(), GenerateBuildTargetName(i)); + } + } + + TYPED_TEST(TargetListFixture, FindNonExistantTargets_ExpectEmptyResults) + { + // Given a valid set of target descriptor + AZStd::vector descriptors; + descriptors.reserve(m_numTargets); + for (size_t i = 0; i < m_numTargets; i++) + { + descriptors.push_back(GenerateTargetDescriptor(i)); + } + + // When constructing the target list containing the valid target descriptors + TargetListType targetList(AZStd::move(descriptors)); + + // Expect the number of targets in the list to match the number of target descriptors used to construct the list + EXPECT_EQ(targetList.GetNumTargets(), m_numTargets); + + for (size_t i = 0; i < targetList.GetNumTargets(); i++) + { + // When attempting to find a target that does not exist + auto target = targetList.GetTarget(GenerateBuildTargetName(i + targetList.GetNumTargets())); + + // Expect an empty result + EXPECT_FALSE(target); + } + } + + TYPED_TEST(TargetListFixture, FindNonExistantTargetsAndThrow_ExpectTargetExceptionss) + { + // Given a valid set of target descriptor + AZStd::vector descriptors; + descriptors.reserve(m_numTargets); + for (size_t i = 0; i < m_numTargets; i++) + { + descriptors.push_back(GenerateTargetDescriptor(i)); + } + + // When constructing the target list containing the valid target descriptors + TargetListType targetList(AZStd::move(descriptors)); + + // Expect the number of targets in the list to match the number of target descriptors used to construct the list + EXPECT_EQ(targetList.GetNumTargets(), m_numTargets); + + for (size_t i = 0; i < targetList.GetNumTargets(); i++) + { + try + { + // When attempting to find a target that does not exist + auto target = targetList.GetTargetOrThrow(GenerateBuildTargetName(i + targetList.GetNumTargets())); + + // Do not expect the target list construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::TargetException& e) + { + // Expect a target exception to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + } +} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactInstrumentedTestRunnerTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactInstrumentedTestRunnerTest.cpp new file mode 100644 index 0000000000..897eda012c --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactInstrumentedTestRunnerTest.cpp @@ -0,0 +1,822 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace UnitTest +{ + using JobExceptionPolicy = TestImpact::InstrumentedTestRunner::JobExceptionPolicy; + using CoverageExceptionPolicy = TestImpact::InstrumentedTestRunner::CoverageExceptionPolicy; + + struct TargetPaths + { + AZ::IO::Path m_targetBinary; + AZ::IO::Path m_testRunArtifact; + AZ::IO::Path m_testCoverageArtifact; + }; + + // Indices for looking up job command arguments for the different coverage levels + enum CoverageLevel : uint8_t + { + LineLevel = 0, + SourceLevel + }; + + // Get the job command for an instrumented test run + AZStd::string GetRunCommandForTarget(const TargetPaths& testTarget, CoverageLevel coverageLevel, const char* sourcesFilter) + { + AZStd::string args = AZStd::string::format( + "%s " // 1. Instrumented test runner + "--coverage_level %s " // 2. Coverage level + "--export_type cobertura:\"%s\" " // 3. Test coverage artifact path + "--modules \"%s\" " // 4. Modules path + "--excluded_modules \"%s\" " // 5. Exclude modules + "--sources \"%s\" -- " // 6. Sources path + "\"%s\" " // 7. Test runner binary + "\"%s\" " // 8. Test target bin + "AzRunUnitTests " + "--gtest_output=xml:%s", // 9. Test run result artifact + + LY_TEST_IMPACT_INSTRUMENTATION_BIN, // 1. + (coverageLevel == CoverageLevel::LineLevel ? "line" : "source"), // 2. + testTarget.m_testCoverageArtifact.c_str(), // 3. + LY_TEST_IMPACT_MODULES_DIR, // 4. + LY_TEST_IMPACT_AZ_TESTRUNNER_BIN, // 5. + sourcesFilter, // 6. + LY_TEST_IMPACT_AZ_TESTRUNNER_BIN, // 7. + testTarget.m_targetBinary.c_str(), // 8. + testTarget.m_testRunArtifact.c_str() // 9. + ); + + // OpenCppCoverage doesn't support forward slash directory separators so replace all with escaped backslashes + return AZStd::regex_replace(args, AZStd::regex("/"), "\\"); + } + + // Get the job command for an instrumented test run with valid source filters to produce coverage artifact + AZStd::string GetRunCommandForTargetWithSources(const TargetPaths& testTarget, CoverageLevel coverageLevel) + { + return GetRunCommandForTarget(testTarget, coverageLevel, LY_TEST_IMPACT_COVERAGE_SOURCES_DIR); + } + + // Get the job command for an instrumented test run without valid source filters to produce empty coverage artifact + AZStd::string GetRunCommandForTargetWithoutSources(const TargetPaths& testTarget, CoverageLevel coverageLevel) + { + return GetRunCommandForTarget(testTarget, coverageLevel, "C:\\No\\Sources\\Here\\At\\All\\Ever\\Ever\\Ever"); + } + + class InstrumentedTestRunnerFixture + : public AllocatorsTestFixture + { + public: + void SetUp() override; + void TearDown() override; + + protected: + using JobInfo = TestImpact::InstrumentedTestRunner::JobInfo; + using JobData = TestImpact::InstrumentedTestRunner::JobData; + + AZStd::vector m_jobInfos; + AZStd::unique_ptr m_testRunner; + AZStd::vector> m_testTargetJobArgs; + AZStd::vector m_testTargetPaths; + AZStd::vector m_expectedTestTargetRuns; + AZStd::vector> m_expectedTestTargetCoverages; + AZStd::vector m_expectedTestTargetResult; + size_t m_maxConcurrency = 0; + CoverageLevel m_coverageLevel = CoverageLevel::LineLevel; + inline static AZ::u32 s_uniqueTestCaseId = 0; // Unique id for each test case to be used as the suffix for the written files + }; + + void InstrumentedTestRunnerFixture::SetUp() + { + AllocatorsTestFixture::SetUp(); + + // Suffix each artifact file with the unique test case id to ensure that there are no possible race conditions between cleaning + // up the artifact files after each test case and those artifact files being written and read again in future test cases + const AZStd::string fileExtension = AZStd::string::format(".%u.xml", s_uniqueTestCaseId++); + + const AZStd::string runPath = AZStd::string(LY_TEST_IMPACT_TEST_TARGET_RESULTS_DIR) + "/%s.Run" + fileExtension; + const AZStd::string coveragePath = AZStd::string(LY_TEST_IMPACT_TEST_TARGET_COVERAGE_DIR) + "/%s.Coverage" + fileExtension; + + // TestTargetA + m_testTargetPaths.emplace_back(TargetPaths{ + LY_TEST_IMPACT_TEST_TARGET_A_BIN, AZStd::string::format(runPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_A_BASE_NAME), + AZStd::string::format(coveragePath.c_str(), LY_TEST_IMPACT_TEST_TARGET_A_BASE_NAME)}); + + // TestTargetB + m_testTargetPaths.emplace_back(TargetPaths{ + LY_TEST_IMPACT_TEST_TARGET_B_BIN, AZStd::string::format(runPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_B_BASE_NAME), + AZStd::string::format(coveragePath.c_str(), LY_TEST_IMPACT_TEST_TARGET_B_BASE_NAME)}); + + // TestTargetC + m_testTargetPaths.emplace_back(TargetPaths{ + LY_TEST_IMPACT_TEST_TARGET_C_BIN, AZStd::string::format(runPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_C_BASE_NAME), + AZStd::string::format(coveragePath.c_str(), LY_TEST_IMPACT_TEST_TARGET_C_BASE_NAME)}); + + // TestTargetD + m_testTargetPaths.emplace_back(TargetPaths{ + LY_TEST_IMPACT_TEST_TARGET_D_BIN, AZStd::string::format(runPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_D_BASE_NAME), + AZStd::string::format(coveragePath.c_str(), LY_TEST_IMPACT_TEST_TARGET_D_BASE_NAME)}); + + m_expectedTestTargetRuns.emplace_back(GetTestTargetATestRunSuites(), AZStd::chrono::milliseconds{500}); // TestTargetA + m_expectedTestTargetRuns.emplace_back(GetTestTargetBTestRunSuites(), AZStd::chrono::milliseconds{500}); // TestTargetB + m_expectedTestTargetRuns.emplace_back(GetTestTargetCTestRunSuites(), AZStd::chrono::milliseconds{500}); // TestTargetC + m_expectedTestTargetRuns.emplace_back(GetTestTargetDTestRunSuites(), AZStd::chrono::milliseconds{500}); // TestTargetD + + // TestTargetA + m_expectedTestTargetCoverages.emplace_back(AZStd::array{ + TestImpact::TestCoverage(GetTestTargetALineModuleCoverages()), + TestImpact::TestCoverage(GetTestTargetASourceModuleCoverages())}); + + // TestTargetB + m_expectedTestTargetCoverages.emplace_back(AZStd::array{ + TestImpact::TestCoverage(GetTestTargetBLineModuleCoverages()), + TestImpact::TestCoverage(GetTestTargetBSourceModuleCoverages())}); + + // TestTargetC + m_expectedTestTargetCoverages.emplace_back(AZStd::array{ + TestImpact::TestCoverage(GetTestTargetCLineModuleCoverages()), + TestImpact::TestCoverage(GetTestTargetCSourceModuleCoverages())}); + + // TestTargetD + m_expectedTestTargetCoverages.emplace_back(AZStd::array{ + TestImpact::TestCoverage(GetTestTargetDLineModuleCoverages()), + TestImpact::TestCoverage(GetTestTargetDSourceModuleCoverages())}); + + m_expectedTestTargetResult.emplace_back(TestImpact::TestRunResult::Failed); // TestTargetA + m_expectedTestTargetResult.emplace_back(TestImpact::TestRunResult::Passed); // TestTargetB + m_expectedTestTargetResult.emplace_back(TestImpact::TestRunResult::Passed); // TestTargetC + m_expectedTestTargetResult.emplace_back(TestImpact::TestRunResult::Passed); // TestTargetD + + // Generate the job command arguments for both line level and source level coverage permutations + for (const auto& testTarget : m_testTargetPaths) + { + m_testTargetJobArgs.emplace_back(AZStd::array{ + GetRunCommandForTargetWithSources(testTarget, CoverageLevel::LineLevel), + GetRunCommandForTargetWithSources(testTarget, CoverageLevel::SourceLevel)}); + } + } + + void InstrumentedTestRunnerFixture::TearDown() + { + DeleteFiles(LY_TEST_IMPACT_TEST_TARGET_COVERAGE_DIR, "*.xml"); + DeleteFiles(LY_TEST_IMPACT_TEST_TARGET_RESULTS_DIR, "*.xml"); + + AllocatorsTestFixture::TearDown(); + } + + using ConcurrencyAndCoveragePermutation = AZStd::tuple + < + size_t, // Max number of concurrent processes + CoverageLevel // Coverage level + >; + + // Fixture parameterized for different max number of concurrent jobs and coverage levels + class InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageParams + : public InstrumentedTestRunnerFixture + , public ::testing::WithParamInterface + { + public: + void SetUp() override + { + InstrumentedTestRunnerFixture::SetUp(); + auto [maxConcurrency, coverageLevel] = GetParam(); + m_maxConcurrency = maxConcurrency; + m_coverageLevel = coverageLevel; + } + }; + + using ConcurrencyAndJobExceptionPermutation = AZStd::tuple + < + size_t, // Max number of concurrent processes + CoverageLevel, // Coverage level + JobExceptionPolicy // Test job exception policy + >; + + // Fixture parameterized for different max number of concurrent jobs, coverage levels and different job exception policies + class InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndJobExceptionParams + : public InstrumentedTestRunnerFixture + , public ::testing::WithParamInterface + { + public: + void SetUp() override + { + InstrumentedTestRunnerFixture::SetUp(); + const auto& [maxConcurrency, coverageLevel, jobExceptionPolicy] = GetParam(); + m_maxConcurrency = maxConcurrency; + m_coverageLevel = coverageLevel; + m_jobExceptionPolicy = jobExceptionPolicy; + } + + protected: + JobExceptionPolicy m_jobExceptionPolicy = JobExceptionPolicy::Never; + }; + + class InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndFailedToLaunchExceptionParams + : public InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndJobExceptionParams + { + }; + + class InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndExecutedWithFailureExceptionParams + : public InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndJobExceptionParams + { + }; + + using ConcurrencyAndCoverageExceptionPermutation = AZStd::tuple + < + size_t, // Max number of concurrent processes + CoverageExceptionPolicy // Test coverage exception policy + >; + + // Fixture parameterized for different max number of concurrent jobs and different coverage exception policies + class InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndCoverageExceptionParams + : public InstrumentedTestRunnerFixture + , public ::testing::WithParamInterface + { + public: + void SetUp() override + { + InstrumentedTestRunnerFixture::SetUp(); + const auto& [maxConcurrency, coverageExceptionPolicy] = GetParam(); + m_maxConcurrency = maxConcurrency; + m_coverageExceptionPolicy = coverageExceptionPolicy; + } + + protected: + CoverageExceptionPolicy m_coverageExceptionPolicy = CoverageExceptionPolicy::Never; + }; + + namespace + { + AZStd::array MaxConcurrentRuns = {1, 2, 3, 4}; + + AZStd::array CoverageLevels = {CoverageLevel::LineLevel, CoverageLevel::SourceLevel}; + + AZStd::array FailedToLaunchExceptionPolicies = { + JobExceptionPolicy::Never, JobExceptionPolicy::OnFailedToExecute}; + + AZStd::array ExecutedWithFailureExceptionPolicies = { + JobExceptionPolicy::Never, JobExceptionPolicy::OnExecutedWithFailure}; + } // namespace + + // Validates that the specified test coverage matches the expected output + void ValidateTestTargetCoverage(const TestImpact::TestCoverage& actualResult, const TestImpact::TestCoverage& expectedResult) + { + EXPECT_TRUE(actualResult == expectedResult); + } + + // Validates that the specified test coverage is empty + void ValidateEmptyTestTargetCoverage(const TestImpact::TestCoverage& actualResult) + { + EXPECT_TRUE(actualResult.GetSourcesCovered().empty()); + EXPECT_TRUE(actualResult.GetModuleCoverages().empty()); + EXPECT_EQ(actualResult.GetNumSourcesCovered(), 0); + EXPECT_EQ(actualResult.GetNumModulesCovered(), 0); + } + + TEST_P( + InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndCoverageExceptionParams, + EmptyTestCoverages_ExpectEmptyTestCoveragesOrTestRunException) + { + // Given a test runner with no client callback or run timeout or runner timeout + m_testRunner = + AZStd::make_unique(AZStd::nullopt, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt); + + // Given a mixture of instrumented test run jobs with and without coverage sources + for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) + { + const AZStd::string args = (jobId % 2) ? GetRunCommandForTargetWithoutSources(m_testTargetPaths[jobId], m_coverageLevel) + : m_testTargetJobArgs[jobId][m_coverageLevel]; + + JobData jobData(m_testTargetPaths[jobId].m_testRunArtifact, m_testTargetPaths[jobId].m_testCoverageArtifact); + m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData))); + } + + try + { + // When the instrumented test run jobs are executed with different exception policies + const auto runnerJobs = m_testRunner->RunInstrumentedTests(m_jobInfos, m_coverageExceptionPolicy, JobExceptionPolicy::Never); + + // Expect this statement to be reachable only if no exception policy for empty coverages + EXPECT_FALSE(::IsFlagSet(m_coverageExceptionPolicy, CoverageExceptionPolicy::OnEmptyCoverage)); + + for (const auto& job : runnerJobs) + { + const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; + ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]); + ValidateTestTargetRun(job.GetPayload().value().first, m_expectedTestTargetRuns[jobId]); + + if (jobId % 2) + { + // Expect jobs have a empty test coverages + ValidateEmptyTestTargetCoverage(job.GetPayload().value().second); + } + else + { + // Expect the jobs to successfully result in a test run and coverage that matches the expected test run data + ValidateTestTargetCoverage(job.GetPayload().value().second, m_expectedTestTargetCoverages[jobId][m_coverageLevel]); + } + } + } + catch ([[maybe_unused]] const TestImpact::TestRunException& e) + { + // Expect this statement to be reachable only if there is an exception policy for empty coverages + EXPECT_TRUE(::IsFlagSet(m_coverageExceptionPolicy, CoverageExceptionPolicy::OnEmptyCoverage)); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_P( + InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndFailedToLaunchExceptionParams, + InvalidCommandArgument_ExpectJobResulFailedToExecuteeOrTestJobException) + { + // Given a test runner with no client callback or run timeout or runner timeout + m_testRunner = + AZStd::make_unique(AZStd::nullopt, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt); + + // Given a mixture of instrumented test run jobs with valid and invalid command arguments + for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) + { + const AZStd::string args = (jobId % 2) ? InvalidProcessPath : m_testTargetJobArgs[jobId][m_coverageLevel]; + JobData jobData(m_testTargetPaths[jobId].m_testRunArtifact, m_testTargetPaths[jobId].m_testCoverageArtifact); + m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData))); + } + + try + { + // When the instrumented test run jobs are executed with different exception policies + const auto runnerJobs = m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, m_jobExceptionPolicy); + + // Expect this statement to be reachable only if no exception policy for launch failures + EXPECT_FALSE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnFailedToExecute)); + + for (const auto& job : runnerJobs) + { + const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; + if (jobId % 2) + { + // Expect invalid jobs have a job result of FailedToExecute + ValidateJobFailedToExecute(job); + } + else + { + // Expect the valid jobs to successfully result in a test run that matches the expected test run data + ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]); + ValidateTestTargetRun(job.GetPayload().value().first, m_expectedTestTargetRuns[jobId]); + ValidateTestTargetCoverage(job.GetPayload().value().second, m_expectedTestTargetCoverages[jobId][m_coverageLevel]); + } + } + } + catch ([[maybe_unused]] const TestImpact::TestJobException& e) + { + // Expect this statement to be reachable only if there is an exception policy for launch failures + EXPECT_TRUE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnFailedToExecute)); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_P( + InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndExecutedWithFailureExceptionParams, + ErroneousReturnCode_ExpectJobResultExecutedWithFailureOrTestJobException) + { + // Given a test runner with no client callback or run timeout or runner timeout + m_testRunner = + AZStd::make_unique(AZStd::nullopt, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt); + + // Given a mixture of instrumented test run jobs that execute and return either successfully or with failure + for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) + { + JobData jobData(m_testTargetPaths[jobId].m_testRunArtifact, m_testTargetPaths[jobId].m_testCoverageArtifact); + m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId][m_coverageLevel], AZStd::move(jobData))); + } + + try + { + // When the instrumented test run jobs are executed with different exception policies + const auto runnerJobs = m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, m_jobExceptionPolicy); + + // Expect this statement to be reachable only if no exception policy for jobs that return with error + EXPECT_FALSE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnExecutedWithFailure)); + + for (const auto& job : runnerJobs) + { + const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; + + // Expect the valid jobs to successfully result in a test run that matches the expected test run data + ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]); + ValidateTestTargetRun(job.GetPayload().value().first, m_expectedTestTargetRuns[jobId]); + ValidateTestTargetCoverage(job.GetPayload().value().second, m_expectedTestTargetCoverages[jobId][m_coverageLevel]); + } + } + catch ([[maybe_unused]] const TestImpact::TestJobException& e) + { + // Expect this statement to be reachable only if there is an exception policy for jobs that return with error + EXPECT_TRUE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnExecutedWithFailure)); + } + catch ([[maybe_unused]] const TestImpact::Exception& e) + { + FAIL(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(InstrumentedTestRunnerFixture, EmptyRunRawData_ExpectTestRunnerException) + { + // Given a test runner with no client callback, concurrency, run timeout or runner timeout + m_testRunner = + AZStd::make_unique(AZStd::nullopt, OneConcurrentProcess, AZStd::nullopt, AZStd::nullopt); + + // Given an test runner job that will return successfully but with an empty artifact string + JobData jobData("", m_testTargetPaths[TestTargetA].m_testCoverageArtifact); + m_jobInfos.emplace_back(JobInfo({TestTargetA}, m_testTargetJobArgs[TestTargetA][LineLevel], AZStd::move(jobData))); + + try + { + // When the test runner job is executed + const auto runnerJobs = + m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, JobExceptionPolicy::Never); + + // Do not expect this statement to be reachable + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::TestRunException& e) + { + // Expect an runner exception + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(InstrumentedTestRunnerFixture, EmptyCoverageRawData_ExpectTestRunnerException) + { + // Given a test runner with no client callback, concurrency, run timeout or runner timeout + m_testRunner = + AZStd::make_unique(AZStd::nullopt, OneConcurrentProcess, AZStd::nullopt, AZStd::nullopt); + + // Given an test runner job that will return successfully but with an empty artifact string + JobData jobData(m_testTargetPaths[TestTargetA].m_testRunArtifact, ""); + m_jobInfos.emplace_back(JobInfo({TestTargetA}, m_testTargetJobArgs[TestTargetA][LineLevel], AZStd::move(jobData))); + + try + { + // When the test runner job is executed + const auto runnerJobs = + m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, JobExceptionPolicy::Never); + + // Do not expect this statement to be reachable + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::TestRunException& e) + { + // Expect an runner exception + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(InstrumentedTestRunnerFixture, InvalidRunArtifact_ExpectArtifactException) + { + // Given a run artifact with invalid contents + WriteTextToFile("There is nothing valid here", m_testTargetPaths[TestTargetA].m_testRunArtifact); + + // Given a job command that will write the run artifact to a different location that what we will read from + TargetPaths invalidRunArtifact = m_testTargetPaths[TestTargetA]; + invalidRunArtifact.m_testRunArtifact /= ".xml"; + const AZStd::string args = GetRunCommandForTargetWithSources(invalidRunArtifact, LineLevel); + + // Given a test runner with no client callback, concurrency, run timeout or runner timeout + m_testRunner = + AZStd::make_unique(AZStd::nullopt, OneConcurrentProcess, AZStd::nullopt, AZStd::nullopt); + + // Given an test runner job that will return successfully but not produce a run artifact + JobData jobData(m_testTargetPaths[TestTargetA].m_testRunArtifact, m_testTargetPaths[TestTargetA].m_testCoverageArtifact); + m_jobInfos.emplace_back(JobInfo({TestTargetA}, args, AZStd::move(jobData))); + + try + { + // When the test runner job is executed + const auto runnerJobs = + m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, JobExceptionPolicy::Never); + + // Do not expect this statement to be reachable + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::ArtifactException& e) + { + // Expect an runner exception + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(InstrumentedTestRunnerFixture, InvalidCoverageArtifact_ExpectArtifactException) + { + // Given a coverage artifact with invalid contents + WriteTextToFile("There is nothing valid here", m_testTargetPaths[TestTargetA].m_testCoverageArtifact); + + // Given a job command that will write the coverage artifact to a different location that what we will read from + TargetPaths invalidCoverageArtifact = m_testTargetPaths[TestTargetA]; + invalidCoverageArtifact.m_testCoverageArtifact /= ".xml"; + const AZStd::string args = GetRunCommandForTargetWithSources(invalidCoverageArtifact, LineLevel); + + // Given a test runner with no client callback, concurrency, run timeout or runner timeout + m_testRunner = + AZStd::make_unique(AZStd::nullopt, OneConcurrentProcess, AZStd::nullopt, AZStd::nullopt); + + // Given an test runner job that will return successfully but not produce a run artifact + JobData jobData(m_testTargetPaths[TestTargetA].m_testRunArtifact, m_testTargetPaths[TestTargetA].m_testCoverageArtifact); + m_jobInfos.emplace_back(JobInfo({TestTargetA}, args, AZStd::move(jobData))); + + try + { + // When the test runner job is executed + const auto runnerJobs = + m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, JobExceptionPolicy::Never); + + // Do not expect this statement to be reachable + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::ArtifactException& e) + { + // Expect an runner exception + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_P(InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageParams, RunTestTargets_RunsAndCoverageMatchTestSuitesInTarget) + { + // Given a test runner with no client callback, runner timeout or runner timeout + m_testRunner = + AZStd::make_unique(AZStd::nullopt, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt); + + // Given an test runner job for each test target with no runner caching + for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) + { + JobData jobData(m_testTargetPaths[jobId].m_testRunArtifact, m_testTargetPaths[jobId].m_testCoverageArtifact); + m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId][m_coverageLevel], AZStd::move(jobData))); + } + + // When the test runner jobs are executed + const auto runnerJobs = m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, JobExceptionPolicy::Never); + + // Expect each job to successfully result in a test runner that matches the expected test runner data for that test target + for (const auto& job : runnerJobs) + { + const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; + ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]); + ValidateTestTargetRun(job.GetPayload().value().first, m_expectedTestTargetRuns[jobId]); + ValidateTestTargetCoverage(job.GetPayload().value().second, m_expectedTestTargetCoverages[jobId][m_coverageLevel]); + } + } + + TEST_P( + InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageParams, + RunTestTargetsWithArbitraryJobIds_RunsAndCoverageMatchTestSuitesInTarget) + { + // Given a set of arbitrary job ids to be used for the test target jobs + enum + { + ArbitraryA = 36, + ArbitraryB = 890, + ArbitraryC = 19, + ArbitraryD = 1 + }; + + const AZStd::unordered_map sequentialToArbitrary = + { + {TestTargetA, ArbitraryA}, + {TestTargetB, ArbitraryB}, + {TestTargetC, ArbitraryC}, + {TestTargetD, ArbitraryD}, + }; + + const AZStd::unordered_map arbitraryToSequential = + { + {ArbitraryA, TestTargetA}, + {ArbitraryB, TestTargetB}, + {ArbitraryC, TestTargetC}, + {ArbitraryD, TestTargetD}, + }; + + // Given a test runner with no client callback, run timeout or runner timeout + m_testRunner = + AZStd::make_unique(AZStd::nullopt, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt); + + // Given an test run job for each test target + for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) + { + JobData jobData(m_testTargetPaths[jobId].m_testRunArtifact, m_testTargetPaths[jobId].m_testCoverageArtifact); + m_jobInfos.emplace_back( + JobInfo({sequentialToArbitrary.at(jobId)}, m_testTargetJobArgs[jobId][m_coverageLevel], AZStd::move(jobData))); + } + + // When the instrumented test run jobs are executed + const auto runnerJobs = m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, JobExceptionPolicy::Never); + + // Expect each job to successfully result in a test run that matches the expected test run data for that test target + for (const auto& job : runnerJobs) + { + const JobInfo::IdType jobId = arbitraryToSequential.at(job.GetJobInfo().GetId().m_value); + ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]); + ValidateTestTargetRun(job.GetPayload().value().first, m_expectedTestTargetRuns[jobId]); + ValidateTestTargetCoverage(job.GetPayload().value().second, m_expectedTestTargetCoverages[jobId][m_coverageLevel]); + } + } + + TEST_P(InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageParams, RunTestTargetsWithCallback_RunsAndCoverageMatchTestSuitesInTarget) + { + // Given a client callback function that tracks the number of successful runs + size_t numSuccesses = 0; + const auto jobCallback = + [&numSuccesses]([[maybe_unused]] const TestImpact::InstrumentedTestRunner::JobInfo& jobInfo, const TestImpact::JobMeta& meta) + { + if (meta.m_result == TestImpact::JobResult::ExecutedWithSuccess) + { + numSuccesses++; + } + }; + + // Given a test runner with no run timeout or runner timeout + m_testRunner = + AZStd::make_unique(jobCallback, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt); + + // Given an test run job for each test target + for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) + { + JobData jobData(m_testTargetPaths[jobId].m_testRunArtifact, m_testTargetPaths[jobId].m_testCoverageArtifact); + m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId][m_coverageLevel], AZStd::move(jobData))); + } + + // When the instrumented test run jobs are executed + const auto runnerJobs = m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, JobExceptionPolicy::Never); + + // Expect the number of successful runs tracked in the callback to match the number of test targets run with no failures + EXPECT_EQ( + numSuccesses, + AZStd::count_if(m_expectedTestTargetResult.begin(), m_expectedTestTargetResult.end(), [](TestImpact::TestRunResult result) { + return result == TestImpact::TestRunResult::Passed; + })); + + // Expect each job to successfully result in a test run that matches the expected test run data for that test target + for (const auto& job : runnerJobs) + { + const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; + ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]); + ValidateTestTargetRun(job.GetPayload().value().first, m_expectedTestTargetRuns[jobId]); + ValidateTestTargetCoverage(job.GetPayload().value().second, m_expectedTestTargetCoverages[jobId][m_coverageLevel]); + } + } + + TEST_P(InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageParams, JobRunnerTimeout_InFlightJobsTimeoutAndQueuedJobsUnlaunched) + { + // Given a test runner with no client callback or runner timeout and 2 second run timeout + m_testRunner = AZStd::make_unique( + AZStd::nullopt, m_maxConcurrency, AZStd::chrono::seconds(2), AZStd::nullopt); + + // Given an test run job for each test target where half will sleep indefinitely + for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) + { + JobData jobData(m_testTargetPaths[jobId].m_testRunArtifact, m_testTargetPaths[jobId].m_testCoverageArtifact); + const AZStd::string args = (jobId % 2) + ? AZStd::string::format("%s %s", ValidProcessPath, ConstructTestProcessArgs(jobId, LongSleep).c_str()) + : m_testTargetJobArgs[jobId][m_coverageLevel]; + m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData))); + } + + // When the instrumented test run jobs are executed + const auto runnerJobs = m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, JobExceptionPolicy::Never); + + // Expect half the jobs to successfully result in a test run that matches the expected test run data for that test target + // with the other half having timed out + for (const auto& job : runnerJobs) + { + const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; + if (jobId % 2) + { + ValidateJobTimeout(job); + } + else + { + ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]); + ValidateTestTargetRun(job.GetPayload().value().first, m_expectedTestTargetRuns[jobId]); + ValidateTestTargetCoverage(job.GetPayload().value().second, m_expectedTestTargetCoverages[jobId][m_coverageLevel]); + } + } + } + + TEST_F(InstrumentedTestRunnerFixture, JobTimeout_InFlightJobTimeoutAndQueuedJobsUnlaunched) + { + // Given a test runner with no client callback or run timeout and a 5 second runner timeout + m_testRunner = AZStd::make_unique( + AZStd::nullopt, FourConcurrentProcesses, AZStd::nullopt, AZStd::chrono::seconds(5)); + + // Given an test run job for each test target where half will sleep indefinitely + for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) + { + JobData jobData(m_testTargetPaths[jobId].m_testRunArtifact, m_testTargetPaths[jobId].m_testCoverageArtifact); + const AZStd::string args = (jobId % 2) + ? AZStd::string::format("%s %s", ValidProcessPath, ConstructTestProcessArgs(jobId, LongSleep).c_str()) + : m_testTargetJobArgs[jobId][m_coverageLevel]; + m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData))); + } + + // When the instrumented test run jobs are executed + const auto runnerJobs = m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, JobExceptionPolicy::Never); + + // Expect half the jobs to successfully result in a test run that matches the expected test run data for that test target + // with the other half having timed out + for (const auto& job : runnerJobs) + { + const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; + if (jobId % 2) + { + ValidateJobTimeout(job); + } + else + { + ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]); + ValidateTestTargetRun(job.GetPayload().value().first, m_expectedTestTargetRuns[jobId]); + ValidateTestTargetCoverage(job.GetPayload().value().second, m_expectedTestTargetCoverages[jobId][m_coverageLevel]); + } + } + } + + INSTANTIATE_TEST_CASE_P( + , + InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndCoverageExceptionParams, + ::testing::Combine( + ::testing::ValuesIn(MaxConcurrentRuns), + ::testing::Values(CoverageExceptionPolicy::Never, CoverageExceptionPolicy::OnEmptyCoverage))); + + INSTANTIATE_TEST_CASE_P( + , + InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndFailedToLaunchExceptionParams, + ::testing::Combine( + ::testing::ValuesIn(MaxConcurrentRuns), ::testing::ValuesIn(CoverageLevels), + ::testing::ValuesIn(FailedToLaunchExceptionPolicies))); + + INSTANTIATE_TEST_CASE_P( + , + InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndExecutedWithFailureExceptionParams, + ::testing::Combine( + ::testing::ValuesIn(MaxConcurrentRuns), ::testing::ValuesIn(CoverageLevels), + ::testing::ValuesIn(ExecutedWithFailureExceptionPolicies))); + + INSTANTIATE_TEST_CASE_P( + , + InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageParams, + ::testing::Combine(::testing::ValuesIn(MaxConcurrentRuns), ::testing::ValuesIn(CoverageLevels))); +} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestCoverageTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestCoverageTest.cpp new file mode 100644 index 0000000000..538097d11e --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestCoverageTest.cpp @@ -0,0 +1,208 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include + +#include +#include + +namespace UnitTest +{ + namespace + { + AZ::IO::Path GenerateSourcePath(AZ::u32 index) + { + return AZStd::string::format("SourceFile%u", index); + } + + AZ::IO::Path GenerateModulePath(AZ::u32 index) + { + return AZStd::string::format("Module%u", index); + } + + AZStd::vector GenerateLineCoverages(AZ::u32 numLines) + { + AZStd::vector lineCoverages; + for (size_t i = 0; i < numLines; i++) + { + // Fudge some superficially different but trivially checkable line coverage data + lineCoverages.emplace_back(TestImpact::LineCoverage{i, i * 2}); + } + + return lineCoverages; + } + + TestImpact::SourceCoverage GenerateSourceCoverage(AZ::u32 index, TestImpact::CoverageLevel coverageLevel) + { + TestImpact::SourceCoverage sourceCoverage; + sourceCoverage.m_path = GenerateSourcePath(index); + if (coverageLevel == TestImpact::CoverageLevel::Line) + { + sourceCoverage.m_coverage.emplace(GenerateLineCoverages(index + 1)); + } + + return sourceCoverage; + } + + AZStd::vector GenerateSourceCoverages(AZ::u32 numSources, TestImpact::CoverageLevel coverageLevel) + { + AZStd::vector sourceCoverages; + for (AZ::u32 i = 0; i < numSources; i++) + { + sourceCoverages.emplace_back(GenerateSourceCoverage(i, coverageLevel)); + } + + return sourceCoverages; + } + + TestImpact::ModuleCoverage GenerateModuleCoverage(AZ::u32 index, AZ::u32 numSources, TestImpact::CoverageLevel coverageLevel) + { + TestImpact::ModuleCoverage moduleCoverage; + moduleCoverage.m_path = GenerateModulePath(index); + moduleCoverage.m_sources = GenerateSourceCoverages(numSources, coverageLevel); + return moduleCoverage; + } + + AZStd::vector GenerateModuleCoverages(AZ::u32 numModules, TestImpact::CoverageLevel coverageLevel) + { + AZStd::vector moduleCoverages; + for (AZ::u32 i = 0; i < numModules; i++) + { + // Fudge some superficially different but trivially deducible module coverage data + moduleCoverages.emplace_back(GenerateModuleCoverage(i, i + 1, coverageLevel)); + } + + return moduleCoverages; + } + } // namespace + + using CoveragePermutation = AZStd::tuple + < + unsigned, // Number of modules covered + TestImpact::CoverageLevel // Test coverage level + >; + + // Fixture parameterized for different max number of concurrent jobs + class TestCoverageFixtureWithCoverageParams + : public AllocatorsTestFixture + , public ::testing::WithParamInterface + { + public: + void SetUp() override; + + protected: + void ValidateTestCoverage(const TestImpact::TestCoverage& testCoverage); + + size_t m_numModulesCovered; + TestImpact::CoverageLevel m_coverageLevel; + }; + + void TestCoverageFixtureWithCoverageParams::SetUp() + { + AllocatorsTestFixture::SetUp(); + const auto& [numModulesCovered, coverageLevel] = GetParam(); + m_numModulesCovered = numModulesCovered; + m_coverageLevel = coverageLevel; + } + + void TestCoverageFixtureWithCoverageParams::ValidateTestCoverage(const TestImpact::TestCoverage& testCoverage) + { + // Expect the coverage level to match that which was used to generate the module coverages generated + EXPECT_EQ(testCoverage.GetCoverageLevel(), m_coverageLevel); + + // Expect the number of modules covered to match the number of module coverages generated + EXPECT_EQ(testCoverage.GetNumModulesCovered(), m_numModulesCovered); + + // Expect the number of unique sources covered to match the number of modules coverages generated + EXPECT_EQ(testCoverage.GetNumSourcesCovered(), m_numModulesCovered); + + // Expect the unique sources covered to match the procedurally generated source paths + for (size_t sourceIndex = 0; sourceIndex < testCoverage.GetNumSourcesCovered(); sourceIndex++) + { + EXPECT_EQ(testCoverage.GetSourcesCovered()[sourceIndex], GenerateSourcePath(sourceIndex)); + } + + // Expect each module covered to match that of the corresponding procedurally generated modules + for (size_t moduleIndex = 0; moduleIndex < testCoverage.GetNumModulesCovered(); moduleIndex++) + { + const TestImpact::ModuleCoverage& moduleCoverage = testCoverage.GetModuleCoverages()[moduleIndex]; + + // Expect the module path to match that of the corresponding procedurally generated module + EXPECT_EQ(moduleCoverage.m_path, GenerateModulePath(moduleIndex)); + + // Expect the module's number of sources to match that of the corresponding procedurally generated module + EXPECT_EQ(moduleCoverage.m_sources.size(), moduleIndex + 1); + + for (size_t sourceIndex = 0; sourceIndex < moduleCoverage.m_sources.size(); sourceIndex++) + { + const TestImpact::SourceCoverage& sourceCoverage = moduleCoverage.m_sources[sourceIndex]; + + // Expect the source path to match the procedurally generated source path + EXPECT_EQ(sourceCoverage.m_path, GenerateSourcePath(sourceIndex)); + + if (m_coverageLevel == TestImpact::CoverageLevel::Line) + { + // Expect there to actually be line coverage data if this coverage was procedurally generated with line data + EXPECT_TRUE(sourceCoverage.m_coverage.has_value()); + + const AZStd::vector& lineCoverages = sourceCoverage.m_coverage.value(); + + // Expect the source's number of lines to match that of the corresponding procedurally generated source + EXPECT_EQ(lineCoverages.size(), sourceIndex + 1); + + for (size_t lineIndex = 0; lineIndex < lineCoverages.size(); lineIndex++) + { + // The expected line number and hit count are deduced as follows: + // Line number: line index + // Hit count: 2x line index + EXPECT_EQ(lineCoverages[lineIndex].m_lineNumber, lineIndex); + EXPECT_EQ(lineCoverages[lineIndex].m_hitCount, lineIndex * 2); + } + } + else + { + // Do not expect there to actually be line coverage data if this coverage was not procedurally generated with line data + EXPECT_FALSE(sourceCoverage.m_coverage.has_value()); + } + } + } + } + + TEST(TestCoverage, EmptyCoverage_ExpectTestRunException) + { + // When constructing a test coverage from the empty module coverages + TestImpact::TestCoverage testCoverage(AZStd::vector{}); + + // Expect the test coverage fields to be empty + EXPECT_EQ(testCoverage.GetNumModulesCovered(), 0); + EXPECT_EQ(testCoverage.GetNumSourcesCovered(), 0); + EXPECT_TRUE(testCoverage.GetModuleCoverages().empty()); + EXPECT_TRUE(testCoverage.GetSourcesCovered().empty()); + } + + TEST_P(TestCoverageFixtureWithCoverageParams, AllCoveragePermutations_ExpectTestCoverageMetaDatasToMatchPermutations) + { + // Given a procedurally generated test coverage + const TestImpact::TestCoverage testCoverage(GenerateModuleCoverages(m_numModulesCovered, m_coverageLevel)); + + // Expect the test coverage data and meta-data to match that of the rules used to procedurally generate the coverage data + ValidateTestCoverage(testCoverage); + } + + INSTANTIATE_TEST_CASE_P( + , + TestCoverageFixtureWithCoverageParams, + ::testing::Combine( + ::testing::Range(1u, 11u), + ::testing::Values(TestImpact::CoverageLevel::Line, TestImpact::CoverageLevel::Source)) + ); +} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestEnumeratorTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestEnumeratorTest.cpp new file mode 100644 index 0000000000..9b67eafa54 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestEnumeratorTest.cpp @@ -0,0 +1,890 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace UnitTest +{ + using JobExceptionPolicy = TestImpact::TestEnumerator::JobExceptionPolicy; + using CacheExceptionPolicy = TestImpact::TestEnumerator::CacheExceptionPolicy; + + // Generates the command to run the given test target through AzTestRunner and get gtest to output the enumeration file + AZStd::string GetEnumerateCommandForTarget(AZStd::pair testTarget) + { + return AZStd::string::format( + "%s %s AzRunUnitTests --gtest_list_tests --gtest_output=xml:%s", + LY_TEST_IMPACT_AZ_TESTRUNNER_BIN, + testTarget.first.c_str(), // Path to test target bin + testTarget.second.c_str()); // Path to test target gtest enumeration file + } + + class TestEnumeratorFixture + : public AllocatorsTestFixture + { + public: + void SetUp() override; + + protected: + using JobInfo = TestImpact::TestEnumerator::JobInfo; + using JobData = TestImpact::TestEnumerator::JobData; + + AZStd::vector m_jobInfos; + AZStd::unique_ptr m_testEnumerator; + AZStd::vector m_testTargetJobArgs; + AZStd::vector> m_testTargetPaths; + AZStd::vector m_expectedTestTargetEnumerations; + AZStd::vector m_cacheFiles; + size_t m_maxConcurrency = 0; + }; + + void TestEnumeratorFixture::SetUp() + { + UnitTest::AllocatorsTestFixture::SetUp(); + + DeleteFiles(LY_TEST_IMPACT_TEST_TARGET_ENUMERATION_DIR, "*.cache"); + DeleteFiles(LY_TEST_IMPACT_TEST_TARGET_ENUMERATION_DIR, "*.xml"); + + // first: path to test target bin + // second: path to test target gtest enumeration file in XML format + const AZStd::string enumPath = AZStd::string(LY_TEST_IMPACT_TEST_TARGET_ENUMERATION_DIR) + "/%s.Enumeration.xml"; + m_testTargetPaths.emplace_back( + LY_TEST_IMPACT_TEST_TARGET_A_BIN, AZStd::string::format(enumPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_A_BASE_NAME)); + m_testTargetPaths.emplace_back( + LY_TEST_IMPACT_TEST_TARGET_B_BIN, AZStd::string::format(enumPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_B_BASE_NAME)); + m_testTargetPaths.emplace_back( + LY_TEST_IMPACT_TEST_TARGET_C_BIN, AZStd::string::format(enumPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_C_BASE_NAME)); + m_testTargetPaths.emplace_back( + LY_TEST_IMPACT_TEST_TARGET_D_BIN, AZStd::string::format(enumPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_D_BASE_NAME)); + + m_expectedTestTargetEnumerations.emplace_back(GetTestTargetATestEnumerationSuites()); + m_expectedTestTargetEnumerations.emplace_back(GetTestTargetBTestEnumerationSuites()); + m_expectedTestTargetEnumerations.emplace_back(GetTestTargetCTestEnumerationSuites()); + m_expectedTestTargetEnumerations.emplace_back(GetTestTargetDTestEnumerationSuites()); + + // Path to enumeration file in TIAF internal JSON format + m_cacheFiles.emplace_back( + AZStd::string::format("%s/%s.cache", LY_TEST_IMPACT_TEST_TARGET_ENUMERATION_DIR, LY_TEST_IMPACT_TEST_TARGET_A_BASE_NAME)); + m_cacheFiles.emplace_back( + AZStd::string::format("%s/%s.cache", LY_TEST_IMPACT_TEST_TARGET_ENUMERATION_DIR, LY_TEST_IMPACT_TEST_TARGET_B_BASE_NAME)); + m_cacheFiles.emplace_back( + AZStd::string::format("%s/%s.cache", LY_TEST_IMPACT_TEST_TARGET_ENUMERATION_DIR, LY_TEST_IMPACT_TEST_TARGET_C_BASE_NAME)); + m_cacheFiles.emplace_back( + AZStd::string::format("%s/%s.cache", LY_TEST_IMPACT_TEST_TARGET_ENUMERATION_DIR, LY_TEST_IMPACT_TEST_TARGET_D_BASE_NAME)); + + for (const auto& testTarget : m_testTargetPaths) + { + m_testTargetJobArgs.emplace_back(GetEnumerateCommandForTarget(testTarget)); + } + } + + // Fixture parameterized for different max number of concurrent jobs + class TestEnumeratorFixtureWithConcurrencyParams + : public TestEnumeratorFixture + , public ::testing::WithParamInterface + { + public: + void SetUp() override + { + TestEnumeratorFixture::SetUp(); + m_maxConcurrency = GetParam(); + } + }; + + using ConcurrencyAndJobExceptionPermutation = AZStd::tuple + < + size_t, // Max number of concurrent processes + JobExceptionPolicy // Test job exception policy + >; + + // Fixture parameterized for different max number of concurrent jobs and different job exception policies + class TestEnumeratorFixtureWithConcurrencyAndJobExceptionParams + : public TestEnumeratorFixture + , public ::testing::WithParamInterface + { + public: + void SetUp() override + { + TestEnumeratorFixture::SetUp(); + const auto& [maxConcurrency, jobExceptionPolicy] = GetParam(); + m_maxConcurrency = maxConcurrency; + m_jobExceptionPolicy = jobExceptionPolicy; + } + + protected: + JobExceptionPolicy m_jobExceptionPolicy = JobExceptionPolicy::Never; + }; + + using ConcurrencyAndCacheExceptionPermutation = AZStd::tuple + < + size_t, // Max number of concurrent processes + CacheExceptionPolicy // Test enumeration exception policy + >; + + // Fixture parameterized for different max number of concurrent jobs and different enumeration exception policies + class TestEnumeratorFixtureWithConcurrencyAndCacheExceptionParams + : public TestEnumeratorFixture + , public ::testing::WithParamInterface + { + public: + void SetUp() override + { + TestEnumeratorFixture::SetUp(); + const auto& [maxConcurrency, cacheExceptionPolicy] = GetParam(); + m_maxConcurrency = maxConcurrency; + m_cacheExceptionPolicy = cacheExceptionPolicy; + } + + protected: + CacheExceptionPolicy m_cacheExceptionPolicy = CacheExceptionPolicy::Never; + }; + + namespace + { + AZStd::array MaxConcurrentEnumerations = {{1, 2, 3, 4}}; + + AZStd::array JobExceptionPolicies = + { + JobExceptionPolicy::Never, + JobExceptionPolicy::OnExecutedWithFailure, + JobExceptionPolicy::OnFailedToExecute + }; + + AZStd::array CacheExceptionPolicies = + { + CacheExceptionPolicy::Never, + CacheExceptionPolicy::OnCacheNotExist, + CacheExceptionPolicy::OnCacheReadFailure, + CacheExceptionPolicy::OnCacheWriteFailure + }; + } + + // Validates that the specified job successfully read from its test enumeration cache + void ValidateJobSuccessfulCacheRead(const TestImpact::TestEnumerator::Job& job) + { + EXPECT_EQ(job.GetResult(), TestImpact::JobResult::NotExecuted); + EXPECT_EQ(job.GetStartTime(), AZStd::chrono::high_resolution_clock::time_point()); + EXPECT_EQ(job.GetEndTime(), AZStd::chrono::high_resolution_clock::time_point()); + EXPECT_EQ(job.GetDuration(), AZStd::chrono::milliseconds(0)); + EXPECT_FALSE(job.GetReturnCode().has_value()); + EXPECT_TRUE(job.GetPayload().has_value()); + } + + // Validates that the specified test enumeration matched the expected output + void ValidateTestTargetEnumeration(const TestImpact::TestEnumeration& actualResult, const TestImpact::TestEnumeration& expectedResult) + { + EXPECT_TRUE(actualResult == expectedResult); + EXPECT_EQ(actualResult.GetNumTestSuites(), CalculateNumTestSuites(expectedResult.GetTestSuites())); + EXPECT_EQ(actualResult.GetNumTests(), CalculateNumTests(expectedResult.GetTestSuites())); + EXPECT_EQ(actualResult.GetNumEnabledTests(), CalculateNumEnabledTests(expectedResult.GetTestSuites())); + EXPECT_EQ(actualResult.GetNumDisabledTests(), CalculateNumDisabledTests(expectedResult.GetTestSuites())); + } + + // Validates that the specified test enumeration cache matches the expected output + void ValidateTestEnumerationCache(const AZ::IO::Path& cacheFile, const TestImpact::TestEnumeration& expectedEnumeration) + { + // Cache file must exist + const auto fileSize = AZ::IO::SystemFile::Length(cacheFile.c_str()); + EXPECT_GT(fileSize, 0); + + // Read raw byte data from cache + AZStd::vector buffer(fileSize + 1); + buffer[fileSize] = 0; + EXPECT_TRUE(AZ::IO::SystemFile::Read(cacheFile.c_str(), buffer.data())); + + // Transform raw byte data to raw string data and attempt to construct the test enumeration + AZStd::string rawEnum(buffer.begin(), buffer.end()); + TestImpact::TestEnumeration actualEnumeration = TestImpact::DeserializeTestEnumeration(rawEnum); + + // Check that the constructed test enumeration matches the expected enumeration + ValidateTestTargetEnumeration(actualEnumeration, expectedEnumeration); + } + + // Validates that the specified cache file does not exist + void ValidateInvalidTestEnumerationCache(const AZ::IO::Path& cacheFile) + { + EXPECT_FALSE(AZ::IO::SystemFile::Exists(cacheFile.c_str())); + } + + TEST_P( + TestEnumeratorFixtureWithConcurrencyAndJobExceptionParams, InvalidCommandArgument_ExpectJobResulFailedToExecuteeOrTestJobException) + { + // Given a test enumerator with no client callback or enumeration timeout or enumerator timeout + m_testEnumerator = AZStd::make_unique( + AZStd::nullopt, + m_maxConcurrency, + AZStd::nullopt, + AZStd::nullopt); + + // Given a mixture of test enumeration jobs with valid and invalid command arguments + for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) + { + const AZStd::string args = (jobId % 2) ? InvalidProcessPath : m_testTargetJobArgs[jobId]; + JobData jobData(m_testTargetPaths[jobId].second, AZStd::nullopt); + m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData))); + } + + try + { + // When the test enumeration jobs are executed with different exception policies + const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, m_jobExceptionPolicy); + + // Expect this statement to be reachable only if no exception policy for launch failures + EXPECT_FALSE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnFailedToExecute)); + + for (const auto& job : enumerationJobs) + { + const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; + if (jobId % 2) + { + // Expect invalid jobs have a job result of FailedToExecute + ValidateJobFailedToExecute(job); + } + else + { + // Expect the valid jobs job to successfully result in a test enumeration that matches the expected test enumeration data + ValidateJobExecutedSuccessfully(job); + ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]); + } + } + } + catch ([[maybe_unused]] const TestImpact::TestJobException& e) + { + // Expect this statement to be reachable only if there is an exception policy for launch failures + EXPECT_TRUE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnFailedToExecute)); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_P( + TestEnumeratorFixtureWithConcurrencyAndJobExceptionParams, ErroneousReturnCode_ExpectJobResultExecutedWithFailureOrTestJobException) + { + // Given a test enumerator with no client callback or enumeration timeout or enumerator timeout + m_testEnumerator = AZStd::make_unique( + AZStd::nullopt, + m_maxConcurrency, + AZStd::nullopt, + AZStd::nullopt); + + // Given a mixture of test enumeration jobs that execute and return either successfully or with failure + for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) + { + JobData jobData(m_testTargetPaths[jobId].second, AZStd::nullopt); + const AZStd::string args = (jobId % 2) + ? AZStd::string::format("%s %s", ValidProcessPath, ConstructTestProcessArgs(jobId, AZStd::chrono::milliseconds(0)).c_str()) + : m_testTargetJobArgs[jobId]; + m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData))); + } + + try + { + // When the test enumeration jobs are executed with different exception policies + const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, m_jobExceptionPolicy); + + // Expect this statement to be reachable only if no exception policy for jobs that return with error + EXPECT_FALSE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnExecutedWithFailure)); + + for (const auto& job : enumerationJobs) + { + const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; + if (jobId % 2) + { + // Expect failed jobs to have job result ExecutedWithFailure and a non-zero return code + ValidateJobExecutedWithFailure(job); + } + else + { + // Expect the valid jobs job to successfully result in a test enumeration that matches the expected test enumeration data + ValidateJobExecutedSuccessfully(job); + ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]); + } + } + } + catch ([[maybe_unused]] const TestImpact::TestJobException& e) + { + // Expect this statement to be reachable only if there is an exception policy for jobs that return with error + EXPECT_TRUE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnExecutedWithFailure)); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_P(TestEnumeratorFixtureWithConcurrencyAndCacheExceptionParams, EmptyCacheRead_NoCacheDataButEnumerationsMatchTestSuitesInTarget) + { + // Given a test enumerator with no client callback, enumeration timeout or enumerator timeout + m_testEnumerator = AZStd::make_unique( + AZStd::nullopt, + m_maxConcurrency, + AZStd::nullopt, + AZStd::nullopt); + + // Given an test enumeration job for each test target that reads from an enumeration caching + for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) + { + JobData jobData(m_testTargetPaths[jobId].second, JobData::Cache{JobData::CachePolicy::Read, m_cacheFiles[jobId]}); + m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId], AZStd::move(jobData))); + } + + try + { + // When the test enumeration jobs are executed with different exception policies + const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, m_cacheExceptionPolicy, JobExceptionPolicy::Never); + + // Expect this statement to be reachable only if no exception policy for read attempts of non-existent caches + EXPECT_FALSE(::IsFlagSet(m_cacheExceptionPolicy, CacheExceptionPolicy::OnCacheNotExist)); + + // Expect each job to successfully result in a test enumeration that matches the expected test enumeration data for that test target + // even though the cache files could not be read + for (const auto& job : enumerationJobs) + { + const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; + ValidateJobExecutedSuccessfully(job); + ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]); + ValidateInvalidTestEnumerationCache(job.GetJobInfo().GetCache()->m_file); + } + } + catch ([[maybe_unused]] const TestImpact::TestEnumerationException& e) + { + // Expect this statement to be reachable only if there is an exception policy for read attempts of non-existent caches + EXPECT_TRUE(::IsFlagSet(m_cacheExceptionPolicy, CacheExceptionPolicy::OnCacheNotExist)); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + // Note: this test only cues up one test for enumeration but still runs the permutations for max concurrency so there is duplicated work + TEST_P( + TestEnumeratorFixtureWithConcurrencyAndCacheExceptionParams, + EmptyCacheDataRead_ExpectEnumerationsMatchTestSuitesInTargetOrTestEnumerationException) + { + // Given an enumeration cache for Test Target A with invalid JSON data + WriteTextToFile("", m_cacheFiles[TestTargetA]); + + // Given a test enumerator with no client callback, enumeration timeout or enumerator timeout + m_testEnumerator = AZStd::make_unique( + AZStd::nullopt, + m_maxConcurrency, + AZStd::nullopt, + AZStd::nullopt); + + // Given an test enumeration job that will attempt to read an invalid enumeration cache + JobData jobData(m_testTargetPaths[TestTargetA].second, JobData::Cache{JobData::CachePolicy::Read, m_cacheFiles[TestTargetA]}); + m_jobInfos.emplace_back(JobInfo({TestTargetA}, m_testTargetJobArgs[TestTargetA], AZStd::move(jobData))); + + try + { + // When the test enumeration jobs are executed with different exception policies + const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, m_cacheExceptionPolicy, JobExceptionPolicy::Never); + + // Expect this statement to be reachable only if no exception policy for cache reads that fail + EXPECT_FALSE(::IsFlagSet(m_cacheExceptionPolicy, CacheExceptionPolicy::OnCacheReadFailure)); + + for (const auto& job : enumerationJobs) + { + const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; + + // Expect the valid jobs job to successfully result in a test enumeration that matches the expected test enumeration data + ValidateJobExecutedSuccessfully(job); + ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]); + } + } + catch ([[maybe_unused]] const TestImpact::TestEnumerationException& e) + { + // Expect this statement to be reachable only if there is an exception policy for cache reads that fail + EXPECT_TRUE(::IsFlagSet(m_cacheExceptionPolicy, CacheExceptionPolicy::OnCacheReadFailure)); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_P( + TestEnumeratorFixtureWithConcurrencyAndCacheExceptionParams, + InvalidCacheWrite_ExpectEnumerationsMatchTestSuitesInTargetOrTestEnumerationException) + { + // Given a test enumerator with no client callback,, enumeration timeout or enumerator timeout + m_testEnumerator = AZStd::make_unique( + AZStd::nullopt, + m_maxConcurrency, + AZStd::nullopt, + AZStd::nullopt); + + // Given an test enumeration job that will attempt to write to an invalid enumeration cache + JobData jobData(m_testTargetPaths[TestTargetA].second, JobData::Cache{JobData::CachePolicy::Write, InvalidProcessPath}); + m_jobInfos.emplace_back(JobInfo({TestTargetA}, m_testTargetJobArgs[TestTargetA], AZStd::move(jobData))); + + try + { + // When the test enumeration job is executed + const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, m_cacheExceptionPolicy, JobExceptionPolicy::Never); + + // Expect this statement to be reachable only if no exception policy for cache writes that fail + EXPECT_FALSE(::IsFlagSet(m_cacheExceptionPolicy, CacheExceptionPolicy::OnCacheWriteFailure)); + + for (const auto& job : enumerationJobs) + { + const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; + + // Expect the valid jobs job to successfully result in a test enumeration that matches the expected test enumeration data + ValidateJobExecutedSuccessfully(job); + ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]); + } + } + catch ([[maybe_unused]] const TestImpact::TestEnumerationException& e) + { + // Expect this statement to be reachable only if there is an exception policy for cache writes that fail + EXPECT_TRUE(::IsFlagSet(m_cacheExceptionPolicy, CacheExceptionPolicy::OnCacheWriteFailure)); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_P(TestEnumeratorFixtureWithConcurrencyParams, ValidAndInvalidCacheRead_CachedEnumerationsMatchTestSuitesInTarget) + { + // Given the cache file written for only test target B + WriteTextToFile(TestImpact::SerializeTestEnumeration(m_expectedTestTargetEnumerations[TestTargetB]), m_cacheFiles[TestTargetB]); + + // Given a test enumerator with no client callback, enumeration timeout or enumerator timeout + m_testEnumerator = AZStd::make_unique( + AZStd::nullopt, + m_maxConcurrency, + AZStd::nullopt, + AZStd::nullopt); + + // Given test enumeration jobs test target A and D with no enumeration caching + m_jobInfos.emplace_back( + JobInfo({TestTargetA}, m_testTargetJobArgs[TestTargetA], JobData{m_testTargetPaths[TestTargetA].second, AZStd::nullopt})); + m_jobInfos.emplace_back( + JobInfo({TestTargetD}, m_testTargetJobArgs[TestTargetD], JobData{m_testTargetPaths[TestTargetD].second, AZStd::nullopt})); + + // Given test target B with enumeration cache reading and a valid cache file + m_jobInfos.emplace_back(JobInfo( + {TestTargetB}, m_testTargetJobArgs[TestTargetB], + JobData{m_testTargetPaths[TestTargetB].second, JobInfo::Cache{JobData::CachePolicy::Read, m_cacheFiles[TestTargetB]}})); + + // Given test target C with enumeration cache reading and an invalid cache file + m_jobInfos.emplace_back(JobInfo( + {TestTargetC}, m_testTargetJobArgs[TestTargetC], + JobData{m_testTargetPaths[TestTargetC].second, JobInfo::Cache{JobData::CachePolicy::Read, "nothing"}})); + + // When the test enumeration jobs are executed + const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never); + + // Expect each job to successfully result in a test enumeration that matches the expected test enumeration data for that test target + for (const auto& job : enumerationJobs) + { + const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; + + switch (jobId) + { + case TestTargetA: // No cache read + case TestTargetC: // Cache read, but invalid cache, so re-enumerate anyway + case TestTargetD: // No cache read + { + ValidateJobExecutedSuccessfully(job); + break; + } + case TestTargetB: // Cache read, successful cache read, so job not executed + { + ValidateJobSuccessfulCacheRead(job); + break; + } + default: + { + FAIL(); + } + } + + // Regardless of cache policy and cache failures all targets should still produce the expected test enumerations + ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]); + } + } + + TEST_F(TestEnumeratorFixture, InvalidCacheDataRead_TestEnumerationException) + { + // Given an enumeration cache for Test Target A with invalid JSON data + WriteTextToFile("There is no valid cache data here", m_cacheFiles[TestTargetA]); + + // Given a test enumerator with no client callback, concurrency, enumeration timeout or enumerator timeout + m_testEnumerator = AZStd::make_unique( + AZStd::nullopt, + OneConcurrentProcess, + AZStd::nullopt, + AZStd::nullopt); + + // Given an test enumeration job that will attempt to read an invalid enumeration cache + JobData jobData(m_testTargetPaths[TestTargetA].second, JobData::Cache{JobData::CachePolicy::Read, m_cacheFiles[TestTargetA]}); + m_jobInfos.emplace_back(JobInfo({TestTargetA}, m_testTargetJobArgs[TestTargetA], AZStd::move(jobData))); + + try + { + // When the test enumeration jobs are executed with different exception policies + const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never); + } + catch ([[maybe_unused]] const TestImpact::TestEnumerationException& e) + { + // Expect this statement to be reachable only if there is an exception policy for jobs that return with error + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_P(TestEnumeratorFixtureWithConcurrencyParams, ValidCacheWrite_CachedEnumerationsMatchTestSuitesInTarget) + { + // Given a test enumerator with no client callback, enumeration timeout or enumerator timeout + m_testEnumerator = AZStd::make_unique( + AZStd::nullopt, + m_maxConcurrency, + AZStd::nullopt, + AZStd::nullopt); + + // Given an test enumeration job for each test target with write enumeration caching + for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) + { + JobData jobData(m_testTargetPaths[jobId].second, JobData::Cache{JobData::CachePolicy::Write, m_cacheFiles[jobId]}); + m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId], AZStd::move(jobData))); + } + + // When the test enumeration jobs are executed + const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never); + + // Expect each job to successfully result in a test enumeration and cache that matches the expected test enumeration data for that + // test target + for (const auto& job : enumerationJobs) + { + const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; + ValidateJobExecutedSuccessfully(job); + ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]); + ValidateTestEnumerationCache(job.GetJobInfo().GetCache()->m_file, m_expectedTestTargetEnumerations[jobId]); + } + } + + TEST_F(TestEnumeratorFixture, EmptyArtifact_ExpectTestEnumerationException) + { + // Given a test enumerator with no client callback, concurrency, enumeration timeout or enumerator timeout + m_testEnumerator = AZStd::make_unique( + AZStd::nullopt, + OneConcurrentProcess, + AZStd::nullopt, + AZStd::nullopt); + + // Given an test enumeration job that will return successfully but with an empty artifact string + m_jobInfos.emplace_back(JobInfo({0}, m_testTargetJobArgs[0], JobData("", AZStd::nullopt))); + + try + { + // When the test enumeration job is executed + const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never); + + // Do not expect this statement to be reachable + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::TestEnumerationException& e) + { + // Expect an enumeration exception + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(TestEnumeratorFixture, InvalidArtifact_ExpectTestEnumerationException) + { + // Given a test enumerator with no client callback, concurrency, enumeration timeout or enumerator timeout + m_testEnumerator = AZStd::make_unique( + AZStd::nullopt, + OneConcurrentProcess, + AZStd::nullopt, + AZStd::nullopt); + + // Given an test enumeration job that will return successfully but not produce an artifact + m_jobInfos.emplace_back(JobInfo( + {0}, AZStd::string::format("%s %s", ValidProcessPath, ConstructTestProcessArgs(0, AZStd::chrono::milliseconds(0)).c_str()), + JobData("", AZStd::nullopt))); + + try + { + // When the test enumeration job is executed + const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never); + + // Do not expect this statement to be reachable + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::TestEnumerationException& e) + { + // Expect an enumeration exception + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_P(TestEnumeratorFixtureWithConcurrencyParams, EnumerateTestTargets_EnumerationsMatchTestSuitesInTarget) + { + // Given a test enumerator with no client callback, enumeration timeout or enumerator timeout + m_testEnumerator = AZStd::make_unique( + AZStd::nullopt, + m_maxConcurrency, + AZStd::nullopt, + AZStd::nullopt); + + // Given an test enumeration job for each test target with no enumeration caching + for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) + { + JobData jobData(m_testTargetPaths[jobId].second, AZStd::nullopt); + m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId], AZStd::move(jobData))); + } + + // When the test enumeration jobs are executed + const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never); + + // Expect each job to successfully result in a test enumeration that matches the expected test enumeration data for that test target + for (const auto& job : enumerationJobs) + { + const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; + ValidateJobExecutedSuccessfully(job); + ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]); + } + } + + TEST_P(TestEnumeratorFixtureWithConcurrencyParams, EnumerateTestTargetsWithArbitraryJobIds_EnumerationsMatchTestSuitesInTarget) + { + // Given a set of arbitrary job ids to be used for the test target jobs + enum + { + ArbitraryA = 36, + ArbitraryB = 890, + ArbitraryC = 19, + ArbitraryD = 1 + }; + + const AZStd::unordered_map sequentialToArbitrary = + { + {TestTargetA, ArbitraryA}, + {TestTargetB, ArbitraryB}, + {TestTargetC, ArbitraryC}, + {TestTargetD, ArbitraryD}, + }; + + const AZStd::unordered_map arbitraryToSequential = + { + {ArbitraryA, TestTargetA}, + {ArbitraryB, TestTargetB}, + {ArbitraryC, TestTargetC}, + {ArbitraryD, TestTargetD}, + }; + + // Given a test enumerator with no client callback, enumeration timeout or enumerator timeout + m_testEnumerator = AZStd::make_unique( + AZStd::nullopt, + m_maxConcurrency, + AZStd::nullopt, + AZStd::nullopt); + + // Given an test enumeration job for each test target with no enumeration caching + for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) + { + JobData jobData(m_testTargetPaths[jobId].second, AZStd::nullopt); + m_jobInfos.emplace_back(JobInfo({sequentialToArbitrary.at(jobId)}, m_testTargetJobArgs[jobId], AZStd::move(jobData))); + } + + // When the test enumeration jobs are executed + const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never); + + // Expect each job to successfully result in a test enumeration that matches the expected test enumeration data for that test target + for (const auto& job : enumerationJobs) + { + const JobInfo::IdType jobId = arbitraryToSequential.at(job.GetJobInfo().GetId().m_value); + ValidateJobExecutedSuccessfully(job); + ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]); + } + } + + TEST_P(TestEnumeratorFixtureWithConcurrencyParams, EnumerateTestTargetsWithCallback_EnumerationsMatchTestSuitesInTarget) + { + // Given a client callback function that tracks the number of successful enumerations + size_t numSuccesses = 0; + const auto jobCallback = [&numSuccesses]([[maybe_unused]] const TestImpact::TestEnumerator::JobInfo& jobInfo, const TestImpact::JobMeta& meta) + { + if (meta.m_result == TestImpact::JobResult::ExecutedWithSuccess) + { + numSuccesses++; + } + }; + + // Given a test enumerator with no enumeration timeout or enumerator timeout + m_testEnumerator = AZStd::make_unique( + jobCallback, + m_maxConcurrency, + AZStd::nullopt, + AZStd::nullopt); + + // Given an test enumeration job for each test target with no enumeration caching + for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) + { + JobData jobData(m_testTargetPaths[jobId].second, AZStd::nullopt); + m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId], AZStd::move(jobData))); + } + + // When the test enumeration jobs are executed + const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never); + + // Expect the number of successful enumerations tracked in the callback to match the number of test targets enumerated + EXPECT_EQ(numSuccesses, enumerationJobs.size()); + + // Expect each job to successfully result in a test enumeration that matches the expected test enumeration data for that test target + for (const auto& job : enumerationJobs) + { + const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; + ValidateJobExecutedSuccessfully(job); + ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]); + } + } + + TEST_P(TestEnumeratorFixtureWithConcurrencyParams, JobRunnerTimeout_InFlightJobsTimeoutAndQueuedJobsUnlaunched) + { + // Given a test enumerator with no client callback or enumerator timeout and 500ms enumeration timeout + m_testEnumerator = AZStd::make_unique( + AZStd::nullopt, + m_maxConcurrency, + AZStd::chrono::milliseconds(500), + AZStd::nullopt); + + // Given an test enumeration job for each test target with no enumeration caching where half will sleep indefinitely + for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) + { + JobData jobData(m_testTargetPaths[jobId].second, AZStd::nullopt); + const AZStd::string args = (jobId % 2) + ? AZStd::string::format("%s %s", ValidProcessPath, ConstructTestProcessArgs(jobId, LongSleep).c_str()) + : m_testTargetJobArgs[jobId]; + m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData))); + } + + // When the test enumeration jobs are executed + const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never); + + // Expect half the jobs to successfully result in a test enumeration that matches the expected test enumeration data for that test + // target with the other half having timed out + for (const auto& job : enumerationJobs) + { + const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; + if (jobId % 2) + { + ValidateJobTimeout(job); + } + else + { + ValidateJobExecutedSuccessfully(job); + ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]); + } + } + } + + TEST_F(TestEnumeratorFixture, JobTimeout_InFlightJobTimeoutAndQueuedJobsUnlaunched) + { + // Given a test enumerator with no client callback or enumeration timeout and a 5 second enumerator timeout + m_testEnumerator = AZStd::make_unique( + AZStd::nullopt, + FourConcurrentProcesses, + AZStd::nullopt, + AZStd::chrono::milliseconds(5000)); + + // Given an test enumeration job for each test target with no enumeration caching where half will sleep indefinitely + for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) + { + JobData jobData(m_testTargetPaths[jobId].second, AZStd::nullopt); + const AZStd::string args = (jobId % 2) + ? AZStd::string::format("%s %s", ValidProcessPath, ConstructTestProcessArgs(jobId, LongSleep).c_str()) + : m_testTargetJobArgs[jobId]; + m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData))); + } + + // When the test enumeration jobs are executed + const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never); + + // Expect half the jobs to successfully result in a test enumeration that matches the expected test enumeration data for that test + // target with the other half having timed out + for (const auto& job : enumerationJobs) + { + const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; + if (jobId % 2) + { + ValidateJobTimeout(job); + } + else + { + ValidateJobExecutedSuccessfully(job); + ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]); + } + } + } + + INSTANTIATE_TEST_CASE_P( + , + TestEnumeratorFixtureWithConcurrencyAndJobExceptionParams, + ::testing::Combine( + ::testing::ValuesIn(MaxConcurrentEnumerations), + ::testing::ValuesIn(JobExceptionPolicies)) + ); + + INSTANTIATE_TEST_CASE_P( + , + TestEnumeratorFixtureWithConcurrencyAndCacheExceptionParams, + ::testing::Combine( + ::testing::ValuesIn(MaxConcurrentEnumerations), + ::testing::ValuesIn(CacheExceptionPolicies)) + ); + + INSTANTIATE_TEST_CASE_P( + , + TestEnumeratorFixtureWithConcurrencyParams, + ::testing::ValuesIn(MaxConcurrentEnumerations) + ); +} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestEumerationSerializerTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestEumerationSerializerTest.cpp new file mode 100644 index 0000000000..aa94657378 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestEumerationSerializerTest.cpp @@ -0,0 +1,99 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include + +#include +#include + +#include +#include + +namespace UnitTest +{ + const TestImpact::TestEnumeration EmptyTestEnum(AZStd::vector{}); + + TEST(TestEnumerationSerializer, EmptyTestEnumerationSuites_ExpectTemptyTestEnumeration) + { + // Given an empty set of test enumeration suites + // When the test enumeration is serialized and deserialized back again + const AZStd::string serializedString = TestImpact::SerializeTestEnumeration(EmptyTestEnum); + const TestImpact::TestEnumeration actualEnumeration = TestImpact::DeserializeTestEnumeration(serializedString); + + // Expect the actual test enumeration to match the expected test enumeration + EXPECT_TRUE(actualEnumeration == EmptyTestEnum); + } + + TEST(TestEnumerationSerializer, SerializeAndDeserializeSuitesForTestTargetA_ActualSuiteDataMatchesExpectedSuiteData) + { + // Given the test of test enumeration suites for Test target A + const TestImpact::TestEnumeration expectedEnumeration = TestImpact::TestEnumeration(GetTestTargetATestEnumerationSuites()); + + // When the test enumeration is serialized and deserialized back again + const AZStd::string serializedString = TestImpact::SerializeTestEnumeration(expectedEnumeration); + const TestImpact::TestEnumeration actualEnumeration = TestImpact::DeserializeTestEnumeration(serializedString); + + // Expect the actual test enumeration to match the expected test enumeration + EXPECT_TRUE(actualEnumeration == expectedEnumeration); + + // Do not expect the actual test enumeration to match the empty test enumeration + EXPECT_FALSE(actualEnumeration == EmptyTestEnum); + } + + TEST(TestEnumerationSerializer, SerializeAndDeserializeSuitesForTestTargetB_ActualSuiteDataMatchesExpectedSuiteData) + { + // Given the test of test enumeration suites for Test target B + const TestImpact::TestEnumeration expectedEnumeration = TestImpact::TestEnumeration(GetTestTargetBTestEnumerationSuites()); + + // When the test enumeration is serialized and deserialized back again + const AZStd::string serializedString = TestImpact::SerializeTestEnumeration(expectedEnumeration); + const TestImpact::TestEnumeration actualEnumeration = TestImpact::DeserializeTestEnumeration(serializedString); + + // Expect the actual test enumeration to match the expected test enumeration + EXPECT_TRUE(actualEnumeration == expectedEnumeration); + + // Do not expect the actual test enumeration to match the empty test enumeration + EXPECT_FALSE(actualEnumeration == EmptyTestEnum); + } + + TEST(TestEnumerationSerializer, SerializeAndDeserializeSuitesForTestTargetC_ActualSuiteDataMatchesExpectedSuiteData) + { + // Given the test of test enumeration suites for Test target B + const TestImpact::TestEnumeration expectedEnumeration = TestImpact::TestEnumeration(GetTestTargetCTestEnumerationSuites()); + + // When the test enumeration is serialized and deserialized back again + const AZStd::string serializedString = TestImpact::SerializeTestEnumeration(expectedEnumeration); + const TestImpact::TestEnumeration actualEnumeration = TestImpact::DeserializeTestEnumeration(serializedString); + + // Expect the actual test enumeration to match the expected test enumeration + EXPECT_TRUE(actualEnumeration == expectedEnumeration); + + // Do not expect the actual test enumeration to match the empty test enumeration + EXPECT_FALSE(actualEnumeration == EmptyTestEnum); + } + + TEST(TestEnumerationSerializer, SerializeAndDeserializeSuitesForTestTargetD_ActualSuiteDataMatchesExpectedSuiteData) + { + // Given the test of test enumeration suites for Test target B + const TestImpact::TestEnumeration expectedEnumeration = TestImpact::TestEnumeration(GetTestTargetDTestEnumerationSuites()); + + // When the test enumeration is serialized and deserialized back again + const AZStd::string serializedString = TestImpact::SerializeTestEnumeration(expectedEnumeration); + const TestImpact::TestEnumeration actualEnumeration = TestImpact::DeserializeTestEnumeration(serializedString); + + // Expect the actual test enumeration to match the expected test enumeration + EXPECT_TRUE(actualEnumeration == expectedEnumeration); + + // Do not expect the actual test enumeration to match the empty test enumeration + EXPECT_FALSE(actualEnumeration == EmptyTestEnum); + } +} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestRunSerializerTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestRunSerializerTest.cpp new file mode 100644 index 0000000000..1fc7c62b3d --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestRunSerializerTest.cpp @@ -0,0 +1,99 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include + +#include +#include + +#include +#include + +namespace UnitTest +{ + const TestImpact::TestRun EmptyTestRun(AZStd::vector{}, AZStd::chrono::milliseconds{0}); + + TEST(TestRunSerializer, EmptyTestRunSuites_ExpectTemptyTestRun) + { + // Given an empty set of test run suites + // When the test run is serialized and deserialized back again + const auto serializedString = TestImpact::SerializeTestRun(EmptyTestRun); + const auto actualRun = TestImpact::DeserializeTestRun(serializedString); + + // Expect the actual test run to match the expected test run + EXPECT_TRUE(actualRun == EmptyTestRun); + } + + TEST(TestRunSerializer, SerializeAndDeserializeSuitesForTestTargetA_ActualSuiteDataMatchesExpectedSuiteData) + { + // Given the test of test run suites for Test target A + const auto expectedRun = TestImpact::TestRun(GetTestTargetATestRunSuites(), AZStd::chrono::milliseconds{500}); + + // When the test run is serialized and deserialized back again + const auto serializedString = TestImpact::SerializeTestRun(expectedRun); + const auto actualRun = TestImpact::DeserializeTestRun(serializedString); + + // Expect the actual test run to match the expected test run + EXPECT_TRUE(actualRun == expectedRun); + + // Do not expect the actual test run to match the empty test run + EXPECT_FALSE(actualRun == EmptyTestRun); + } + + TEST(TestRunSerializer, SerializeAndDeserializeSuitesForTestTargetB_ActualSuiteDataMatchesExpectedSuiteData) + { + // Given the test of test run suites for Test target B + const auto expectedRun = TestImpact::TestRun(GetTestTargetBTestRunSuites(), AZStd::chrono::milliseconds{500}); + + // When the test run is serialized and deserialized back again + const auto serializedString = TestImpact::SerializeTestRun(expectedRun); + const auto actualRun = TestImpact::DeserializeTestRun(serializedString); + + // Expect the actual test run to match the expected test run + EXPECT_TRUE(actualRun == expectedRun); + + // Do not expect the actual test run to match the empty test run + EXPECT_FALSE(actualRun == EmptyTestRun); + } + + TEST(TestRunSerializer, SerializeAndDeserializeSuitesForTestTargetC_ActualSuiteDataMatchesExpectedSuiteData) + { + // Given the test of test run suites for Test target B + const auto expectedRun = TestImpact::TestRun(GetTestTargetCTestRunSuites(), AZStd::chrono::milliseconds{500}); + + // When the test run is serialized and deserialized back again + const auto serializedString = TestImpact::SerializeTestRun(expectedRun); + const auto actualRun = TestImpact::DeserializeTestRun(serializedString); + + // Expect the actual test run to match the expected test run + EXPECT_TRUE(actualRun == expectedRun); + + // Do not expect the actual test run to match the empty test run + EXPECT_FALSE(actualRun == EmptyTestRun); + } + + TEST(TestRunSerializer, SerializeAndDeserializeSuitesForTestTargetD_ActualSuiteDataMatchesExpectedSuiteData) + { + // Given the test of test run suites for Test target B + const auto expectedRun = TestImpact::TestRun(GetTestTargetDTestRunSuites(), AZStd::chrono::milliseconds{500}); + + // When the test run is serialized and deserialized back again + const auto serializedString = TestImpact::SerializeTestRun(expectedRun); + const auto actualRun = TestImpact::DeserializeTestRun(serializedString); + + // Expect the actual test run to match the expected test run + EXPECT_TRUE(actualRun == expectedRun); + + // Do not expect the actual test run to match the empty test run + EXPECT_FALSE(actualRun == EmptyTestRun); + } +} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestRunnerTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestRunnerTest.cpp new file mode 100644 index 0000000000..08a0306e7f --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestRunnerTest.cpp @@ -0,0 +1,516 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace UnitTest +{ + using JobExceptionPolicy = TestImpact::TestRunner::JobExceptionPolicy; + + AZStd::string GetRunCommandForTarget(AZStd::pair testTarget) + { + return AZStd::string::format( + "%s %s AzRunUnitTests --gtest_output=xml:%s", LY_TEST_IMPACT_AZ_TESTRUNNER_BIN, testTarget.first.c_str(), + testTarget.second.c_str()); + } + + class TestRunnerFixture + : public AllocatorsTestFixture + { + public: + void SetUp() override; + + protected: + using JobInfo = TestImpact::TestRunner::JobInfo; + using JobData = TestImpact::TestRunner::JobData; + + AZStd::vector m_jobInfos; + AZStd::unique_ptr m_testRunner; + AZStd::vector m_testTargetJobArgs; + AZStd::vector> m_testTargetPaths; + AZStd::vector m_expectedTestTargetRuns; + AZStd::vector m_expectedTestTargetResult; + size_t m_maxConcurrency = 0; + }; + + void TestRunnerFixture::SetUp() + { + UnitTest::AllocatorsTestFixture::SetUp(); + + DeleteFiles(LY_TEST_IMPACT_TEST_TARGET_RESULTS_DIR, "*.xml"); + + // first: path to test target bin + // second: path to test target gtest results file in XML format + const AZStd::string runPath = AZStd::string(LY_TEST_IMPACT_TEST_TARGET_RESULTS_DIR) + "/%s.Run.xml"; + m_testTargetPaths.emplace_back( + LY_TEST_IMPACT_TEST_TARGET_A_BIN, AZStd::string::format(runPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_A_BASE_NAME)); + m_testTargetPaths.emplace_back( + LY_TEST_IMPACT_TEST_TARGET_B_BIN, AZStd::string::format(runPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_B_BASE_NAME)); + m_testTargetPaths.emplace_back( + LY_TEST_IMPACT_TEST_TARGET_C_BIN, AZStd::string::format(runPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_C_BASE_NAME)); + m_testTargetPaths.emplace_back( + LY_TEST_IMPACT_TEST_TARGET_D_BIN, AZStd::string::format(runPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_D_BASE_NAME)); + + m_expectedTestTargetRuns.emplace_back(GetTestTargetATestRunSuites(), AZStd::chrono::milliseconds{500}); + m_expectedTestTargetRuns.emplace_back(GetTestTargetBTestRunSuites(), AZStd::chrono::milliseconds{500}); + m_expectedTestTargetRuns.emplace_back(GetTestTargetCTestRunSuites(), AZStd::chrono::milliseconds{500}); + m_expectedTestTargetRuns.emplace_back(GetTestTargetDTestRunSuites(), AZStd::chrono::milliseconds{500}); + + m_expectedTestTargetResult.emplace_back(TestImpact::TestRunResult::Failed); + m_expectedTestTargetResult.emplace_back(TestImpact::TestRunResult::Passed); + m_expectedTestTargetResult.emplace_back(TestImpact::TestRunResult::Passed); + m_expectedTestTargetResult.emplace_back(TestImpact::TestRunResult::Passed); + + for (const auto& testTarget : m_testTargetPaths) + { + m_testTargetJobArgs.emplace_back(GetRunCommandForTarget(testTarget)); + } + } + + // Fixture parameterized for different max number of concurrent jobs + class TestRunnerFixtureWithConcurrencyParams + : public TestRunnerFixture + , public ::testing::WithParamInterface + { + public: + void SetUp() override + { + TestRunnerFixture::SetUp(); + m_maxConcurrency = GetParam(); + } + }; + + using ConcurrencyAndJobExceptionPermutation = AZStd::tuple + < + size_t, // Max number of concurrent processes + JobExceptionPolicy // Test job exception policy + >; + + // Fixture parameterized for different max number of concurrent jobs and different job exception policies + class TestRunnerFixtureWithConcurrencyAndJobExceptionParams + : public TestRunnerFixture + , public ::testing::WithParamInterface + { + public: + void SetUp() override + { + TestRunnerFixture::SetUp(); + const auto& [maxConcurrency, jobExceptionPolicy] = GetParam(); + m_maxConcurrency = maxConcurrency; + m_jobExceptionPolicy = jobExceptionPolicy; + } + + protected: + JobExceptionPolicy m_jobExceptionPolicy = JobExceptionPolicy::Never; + }; + + class TestRunnerFixtureWithConcurrencyAndFailedToLaunchExceptionParams + : public TestRunnerFixtureWithConcurrencyAndJobExceptionParams + { + }; + + class TestRunnerFixtureWithConcurrencyAndExecutedWithFailureExceptionParams + : public TestRunnerFixtureWithConcurrencyAndJobExceptionParams + { + }; + + namespace + { + AZStd::array MaxConcurrentRuns = {1, 2, 3, 4}; + + AZStd::array FailedToLaunchExceptionPolicies = { + JobExceptionPolicy::Never, JobExceptionPolicy::OnFailedToExecute}; + + AZStd::array ExecutedWithFailureExceptionPolicies = { + JobExceptionPolicy::Never, JobExceptionPolicy::OnExecutedWithFailure}; + } // namespace + + TEST_P( + TestRunnerFixtureWithConcurrencyAndFailedToLaunchExceptionParams, + InvalidCommandArgument_ExpectJobResulFailedToExecuteeOrTestJobException) + { + // Given a test runner with no client callback or run timeout or runner timeout + m_testRunner = AZStd::make_unique(AZStd::nullopt, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt); + + // Given a mixture of test run jobs with valid and invalid command arguments + for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) + { + const AZStd::string args = (jobId % 2) ? InvalidProcessPath : m_testTargetJobArgs[jobId]; + JobData jobData(m_testTargetPaths[jobId].second); + m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData))); + } + + try + { + // When the test run jobs are executed with different exception policies + const auto runnerJobs = m_testRunner->RunTests(m_jobInfos, m_jobExceptionPolicy); + + // Expect this statement to be reachable only if no exception policy for launch failures + EXPECT_FALSE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnFailedToExecute)); + + for (const auto& job : runnerJobs) + { + const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; + if (jobId % 2) + { + // Expect invalid jobs have a job result of FailedToExecute + ValidateJobFailedToExecute(job); + } + else + { + // Expect the valid jobs to successfully result in a test run that matches the expected test run data + ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]); + ValidateTestTargetRun(job.GetPayload().value(), m_expectedTestTargetRuns[jobId]); + } + } + } + catch ([[maybe_unused]] const TestImpact::TestJobException& e) + { + // Expect this statement to be reachable only if there is an exception policy for launch failures + EXPECT_TRUE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnFailedToExecute)); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_P( + TestRunnerFixtureWithConcurrencyAndExecutedWithFailureExceptionParams, + ErroneousReturnCode_ExpectJobResultExecutedWithFailureOrTestJobException) + { + // Given a test runner with no client callback or run timeout or runner timeout + m_testRunner = AZStd::make_unique(AZStd::nullopt, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt); + + // Given a mixture of test run jobs that execute and return either successfully or with failure + for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) + { + JobData jobData(m_testTargetPaths[jobId].second); + m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId], AZStd::move(jobData))); + } + + try + { + // When the test run jobs are executed with different exception policies + const auto runnerJobs = m_testRunner->RunTests(m_jobInfos, m_jobExceptionPolicy); + + // Expect this statement to be reachable only if no exception policy for jobs that return with error + EXPECT_FALSE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnExecutedWithFailure)); + + for (const auto& job : runnerJobs) + { + const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; + + // Expect the valid jobs to successfully result in a test run that matches the expected test run data + ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]); + ValidateTestTargetRun(job.GetPayload().value(), m_expectedTestTargetRuns[jobId]); + } + } + catch ([[maybe_unused]] const TestImpact::TestJobException& e) + { + // Expect this statement to be reachable only if there is an exception policy for jobs that return with error + EXPECT_TRUE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnExecutedWithFailure)); + } + catch ([[maybe_unused]] const TestImpact::Exception& e) + { + FAIL(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(TestRunnerFixture, EmptyArtifact_ExpectTestRunnerException) + { + // Given a test runner with no client callback, concurrency, run timeout or runner timeout + m_testRunner = AZStd::make_unique(AZStd::nullopt, OneConcurrentProcess, AZStd::nullopt, AZStd::nullopt); + + // Given an test runner job that will return successfully but with an empty artifact string + m_jobInfos.emplace_back(JobInfo({0}, m_testTargetJobArgs[0], JobData(""))); + + try + { + // When the test runner job is executed + const auto runnerJobs = m_testRunner->RunTests(m_jobInfos, JobExceptionPolicy::Never); + + // Do not expect this statement to be reachable + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::TestRunException& e) + { + // Expect an runner exception + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(TestRunnerFixture, InvalidRunArtifact_ExpectArtifactException) + { + // Given a test run artifact with invalid contents + WriteTextToFile("There is nothing valid here", m_testTargetPaths[TestTargetA].second); + + // Given a job command that will write the test run artifact to a different location that what we will read from + auto invalidRunArtifact = m_testTargetPaths[TestTargetA]; + invalidRunArtifact.second /= ".xml"; + const AZStd::string args = GetRunCommandForTarget(invalidRunArtifact); + + // Given a test runner with no client callback, concurrency, run timeout or runner timeout + m_testRunner = AZStd::make_unique(AZStd::nullopt, OneConcurrentProcess, AZStd::nullopt, AZStd::nullopt); + + // Given an test runner job that will return successfully but not produce an artifact + JobData jobData(m_testTargetPaths[TestTargetA].second); + m_jobInfos.emplace_back(JobInfo({TestTargetA}, args, AZStd::move(jobData))); + + try + { + // When the test runner job is executed + const auto runnerJobs = m_testRunner->RunTests(m_jobInfos, JobExceptionPolicy::Never); + + // Do not expect this statement to be reachable + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::ArtifactException& e) + { + // Expect an runner exception + SUCCEED(); + } + catch (const TestImpact::Exception& e) + { + std::cout << e.what(); + FAIL(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_P(TestRunnerFixtureWithConcurrencyParams, RunTestTargets_RunsMatchTestSuitesInTarget) + { + // Given a test runner with no client callback, runner timeout or runner timeout + m_testRunner = AZStd::make_unique(AZStd::nullopt, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt); + + // Given an test runner job for each test target with no runner caching + for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) + { + JobData jobData(m_testTargetPaths[jobId].second); + m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId], AZStd::move(jobData))); + } + + // When the test runner jobs are executed + const auto runnerJobs = m_testRunner->RunTests(m_jobInfos, JobExceptionPolicy::Never); + + // Expect each job to successfully result in a test runner that matches the expected test runner data for that test target + for (const auto& job : runnerJobs) + { + const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; + ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]); + ValidateTestTargetRun(job.GetPayload().value(), m_expectedTestTargetRuns[jobId]); + } + } + + TEST_P(TestRunnerFixtureWithConcurrencyParams, RunTestTargetsWithArbitraryJobIds_RunsMatchTestSuitesInTarget) + { + // Given a set of arbitrary job ids to be used for the test target jobs + enum + { + ArbitraryA = 36, + ArbitraryB = 890, + ArbitraryC = 19, + ArbitraryD = 1 + }; + + const AZStd::unordered_map sequentialToArbitrary = + { + {TestTargetA, ArbitraryA}, + {TestTargetB, ArbitraryB}, + {TestTargetC, ArbitraryC}, + {TestTargetD, ArbitraryD}, + }; + + const AZStd::unordered_map arbitraryToSequential = + { + {ArbitraryA, TestTargetA}, + {ArbitraryB, TestTargetB}, + {ArbitraryC, TestTargetC}, + {ArbitraryD, TestTargetD}, + }; + + // Given a test runner with no client callback, run timeout or runner timeout + m_testRunner = AZStd::make_unique(AZStd::nullopt, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt); + + // Given an test run job for each test target + for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) + { + JobData jobData(m_testTargetPaths[jobId].second); + m_jobInfos.emplace_back(JobInfo({sequentialToArbitrary.at(jobId)}, m_testTargetJobArgs[jobId], AZStd::move(jobData))); + } + + // When the test run jobs are executed + const auto runnerJobs = m_testRunner->RunTests(m_jobInfos, JobExceptionPolicy::Never); + + // Expect each job to successfully result in a test run that matches the expected test run data for that test target + for (const auto& job : runnerJobs) + { + const JobInfo::IdType jobId = arbitraryToSequential.at(job.GetJobInfo().GetId().m_value); + ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]); + ValidateTestTargetRun(job.GetPayload().value(), m_expectedTestTargetRuns[jobId]); + } + } + + TEST_P(TestRunnerFixtureWithConcurrencyParams, RunTestTargetsWithCallback_RunsMatchTestSuitesInTarget) + { + // Given a client callback function that tracks the number of successful runs + size_t numSuccesses = 0; + const auto jobCallback = + [&numSuccesses]([[maybe_unused]] const TestImpact::TestRunner::JobInfo& jobInfo, const TestImpact::JobMeta& meta) { + if (meta.m_result == TestImpact::JobResult::ExecutedWithSuccess) + { + numSuccesses++; + } + }; + + // Given a test runner with no run timeout or runner timeout + m_testRunner = AZStd::make_unique(jobCallback, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt); + + // Given an test run job for each test target + for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) + { + JobData jobData(m_testTargetPaths[jobId].second); + m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId], AZStd::move(jobData))); + } + + // When the test run jobs are executed + const auto runnerJobs = m_testRunner->RunTests(m_jobInfos, JobExceptionPolicy::Never); + + // Expect the number of successful runs tracked in the callback to match the number of test targets run with no failures + EXPECT_EQ( + numSuccesses, + AZStd::count_if(m_expectedTestTargetResult.begin(), m_expectedTestTargetResult.end(), [](TestImpact::TestRunResult result) { + return result == TestImpact::TestRunResult::Passed; + })); + + // Expect each job to successfully result in a test run that matches the expected test run data for that test target + for (const auto& job : runnerJobs) + { + const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; + ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]); + ValidateTestTargetRun(job.GetPayload().value(), m_expectedTestTargetRuns[jobId]); + } + } + + TEST_P(TestRunnerFixtureWithConcurrencyParams, JobRunnerTimeout_InFlightJobsTimeoutAndQueuedJobsUnlaunched) + { + // Given a test runner with no client callback or runner timeout and 500ms run timeout + m_testRunner = + AZStd::make_unique(AZStd::nullopt, m_maxConcurrency, AZStd::chrono::milliseconds(500), AZStd::nullopt); + + // Given an test run job for each test target where half will sleep indefinitely + for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) + { + JobData jobData(m_testTargetPaths[jobId].second); + const AZStd::string args = (jobId % 2) + ? AZStd::string::format("%s %s", ValidProcessPath, ConstructTestProcessArgs(jobId, LongSleep).c_str()) + : m_testTargetJobArgs[jobId]; + m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData))); + } + + // When the test run jobs are executed + const auto runnerJobs = m_testRunner->RunTests(m_jobInfos, JobExceptionPolicy::Never); + + // Expect half the jobs to successfully result in a test run that matches the expected test run data for that test target + // with the other half having timed out + for (const auto& job : runnerJobs) + { + const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; + if (jobId % 2) + { + ValidateJobTimeout(job); + } + else + { + ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]); + ValidateTestTargetRun(job.GetPayload().value(), m_expectedTestTargetRuns[jobId]); + } + } + } + + TEST_F(TestRunnerFixture, JobTimeout_InFlightJobTimeoutAndQueuedJobsUnlaunched) + { + // Given a test runner with no client callback or run timeout and a 5 second runner timeout + m_testRunner = AZStd::make_unique( + AZStd::nullopt, FourConcurrentProcesses, AZStd::nullopt, AZStd::chrono::milliseconds(5000)); + + // Given an test run job for each test target where half will sleep indefinitely + for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) + { + JobData jobData(m_testTargetPaths[jobId].second); + const AZStd::string args = (jobId % 2) + ? AZStd::string::format("%s %s", ValidProcessPath, ConstructTestProcessArgs(jobId, LongSleep).c_str()) + : m_testTargetJobArgs[jobId]; + m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData))); + } + + // When the test run jobs are executed + const auto runnerJobs = m_testRunner->RunTests(m_jobInfos, JobExceptionPolicy::Never); + + // Expect half the jobs to successfully result in a test run that matches the expected test run data for that test target + // with the other half having timed out + for (const auto& job : runnerJobs) + { + const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; + if (jobId % 2) + { + ValidateJobTimeout(job); + } + else + { + ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]); + ValidateTestTargetRun(job.GetPayload().value(), m_expectedTestTargetRuns[jobId]); + } + } + } + + INSTANTIATE_TEST_CASE_P( + , + TestRunnerFixtureWithConcurrencyAndFailedToLaunchExceptionParams, + ::testing::Combine(::testing::ValuesIn(MaxConcurrentRuns), ::testing::ValuesIn(FailedToLaunchExceptionPolicies))); + + INSTANTIATE_TEST_CASE_P( + , + TestRunnerFixtureWithConcurrencyAndExecutedWithFailureExceptionParams, + ::testing::Combine(::testing::ValuesIn(MaxConcurrentRuns), ::testing::ValuesIn(ExecutedWithFailureExceptionPolicies))); + + INSTANTIATE_TEST_CASE_P(, TestRunnerFixtureWithConcurrencyParams, ::testing::ValuesIn(MaxConcurrentRuns)); +} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactExceptionTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactExceptionTest.cpp new file mode 100644 index 0000000000..bf2ccb6bc7 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactExceptionTest.cpp @@ -0,0 +1,150 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include +#include + +namespace UnitTest +{ + constexpr const char* Message = "String Constructor"; + + TEST(ExceptionTest, DefaultConstructor_HasEmptyMessageString) + { + // Given an exception instantiated with the default constructor + const TestImpact::Exception e; + + // Expect the message to be an empty string + EXPECT_STREQ(e.what(), "\0"); + } + + TEST(ExceptionTest, StringConstructor_HasSpecifiedMessageString) + { + // Given an exception instantiated with a string + const AZStd::string msg(Message); + const TestImpact::Exception e(msg); + + // Expect the message to be the specified string + EXPECT_STREQ(e.what(), Message); + } + + TEST(ExceptionTest, StringLiteralConstructor_HasSpecifiedMessageString) + { + // Given an exception instantiated with a string literal + const TestImpact::Exception e(Message); + + // Expect the message to be the specified string + EXPECT_STREQ(e.what(), Message); + } + + TEST(ExceptionTest, InitializedWithLocalString_HasCopyOfLocalStringString) + { + try + { + // Given an exception instantiated with a string in local scope + throw(TestImpact::Exception(AZStd::string::format("%s", Message))); + + // Do not expect this code to be reachable + FAIL(); + } + catch (const TestImpact::Exception& e) + { + // Expect the message to be the specified out-of-scope string + EXPECT_STREQ(e.what(), Message); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST(ExceptionTest, InitializedWithLocalStringLiteral_HasCopyOfLocalStringString) + { + try + { + // Given an exception instantiated with a copy of the message string in local scope + throw(TestImpact::Exception(AZStd::string::format("%s", Message).c_str())); + + // Do not expect this code to be reachable + FAIL(); + } + catch (const TestImpact::Exception& e) + { + // Expect the message to be the specified out-of-scope string literal + EXPECT_STREQ(e.what(), Message); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST(ExceptionTest, EvalMacroFails_ExceptionTestThrown) + { + try + { + // Given an exception instantiated with a string literal in local scope + AZ_TestImpact_Eval(false, TestImpact::Exception, Message); + + // Do not expect this code to be reachable + FAIL(); + } + catch (const TestImpact::Exception& e) + { + // Expect the message contain the specified string + EXPECT_STREQ(e.what(), Message); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST(ExceptionTest, EvalMacroSucceeds_ExceptionTestNotThrown) + { + try + { + // Given an exception instantiated with a string literal in local scope + AZ_TestImpact_Eval(true, TestImpact::Exception, Message); + + // Expect this code to be reachable + SUCCEED(); + } + catch (...) + { + // Do not expect any exceptions + FAIL(); + } + } + + TEST(ExceptionTest, Throw_ExceptionTestThrown) + { + try + { + // Given an exception instantiated with a string literal in local scope + throw(TestImpact::Exception(Message)); + } + catch (const TestImpact::Exception& e) + { + // Expect the message contain the specified string + EXPECT_STREQ(e.what(), Message); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } +} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactFrameworkPathTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactFrameworkPathTest.cpp new file mode 100644 index 0000000000..2982cb0551 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactFrameworkPathTest.cpp @@ -0,0 +1,137 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include +#include + +namespace UnitTest +{ + class FrameworkPathTestFixture + : public AllocatorsTestFixture + { + public: + void SetUp() override + { + UnitTest::AllocatorsTestFixture::SetUp(); + + m_parentPathAbs = AZStd::string(AZStd::string("parent") + AZ_TRAIT_OS_PATH_SEPARATOR + "path"); + m_childPathRel = AZStd::string(AZStd::string("child") + AZ_TRAIT_OS_PATH_SEPARATOR + "path"); + m_childPathAbs = AZStd::string(m_parentPathAbs + AZ_TRAIT_OS_PATH_SEPARATOR + m_childPathRel); + + m_pathComponentA = AZStd::string("DirA"); + m_pathComponentB = AZStd::string("DirB"); + m_pathComponentC = AZStd::string("DirC"); + + m_posixPath = m_pathComponentA + AZ::IO::PosixPathSeparator + m_pathComponentB + AZ::IO::PosixPathSeparator + m_pathComponentC; + + m_windowsPath = + m_pathComponentA + AZ::IO::WindowsPathSeparator + m_pathComponentB + AZ::IO::WindowsPathSeparator + m_pathComponentC; + + m_mixedPath = + m_pathComponentA + AZ::IO::WindowsPathSeparator + m_pathComponentB + AZ::IO::PosixPathSeparator + m_pathComponentC; + + m_referredPath = + m_pathComponentA + AZ_TRAIT_OS_PATH_SEPARATOR + m_pathComponentB + AZ_TRAIT_OS_PATH_SEPARATOR + m_pathComponentC; + } + + void TearDown() override + { + UnitTest::AllocatorsTestFixture::TearDown(); + } + + AZStd::string m_parentPathAbs; + AZStd::string m_childPathRel; + AZStd::string m_childPathAbs; + + AZStd::string m_pathComponentA; + AZStd::string m_pathComponentB; + AZStd::string m_pathComponentC; + + AZ::IO::Path m_posixPath; + AZ::IO::Path m_windowsPath; + AZ::IO::Path m_mixedPath; + AZ::IO::Path m_referredPath; + }; + + TEST_F(FrameworkPathTestFixture, DefaultConstructor_HasEmptyAbsAndRelPaths) + { + // Given an empty framework path + TestImpact::FrameworkPath path; + + // Expect the absolute path to be empty + EXPECT_TRUE(path.Absolute().empty()); + + // Expect the relative path to be empty + EXPECT_TRUE(path.Relative().empty()); + } + + TEST_F(FrameworkPathTestFixture, OrphanConstructor_HasAbsAndEmptyRelPaths) + { + // Given an orhpan framework path + TestImpact::FrameworkPath path(m_parentPathAbs); + + // Expect the absolute path to be equal to the specified path + EXPECT_EQ(path.Absolute().String(), m_parentPathAbs); + + // Expect the relative path to be current directory symbol + EXPECT_STREQ(path.Relative().c_str(), "."); + } + + TEST_F(FrameworkPathTestFixture, ParentConstructor_HasAbsAndRelPaths) + { + // Given a child framework path + TestImpact::FrameworkPath path(m_childPathAbs, TestImpact::FrameworkPath(m_parentPathAbs)); + + // Expect the absolute path to be equal to the concatenation of the parent and child path + EXPECT_EQ(path.Absolute().String(), m_childPathAbs); + + // Expect the relative path to equal to the specified path + EXPECT_EQ(path.Relative().String(), m_childPathRel); + } + + TEST_F(FrameworkPathTestFixture, PosixSeperators_HasUniformPreferredSeperators) + { + // Given an orhpan framework path with Posix separators + TestImpact::FrameworkPath path(m_posixPath); + + // Expect the absolute path to be equal to the specified path with preferred separators + EXPECT_EQ(path.Absolute(), m_referredPath); + + // Expect the relative path to be current directory symbol + EXPECT_STREQ(path.Relative().c_str(), "."); + } + + TEST_F(FrameworkPathTestFixture, WindowsSeperators_HasUniformPreferredSeperators) + { + // Given an orhpan framework path with Windows separators + TestImpact::FrameworkPath path(m_windowsPath); + + // Expect the absolute path to be equal to the specified path with preferred separators + EXPECT_EQ(path.Absolute(), m_referredPath); + + // Expect the relative path to be current directory symbol + EXPECT_STREQ(path.Relative().c_str(), "."); + } + + TEST_F(FrameworkPathTestFixture, MixedSeperators_HasUniformPreferredSeperators) + { + // Given an orhpan framework path with mixed separators + TestImpact::FrameworkPath path(m_windowsPath); + + // Expect the absolute path to be equal to the specified path with preferred separators + EXPECT_EQ(path.Absolute(), m_referredPath); + + // Expect the relative path to be current directory symbol + EXPECT_STREQ(path.Relative().c_str(), "."); + } +} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactTestJobRunnerCommon.h b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactTestJobRunnerCommon.h new file mode 100644 index 0000000000..9d5ddd6c8c --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactTestJobRunnerCommon.h @@ -0,0 +1,159 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +#include +#include + +#include +#include +#include + +template +constexpr bool IsFlagSet(Flags flags, Flags flag) +{ + return TestImpact::Bitwise::IsFlagSet(flags, flag); +} + +namespace UnitTest +{ + // Named constants for array of targets lookup + enum + { + TestTargetA, + TestTargetB, + TestTargetC, + TestTargetD + }; + + // Named constants for max concurrency values + enum + { + OneConcurrentProcess = 1, + FourConcurrentProcesses = 4 + }; + + // Validates that the specified job was executed and returned successfully + template + void ValidateJobExecutedSuccessfully(const Job& job) + { + EXPECT_EQ(job.GetResult(), TestImpact::JobResult::ExecutedWithSuccess); + EXPECT_TRUE(job.GetDuration() > AZStd::chrono::milliseconds(0)); + EXPECT_TRUE(job.GetReturnCode().has_value()); + EXPECT_EQ(job.GetReturnCode(), 0); + EXPECT_TRUE(job.GetPayload().has_value()); + } + + // Validates that the specified job has not been executed + template + void ValidateJobNotExecuted(const Job& job) + { + EXPECT_EQ(job.GetResult(), TestImpact::JobResult::NotExecuted); + EXPECT_EQ(job.GetStartTime(), AZStd::chrono::high_resolution_clock::time_point()); + EXPECT_EQ(job.GetEndTime(), AZStd::chrono::high_resolution_clock::time_point()); + EXPECT_EQ(job.GetDuration(), AZStd::chrono::milliseconds(0)); + EXPECT_FALSE(job.GetReturnCode().has_value()); + EXPECT_FALSE(job.GetPayload().has_value()); + } + + // Validates that the specified job failed to execute + template + void ValidateJobFailedToExecute(const Job& job) + { + EXPECT_EQ(job.GetResult(), TestImpact::JobResult::FailedToExecute); + EXPECT_EQ(job.GetStartTime(), AZStd::chrono::high_resolution_clock::time_point()); + EXPECT_EQ(job.GetEndTime(), AZStd::chrono::high_resolution_clock::time_point()); + EXPECT_EQ(job.GetDuration(), AZStd::chrono::milliseconds(0)); + EXPECT_FALSE(job.GetReturnCode().has_value()); + EXPECT_FALSE(job.GetPayload().has_value()); + } + + // Validates that the specified job executed but returned with error + template + void ValidateJobExecutedWithFailure(const Job& job) + { + EXPECT_EQ(job.GetResult(), TestImpact::JobResult::ExecutedWithFailure); + EXPECT_TRUE(job.GetDuration() > AZStd::chrono::milliseconds(0)); + EXPECT_TRUE(job.GetReturnCode().has_value()); + EXPECT_GT(job.GetReturnCode().value(), 0); + EXPECT_FALSE(job.GetPayload().has_value()); + } + + // Validates that the specified job was executed but was terminated by the job runner + template + void ValidateJobTimeout(const Job& job) + { + EXPECT_EQ(job.GetResult(), TestImpact::JobResult::Terminated); + EXPECT_TRUE(job.GetDuration() > AZStd::chrono::milliseconds(0)); + EXPECT_TRUE(job.GetReturnCode().has_value()); + EXPECT_EQ(job.GetReturnCode().value(), TestImpact::ProcessTimeoutErrorCode); + EXPECT_FALSE(job.GetPayload().has_value()); + } + + // Validates that the specified job executed but returned with error + template + void ValidateJobExecutedWithFailedTests(const Job& job) + { + EXPECT_EQ(job.GetResult(), TestImpact::JobResult::ExecutedWithFailure); + EXPECT_TRUE(job.GetDuration() > AZStd::chrono::milliseconds(0)); + EXPECT_TRUE(job.GetReturnCode().has_value()); + EXPECT_GT(job.GetReturnCode().value(), 0); + EXPECT_TRUE(job.GetPayload().has_value()); + } + + // Validates whether a test run completed (passed/failed) + template + void ValidateTestRunCompleted(const Job& job, TestImpact::TestRunResult result) + { + if (result == TestImpact::TestRunResult::Passed) + { + ValidateJobExecutedSuccessfully(job); + } + else + { + ValidateJobExecutedWithFailedTests(job); + } + } + + // Validates that the specified test run matches the expected output + inline void ValidateTestTargetRun(const TestImpact::TestRun& actualResult, const TestImpact::TestRun& expectedResult) + { + EXPECT_TRUE(CheckTestRunsAreEqualIgnoreDurations(actualResult, expectedResult)); + EXPECT_EQ(actualResult.GetNumTestSuites(), CalculateNumTestSuites(expectedResult.GetTestSuites())); + EXPECT_EQ(actualResult.GetNumTests(), CalculateNumTests(expectedResult.GetTestSuites())); + EXPECT_EQ(actualResult.GetNumEnabledTests(), CalculateNumEnabledTests(expectedResult.GetTestSuites())); + EXPECT_EQ(actualResult.GetNumDisabledTests(), CalculateNumDisabledTests(expectedResult.GetTestSuites())); + EXPECT_GT(actualResult.GetDuration(), AZStd::chrono::milliseconds{0}); + EXPECT_EQ(actualResult.GetNumPasses(), CalculateNumPassedTests(expectedResult.GetTestSuites())); + EXPECT_EQ(actualResult.GetNumFailures(), CalculateNumFailedTests(expectedResult.GetTestSuites())); + EXPECT_EQ(actualResult.GetNumRuns(), CalculateNumRunTests(expectedResult.GetTestSuites())); + EXPECT_EQ(actualResult.GetNumNotRuns(), CalculateNumNotRunTests(expectedResult.GetTestSuites())); + } + + // Delete any existing data in the test run folder as not to pollute tests with data from previous test runs + // Note: the file IO operations of this fixture means it cannot be sharded by the test sharder due to file race conditions + inline void DeleteFiles(const AZStd::string& path, const AZStd::string& pattern) + { + AZ::IO::SystemFile::FindFiles(AZStd::string::format("%s/%s", path.c_str(), pattern.c_str()).c_str(), [&path](const char* file, bool isFile) + { + if (isFile) + { + AZ::IO::SystemFile::Delete(AZStd::string::format("%s/%s", path.c_str(), file).c_str()); + } + + return true; + }); + }; +} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactTestMain.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactTestMain.cpp new file mode 100644 index 0000000000..632e84ad9f --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactTestMain.cpp @@ -0,0 +1,33 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include +#include + +class TestImpactTestEnvironment : public AZ::Test::ITestEnvironment +{ +protected: + void SetupEnvironment() override + { + AZ::AllocatorInstance::Create(); + AZ::AllocatorInstance::Create(); + } + + void TeardownEnvironment() override + { + AZ::AllocatorInstance::Destroy(); + AZ::AllocatorInstance::Destroy(); + } +}; + +AZ_UNIT_TEST_HOOK(new TestImpactTestEnvironment); diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactTestUtils.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactTestUtils.cpp new file mode 100644 index 0000000000..90f384de85 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactTestUtils.cpp @@ -0,0 +1,1843 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include + +#include + +#include +#include + +namespace UnitTest +{ + void WriteTextToFile(const AZStd::string& text, const AZ::IO::Path& path) + { + const AZStd::vector textBytes(text.begin(), text.end()); + AZ::IO::SystemFile textFile; + AZ_TestImpact_Eval( + textFile.Open( + path.c_str(), + AZ::IO::SystemFile::SF_OPEN_CREATE | AZ::IO::SystemFile::SF_OPEN_CREATE_PATH | AZ::IO::SystemFile::SF_OPEN_WRITE_ONLY), + TestImpact::Exception, "Couldn't open cache file for writing"); + const auto bytesWritten = textFile.Write(textBytes.data(), textBytes.size()); + AZ_TestImpact_Eval(bytesWritten == textBytes.size(), TestImpact::Exception, "Couldn't write text file data"); + } + + AZStd::string ConstructTestProcessArgs(TestImpact::ProcessId pid, AZStd::chrono::milliseconds sleepTime) + { + return AZStd::string::format("--id %i --sleep %u", pid, sleepTime.count()); + } + + AZStd::string ConstructTestProcessArgsLargeText(TestImpact::ProcessId pid, AZStd::chrono::milliseconds sleepTime) + { + return ConstructTestProcessArgs(pid, sleepTime) + " large"; + } + + AZStd::string KnownTestProcessOutputString(TestImpact::ProcessId pid) + { + return AZStd::string("TestProcessMainStdOut" + AZStd::to_string(pid)); + } + + AZStd::string KnownTestProcessErrorString(TestImpact::ProcessId pid) + { + return AZStd::string("TestProcessMainStdErr" + AZStd::to_string(pid)); + } + + AZStd::string GenerateSuiteOrFixtureName(const AZStd::string& name) + { + return name; + } + + AZStd::string GenerateTypedFixtureName(const AZStd::string& name, size_t typeNum) + { + return AZStd::string::format("%s/%u", name.c_str(), typeNum); + } + + AZStd::string GenerateParameterizedFixtureName(const AZStd::string& name, const AZStd::optional& prefix) + { + if (prefix.has_value()) + { + return AZStd::string::format("%s/%s", prefix->c_str(), name.c_str()); + } + + return AZStd::string::format("%s", name.c_str()); + } + + AZStd::string GenerateParameterizedTestName(const AZStd::string& name, size_t testNum) + { + return AZStd::string::format("%s/%u", name.c_str(), testNum); + } + + AZStd::string StringVectorToJSONElements(const AZStd::vector strings) + { + AZStd::string output; + + for (size_t i = 0, last = strings.size() - 1; i < strings.size(); i++) + { + output += AZStd::string::format("\"%s\"", strings[i].c_str()); + + if (i != last) + { + output += ","; + } + + output += "\n"; + } + + return output; + } + + AZStd::string GenerateBuildTargetDescriptorString( + const AZStd::string& name, + const AZStd::string& outputName, + const AZStd::string& path, + const AZStd::vector& staticSources, + const AZStd::vector& autogenInputs, + const AZStd::vector& autogenOutputs) + { + constexpr const char* const targetTemplate = + "{\n" + " \"sources\": {\n" + " \"input\": [\n" + "%s\n" + " ],\n" + " \"output\": [\n" + "%s\n" + " ],\n" + " \"static\": [\n" + "%s\n" + " ]\n" + " },\n" + " \"target\": {\n" + " \"name\": \"%s\",\n" + " \"output_name\": \"%s\",\n" + " \"path\": \"%s\"\n" + " }\n" + "}\n" + "\n"; + + AZStd::string output = AZStd::string::format(targetTemplate, + StringVectorToJSONElements(autogenInputs).c_str(), + StringVectorToJSONElements(autogenOutputs).c_str(), + StringVectorToJSONElements(staticSources).c_str(), + name.c_str(), + outputName.c_str(), + path.c_str()); + + return output; + } + + TestImpact::BuildTargetDescriptor GenerateBuildTargetDescriptor( + const AZStd::string& name, + const AZStd::string& outputName, + const AZStd::string& path, + const AZStd::vector& staticSources, + const TestImpact::AutogenSources& autogenSources) + { + TestImpact::BuildTargetDescriptor targetDescriptor; + targetDescriptor.m_buildMetaData = {name, outputName, path}; + targetDescriptor.m_sources = {staticSources, autogenSources}; + return targetDescriptor; + } + + TestImpact::TestEnumerationSuite GenerateParamterizedSuite( + const AZStd::pair& fixture, + const AZStd::optional& permutation, + const AZStd::vector> tests, + size_t permutationCount) + { + TestImpact::TestEnumerationSuite suite = + TestImpact::TestEnumerationSuite{GenerateParameterizedFixtureName(fixture.first, permutation), fixture.second, {}}; + + for (const auto& test : tests) + { + for (auto i = 0; i < permutationCount; i++) + { + suite.m_tests.push_back(TestImpact::TestEnumerationCase{GenerateParameterizedTestName(test.first, i), test.second}); + } + } + + return suite; + } + + void GenerateTypedSuite( + const AZStd::pair& fixture, + const AZStd::vector> tests, + size_t permutationCount, + AZStd::vector& parentSuiteList) + { + for (auto i = 0; i < permutationCount; i++) + { + parentSuiteList.push_back(TestImpact::TestEnumerationSuite{GenerateTypedFixtureName(fixture.first, i), fixture.second, {}}); + + for (const auto& test : tests) + { + parentSuiteList.back().m_tests.push_back(TestImpact::TestEnumerationCase{test.first, test.second}); + } + } + } + + size_t CalculateNumPassedTests(const AZStd::vector& suites) + { + size_t numPassedTests = 0; + for (const auto& suite : suites) + { + numPassedTests += AZStd::count_if(suite.m_tests.begin(), suite.m_tests.end(), [](const auto& test) { + return test.m_result.has_value() && test.m_result.value() == TestImpact::TestRunResult::Passed; + }); + } + + return numPassedTests; + } + + size_t CalculateNumFailedTests(const AZStd::vector& suites) + { + size_t numFailedTests = 0; + for (const auto& suite : suites) + { + for (const auto& test : suite.m_tests) + { + if (test.m_result.has_value() && test.m_result.value() == TestImpact::TestRunResult::Failed) + { + numFailedTests++; + } + } + } + + return numFailedTests; + } + + size_t CalculateNumRunTests(const AZStd::vector& suites) + { + size_t numRunTests = 0; + for (const auto& suite : suites) + { + for (const auto& test : suite.m_tests) + { + if (test.m_status == TestImpact::TestRunStatus::Run) + { + numRunTests++; + } + } + } + + return numRunTests; + } + + size_t CalculateNumNotRunTests(const AZStd::vector& suites) + { + size_t numNotRunTests = 0; + for (const auto& suite : suites) + { + for (const auto& test : suite.m_tests) + { + if (test.m_status == TestImpact::TestRunStatus::NotRun) + { + numNotRunTests++; + } + } + } + + return numNotRunTests; + } + + AZStd::vector GetTestTargetATestEnumerationSuites() + { + AZStd::vector suites; + + suites.push_back(TestImpact::TestEnumerationSuite{GenerateSuiteOrFixtureName("TestCase"), true, {}}); + suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test1_WillPass", true}); + suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test2_WillPass", true}); + suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test3_WillPass", true}); + suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test4_WillPass", true}); + suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test5_WillPass", true}); + suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test6_WillPass", true}); + suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test7_WillFail", true}); + + suites.push_back(TestImpact::TestEnumerationSuite{GenerateSuiteOrFixtureName("TestFixture"), true, {}}); + suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test1_WillPass", true}); + suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test2_WillPass", true}); + suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test3_WillPass", true}); + + return suites; + } + + AZStd::vector GetTestTargetBTestEnumerationSuites() + { + AZStd::vector suites; + + suites.push_back(TestImpact::TestEnumerationSuite{GenerateSuiteOrFixtureName("TestCase"), true, {}}); + suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test1_WillPass", true}); + suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test2_WillPass", true}); + suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test3_WillPass", true}); + + suites.push_back(TestImpact::TestEnumerationSuite{GenerateSuiteOrFixtureName("TestFixture"), true, {}}); + suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test1_WillPass", true}); + + const size_t numParams = 27; + suites.push_back(GenerateParamterizedSuite( + {"TestFixtureWithParams", true}, "PermutationA", {{"Test1_WillPass", true}, {"Test2_WillPass", true}}, numParams)); + suites.push_back(GenerateParamterizedSuite( + {"TestFixtureWithParams", true}, AZStd::nullopt, {{"Test1_WillPass", true}, {"Test2_WillPass", true}}, numParams)); + + return suites; + } + + AZStd::vector GetTestTargetCTestEnumerationSuites() + { + AZStd::vector suites; + + suites.push_back(TestImpact::TestEnumerationSuite{GenerateSuiteOrFixtureName("TestFixture"), true, {}}); + suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test1_WillPass", true}); + suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test2_WillPass", true}); + + const size_t numTypes = 4; + for (auto i = 0; i < numTypes; i++) + { + suites.push_back(TestImpact::TestEnumerationSuite{GenerateTypedFixtureName("TestFixtureWithTypes", i), true, {}}); + for (auto j = 1; j < numTypes + 1; j++) + { + suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{AZStd::string::format("Test%u_WillPass", j), true}); + } + } + + return suites; + } + + AZStd::vector GetTestTargetDTestEnumerationSuites() + { + AZStd::vector suites; + + suites.push_back(TestImpact::TestEnumerationSuite{GenerateSuiteOrFixtureName("TestCase"), true, {}}); + suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test1_WillPass", true}); + suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"DISABLED_Test2_WillPass", false}); + suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test3_WillPass", true}); + suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test4_WillPass", true}); + suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test5_WillPass", true}); + + suites.push_back(TestImpact::TestEnumerationSuite{GenerateSuiteOrFixtureName("TestFixture1"), true, {}}); + suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test1_WillPass", true}); + suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test2_WillPass", true}); + + suites.push_back(TestImpact::TestEnumerationSuite{GenerateSuiteOrFixtureName("DISABLED_TestFixture2"), false, {}}); + suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test1_WillPass", true}); + suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test2_WillPass", true}); + + const size_t numTypes = 4; + GenerateTypedSuite( + {"TestFixtureWithTypes1", true}, {{"Test1_WillPass", true}, {"DISABLED_Test2_WillPass", false}, {"Test3_WillPass", true}}, + numTypes, suites); + + GenerateTypedSuite( + {"DISABLED_TestFixtureWithTypes2", false}, + {{"Test1_WillPass", true}, {"DISABLED_Test2_WillPass", false}, {"Test3_WillPass", true}}, numTypes, suites); + + const size_t numParams = 27; + suites.push_back(GenerateParamterizedSuite( + {"TestFixtureWithParams1", true}, "PermutationA", {{"Test1_WillPass", true}, {"DISABLED_Test2_WillPass", false}}, numParams)); + suites.push_back(GenerateParamterizedSuite( + {"TestFixtureWithParams1", true}, AZStd::nullopt, {{"Test1_WillPass", true}, {"DISABLED_Test2_WillPass", false}}, numParams)); + suites.push_back(GenerateParamterizedSuite( + {"DISABLED_TestFixtureWithParams2", false}, "PermutationA", {{"Test1_WillPass", true}, {"DISABLED_Test2_WillPass", false}}, + numParams)); + suites.push_back(GenerateParamterizedSuite( + {"DISABLED_TestFixtureWithParams2", false}, AZStd::nullopt, {{"Test1_WillPass", true}, {"DISABLED_Test2_WillPass", false}}, + numParams)); + + return suites; + } + + TestImpact::TestRunCase TestRunCaseFromTestEnumerationCase(const TestImpact::TestEnumerationCase& enumCase) + { + TestImpact::TestRunCase runCase; + runCase.m_name = enumCase.m_name; + runCase.m_enabled = enumCase.m_enabled; + return runCase; + } + + TestImpact::TestRunSuite TestRunSuiteFromTestEnumerationSuite(const TestImpact::TestEnumerationSuite& enumSuite) + { + TestImpact::TestRunSuite runSuite; + runSuite.m_name = enumSuite.m_name; + runSuite.m_enabled = enumSuite.m_enabled; + + for (const auto& enumCase : enumSuite.m_tests) + { + runSuite.m_tests.push_back(TestRunCaseFromTestEnumerationCase(enumCase)); + } + + return runSuite; + } + + AZStd::vector TestRunSuitesFromTestEnumerationSuites( + const AZStd::vector& enumSuites) + { + AZStd::vector runSuites; + runSuites.reserve(enumSuites.size()); + + for (const auto& enumSuite : enumSuites) + { + runSuites.push_back(TestRunSuiteFromTestEnumerationSuite(enumSuite)); + } + + return runSuites; + } + + void SetTestRunSuiteData(TestImpact::TestRunSuite& testSuite, AZStd::chrono::milliseconds duration) + { + testSuite.m_duration = duration; + } + + void SetTestRunCaseData( + TestImpact::TestRunCase& testCase, AZStd::chrono::milliseconds duration, TestImpact::TestRunStatus status, + AZStd::optional result = AZStd::nullopt) + { + testCase.m_duration = duration; + testCase.m_status = status; + testCase.m_result = result; + } + + AZStd::vector GetTestTargetATestRunSuites() + { + AZStd::vector suites(TestRunSuitesFromTestEnumerationSuites(GetTestTargetATestEnumerationSuites())); + + enum + { + TestCaseIndex = 0, + TestFixtureIndex + }; + + { + auto& suite = suites[TestCaseIndex]; + SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{3}); + SetTestRunCaseData( + suite.m_tests[0], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[1], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[2], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[3], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[4], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[5], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[6], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Failed); + } + { + auto& suite = suites[TestFixtureIndex]; + SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{38}); + SetTestRunCaseData( + suite.m_tests[0], AZStd::chrono::milliseconds{4}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[1], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[2], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + } + + return suites; + } + + AZStd::vector GetTestTargetBTestRunSuites() + { + AZStd::vector suites(TestRunSuitesFromTestEnumerationSuites(GetTestTargetBTestEnumerationSuites())); + + enum + { + TestCaseIndex = 0, + TestFixtureIndex, + PermutationATestFixtureWithParamsIndex, + TestFixtureWithParamsIndex + }; + + { + auto& suite = suites[TestCaseIndex]; + SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{202}); + SetTestRunCaseData( + suite.m_tests[0], AZStd::chrono::milliseconds{3}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[1], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[2], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + } + { + auto& suite = suites[TestFixtureIndex]; + SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{62}); + SetTestRunCaseData( + suite.m_tests[0], AZStd::chrono::milliseconds{5}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + } + { + auto& suite = suites[PermutationATestFixtureWithParamsIndex]; + SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{3203}); + SetTestRunCaseData( + suite.m_tests[0], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[1], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[2], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[3], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[4], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[5], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[6], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[7], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[8], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[9], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[10], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[11], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[12], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[13], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[14], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[15], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[16], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[17], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[18], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[19], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[20], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[21], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[22], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[23], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[24], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[25], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[26], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[27], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[28], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[29], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[30], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[31], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[32], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[33], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[34], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[35], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[36], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[37], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[38], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[39], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[40], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[41], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[42], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[43], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[44], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[45], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[46], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[47], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[48], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[49], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[50], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[51], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[52], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[53], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + } + { + auto& suite = suites[TestFixtureWithParamsIndex]; + SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{3360}); + SetTestRunCaseData( + suite.m_tests[0], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[1], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[2], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[3], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[4], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[5], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[6], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[7], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[8], AZStd::chrono::milliseconds{2}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[9], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[10], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[11], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[12], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[13], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[14], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[15], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[16], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[17], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[18], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[19], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[20], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[21], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[22], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[23], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[24], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[25], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[26], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[27], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[28], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[29], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[30], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[31], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[32], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[33], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[34], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[35], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[36], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[37], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[38], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[39], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[40], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[41], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[42], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[43], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[44], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[45], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[46], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[47], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[48], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[49], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[50], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[51], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[52], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[53], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + } + + return suites; + } + + AZStd::vector GetTestTargetCTestRunSuites() + { + AZStd::vector suites(TestRunSuitesFromTestEnumerationSuites(GetTestTargetCTestEnumerationSuites())); + + enum + { + TestCaseIndex = 0, + TestFixtureWithTypes0Index, + TestFixtureWithTypes1Index, + TestFixtureWithTypes2Index, + TestFixtureWithTypes3Index + }; + + { + auto& suite = suites[TestCaseIndex]; + SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{125}); + SetTestRunCaseData( + suite.m_tests[0], AZStd::chrono::milliseconds{4}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[1], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + } + { + auto& suite = suites[TestFixtureWithTypes0Index]; + SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{210}); + SetTestRunCaseData( + suite.m_tests[0], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[1], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[2], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[3], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + } + { + auto& suite = suites[TestFixtureWithTypes1Index]; + SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{208}); + SetTestRunCaseData( + suite.m_tests[0], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[1], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[2], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[3], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + } + { + auto& suite = suites[TestFixtureWithTypes2Index]; + SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{199}); + SetTestRunCaseData( + suite.m_tests[0], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[1], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[2], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[3], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + } + { + auto& suite = suites[TestFixtureWithTypes3Index]; + SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{49}); + SetTestRunCaseData( + suite.m_tests[0], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[1], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[2], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[3], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + } + + return suites; + } + + AZStd::vector GetTestTargetDTestRunSuites() + { + AZStd::vector suites(TestRunSuitesFromTestEnumerationSuites(GetTestTargetDTestEnumerationSuites())); + + enum + { + TestCaseIndex = 0, + TestFixture1Index, + DISABLEDTestFixture2Index, + TestFixtureWithTypes10Index, + TestFixtureWithTypes11Index, + TestFixtureWithTypes12Index, + TestFixtureWithTypes13Index, + DISABLEDTestFixtureWithTypes20Index, + DISABLEDTestFixtureWithTypes21Index, + DISABLEDTestFixtureWithTypes22Index, + DISABLEDTestFixtureWithTypes23Index, + PermutationATestFixtureWithParams1Index, + TestFixtureWithParams1Index, + PermutationADISABLED_TestFixtureWithParams2Index, + DISABLED_TestFixtureWithParams2Index, + }; + + { + auto& suite = suites[TestCaseIndex]; + SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{3}); + SetTestRunCaseData( + suite.m_tests[0], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData(suite.m_tests[1], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData( + suite.m_tests[2], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[3], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[4], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + } + { + auto& suite = suites[TestFixture1Index]; + SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{4}); + SetTestRunCaseData( + suite.m_tests[0], AZStd::chrono::milliseconds{2}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[1], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + } + { + auto& suite = suites[DISABLEDTestFixture2Index]; + SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{0}); + SetTestRunCaseData(suite.m_tests[0], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[1], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + } + { + auto& suite = suites[TestFixtureWithTypes10Index]; + SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{1}); + SetTestRunCaseData( + suite.m_tests[0], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData(suite.m_tests[1], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData( + suite.m_tests[2], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + } + { + auto& suite = suites[TestFixtureWithTypes11Index]; + SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{3}); + SetTestRunCaseData( + suite.m_tests[0], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData(suite.m_tests[1], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData( + suite.m_tests[2], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + } + { + auto& suite = suites[TestFixtureWithTypes12Index]; + SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{0}); + SetTestRunCaseData( + suite.m_tests[0], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData(suite.m_tests[1], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData( + suite.m_tests[2], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + } + { + auto& suite = suites[TestFixtureWithTypes13Index]; + SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{1}); + SetTestRunCaseData( + suite.m_tests[0], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData(suite.m_tests[1], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData( + suite.m_tests[2], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + } + { + auto& suite = suites[DISABLEDTestFixtureWithTypes20Index]; + SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{0}); + SetTestRunCaseData(suite.m_tests[0], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[1], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[2], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + } + { + auto& suite = suites[DISABLEDTestFixtureWithTypes21Index]; + SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{0}); + SetTestRunCaseData(suite.m_tests[0], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[1], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[2], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + } + { + auto& suite = suites[DISABLEDTestFixtureWithTypes22Index]; + SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{0}); + SetTestRunCaseData(suite.m_tests[0], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[1], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[2], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + } + { + auto& suite = suites[DISABLEDTestFixtureWithTypes23Index]; + SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{0}); + SetTestRunCaseData(suite.m_tests[0], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[1], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[2], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + } + { + auto& suite = suites[PermutationATestFixtureWithParams1Index]; + SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{173}); + SetTestRunCaseData( + suite.m_tests[0], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[1], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[2], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[3], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[4], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[5], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[6], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[7], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[8], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[9], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[10], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[11], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[12], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[13], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[14], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[15], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[16], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[17], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[18], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[19], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[20], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[21], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[22], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[23], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[24], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[25], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[26], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData(suite.m_tests[27], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[28], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[29], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[30], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[31], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[32], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[33], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[34], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[35], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[36], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[37], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[38], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[39], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[40], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[41], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[42], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[43], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[44], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[45], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[46], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[47], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[48], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[49], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[50], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[51], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[52], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[53], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + } + { + auto& suite = suites[TestFixtureWithParams1Index]; + SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{102}); + SetTestRunCaseData( + suite.m_tests[0], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[1], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[2], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[3], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[4], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[5], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[6], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[7], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[8], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[9], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[10], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[11], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[12], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[13], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[14], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[15], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[16], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[17], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[18], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[19], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[20], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[21], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[22], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[23], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[24], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[25], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData( + suite.m_tests[26], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); + SetTestRunCaseData(suite.m_tests[27], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[28], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[29], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[30], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[31], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[32], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[33], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[34], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[35], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[36], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[37], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[38], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[39], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[40], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[41], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[42], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[43], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[44], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[45], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[46], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[47], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[48], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[49], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[50], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[51], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[52], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + SetTestRunCaseData(suite.m_tests[53], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); + } + // The other two fixtures are all disabled (not run) + + return suites; + } + + AZStd::vector GetTestTargetASourceModuleCoverages() + { + AZStd::vector moduleCoverages; + + TestImpact::ModuleCoverage moduleCoverage; + moduleCoverage.m_path = "C:\\Lumberyard\\windows_vs2019\\bin\\debug\\TestImpact.TestTargetA.Tests.dll"; + + TestImpact::SourceCoverage sourceCoverage; + sourceCoverage.m_path = + "C:\\Lumberyard\\Code\\Tools\\TestImpactFramework\\Runtime\\Code\\Tests\\TestTargetA\\Code\\Tests\\TestImpactTestTargetA.cpp"; + + moduleCoverage.m_sources.push_back(AZStd::move(sourceCoverage)); + moduleCoverages.push_back(AZStd::move(moduleCoverage)); + return moduleCoverages; + } + + AZStd::vector GetTestTargetBSourceModuleCoverages() + { + AZStd::vector moduleCoverages; + + TestImpact::ModuleCoverage moduleCoverage; + moduleCoverage.m_path = "C:\\Lumberyard\\windows_vs2019\\bin\\debug\\TestImpact.TestTargetB.Tests.dll"; + + TestImpact::SourceCoverage sourceCoverage; + sourceCoverage.m_path = + "C:\\Lumberyard\\Code\\Tools\\TestImpactFramework\\Runtime\\Code\\Tests\\TestTargetB\\Code\\Tests\\TestImpactTestTargetB.cpp"; + + moduleCoverage.m_sources.push_back(AZStd::move(sourceCoverage)); + moduleCoverages.push_back(AZStd::move(moduleCoverage)); + return moduleCoverages; + } + + AZStd::vector GetTestTargetCSourceModuleCoverages() + { + AZStd::vector moduleCoverages; + + TestImpact::ModuleCoverage moduleCoverage; + moduleCoverage.m_path = "C:\\Lumberyard\\windows_vs2019\\bin\\debug\\TestImpact.TestTargetC.Tests.dll"; + + TestImpact::SourceCoverage sourceCoverage; + sourceCoverage.m_path = + "C:\\Lumberyard\\Code\\Tools\\TestImpactFramework\\Runtime\\Code\\Tests\\TestTargetC\\Code\\Tests\\TestImpactTestTargetC.cpp"; + + moduleCoverage.m_sources.push_back(AZStd::move(sourceCoverage)); + moduleCoverages.push_back(AZStd::move(moduleCoverage)); + return moduleCoverages; + } + + AZStd::vector GetTestTargetDSourceModuleCoverages() + { + AZStd::vector moduleCoverages; + + TestImpact::ModuleCoverage moduleCoverage; + moduleCoverage.m_path = "C:\\Lumberyard\\windows_vs2019\\bin\\debug\\TestImpact.TestTargetD.Tests.dll"; + + TestImpact::SourceCoverage sourceCoverage; + sourceCoverage.m_path = + "C:\\Lumberyard\\Code\\Tools\\TestImpactFramework\\Runtime\\Code\\Tests\\TestTargetD\\Code\\Tests\\TestImpactTestTargetD.cpp"; + + moduleCoverage.m_sources.push_back(AZStd::move(sourceCoverage)); + moduleCoverages.push_back(AZStd::move(moduleCoverage)); + return moduleCoverages; + } + + AZStd::vector GetTestTargetALineModuleCoverages() + { + AZStd::vector moduleCoverages; + + TestImpact::ModuleCoverage moduleCoverage; + moduleCoverage.m_path = "C:\\Lumberyard\\windows_vs2019\\bin\\debug\\TestImpact.TestTargetA.Tests.dll"; + + TestImpact::SourceCoverage sourceCoverage; + sourceCoverage.m_path = + "C:\\Lumberyard\\Code\\Tools\\TestImpactFramework\\Runtime\\Code\\Tests\\TestTargetA\\Code\\Tests\\TestImpactTestTargetA.cpp"; + sourceCoverage.m_coverage = AZStd::vector(); + + auto& lines = sourceCoverage.m_coverage.value(); + lines.push_back(TestImpact::LineCoverage{22, 1}); + lines.push_back(TestImpact::LineCoverage{23, 1}); + lines.push_back(TestImpact::LineCoverage{24, 1}); + lines.push_back(TestImpact::LineCoverage{25, 1}); + lines.push_back(TestImpact::LineCoverage{27, 1}); + lines.push_back(TestImpact::LineCoverage{28, 1}); + lines.push_back(TestImpact::LineCoverage{29, 1}); + lines.push_back(TestImpact::LineCoverage{30, 1}); + lines.push_back(TestImpact::LineCoverage{32, 1}); + lines.push_back(TestImpact::LineCoverage{33, 1}); + lines.push_back(TestImpact::LineCoverage{34, 1}); + lines.push_back(TestImpact::LineCoverage{35, 1}); + lines.push_back(TestImpact::LineCoverage{37, 1}); + lines.push_back(TestImpact::LineCoverage{38, 1}); + lines.push_back(TestImpact::LineCoverage{39, 1}); + lines.push_back(TestImpact::LineCoverage{40, 1}); + lines.push_back(TestImpact::LineCoverage{42, 1}); + lines.push_back(TestImpact::LineCoverage{43, 1}); + lines.push_back(TestImpact::LineCoverage{44, 1}); + lines.push_back(TestImpact::LineCoverage{45, 1}); + lines.push_back(TestImpact::LineCoverage{47, 1}); + lines.push_back(TestImpact::LineCoverage{48, 1}); + lines.push_back(TestImpact::LineCoverage{49, 1}); + lines.push_back(TestImpact::LineCoverage{50, 1}); + lines.push_back(TestImpact::LineCoverage{52, 1}); + lines.push_back(TestImpact::LineCoverage{53, 1}); + lines.push_back(TestImpact::LineCoverage{54, 1}); + lines.push_back(TestImpact::LineCoverage{55, 1}); + lines.push_back(TestImpact::LineCoverage{57, 1}); + lines.push_back(TestImpact::LineCoverage{58, 1}); + lines.push_back(TestImpact::LineCoverage{59, 1}); + lines.push_back(TestImpact::LineCoverage{60, 1}); + lines.push_back(TestImpact::LineCoverage{62, 1}); + lines.push_back(TestImpact::LineCoverage{63, 1}); + lines.push_back(TestImpact::LineCoverage{64, 1}); + lines.push_back(TestImpact::LineCoverage{65, 1}); + lines.push_back(TestImpact::LineCoverage{67, 1}); + lines.push_back(TestImpact::LineCoverage{68, 1}); + lines.push_back(TestImpact::LineCoverage{69, 1}); + lines.push_back(TestImpact::LineCoverage{70, 1}); + lines.push_back(TestImpact::LineCoverage{73, 1}); + + moduleCoverage.m_sources.push_back(AZStd::move(sourceCoverage)); + moduleCoverages.push_back(AZStd::move(moduleCoverage)); + return moduleCoverages; + } + + AZStd::vector GetTestTargetBLineModuleCoverages() + { + AZStd::vector moduleCoverages; + + TestImpact::ModuleCoverage moduleCoverage; + moduleCoverage.m_path = "C:\\Lumberyard\\windows_vs2019\\bin\\debug\\TestImpact.TestTargetB.Tests.dll"; + + TestImpact::SourceCoverage sourceCoverage; + sourceCoverage.m_path = + "C:\\Lumberyard\\Code\\Tools\\TestImpactFramework\\Runtime\\Code\\Tests\\TestTargetB\\Code\\Tests\\TestImpactTestTargetB.cpp"; + sourceCoverage.m_coverage = AZStd::vector(); + + auto& lines = sourceCoverage.m_coverage.value(); + lines.push_back(TestImpact::LineCoverage{29, 1}); + lines.push_back(TestImpact::LineCoverage{30, 1}); + lines.push_back(TestImpact::LineCoverage{31, 1}); + lines.push_back(TestImpact::LineCoverage{32, 1}); + lines.push_back(TestImpact::LineCoverage{34, 1}); + lines.push_back(TestImpact::LineCoverage{35, 1}); + lines.push_back(TestImpact::LineCoverage{36, 1}); + lines.push_back(TestImpact::LineCoverage{37, 1}); + lines.push_back(TestImpact::LineCoverage{39, 1}); + lines.push_back(TestImpact::LineCoverage{40, 1}); + lines.push_back(TestImpact::LineCoverage{41, 1}); + lines.push_back(TestImpact::LineCoverage{42, 1}); + lines.push_back(TestImpact::LineCoverage{44, 1}); + lines.push_back(TestImpact::LineCoverage{45, 1}); + lines.push_back(TestImpact::LineCoverage{46, 1}); + lines.push_back(TestImpact::LineCoverage{47, 1}); + lines.push_back(TestImpact::LineCoverage{49, 1}); + lines.push_back(TestImpact::LineCoverage{50, 1}); + lines.push_back(TestImpact::LineCoverage{51, 1}); + lines.push_back(TestImpact::LineCoverage{52, 1}); + lines.push_back(TestImpact::LineCoverage{54, 1}); + lines.push_back(TestImpact::LineCoverage{55, 1}); + lines.push_back(TestImpact::LineCoverage{56, 1}); + lines.push_back(TestImpact::LineCoverage{57, 1}); + lines.push_back(TestImpact::LineCoverage{59, 1}); + lines.push_back(TestImpact::LineCoverage{66, 1}); + lines.push_back(TestImpact::LineCoverage{68, 1}); + lines.push_back(TestImpact::LineCoverage{75, 1}); + lines.push_back(TestImpact::LineCoverage{78, 1}); + + moduleCoverage.m_sources.push_back(AZStd::move(sourceCoverage)); + moduleCoverages.push_back(AZStd::move(moduleCoverage)); + return moduleCoverages; + } + + AZStd::vector GetTestTargetCLineModuleCoverages() + { + AZStd::vector moduleCoverages; + + TestImpact::ModuleCoverage moduleCoverage; + moduleCoverage.m_path = "C:\\Lumberyard\\windows_vs2019\\bin\\debug\\TestImpact.TestTargetC.Tests.dll"; + + TestImpact::SourceCoverage sourceCoverage; + sourceCoverage.m_path = + "C:\\Lumberyard\\Code\\Tools\\TestImpactFramework\\Runtime\\Code\\Tests\\TestTargetC\\Code\\Tests\\TestImpactTestTargetC.cpp"; + sourceCoverage.m_coverage = AZStd::vector(); + + auto& lines = sourceCoverage.m_coverage.value(); + lines.push_back(TestImpact::LineCoverage{32, 1}); + lines.push_back(TestImpact::LineCoverage{33, 1}); + lines.push_back(TestImpact::LineCoverage{34, 1}); + lines.push_back(TestImpact::LineCoverage{35, 1}); + lines.push_back(TestImpact::LineCoverage{37, 1}); + lines.push_back(TestImpact::LineCoverage{38, 1}); + lines.push_back(TestImpact::LineCoverage{39, 1}); + lines.push_back(TestImpact::LineCoverage{40, 1}); + lines.push_back(TestImpact::LineCoverage{42, 1}); + lines.push_back(TestImpact::LineCoverage{43, 1}); + lines.push_back(TestImpact::LineCoverage{44, 1}); + lines.push_back(TestImpact::LineCoverage{45, 1}); + lines.push_back(TestImpact::LineCoverage{47, 1}); + lines.push_back(TestImpact::LineCoverage{48, 1}); + lines.push_back(TestImpact::LineCoverage{49, 1}); + lines.push_back(TestImpact::LineCoverage{50, 1}); + lines.push_back(TestImpact::LineCoverage{52, 1}); + lines.push_back(TestImpact::LineCoverage{53, 1}); + lines.push_back(TestImpact::LineCoverage{54, 1}); + lines.push_back(TestImpact::LineCoverage{55, 1}); + lines.push_back(TestImpact::LineCoverage{57, 1}); + lines.push_back(TestImpact::LineCoverage{58, 1}); + lines.push_back(TestImpact::LineCoverage{59, 1}); + lines.push_back(TestImpact::LineCoverage{60, 1}); + lines.push_back(TestImpact::LineCoverage{63, 1}); + + moduleCoverage.m_sources.push_back(AZStd::move(sourceCoverage)); + moduleCoverages.push_back(AZStd::move(moduleCoverage)); + return moduleCoverages; + } + + AZStd::vector GetTestTargetDLineModuleCoverages() + { + AZStd::vector moduleCoverages; + + TestImpact::ModuleCoverage moduleCoverage; + moduleCoverage.m_path = "C:\\Lumberyard\\windows_vs2019\\bin\\debug\\TestImpact.TestTargetD.Tests.dll"; + + TestImpact::SourceCoverage sourceCoverage; + sourceCoverage.m_path = + "C:\\Lumberyard\\Code\\Tools\\TestImpactFramework\\Runtime\\Code\\Tests\\TestTargetD\\Code\\Tests\\TestImpactTestTargetD.cpp"; + sourceCoverage.m_coverage = AZStd::vector(); + + auto& lines = sourceCoverage.m_coverage.value(); + lines.push_back(TestImpact::LineCoverage{56, 1}); + lines.push_back(TestImpact::LineCoverage{57, 1}); + lines.push_back(TestImpact::LineCoverage{58, 1}); + lines.push_back(TestImpact::LineCoverage{59, 1}); + lines.push_back(TestImpact::LineCoverage{61, 1}); + lines.push_back(TestImpact::LineCoverage{62, 0}); + lines.push_back(TestImpact::LineCoverage{63, 0}); + lines.push_back(TestImpact::LineCoverage{64, 0}); + lines.push_back(TestImpact::LineCoverage{66, 1}); + lines.push_back(TestImpact::LineCoverage{67, 1}); + lines.push_back(TestImpact::LineCoverage{68, 1}); + lines.push_back(TestImpact::LineCoverage{69, 1}); + lines.push_back(TestImpact::LineCoverage{71, 1}); + lines.push_back(TestImpact::LineCoverage{72, 1}); + lines.push_back(TestImpact::LineCoverage{73, 1}); + lines.push_back(TestImpact::LineCoverage{74, 1}); + lines.push_back(TestImpact::LineCoverage{76, 1}); + lines.push_back(TestImpact::LineCoverage{77, 1}); + lines.push_back(TestImpact::LineCoverage{78, 1}); + lines.push_back(TestImpact::LineCoverage{79, 1}); + lines.push_back(TestImpact::LineCoverage{81, 1}); + lines.push_back(TestImpact::LineCoverage{82, 1}); + lines.push_back(TestImpact::LineCoverage{83, 1}); + lines.push_back(TestImpact::LineCoverage{84, 1}); + lines.push_back(TestImpact::LineCoverage{86, 1}); + lines.push_back(TestImpact::LineCoverage{87, 1}); + lines.push_back(TestImpact::LineCoverage{88, 1}); + lines.push_back(TestImpact::LineCoverage{89, 1}); + lines.push_back(TestImpact::LineCoverage{91, 1}); + lines.push_back(TestImpact::LineCoverage{92, 0}); + lines.push_back(TestImpact::LineCoverage{93, 0}); + lines.push_back(TestImpact::LineCoverage{94, 0}); + lines.push_back(TestImpact::LineCoverage{96, 1}); + lines.push_back(TestImpact::LineCoverage{97, 0}); + lines.push_back(TestImpact::LineCoverage{98, 0}); + lines.push_back(TestImpact::LineCoverage{99, 0}); + lines.push_back(TestImpact::LineCoverage{101, 1}); + lines.push_back(TestImpact::LineCoverage{102, 1}); + lines.push_back(TestImpact::LineCoverage{103, 1}); + lines.push_back(TestImpact::LineCoverage{104, 1}); + lines.push_back(TestImpact::LineCoverage{106, 1}); + lines.push_back(TestImpact::LineCoverage{107, 0}); + lines.push_back(TestImpact::LineCoverage{108, 0}); + lines.push_back(TestImpact::LineCoverage{109, 0}); + lines.push_back(TestImpact::LineCoverage{111, 1}); + lines.push_back(TestImpact::LineCoverage{112, 0}); + lines.push_back(TestImpact::LineCoverage{113, 0}); + lines.push_back(TestImpact::LineCoverage{114, 0}); + lines.push_back(TestImpact::LineCoverage{116, 1}); + lines.push_back(TestImpact::LineCoverage{117, 0}); + lines.push_back(TestImpact::LineCoverage{118, 0}); + lines.push_back(TestImpact::LineCoverage{119, 0}); + lines.push_back(TestImpact::LineCoverage{121, 1}); + lines.push_back(TestImpact::LineCoverage{128, 1}); + lines.push_back(TestImpact::LineCoverage{130, 1}); + lines.push_back(TestImpact::LineCoverage{137, 1}); + lines.push_back(TestImpact::LineCoverage{139, 1}); + lines.push_back(TestImpact::LineCoverage{146, 1}); + lines.push_back(TestImpact::LineCoverage{148, 1}); + lines.push_back(TestImpact::LineCoverage{155, 1}); + lines.push_back(TestImpact::LineCoverage{157, 1}); + lines.push_back(TestImpact::LineCoverage{158, 1}); + lines.push_back(TestImpact::LineCoverage{159, 1}); + lines.push_back(TestImpact::LineCoverage{160, 1}); + lines.push_back(TestImpact::LineCoverage{162, 1}); + lines.push_back(TestImpact::LineCoverage{163, 0}); + lines.push_back(TestImpact::LineCoverage{164, 0}); + lines.push_back(TestImpact::LineCoverage{165, 0}); + lines.push_back(TestImpact::LineCoverage{167, 1}); + lines.push_back(TestImpact::LineCoverage{168, 1}); + lines.push_back(TestImpact::LineCoverage{169, 1}); + lines.push_back(TestImpact::LineCoverage{170, 1}); + lines.push_back(TestImpact::LineCoverage{172, 1}); + lines.push_back(TestImpact::LineCoverage{173, 0}); + lines.push_back(TestImpact::LineCoverage{174, 0}); + lines.push_back(TestImpact::LineCoverage{175, 0}); + lines.push_back(TestImpact::LineCoverage{177, 1}); + lines.push_back(TestImpact::LineCoverage{178, 0}); + lines.push_back(TestImpact::LineCoverage{179, 0}); + lines.push_back(TestImpact::LineCoverage{180, 0}); + lines.push_back(TestImpact::LineCoverage{182, 1}); + lines.push_back(TestImpact::LineCoverage{183, 0}); + lines.push_back(TestImpact::LineCoverage{184, 0}); + lines.push_back(TestImpact::LineCoverage{185, 0}); + lines.push_back(TestImpact::LineCoverage{188, 1}); + + moduleCoverage.m_sources.push_back(AZStd::move(sourceCoverage)); + moduleCoverages.push_back(AZStd::move(moduleCoverage)); + return moduleCoverages; + } + + template + bool CheckTestCasesAreEqual(const TestCase& lhs, const TestCase& rhs) + { + if (lhs.m_enabled != rhs.m_enabled) + { + AZ_Error("CheckTestCasesAreEqual", false, "lhs.m_enabled: %u, rhs.m_enabled: %u", lhs.m_enabled, rhs.m_enabled); + return false; + } + + if (lhs.m_name != rhs.m_name) + { + AZ_Error("CheckTestCasesAreEqual", false, "lhs.m_name: %s, rhs.m_name: %s", lhs.m_name.c_str(), rhs.m_name.c_str()); + return false; + } + + return true; + } + + template + bool CheckTestSuitesAreEqual(const TestSuite& lhs, const TestSuite& rhs) + { + if (lhs.m_enabled != rhs.m_enabled) + { + AZ_Error("CheckTestSuitesAreEqual", false, "lhs.m_enabled: %u, rhs.m_enabled: %u", lhs.m_enabled, rhs.m_enabled); + return false; + } + + if (lhs.m_name != rhs.m_name) + { + AZ_Error("CheckTestSuitesAreEqual", false, "lhs.m_name: %s, rhs.m_name: %s", lhs.m_name.c_str(), rhs.m_name.c_str()); + return false; + } + + return AZStd::equal(lhs.m_tests.begin(), lhs.m_tests.end(), rhs.m_tests.begin(), [](const auto& left, const auto& right) { + return left == right; + }); + } + + template + bool CheckTestSuiteVectorsAreEqual(const AZStd::vector& lhs, const AZStd::vector& rhs) + { + return AZStd::equal(lhs.begin(), lhs.end(), rhs.begin(), [](const TestSuite& left, const TestSuite& right) { + return left == right; + }); + } + + template + bool CheckTestContainersAreEqual(const TestContainer& lhs, const TestContainer& rhs) + { + if (lhs.GetTestSuites().size() != rhs.GetTestSuites().size()) + { + return false; + } + + return AZStd::equal( + lhs.GetTestSuites().begin(), lhs.GetTestSuites().end(), rhs.GetTestSuites().begin(), [](const auto& left, const auto& right) { + return left == right; + }); + } + + bool operator==(const TestImpact::TestEnumerationCase& lhs, const TestImpact::TestEnumerationCase& rhs) + { + return CheckTestCasesAreEqual(lhs, rhs); + } + + bool operator==(const TestImpact::TestRunCase& lhs, const TestImpact::TestRunCase& rhs) + { + if (!CheckTestCasesAreEqual(lhs, rhs)) + { + return false; + } + + if (lhs.m_status != rhs.m_status) + { + AZ_Error( + "TestRunCase ==", false, "lhs.m_status: %u, rhs.m_status: %u", static_cast(lhs.m_status), + static_cast(rhs.m_status)); + return false; + } + + if (lhs.m_duration != rhs.m_duration) + { + AZ_Error("TestRunCase ==", false, "lhs.m_duration: %u, rhs.m_duration: %u", lhs.m_duration.count(), rhs.m_duration.count()); + return false; + } + + if (lhs.m_result != rhs.m_result) + { + if (!lhs.m_result.has_value() && rhs.m_result.has_value()) + { + AZ_Error("TestRunCase ==", false, "lhs.m_result: null, rhs.m_result: %u", static_cast(rhs.m_result.value())); + } + else if (lhs.m_result.has_value() && !rhs.m_result.has_value()) + { + AZ_Error("TestRunCase ==", false, "lhs.m_result: %u, rhs.m_result: null", static_cast(lhs.m_result.value())); + } + else + { + AZ_Error( + "TestRunCase ==", false, "lhs.m_result: %u, rhs.m_result: %u", static_cast(lhs.m_result.value()), + static_cast(rhs.m_result.value())); + } + + return false; + } + + return true; + } + + bool operator==(const TestImpact::TestEnumerationSuite& lhs, const TestImpact::TestEnumerationSuite& rhs) + { + return CheckTestSuitesAreEqual(lhs, rhs); + } + + bool operator==(const TestImpact::TestRunSuite& lhs, const TestImpact::TestRunSuite& rhs) + { + if (!CheckTestSuitesAreEqual(lhs, rhs)) + { + return false; + } + + if (lhs.m_duration != rhs.m_duration) + { + AZ_Error( + "TestEnumerationSuite ==", false, "lhs.m_duration: %u, rhs.m_duration: %u", lhs.m_duration.count(), rhs.m_duration.count()); + return false; + } + + return true; + } + + bool operator==(const AZStd::vector& lhs, const AZStd::vector& rhs) + { + return CheckTestSuiteVectorsAreEqual(lhs, rhs); + } + + bool operator==(const AZStd::vector& lhs, const AZStd::vector& rhs) + { + return CheckTestSuiteVectorsAreEqual(lhs, rhs); + } + + bool operator==(const TestImpact::TestEnumeration& lhs, const TestImpact::TestEnumeration& rhs) + { + return CheckTestContainersAreEqual(lhs, rhs); + } + + bool operator==(const TestImpact::TestRun& lhs, const TestImpact::TestRun& rhs) + { + if (lhs.GetDuration() != rhs.GetDuration()) + { + AZ_Error("TestRun ==", false, "lhs.GetDuration(): %u, rhs.GetDuration(): %u", lhs.GetDuration(), rhs.GetDuration()); + return false; + } + + if (lhs.GetNumDisabledTests() != rhs.GetNumDisabledTests()) + { + AZ_Error( + "TestRun ==", false, "lhs.GetNumDisabledTests(): %u, rhs.GetNumDisabledTests(): %u", lhs.GetNumDisabledTests(), + rhs.GetNumDisabledTests()); + return false; + } + + if (lhs.GetNumEnabledTests() != rhs.GetNumEnabledTests()) + { + AZ_Error( + "TestRun ==", false, "lhs.GetNumEnabledTests(): %u, rhs.GetNumEnabledTests(): %u", lhs.GetNumEnabledTests(), + rhs.GetNumEnabledTests()); + return false; + } + + if (lhs.GetNumFailures() != rhs.GetNumFailures()) + { + AZ_Error("TestRun ==", false, "lhs.GetNumFailures(): %u, rhs.GetNumFailures(): %u", lhs.GetNumFailures(), rhs.GetNumFailures()); + return false; + } + + if (lhs.GetNumNotRuns() != rhs.GetNumNotRuns()) + { + AZ_Error("TestRun ==", false, "lhs.GetNumNotRuns(): %u, rhs.GetNumNotRuns(): %u", lhs.GetNumNotRuns(), rhs.GetNumNotRuns()); + return false; + } + + if (lhs.GetNumPasses() != rhs.GetNumPasses()) + { + AZ_Error("TestRun ==", false, "lhs.GetNumPasses(): %u, rhs.GetNumPasses(): %u", lhs.GetNumPasses(), rhs.GetNumPasses()); + return false; + } + + if (lhs.GetNumRuns() != rhs.GetNumRuns()) + { + AZ_Error("TestRun ==", false, "lhs.GetNumRuns(): %u, rhs.GetNumRuns(): %u", lhs.GetNumRuns(), rhs.GetNumRuns()); + return false; + } + + return CheckTestContainersAreEqual(lhs, rhs); + } + + bool CheckTestRunCaseVectorsAreEqual( + const AZStd::vector& lhs, const AZStd::vector& rhs) + { + return AZStd::equal( + lhs.begin(), lhs.end(), rhs.begin(), [](const TestImpact::TestRunCase& leftCase, const TestImpact::TestRunCase& rightCase) { + if (!CheckTestCasesAreEqual(leftCase, rightCase)) + { + return false; + } + + if (leftCase.m_status != rightCase.m_status) + { + AZ_Error( + "CheckTestRunsAreEqualIgnoreDurations", false, "leftCase.m_status: %u, rightCase.m_status: %u", + static_cast(leftCase.m_status), static_cast(rightCase.m_status)); + return false; + } + + if (leftCase.m_result != rightCase.m_result) + { + if (!leftCase.m_result.has_value() && rightCase.m_result.has_value()) + { + AZ_Error( + "CheckTestRunsAreEqualIgnoreDurations", false, "leftCase.m_result: null, rightCase.m_result: %u", + static_cast(rightCase.m_result.value())); + } + else if (leftCase.m_result.has_value() && !rightCase.m_result.has_value()) + { + AZ_Error( + "CheckTestRunsAreEqualIgnoreDurations", false, "leftCase.m_result: %u, rightCase.m_result: null", + static_cast(leftCase.m_result.value())); + } + else + { + AZ_Error( + "CheckTestRunsAreEqualIgnoreDurations", false, "leftCase.m_result: %u, rightCase.m_result: %u", + static_cast(leftCase.m_result.value()), static_cast(rightCase.m_result.value())); + } + + return false; + } + + return true; + }); + } + + bool CheckTestRunsAreEqualIgnoreDurations(const TestImpact::TestRun& lhs, const TestImpact::TestRun& rhs) + { + return AZStd::equal( + lhs.GetTestSuites().begin(), lhs.GetTestSuites().end(), rhs.GetTestSuites().begin(), + [](const TestImpact::TestRunSuite& leftSuite, const TestImpact::TestRunSuite& rightSuite) { + if (leftSuite.m_enabled != rightSuite.m_enabled) + { + AZ_Error( + "CheckTestRunsAreEqualIgnoreDurations", false, "leftSuite.m_enabled: %u, rightSuite.m_enabled: %u", + leftSuite.m_enabled, rightSuite.m_enabled); + return false; + } + + if (leftSuite.m_name != rightSuite.m_name) + { + AZ_Error( + "CheckTestRunsAreEqualIgnoreDurations", false, "leftSuite.m_name: %s, rightSuite.m_name: %s", + leftSuite.m_name.c_str(), rightSuite.m_name.c_str()); + return false; + } + + return CheckTestRunCaseVectorsAreEqual(leftSuite.m_tests, rightSuite.m_tests); + }); + } + + bool operator==(const TestImpact::BuildMetaData& lhs, const TestImpact::BuildMetaData& rhs) + { + if (lhs.m_name != rhs.m_name) + { + return false; + } + else if (lhs.m_outputName != rhs.m_outputName) + { + return false; + } + else if (lhs.m_path != rhs.m_path) + { + return false; + } + + return true; + } + + bool operator==(const TestImpact::TargetSources& lhs, const TestImpact::TargetSources& rhs) + { + if (lhs.m_staticSources != rhs.m_staticSources) + { + return false; + } + else if (lhs.m_autogenSources.size() != rhs.m_autogenSources.size()) + { + return false; + } + else + { + for (size_t i = 0; i < lhs.m_autogenSources.size(); i++) + { + if (lhs.m_autogenSources[i].m_input != rhs.m_autogenSources[i].m_input) + { + return false; + } + else if (lhs.m_autogenSources[i].m_outputs.size() != rhs.m_autogenSources[i].m_outputs.size()) + { + return false; + } + + for (size_t j = 0; j < lhs.m_autogenSources[i].m_outputs.size(); j++) + { + if (lhs.m_autogenSources[i].m_outputs[j] != rhs.m_autogenSources[i].m_outputs[j]) + { + return false; + } + } + } + } + + return true; + } + + bool operator==(const TestImpact::BuildTargetDescriptor& lhs, const TestImpact::BuildTargetDescriptor& rhs) + { + return lhs.m_buildMetaData == rhs.m_buildMetaData && lhs.m_sources == rhs.m_sources; + } + + bool operator==(const TestImpact::TestTargetMeta& lhs, const TestImpact::TestTargetMeta& rhs) + { + if (lhs.m_suite != rhs.m_suite) + { + return false; + } + else if (lhs.m_launchMethod != rhs.m_launchMethod) + { + return false; + } + + return true; + } + + bool operator==(const TestImpact::ProductionTargetDescriptor& lhs, const TestImpact::ProductionTargetDescriptor& rhs) + { + return lhs.m_buildMetaData == rhs.m_buildMetaData; + } + + bool operator==(const TestImpact::TestTargetDescriptor& lhs, const TestImpact::TestTargetDescriptor& rhs) + { + return lhs.m_buildMetaData == rhs.m_buildMetaData && lhs.m_sources == rhs.m_sources && lhs.m_testMetaData == rhs.m_testMetaData; + } + + bool operator==(const TestImpact::LineCoverage& lhs, const TestImpact::LineCoverage& rhs) + { + if (lhs.m_hitCount != rhs.m_hitCount) + { + AZ_Error("LineCoverage ==", false, "lhs.m_hitCount: %u, rhs.m_hitCount: %u", lhs.m_hitCount, rhs.m_hitCount); + return false; + } + + if (lhs.m_lineNumber != rhs.m_lineNumber) + { + AZ_Error("LineCoverage ==", false, "lhs.m_lineNumber: %u, rhs.m_lineNumber: %u", lhs.m_lineNumber, rhs.m_lineNumber); + return false; + } + + return true; + } + + bool operator==(const TestImpact::SourceCoverage& lhs, const TestImpact::SourceCoverage& rhs) + { + if (lhs.m_path != rhs.m_path) + { + AZ_Error("LineCoverage ==", false, "lhs.m_path: %s, rhs.m_path: %s", lhs.m_path.c_str(), rhs.m_path.c_str()); + return false; + } + + if (lhs.m_coverage.has_value() != rhs.m_coverage.has_value()) + { + AZ_Error( + "LineCoverage ==", false, "lhs.m_coverage.has_value(): %u, rhs.m_coverage.has_value(): %u", lhs.m_coverage.has_value(), + rhs.m_coverage.has_value()); + return false; + } + + if (lhs.m_coverage.has_value()) + { + return AZStd::equal( + lhs.m_coverage.value().begin(), lhs.m_coverage.value().end(), rhs.m_coverage.value().begin(), + [](const TestImpact::LineCoverage& left, const TestImpact::LineCoverage& right) { + return left == right; + }); + } + + return true; + } + + bool operator==(const TestImpact::ModuleCoverage& lhs, const TestImpact::ModuleCoverage& rhs) + { + if (lhs.m_path != rhs.m_path) + { + AZ_Error("ModuleCoverage ==", false, "lhs.m_path: %s, rhs.m_path: %s", lhs.m_path.c_str(), rhs.m_path.c_str()); + return false; + } + + return AZStd::equal( + lhs.m_sources.begin(), lhs.m_sources.end(), rhs.m_sources.begin(), + [](const TestImpact::SourceCoverage& left, const TestImpact::SourceCoverage& right) { + return left == right; + }); + } + + bool operator==(const AZStd::vector& lhs, const AZStd::vector& rhs) + { + if (lhs.size() != rhs.size()) + { + AZ_Error("ModuleCoverage ==", false, "lhs.size(): %u, rhs.size(): %u", lhs.size(), rhs.size()); + return false; + } + + return AZStd::equal( + lhs.begin(), lhs.end(), rhs.begin(), [](const TestImpact::ModuleCoverage& left, const TestImpact::ModuleCoverage& right) { + return left == right; + }); + } + + bool operator!=(const AZStd::vector& lhs, const AZStd::vector& rhs) + { + return !(lhs == rhs); + } + + bool operator==(const TestImpact::TestCoverage& lhs, const TestImpact::TestCoverage& rhs) + { + if (lhs.GetNumModulesCovered() != rhs.GetNumModulesCovered()) + { + return false; + } + + if (lhs.GetNumSourcesCovered() != rhs.GetNumSourcesCovered()) + { + return false; + } + + if (lhs.GetModuleCoverages() != rhs.GetModuleCoverages()) + { + return false; + } + + if (lhs.GetSourcesCovered().size() != rhs.GetSourcesCovered().size()) + { + return false; + } + + return true; + } +} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactTestUtils.h b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactTestUtils.h new file mode 100644 index 0000000000..cc8d4190ec --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactTestUtils.h @@ -0,0 +1,220 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace UnitTest +{ + // Common parameters for process related tests + inline constexpr const char* ValidProcessPath = LY_TEST_IMPACT_TEST_PROCESS_BIN; + inline constexpr const char* InvalidProcessPath = "!!!@@@---???"; + inline constexpr const AZStd::chrono::milliseconds LongSleep = AZStd::chrono::minutes(60); + inline constexpr const size_t LargeTextSize = 0xFFFF - 1; // 65,535 chars less the null terminator + inline constexpr const AZStd::chrono::milliseconds ShortSleep = AZStd::chrono::milliseconds(500); + inline constexpr const AZStd::chrono::milliseconds NoSleep = AZStd::chrono::milliseconds(0); + + // Writes the specified text string to the specified file + void WriteTextToFile(const AZStd::string& text, const AZ::IO::Path& path); + + // Construct the arguments for launcing the test process + AZStd::string ConstructTestProcessArgs(TestImpact::ProcessId pid, AZStd::chrono::milliseconds sleepTime); + + // Construct the arguments for launcing the test process with large text dump + AZStd::string ConstructTestProcessArgsLargeText(TestImpact::ProcessId pid, AZStd::chrono::milliseconds sleepTime); + + // Known standard output string of the test process + AZStd::string KnownTestProcessOutputString(TestImpact::ProcessId pid); + + // Known standard error string of the test process + AZStd::string KnownTestProcessErrorString(TestImpact::ProcessId pid); + + // Generate a gtest typed test fixture name string based on the specified name and type + AZStd::string GenerateTypedFixtureName(const AZStd::string& name, size_t typeNum); + + // Generate a gtest parameterized test fixture name string based on the specified name and permutation number + AZStd::string GenerateParameterizedFixtureName( + const AZStd::string& name, + const AZStd::optional& prefix = AZStd::nullopt); + + // Generate a gtest parameterized test name string based on the specified name and permutation number + AZStd::string GenerateParameterizedTestName(const AZStd::string& name, size_t testNum); + + // Generate a JSON string of array elements from the specified vector + AZStd::string StringVectorToJSONElements(const AZStd::vector strings); + + // Generate a build target descriptor string in JSON format from the specified build target description + AZStd::string GenerateBuildTargetDescriptorString( + const AZStd::string& name, + const AZStd::string& outputName, + const AZStd::string& path, + const AZStd::vector& staticSources, + const AZStd::vector& autogenInputs, + const AZStd::vector& autogenOutputs); + + // Generate a build target descriptor from the specified build target description + // Note: no check for correctness of arguments is peformed + TestImpact::BuildTargetDescriptor GenerateBuildTargetDescriptor( + const AZStd::string& name, + const AZStd::string& outputName, + const AZStd::string& path, + const AZStd::vector& staticSources, + const TestImpact::AutogenSources& autogenSources); + + // Procedurally generate a parameterized test suite based on the supplied parameters + TestImpact::TestEnumerationSuite GenerateParamterizedSuite( + const AZStd::pair& fixture, + const AZStd::optional& permutation, + const AZStd::vector> tests, + size_t permutationCount); + + // Procedurally generate a typed test suite based on the supplied parameters + void GenerateTypedSuite( + const AZStd::pair& fixture, + const AZStd::vector> tests, + size_t permutationCount, + AZStd::vector& parentSuiteList); + + // Helper functions for calculating test suite meta-data + size_t CalculateNumPassedTests(const AZStd::vector& suites); + size_t CalculateNumFailedTests(const AZStd::vector& suites); + size_t CalculateNumRunTests(const AZStd::vector& suites); + size_t CalculateNumNotRunTests(const AZStd::vector& suites); + + template + size_t CalculateNumTestSuites(const AZStd::vector& suites) + { + return suites.size(); + } + + template + size_t CalculateNumTests(const AZStd::vector& suites) + { + size_t numTests = 0; + for (const auto& suite : suites) + { + numTests += suite.m_tests.size(); + } + + return numTests; + } + + template + size_t CalculateNumEnabledTests(const AZStd::vector& suites) + { + size_t numEnabledTests = 0; + for (const auto& suite : suites) + { + if (!suite.m_enabled) + { + continue; + } + + for (const auto& test : suite.m_tests) + { + if (test.m_enabled) + { + numEnabledTests++; + } + } + } + + return numEnabledTests; + } + + template + size_t CalculateNumDisabledTests(const AZStd::vector& suites) + { + size_t numDisabledTests = 0; + for (const auto& suite : suites) + { + if (!suite.m_enabled) + { + numDisabledTests += suite.m_tests.size(); + continue; + } + + for (const auto& test : suite.m_tests) + { + if (!test.m_enabled) + { + numDisabledTests++; + } + } + } + + return numDisabledTests; + } + + // Test enumeration suite representation of the test targets used for testing + AZStd::vector GetTestTargetATestEnumerationSuites(); + AZStd::vector GetTestTargetBTestEnumerationSuites(); + AZStd::vector GetTestTargetCTestEnumerationSuites(); + AZStd::vector GetTestTargetDTestEnumerationSuites(); + + // Test run suite representation of the test targets used for testing + AZStd::vector GetTestTargetATestRunSuites(); + AZStd::vector GetTestTargetBTestRunSuites(); + AZStd::vector GetTestTargetCTestRunSuites(); + AZStd::vector GetTestTargetDTestRunSuites(); + + // Line coverage representation of the test targets used for testing + AZStd::vector GetTestTargetALineModuleCoverages(); + AZStd::vector GetTestTargetBLineModuleCoverages(); + AZStd::vector GetTestTargetCLineModuleCoverages(); + AZStd::vector GetTestTargetDLineModuleCoverages(); + + // Source coverage representation of the test targets used for testing + AZStd::vector GetTestTargetASourceModuleCoverages(); + AZStd::vector GetTestTargetBSourceModuleCoverages(); + AZStd::vector GetTestTargetCSourceModuleCoverages(); + AZStd::vector GetTestTargetDSourceModuleCoverages(); + + // Helper comparisons for test validation (could potentially be moved to production source in the future) + bool operator==(const TestImpact::TestEnumerationCase& lhs, const TestImpact::TestEnumerationCase& rhs); + bool operator==(const TestImpact::TestEnumerationSuite& lhs, const TestImpact::TestEnumerationSuite& rhs); + bool operator==(const AZStd::vector& lhs, const AZStd::vector& rhs); + bool operator==(const TestImpact::TestEnumeration& lhs, const TestImpact::TestEnumeration& rhs); + + bool operator==(const TestImpact::TestRunCase& lhs, const TestImpact::TestRunCase& rhs); + bool operator==(const TestImpact::TestRunSuite& lhs, const TestImpact::TestRunSuite& rhs); + bool operator==(const AZStd::vector& lhs, const AZStd::vector& rhs); + bool operator==(const TestImpact::TestRun& lhs, const TestImpact::TestRun& rhs); + bool CheckTestRunsAreEqualIgnoreDurations(const TestImpact::TestRun& lhs, const TestImpact::TestRun& rhs); + + bool operator==(const TestImpact::BuildMetaData& lhs, const TestImpact::BuildMetaData& rhs); + bool operator==(const TestImpact::TargetSources& lhs, const TestImpact::TargetSources& rhs); + bool operator==(const TestImpact::BuildTargetDescriptor& lhs, const TestImpact::BuildTargetDescriptor& rhs); + bool operator==(const TestImpact::TestTargetMeta& lhs, const TestImpact::TestTargetMeta& rhs); + bool operator==(const TestImpact::ProductionTargetDescriptor& lhs, const TestImpact::ProductionTargetDescriptor& rhs); + bool operator==(const TestImpact::TestTargetDescriptor& lhs, const TestImpact::TestTargetDescriptor& rhs); + + bool operator==(const TestImpact::LineCoverage& lhs, const TestImpact::LineCoverage& rhs); + bool operator==(const TestImpact::SourceCoverage& lhs, const TestImpact::SourceCoverage& rhs); + bool operator==(const TestImpact::ModuleCoverage& lhs, const TestImpact::ModuleCoverage& rhs); + bool operator==(const AZStd::vector& lhs, const AZStd::vector& rhs); + bool operator==(const TestImpact::TestCoverage& lhs, const TestImpact::TestCoverage& rhs); +} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/CMakeLists.txt b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/CMakeLists.txt new file mode 100644 index 0000000000..8298bb7123 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/CMakeLists.txt @@ -0,0 +1,12 @@ +# +# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +# its licensors. +# +# For complete copyright and license terms please see the LICENSE at the root of this +# distribution (the "License"). All use of this software is governed by the License, +# or, if provided, by the license below or the license accompanying this file. Do not +# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# + +add_subdirectory(Code) \ No newline at end of file diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/CMakeLists.txt b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/CMakeLists.txt new file mode 100644 index 0000000000..4b8e9d2809 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/CMakeLists.txt @@ -0,0 +1,24 @@ +# +# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +# its licensors. +# +# For complete copyright and license terms please see the LICENSE at the root of this +# distribution (the "License"). All use of this software is governed by the License, +# or, if provided, by the license below or the license accompanying this file. Do not +# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# + +ly_add_target( + NAME TestImpact.TestProcess.Console EXECUTABLE + NAMESPACE AZ + FILES_CMAKE + testimpactframework_testprocess_files.cmake + INCLUDE_DIRECTORIES + PRIVATE + Source + BUILD_DEPENDENCIES + PRIVATE + AZ::AzCore + AZ::AzFramework +) \ No newline at end of file diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/Source/TestImpactTestProcess.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/Source/TestImpactTestProcess.cpp new file mode 100644 index 0000000000..cdfad1404f --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/Source/TestImpactTestProcess.cpp @@ -0,0 +1,93 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include "TestImpactTestProcess.h" +#include "TestImpactTestProcessLargeText.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace TestImpact +{ + TestProcess::TestProcess(int argc, char* argv[]) + { + StartupEnvironment(); + ParseArgs(argc, argv); + } + + TestProcess::~TestProcess() + { + TeardownEnvironment(); + } + + void TestProcess::StartupEnvironment() + { + AZ::AllocatorInstance::Create(); + } + + void TestProcess::TeardownEnvironment() + { + AZ::AllocatorInstance::Destroy(); + } + + void TestProcess::ParseArgs(int argc, char* argv[]) + { + const AZStd::string idArg = "id"; + const AZStd::string sleepArg = "sleep"; + const AZStd::string largeArg = "large"; + + AZ::CommandLine commandLine; + commandLine.Parse(argc, argv); + + m_id = AZStd::stoi(commandLine.GetSwitchValue(idArg, 0)); + m_sleep = AZStd::stoi(commandLine.GetSwitchValue(sleepArg, 0)); + + m_dumpLargeText = false; + for (auto i = 0; i < commandLine.GetNumMiscValues(); i++) + { + const auto& miscValue = commandLine.GetMiscValue(i); + if (miscValue == largeArg) + { + m_dumpLargeText = true; + break; + } + } + } + + int TestProcess::MainFunc() + { + if (m_dumpLargeText) + { + // Dump the large text blob to stdout and stderr + std::cout << LongText; + std::cerr << LongText; + } + else + { + // Dump the short known output string with id appended to stdout and stderr + std::cout << "TestProcessMainStdOut" << m_id; + std::cerr << "TestProcessMainStdErr" << m_id; + } + + AZStd::this_thread::sleep_for(AZStd::chrono::milliseconds(m_sleep)); + + return m_id; + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/Source/TestImpactTestProcess.h b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/Source/TestImpactTestProcess.h new file mode 100644 index 0000000000..cfe18aa3df --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/Source/TestImpactTestProcess.h @@ -0,0 +1,38 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include +#include + +namespace TestImpact +{ + class TestProcess + { + public: + TestProcess(int argc, char* argv[]); + ~TestProcess(); + + int MainFunc(); + + private: + void StartupEnvironment(); + void TeardownEnvironment(); + void ParseArgs(int argc, char* argv[]); + + int m_id; + int m_sleep; + bool m_dumpLargeText = false; + }; + +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/Source/TestImpactTestProcessLargeText.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/Source/TestImpactTestProcessLargeText.cpp new file mode 100644 index 0000000000..a4b899dd65 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/Source/TestImpactTestProcessLargeText.cpp @@ -0,0 +1,531 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include "TestImpactTestProcessLargeText.h" + +namespace TestImpact +{ + // Long text blob measuring 65,535 in length (inc. null terminator) + const char* const LongText = + "m65NIFtFmZV9CiZj1go6fPV6xo2NcBeEHoJ1JktsoT9mHUclADmJhjTMT5qaKd0FD8GiredOA6wSUiEGFkc62QwNxuEFUQIuOeQAH8NE4MuhvjSWnw0MCXnf5TppuCfU" + "5Xx7fimsd8wYoxDiPSatXSaXWcCdn7QNttzmGbwuldr3Fmbntiz5aliAcVrlB82RCVQ4v4aT41wCivgPJ0CNbfQyiPDBzZd3cGwLmvt6Nu9pP4kbTVHRsdif0tvs8j2V" + "93fotwV7Q50Xl3jniX2AUnwO6OgZdl97lPHn9wZbO7V5vRGQe1n9aweXK4u3qlBfplvsZAVOoC1RuY1JmCz2V2iOfucmIbpxL5jR6qobgFZI0Z2ioi34ssqqyH9xFgwG" + "muF69vqJjYUzJGgneGQUqWTH1yr4AIFyqfBFvBe04UAsYfo7jPQWxCpCRXNQfTuN2609HPafOQ4IJh8IdL77CtRd8B1y1Ah8oqWzQZ54HZFOWQ1FZ1yCIxcnaBNAhauS" + "RJnYB4gcTmdtrZlFysea28CjyjroTKTzeyzqfNhfteIs3urS3IyspinKEFIAWQ8z1NYS0ks6o47r777gzQcQZ51T6fAdHO346PBHyeZBdXZxCXK615SKctxlG1p2TvhQ" + "2TMOtb9xTr24JrlijnR9WFGzG1CuoLeTN7cgm6rk86KVbFnG74XwKNvIZ5xbHaiD0qaSrZzoGtBxrDdldw8BjcXLZS9FFOToOxdUASqX8TkWEP9oA4CDImFRr3AlUtsS" + "g6m4nzc7Lp4wdM8Jd9pPl2rmF7HgZ5yjO3dew9G83ewcAyN3XhLjVH68ExbdN4RTKbHsczWEPKDWQAoEzGLn0YILmrKh1f2GaobAcnqbyPkN7pMax7CqYfUJao1DvedD" + "gfqcRzqEb9F90Xn4Il3SaW0Qmsxy6OGTJDSSOJr7hBsthALqpKdFpMbFYeAvh2PYhS6G8KvzcNeKLcZfsWdONwVWnenuyQDRxQjOzp87pHMEtWGDJ2UxTsIKGvvxjrqC" + "bBalQhDZk0mIwUJdb5hf5qkJLD5tPdozFDe53FkFYOKDlsWdRpSvTnYeBk1WMxGD31yywHFUlOfUf2a17A76dBYu6Jb2KMfHcNfxJhIJOXWfT8nnIAJwvir4SHqlzPbr" + "IEh35MHMM5XpbreQZjD7sB3ovEmZfXx4dCbztrIXs17zpskuW36cVQDxR4Mz1ZEbdUtdZg7r8AxUNHZJhe0v96QmodqTp8qh7WP8EUmmaTr8F3PMyyMXgJfuAczpmMfr" + "rXvcFE48Cr0ZRcMDuWpPthkVPFxXAcaOAn6eqaJdHRQlioODCbjjw1kEcV2VcONDXtS7pV3YguletmZOUxYIcqVqWYAyztnam06QtmuCTTjg0l4a6R3obuugJZAt1gBS" + "IN6gaVPKQwJQLvLM7ChKDB0jmYE3nIRliYjIZ02blecFbvqCxJMakyc818tULWFQNV3msn43rNLCO3VAUKaQrXGn3zWjuqeur9WFoUnANTZhndyiuspIx0mxrCuRXqpO" + "h4zNhN0d7Rh0v22PrkzgLvB23UcgxbX6xcw7UQ5g3Cq77sq1ko191ZpKuxrBXhDLye9GXv7BKh4Wcfc8qHAP3qjKmHZInIVlpholPrxgrxfPW9MZSGDFPxJfhkyb6VSG" + "SNcyMPMhdtoCtUUAfDXoqLCbQmN7On7eJ0NJZdZA9BWtzjbFIxYYmnxl39QLFmpEkgZaq0AtAsYh8L7sbm8v0yGB0wTCWLFYqAx8NBsooS2YlgT9SX4XFgG29HibgsX3" + "VkCoJ3m4nzukHgnpzI6WIoiNjj4Z8sHR5NLWYWfN3ZOCj9dw23dhcTnRVBbUxkpyOMybV2S7ytwd0vb3ZJ6hlOh6dYZNdw2M3iN0NaKYs55a5zxNAMyKBHfemBFLN18E" + "zVEQgKz9qJv8EEBmgZ2p8iKoeeQrSRLz8blVW1OPilcpKGvIh9kW4lgssMZ7qRju2oJ0DyJ3joEnFOvtGwljLgoy7AzHLpN0dF5lrD94IL3Kn7b5g2h8yAhwRAV3Pmtf" + "GHLnb0gKzLbyJVEoBhvH41B5MOHtLfiAtcyJ2zMV5R4ldJV1vJm0t79xK0X99YykIlUuguwxnzWwd7tQO6F00gaBGvpVcz1zEwJEenBbjncVzXqAQwnO9zKtzUOcC5nR" + "7CfvLz0l4dMfAAMHZ7wGlCnWdBVKO6L6OyUSLGvrpz6jhdlhS45QJKkzTB50ZgOmK7owJwPrTnvDvH3Dxq8SDbmynSfvESmx4utROmawwY9R3fySBFKsLSB3lYlmg8Q8" + "xjabKm0PCDMjeslYXgohxklk7XMCOzkutm2o72sOGgqvFNXFQz9BJAnid2f9KpGc29oIUobf1RyIcTXpnPMedrM8brEd4v1aGdcb9rJv0eE3v9xt3eznh85sf2AN7PwS" + "tROP09CXfnq0qj6SOr2Wop7uCypqKx3gRSTggIZJL2Q7bFvwhYmIIpxuGMuKOBf2xxzo4pqME0HfIuXNkElZpj729pyxas04iOheSqurWhM2EKKmaBrkvFVK0mEa2T9I" + "9Rjh1ZNYvJU59S1lrD0DP4jkOdQ9bHYA9jgpXmFFN8jgWRRu14eKdrTYsZ6w3HNAiN0nqpmQUl30d10UAeMJHkv4xD889zQRMmmPCgLLJfvhZYfiwlqzBjbh7OQhydxm" + "dHPaOcFZpEQwbvR3KhHSVcbhKewJGHLSbHNWsRRsJrfOBnwSm0TfU3FD3xEqkPEl8mmsgQvakUAxI49RV9B3ueJJ9K8mvsZBBIRhnSZoqwipEskXqIc2VIW1CnW9qi0y" + "OObnoJ7qBpu4aueF3lGn0rgPXiXpcw0KaFxoq3kUTSEJXsj7mwHVYxKJfoPdaTuWDfJZ5Hk4Xgy16pUiD11SxMl7GZcXuAGl319LqX3nIAiahaoHTAhYimesTaf8x5h6" + "kgPxITKZXkqdD21Buykh4jKvcPLVivIr7yM8NOKvDf6zgQmm6VHihI4XFLgvEUe6EoYoE0RJS7Luc8dVFdNsVcGICayaaznjL3sFV6J7d4TVd35PEGmXtvLKKSvQ6Be9" + "gb5KZEk8pcHvYqpqzyrX4ZFve3cpv0IyLAlsWSawYV5L7eluYJNsC0vBImelV5oCecW7J6WmYNxTBpQt4M1MWcFzw920KaE9mXf4gkNCQb14FrbT8sNzt5JVFgz9Fjy7" + "aoRbG9xGCqyZkUa7RwmiTgFuk5CRSCgMu2lUNydfnAHbapUpkFyYRAShQr5Hjnu9H0OPhWUSTM6tz5Zzj8OzAwTTTb3Zp5DBywqQXZZPVcUutswpPVHgRRI6t65rd9nB" + "7XGXuJNfSQHotH5V3smIAk5gJezFb31PsfLkJ898DYGkmCwWtOvJqCglscSDSLv0EUoQGIRpJu9W9PQR9pU2IKpJhYks388vSwtAnYrUNOxEJAmUfnKlUZsHbJtFezAM" + "YtcS1I1kBK4eaP61zPWBWQ0uTR2UMXE6oVNuHXKLlQZFVADVTNBNRFvRO8s7LTmE4iMEeuRiOHbOkzexztJuxi4NDnyG964iJ580zHsehJckVyRh9w8AMaRs7HE7BwKx" + "gGwKY9DmZ8V2CyvqDLjxtdt9yIaPxg4ZtsAHTzYY4gXCQGoZRp9AsWaoNSJKt2JRmZE8CwJ6kPUyN9DCqPH33c8mu1zU2JHLBrsjNpgAaOlcr95W9YRYi6eBHoE9sT4i" + "a1zNr1veiFCh281PLhH52Bh3L4eXj9Z9vitWE9nJ6U58aDGj7zP3ITxRBni1UVlvVOYkInOQysSB6Qc3kjKq2dpYjCUQBZRaJO4fGyK8GwsXkBJAVJ5cFFCY2DARgsB4" + "9vksxVhIC8iNCUHeuzaIqqoDVD4EABtN5sgbjqsTmA76iojm99w0sfOGPdDmqKfxybKi8Lyg7xOQUu0HIIF4HYOcnsG6xuHfW1CNqvkcZR9tnOeyax19PgZaAX0Ulo2L" + "0HNi98GXoKbmuD9l1eFWPRQVVFfnDDH84KuPZcciXWWAIMDhGwtfUd72GcgeUju0NqI6DsEf9WPDDAw8VS5U6rHeXmhp9hR7vbOnyAstoarOqPlx7L3q9P2jGJu7Lhzx" + "fmUY2iGty6xoG7MM4jnrywA15ErsQXjQbe1JOwVx7YycaIH3pLjd72qLp0Oy1lINFHQnbqDqdnWWYHHNFnnYrndj9fXueBzctmlxoUM2duxpZJQ6BB2CXQ7y9guWWx9N" + "PrrYO7QpMAzi5MOLLIqkbkQbta7SlxHh7eouqkBHOvWaAJR6iI52rEJkMUudh5VevD5Sh4QMXUR3OsfPTLyENMhT50zwLzBzSHKQ3EaT3MuyERp33j3KXvME05lZpUfy" + "g7UkcSCAm0duYh7RxWBYykoXmLxBtlGLs7XtqjGvzvuQohfJGMnUnYCxNTpxjIlmhORhc8lUBPZ8Flq51MhnYp6sYztV3hvyDKY1wSgCvauNNNxjdguRzMOyBl8PcNcV" + "a02kDeE2wqVZvhv48vyzMITPZvMKOzhA2U1giReSYereQOehKtyeXXqhTGooUxUfOfE5dKCbfLyBW5wuptRtY0LZ1VdxWQluqErsSNJvX6JVXic4n3vwc7ockTlU8iFk" + "UCG2tfegjf8EpSlydI4vTjphHnIkiTTG4g26ILlGEDnoxJL3U8jZhWgdf5aGNaJ2wiOLy7FZlJkD8StKWRFjyGyNM3LEDOFAmkyzWwwWOV0hBjAKwkne0Wdbn1vV88b3" + "w2u4Cfi1ZK3n7NeZwI96qpzWV4oA0aj7yl4mkjFtouALOD09aKqspPNBMkY80pMwNrwkp9EIZNDvAslGAdCs4LmmsAsXMNQqfDo9ubMeDGqgjV4FhhrA26UljfKbCKCe" + "05jYo8VhtjfRUAutYjCE3H4VSVGEs8OIkQyNdvPcI8o1kqDoG0OdadaId1v4z9Qos5KCvPh6wx1mh8OW6x8MOCUwWKy7kDbIk81zjfMPADNTHe4u5xRsir5VSiaBGb3G" + "TADf5NpPnxAOpUAJlbiaLRA4TRFAuBmITqBeL8htAJIggtZsuxUZkTIs5acgpRBYgdV3gAN8mmkBb0m78x2buNHIqdbV8ggNCuJICD14AK8PKSuxNvvnzqhEEkUZEr8h" + "dF4bDnZ5kL3djQDEl0C1eDGOOL0LqyspoADQFmxkZV2fJAJQnYU0ewqvRqZTCBNtRtg1vneRzDUDxlSkMZP6PddDOkJ03VxNDnttG250htR0M8elz9sHCzMxBh6T4k16" + "lwOGctboSL0GhW7ZFjeE7sfXgCfdH80r0K28pw2sm8l1cdY14ZWuqpRP25xI6DYlWAx78T6yQ9IdVIrF5eVYbAkVs7pUCgNgOWdJmhet21WJgc25sRRcMpy9Gxq7rugS" + "EhhVAtfM4HiA3ofo3WwDuWcsUCjqVcQjh209w0O4vRDhHediAccuO62DwZRG4Gh4xAG0yFEOsrrk9znlSV4XMc3HK2ARPEguqVCVgpjJgfr5jCZnwfwmTz4L6DAAVb9f" + "ctBNSpYhr22klJgG96CHGnnI5ybpv1BGTYlJwRS1DUDeoWh537pcRmD2l5LgyFu2aWuwPxEypG9hpPmf5Gj3uwdyL0ABlKTUleyBeEVCu5B1RgnzKe6BTnInP3XtQKJu" + "l49rhKfDo8JOtcyyQ4LXEVaWnOrx4hs6Atu1L0mWEZ2M1UGwfYrv4Cd1ORt3WU30ZWwr5EsVBj4i9WtVogL3DL66CtYNM4HTUdU9p8VUjEMiKoxt7XkligvrvgcDxqe4" + "gGRAsMd1tDSUXCxYOv9aMO64DEYOAEkODj6neQg2zZwANgRY4wAagV4vzL4YomGQmSA5xE5mE0InfMJJdSVLPdamy2wNshqwe5ow7aj5c0itCMyL0EZek5NmMBqBAn5F" + "9pULD0wljEo8TgcvPkVSmIWrA4iTMraEHeHS7KgzDYC0FtM8KHnE5j7SvGxvykldyoXfjHn8HMe5XdHqNxNREZarZcZ6nkP9CJYCaWpuYoCDza4RBp80HTTDSWZXjKpY" + "H2PTPsztToaX8GjgAfjjICrUBiuKdOUUmznqa4SUeKvKoKbLK1GIRnOOk38aS69xB8t89iuhkymMZrczCWKdVTdEaDMFKI0ILTy3wyfkbkiXjEVQ62byURuHDr3tF5K5" + "aqLl11VLvZzNsTbJ6DhWaKD0HzSD0C2r4LtRzSNtsUkWIqETP1tkqCXAzD1KeP90KAVsj9TLOa8AagEau3tF041igctfjTa0vrrXm47zncAKn6MpklJjoBrtE8c8r3p6" + "rOPEtDdM2dZxaWwukrtLao3F5aezSu1462z2PgnYMmv1FlMNe3eETZ7thoyChiQrxyxrkzTKjwj6ra1NNYok5dZE6bgUZTxhC6Y44eAOIqWLn0tTHb8mryBSwiBkrPAx" + "RzVkYak80HfVgxFyL9Rb1nsq39QZLxrWuniLssgEFrutQA89B3nvfCV13NacROJdL309VUPMbDAZXxc193UYtIEZngCj7H8DGdezs2OQ8j8eIvSizweHcbFSPanqtlKN" + "BJ4rQYsdh3Zfq5OYMdNytxp2CNoUHg36oI4zxSMwyYKPC4KicGSQnwLMEwmyXKu57CazUHdc7PZkK3Lea7kNWA5hUdnBmrxd0xIOVjhFi4e3wSgsHnGphH4UF2rDPvC2" + "GRvuPSbEwEMyXJL159oeR0O5z7DG683VtOuLQPCyLm1tgqAGUaVg2iZULOfOLZuHlC8ByzwKQnQGvTNL1g6lHxh1ygaDdO3weStxgpMe9O2mV9mfj8CsOZSbx12NtDvR" + "46yqInqtHCT5nIbzTYQloULXEuXWL5AQ67g7rUsInEqzO2iwh1LwxqLMvoB42DPBEvpb7fmVOFddxzaEE0kd4p6i7t9cUi0JR1e2ZMrMNK7zFyLmx9gvAzarDSjGOY3n" + "EvKdBn63lLSANMwoInAt4Aept7tWydGScGtaj1yusS5iwO0THRP177dKlAfCjkFGx6mR942AnCxo6oMmJ91gATIKWm2fgBaW5BUKy1fQ5MqQDnQzEzRYEjMi9k6vQEur" + "VkVM5059sI5UJxlVr9yaKGHt3KTeu1F7PGNWQCwPIMqpVMlx7CbQ3bhQWSJRNISWUWFrWCOY1n8jbtPdwojNG23ugMCGwZYedB29EHuBfHIDX2RPdKt2dJAHg4aXjx6i" + "1TRgq5nm4K0Avaq86WlMfK3JkK7VM7Tt4XzfLLx6Uutkv7XU7GGG0bbBbUrtIsug5NFvhvrwcRR9KL5kl9z9nS1iJu7tJdljEcGggHiqjRBe6hoSPxvAa7AF6kDV5iOV" + "Ca9LkWZjg9C2P6Ri4KcmiK3OwZhFzn6P6jFAF8Y0raLbZSZBLJwUUzT5fYLYzpiMMsmgT2HXl3tOzEEZKhdxRjBPR5br51LanQypKBNgFuR7FlRuSEJaNiEPZmXK3GeU" + "Pk6PSpR0AdFaEKbowx4wxsJvzKT96pSzR7LaEEqOa9phydtTh85HoIuPnKkVcFIHGpbynCGL4TUAbqv14rfQfmNrVTqx9hqpJApofNu9tyhBYJUrQ7PrOCPG2ur34lOw" + "sUg5MkHEyzVfq0WGqlC7TTGoO3bA3TWsO5SHGOhHsq5wuo4JXjPrOz5vEI2tQQlG2Pn275cZiCAJ2XZmxG7k3LRUJOQ6NpFg1Rv509bnfB1MBet2KCA05UYQMG5MMaWN" + "WlauS0eN6pka7p1l2D39aDiiIeOjwJvWfc6Tf4FM7muhWABXwDymT41VX5140sKrFZ6h84jdBDKBoTaqANexdXrs9D3Zr2BlgiP0hDX9bqmwvvxP4JACCBFFcA2ZSM0E" + "K9J9P15AjyZg6iWei3FjJ9t3B81dN7NwG2x8bGgw8RZt5m8A9D8F3D6LeVgGVLGuAa55DFnhCwtkCviBSTJn0btERDELpFhQU5H3A19P26dEFpIZIqM5kj9xQwqMUVbC" + "riaOOL3BVc9mGt6jSJn1TI7YyibFoT06lSybiMGB3CO6T2tov9puMTiVoWEQJDGNlUUhqLyokzdbo3d8lBe4aWUFq19Cs0HZt50o9xdbIQcyEtHBMXNZ3gwECJ6QbYC5" + "igpTeTLc2aXk1003QJsZ0qgr6OkeOqfdBFr37zB9EEYw29uP7en85bKbqk3Xdnvo8HbIeCIDyb4A5u3RH9BJbre6CCsWbqP9P6bCkJrtT7JuDcpb2q7sWKuP6vR3fILk" + "eR30Ptmb6XX6C6LL69TRXPDE5jkuWiO4Sb3SJi1zSqztVvDgyfrar1P0LXnE6vkJ4Anqws8raHKz9EYuo9ZLZoYqxkqieyzRiTkqyrz7e3pAk7VLpHItA1HjV5NNBWyB" + "2Z7xnMziRxWtgv6APf4huLiLE7WcTTkkdyv5vm7uPjtZHq5ziUx30IjXW4fexcBcp4kmFPwYDBTyI7yOLHPrXxJIFgcdalZTEbwxSVLdlJpucFkXfG6uIVWSTWR27O5F" + "ClBzl09FLL5w0EON184alTAtonqJou8HJjxPQC9zaqqgshTvdgLw9IVZLFGTQwfs3bgSiFH2foWClTcdns2SnY1BZf2EdPUETwJKaFlAcz9DO1shcPN7Jo1gjm0xz2um" + "vznuMIOu48BSYrkvev0C5P3fTNoaYsAmmA2c0oWfgeJNN0SgEJ1FuNWc00yLWOblaOEakAkIysiRemkijteLN2p7OoW5hzOqwADExvpRsy4gdHbCGfHGQPKf0wvhqXvm" + "OqdkPLqp2gynWmiUuKcWPulZlyhgQXAePzS0oVkYTrbv6yy5mLSvgro9L6Vf7Z0OsEa5fWOM0mC718tmE4vbtBXPgXiiTDQVqqOaUlAMndzA18rzlNLRlZtv7XW4bN4k" + "eOcuSPbNe9tTWo9XABv5tqe94TRBPyakEpX9hQ7Fc3yy0wQLXrFSzzZDPlKTajNnOOb1nBksaoD6N6x8vXBTh5rhsG2DL9jKnfrEveVJt6H5o2HrjYaNFPQuddKJb5pW" + "ns98Xb8puf5QSVs4VN8wyzW5veX5PfTUOHjDRrBAzMtmb0C0a0qCuZq8dRkqigJMRv3LCmjtoYYYs1uIuZMWyaqj1BD8g2oOxNc411hkQpwwxIqzm7cevxMFKm1SBkqb" + "JoZHNX7P91ufuEeStHxRXlwUXep029JUyu6DkxuGCxbQIAEvNTdI8zpThTbOxI0jlVKOpCNEBo9LetsePnoA0Hj7qha4owWWfElKHzdxPB5RBxxkjeVAXOQLL5waKW6V" + "nufDcpSYJkpySbB6cR1n0qeX5lIqMBTxyY2rhCq3lK8mauRd3KxTOoDRuWh06FeR6kUindQ3GmaSDvKDF3aWunZtjcpqFmg7thwPhdsw5QyUlS4JFRbpBlc7YU2PJ8zJ" + "UYY2aecQmncf9ixMiYMNQbFX9k9SPM1T0DdzF8DP7tmgKLDsTYqhOutxBRIYCxAS27jBZPsoxBYAw8oxO1NXeCCJ4SETjJw1sLGz9HcBPzONy5slxvbYpdNZumHg3bp6" + "FRXdNDFYKiHkjnWIHP23HEAc3m3gqO4ZR4t3uMORHPFWkMOPbBwoKf6RMlLJsGCEVi0f3hZ9bAMaLVdGgvNRKMvpHNELMw197TbFZiznwIfwwt8tX07AtcDMQmqatYss" + "WKReaIWsECI1KHLfAehJtSqQYy1XPnhbbE7aSWlrkHl7WIbxt9jY4huOuxxhabKaZ1mbIaeyOOefMXSwzbKY49MrblqAJG2mS5Kuj5njMpSjJXhrNndZwYHIEFaquWqk" + "PEo9mbmcDwQI432dz9hY4wmeL3GxJO2318EzCPTITS0mZ8gkNg2DzFmU7IRSSW1hb9SxjdiBZXEuwlMWLzawO3MhiWQQiQfeWp3eISuZZ84IUuC49kKrmDiWkGRxLjnW" + "6KxvADT097jBN8blD56XtitDKkCgnDVtnmQhiNPOReweIesKxwTYzCnXH7jNkZKQtqXjunfjORabiXUAe6iiWLet7Qls4frgiSSjS8oH27zifNCmgfkvTEjxQXXTTFeI" + "TdrjTU5HYmTQJ348OFFYNNuQkZhdhGKaloaMkfhKiB9zWHY6s2hSO9nAfwBNmPD80WhNQPP9WIwaW0hMTFCAGoZNMRcHrlSgdn9pW10yoErcm2yrSx3NKu8paYUwmtPs" + "1qtEla6PfuAJzyJ0VsLWoegOH3Y53UUhggjqIkfqoqIokAsw798RLLlsuMWUGl0UgGKsViEXmBTjCXGZEQNX6BdDhr32ysNyybnPvWOoHF9pNCYssG6dtdMzyxydLYQj" + "l7vnIwwhi92QKHr4qluQCmFqOHvWEc2Wdo5zVeZVjmSycRaFR08b6SZZS0Q09WyGhXO6jy6Zw45nUCY0KwmtB7vIcp0mq9mXSrvnMZt7rKivuHh8H5jOnDTXHrZa3hxW" + "QbTyE1SBTKavdQWknVhUFxvas6tOZk4H7W7JIHT7DYUoRYHQkDPhD2CbOH3ReIL2QvESfjbUBVej22j79E36JoaPGjukIXPpbiA2jsefOicv4T35jEj0bdSbQ3yEv9vb" + "z0YOZPutnxwLfffZmLnnzgU9ydVXnn5ih27Vzj7Lv9fB3spJcf2j02GbNc0rp9gOpu5PhgW5fbCe6IUyfCL3oNnjSm7BSMuumMVGFFJjIOX2idgXevLQTGbkeMwv0KHl" + "tEcMaVj6HrbKYUKvmAIhQ2lDGBF3l9ox1k2lFCRKzs9VvMZohSTLei3ueO57hBOnVCxmMKMLVEOJRTgv7ZX3yQ8CvlZQT7nTJ18O5k4HingPWGjENFctUQ2VfDQK8Out" + "ZKOoLEzCE5HzlyUEJGIjtEhvbKq8EWtBAx4p8v4O4ufAGcIlTzoOveKb7UyXhjqpp4o9BBMxMLh0mH7mNrNh7cmLPDpd1QaR2PdOF5BGs5YtCdHlw8Sb47B9Q0vWAsBz" + "P39MHbITqKLClcUKgJ8a7AEaKnXKVXJGXul6tjwuiuEqfQtpnmY1I0nj75OhV1H2w7CtoXqEMDjrk9yT3mxFPwwVLgCVBNvdAaPdKSTa9U2zw2sNatYgMHN9ors4Vb9y" + "H1elBvlBbl7HoXyIbV64P19xipFrCtZYV7otEtSFtKDkMygolFbc07bnnJrelGMxOpISP77dRfVMEFqLG3hYr8vW6AojA7p2uZ0sVMEwO45ItjLo68e9nl7jhK9DDou7" + "Aj0v1VV5fxZXB0I09r1fGCM65wR75rE6bGC3vUfclYIspmziq04GApz7XCnUyBv8llsK7nHzIglGUynfnPlHYO6tliqLHEE682nPTH8wbUggvPRcrfve09ALkHsSDUIU" + "5HgRPXwQS34C9xFUTNkoOsgzAW9nsnJz3kwadwRqFSHJR8F7IkRQvGsg4G0FKPUiIs01mJ82Lkm53AJ2a8JBYICsWPEFN7e50BqMioWt9b6OQ0vwvPhZ0NmjBDKSwf3O" + "8QxFuPdsQ8nALXz3MiM5UcEykdOElKHtah3DUwJKjX6I3Ie1rSB8qKPEqjrid2AAwJo89TlwrNjfgvLnjz0hjTUhcB71YhM3BJbl0hW0JWg5wLgNQuFp21gzIl96wEVR" + "yD93LuDNmqsEELurUUllJ2rF929e6gwA8aidNYAFmej1OWw2ULGgCXt5Ib6BjUVeexWxQqMsgOb3I7YKsmX7CNgBq5odN7FEbW4MY4VDpgx26bcWYFoV0FY58al7Y2Um" + "abSKv1jW4zEuCOS7NDaceJtPMXUqi4h9pFw3jdJdKP8JbWhHGCJwFotSc3xe57KPlpzZ08pafdZefJq1t2eMK3Z9tRvLnWNSmcuicTMJvdRbgDSONRRvpwxwyJpwuGxk" + "gqSr6hY5QV9xFO0u3JcY2zuYvaJ8wcB4CxqtNBVVghf5HvvPKOYS188mVw9FmrlWuE5dHCquIVj90gp5vbAYJgUCUrRxNzVaL7sTS6jSA1Li83Q1I1NsaJyZa1SKgyen" + "zjddCeUK8Tn3xHj2HVlK85C0s6drwfyxIrbnCbQc5KGEDBM4W9H3Pe4E7zB58UZCOYDq59JD5horMpFT1Ny4CIZz1heh3n5mb4jH2f5AdXnKLqnAPBCBWB2GW90xYEPO" + "ZvUzpGHELmD6Maf6OOlC86wjj9CjrQqI4ER3m0Fb4StARxGXkV0yvRQvttaCZkrMOXogiRvtlQjimLNfFfcsNBEK8UHwqcH5tB5oQ7ix4dNa3sdCrtSSZIdcfKhefcjF" + "xO0YLbtWKnWttWOUDkZx7RTPMYtdZXDF2jJ4mR71pP1vYcHhSGVjBymb7pN4ZqbiAhi8AqCBCch7pGV87bDnLEkxAXdZRs8ItHIT6ztTM1HEI7pzzKPYzhFjVGzQ41a5" + "xnpWXGGuj0p6cVSV23s8fbNIfv7Gh2M6lS7uOjSlltlD3Gicv1HHRV2MEYMwHbJMFiFSPykXge8BIjhE4cCawxau4rKpJFp84kHu7HPlE6G6fUadIXJmyaDyIZ5TTAPP" + "YiShh1HOghRKpNZLEOkmJpfRUwbV4NnCJalLFQKSR3eTRSBqw932wLS1CUcj4bXmHbKDtNOEIkdUp9lT0lOyVP9IUOOFH709dDTlf3Dd6IZNJpXzi3GScIxsFaZVq7NC" + "6cdcKwipRUsD1aghJqSpAJZFHFIU8nPsPJlu9CpWCUCwKIwV0eSWonw2tSGvJToFqhyHmBeBTXsbymRCXJEcA8WVqjkmiR2nHmh6AnGBhdiTZfBxu5lGotYDfR5b46zx" + "fVGo7Ys1LPj1g5U81hl27NqbgutFQ3maGBu5xZkpeka3JCmdbpaPRIO076emwiJYi03Bb6Ncogv3Eg8Cf9xrNmIdvFRVi3iPws6HynQaq48lDYPTRcVB2l29UiA0nJPv" + "bomNxXa2RTMYDs9PzoI9CLRbxvz51XRRrAP9XzASOu5Y7SqzRFYmoCyS6drTrxMts1lywiY2Mzl3022xsQOcHz1Z6g7urt1UrSCtvQYJzIzgtWuHkVN6m9c7qfor7bbG" + "xctZRuHSrQaZUqeTmFduVLQlJuOZkO6VVwyDRUz48pzPCAfQKtDKUj9uuDSNpBlRnreaNsrIL8U28FDk4RO9TYrn2JgWLbo1bYewmyJd3hPCJg6HlLYyv7YH29SoAVyl" + "haTjaIfnjijq8cOHS7Tm6DANE5iuP2MjomrIwI9XjwDtLViGzWEoM7JC7hspvkTqaGnaaDKVoHnLLfAcEeF6dEeJtzSkGmOmt9LrN7Kg5sWBXRfbyUKBsnPIkCwlrbeK" + "fTHdvlzFgBrgdrkadA6GB6h0HMqWcWvzyZoV3aSJqSQXMg3hAYBwPP1iGxygyAY33kKRQJcpW4738TOBPuEd1GXn6WaIlB9DDBKJomssrefT8TuOeTKUEy3HlWPkyixQ" + "1tvX3AGy13TmbTva6Bg48QncMhRQlpht7xuGJ8TTRSeyvcTWFB68tvA6Is15sVdvnvyiwTy75Q2XHM4K06BKGH4PzOm8WRM26Jp2FMcsyZka7134eZVDjQ7Vb0MG9W8M" + "qioMWg5BM3h3Gz5zYKU2iqLZ9qTsLxESdwetbCSb3kn9pIL39qdEcQ5DPkp49nVHmS1NlPquBDBlD3ZCdDvPHVQ57VKMGZ8JdMU1xOyurJBq53FUpxt0UJStd4aeJwRm" + "i3WD7OYA0TYzarDn9webrHrg5XLJL1Da49wQVFuDcoZK1RL5E2Cqd1xA6Bw3jUZ6kd3hEaPTff6J7yBdkMwSnV7hLLthPCexiOZ0PQuyITyzs7S9qaWXTu8IAm6Z9IsU" + "hoNFxetpiEQkVNkLOhDxmTDcks3Ot1vXYaIVJs7mmEaqHYdDDjvccMi7RJfyh8hRADyEWlMkqoY8io3NcZt4p6FtLv9btEwVkJ7cb6wjrqki8V2jas6vpjXwE26PWxh3" + "yBqfBZMG5YItqZBonduR9cM7xyCVQyocuHvTjOFLThhTgvBnNKgjv3twZL9mxlWIG6AEip0xLt0feP4GZDS7c0ojbOYomc01mZkq5NY9nC8SlUPuk4llPL7ToaA6VSGb" + "rgTx3JUx1LQj3Yq9oSJ1A4FqKxVN5MxuPQ4D7CvClVz09Mw8tjnvnCAv5fJdxGEvPT9Ma4vnivT2wrdA7Wl2HtH3KbakBXd4kfwfHgqdQo30sAr8jxZcRr8AavLgx43g" + "PKe307AulqTgKyRy3iiCrbTUUJKkq2BbptM4lzDi3GljCA9W2t1Qs4W4PWWptuYpMy4r2r3ikAlqLgKooKWwhZ5uhgzGQjb1YYpXlEYHMNDLBgXTDGGoqOO0z4Xlcg1i" + "yXRsy1p8kub5VW8xlWtAwzF58t9w2Wn90P02qNzkaNkw9V2jcKRr8MhtDmCG9PKaRliDmch9F91jnQJ4Hq3JD1B0qAKOB1PYSVh5mHXVEtREztkKYiNRup7DEwpGBRnG" + "qCTZvePHgryl9CIRSH4X2D2unXnfpfIg5PWh9NR1sl9ECu6hCLDprmil2RSQWCna4zbISSAUGurXSJ2m82aU14kseARLymGqBNSDBlPBUp4DCSJsIhk5DvIJNuFzP7cD" + "3O9byQeGI64Q3VzClDGntGdVmb5MZ0ZgGpPGqNM6LyN9b5n7oArfgsRF2zsp0smqLPd5dmqHv43FhzcA7wr3JxbxTcp53vmyDMgCpewPvik2HeemXS9UA6TAC9CwrMf4" + "OqrdXohx5x5TQ1Z3kNieRYeqoJ20sVzWgXK3BZ9n91nGPWUV3BsaIQbT7TbrMkcDQrHE5uKuvkQmiJX0AjUOrJBLq5LarUbze7IzxOhw9HaYaOT5XfimMO81cP0F3WJq" + "0cpVwMECtoIdKHioGUA0ZxwArOGbMqZ6jGVHShE6I6xRh62qdLDwJnN1BPxenXG6rxFA3woQh7eIR4wG81yxycQOx6KS9yaiUPZ0xJcWZgwbP036CLJLW71v35xmvx4w" + "OeMVDMIpDOm7RWddxM45nrx4R75c8oYSgAE8bDGhQGnAfC24prxHTJQo900wtDPKvosOFWWzcPgFTbTkK4QybVCMDdfLly1rAm0SJuLRfvDF3PMOoIm887m9LxX12E29" + "FjQK1nSkR5vO0f6SKew3yNoJ7eDahdjpwwzZIej7N4T8bLliw7gYruoFDqEPlLaS40rlCSTEr5QlCF92omO4mHx76bB5ZUalASqBurhaMFay5k8FSANzmbfhngNfgNos" + "wFAVSc9hZaa1PVPvE1Nef9WwvWPYwIvMPqi2QXu4NVkTYHgQjTi8mxE3L0ozUKAMZ1P2mtm9E7HzwFOIaE81DGVD6I0M32505m5IHXVLW7iCobNezqUgNU2htx9q2w2J" + "UlkLIosIHCgw3aarlE9F8axZ1juvhjNGFLpYwe3UkjgAVzURaXzuYK5Ma6XLbiNKPMZuvkXOQxLB5eOgMR58O3Yr4d5PDRLJeYMqh8lficRaPxeFs0ls0hFg9IAYuYgl" + "2fuhqJu5GloC75zb9NqMuQDIRicNur1AhFkNeGQUBylYBjMxSy4BZNOwl7MC8di37VCsid5oWhoYGumzWcLjbChAqjiW6Ogrn600KR66W6dotVjQ2Q4BWEIwr6CJ7nNF" + "ytapx621nMo2PBBUJFl7Zv2i85j9DuLxRwSplpH0Je1nGZU5YxBDst6f337AhExBD8yfurBLd8KVWhd3NPhulptB6UdQveFAtdfcvyjtZfmLnhQLzKGEkJbZnHckUwP8" + "63dAGYf6Ii737h8doUsxsoGzfRkoYzhw5mWZCMGUAAsX0iWPTJMvEXirBqAclhrVd8EypgPvR10yR9dA7Rc1BBGqiX4WA4vjODVnSblpNc6ubuTQlkQdwnmUbSe6Kk7W" + "gYDclvLrp8FqqVe9oA2OmVJSuVOcJwD5lZKjIhZjxV1lQFdpoce27RNiYgeaoGopE8tpYd6gpts8ZJ3ymzEJEzuM7HoMVQcx2gd9x1YP0zUYMCz2FGTEyteCAJSwUI1j" + "KY3krXyLnF4GzQDvvIdPGJW3gsSVfAl8mNlO2Mc9Ex09Q65D79SWw3MuUJkuCg1efM4utsThbeNcAmTAR98UIAr2auj1xknHmnoSVXoM0T5DVtcGrWkGUveXaotUNSrj" + "GrYu1JqislrdQ4AjrEBxYEvj0ic5eoAaYjVSiiFTY5FJqvUdbqjHyEoBiSjmliKDcQ9kGqnNj2fVzJRMgKeVwiNiz9DxyiPeU1a8pG6aW1CHrWw8J68KmRaiWQ2dxRpb" + "agY0Xlpsp7je3LxnEmBihYMcjX2R2k30bpFj5I1xxwciXIdzvXZSfxBNCBH7xJ220acIIiGvGqihMIIcMiKiotbUw1eqgnCTIDLOmKvzKnTbhuNQhN3bRhcT8XDZOEvT" + "wWSdnAAxsA43qbjPBWbftqGyVuDxNCJZ2qocGlM9YzzXJg2ScsPJFmbdk2Ou9YelpInJiDWpaMndSi8QZdO11nW2MNoertuHskTGZoF42mqgC2YqSpJ2DAlJPlF7h41U" + "uTXuSh3X5zDxa7n8CURmgR998iS4SuwbdJUYqotfvjAR1HtGizaWAVwL5wBm08tawEkWl8Qmo1Rm2h3ZZLlYfi2wvopailUsaSA9QMehQEnV91ZgiI4GuV2hzekuySY7" + "hcyxITjNkcOqNT23te9W8eUAOEvuiXADAU83lm0ehRpEn3V63NV4kAr88NZSFIcg4H4AeY3EVCYTvyojeHTMhb0RsaXMpELqpyRmAkmTALZLoEZsSskkioWF6lZrLn2Z" + "w0BmR024Kzh7m2gNewqWpz3xlvznzi4SQnJBe67EPPugbSltS0BQeg5JxynPYEX5Pu9RArpjihRKSDhzTai5xBA1H9Yv44UkZCdWUq4RaCCxEVzgeabPBhZOhbuUMuom" + "Kg20t1aEtLqIIiaX6KuKr0qjO7a8AvL6gsAqdCzI6Ml8RAOVlfsJHBTRZoDP1WAzHu5DBYJ5zJ5Q19TdAdc4VAfsPb9pNFtZdFdLx4kXiONawnVkzAzcItx236dpifCX" + "4bVCd8hOjhIn4KjksdvclQbNQd6x0bV35N0Km5AjeeSwBcCaXdDTPwH1iQDsNg7S1azrDSh1oqzAO3yamt2QJyRI0k9phTNVEHGXZFo2doqCMYTZf9pF46hzRVlY7EAw" + "Ke1Vbow85IMRNcrWtGte1ld6Sh89uWNSzfVqivSLpYd4zrmmAbbW7KbZxHT94tFgJiqtxW9bjkdkWrU6EBWosdMOlH8AEY6zk35Lr9FcTMzmsbfXEXzsmuD3iWC83cQT" + "LIHGrWc2koDgXZCY3SPXdboUfG8M4uxyUESkw87Q3hB2mfjSrwjbFZyhkBWXE8C1yhl7oG4KnJY9Z00TTNJJFQmG8UMAQqnraoTleI7iUN77eFm65OHEp62q9FpbJZih" + "8GFyFstnUEJZaPmHBGkGD3LNXsNeLr0RNAgdxC1FsjWPfjrSdNQ5JO7mg6DpZoiqnvnbwHMk2kB4wra1csCe5LI82FZrEHliaDPo93hoPvakmUKvCirxehDKHwCbRaR0" + "aR0CeCo23xc6P3Dti8dcdiWXkMa3MFdU1LjlDXeIcS6Ovhat8sgOp04s86She5MwS0n1pMRv8Qr7f15PFF8kVK0PpfQ6WUTMKoya6RtBZYkQ2A0pdrvNEVAWWgck9RWM" + "BXnsBbc0MfrJlDaQ00P9Nf21yd6CTNLZRFZYdWXd9ss4uIaRDQ1pPcwoF5dklMyLEyQinBUY3rWgSVKgRHHHa28Bzo6XXBwN80tE66hIFsxro2q5PXQwaYR7S8absEhX" + "haUb6kGrwQpO7ni0B9m0g8L5Xv6JyBAX2A7vMp2E8sLGTbWcnSdz51DfPJs5pW2GLbQ9AigvRftXWUpgDELvxlFOOWg4tsr2HT890j4PhnyFUxcQHNYN4lF4iFr2Muhl" + "mhYcCifWzrhCP0LZ31lGgNNQE4hyYwAUzI6JZ88Gw9sAKngB7mtmoBtJqxmxjPA5IXmXuwC20MDxYwi71gLFoRRqJy4eq74U9gU9vWZWTDjNAJ4v97ngZN6UvxnJmZMb" + "L7Spv7bOmiwoBeGOtp6kjVvgjzyM699Ljoy4l1TAgR0LNnEf1ue2iJeaBtwciWqvhdEaZcFqO7lGjo20BAPTQugYN7jVU2Oivh49sLLCJptKrhG5ovnPZZdIQnm5hR2s" + "dusQMFP1fi1jlEhecCihahLQnigi78WquMOE0q9NS2N85gn3fmXByTgOlW0QwWRxJIbXLz5AU3E09wpSRU4MGYB41JWPjKrb2FQ8LUBS5rhmtqEV8TMkmeFmFcH48UUL" + "FhNvRLnxXKoaJo5uNqXQ2C7IvJ0UAT5NrjCsMJdMm9uYpw8Mt0reYMvVVXN2sB6lhRbucZ2Piyun9zPjqYuzJIoyvEncHulxVcPQKCmdwfVxsQ4CrFRKYXO7F2WbiFY5" + "jvyfymTKDEX5816guSrltslhnPsivIy3oYVuBU204wOXhuO16iJS2PB5ZNVepUEQvUdYFqiRU9ltVxd3OhdoJZCsLUxMm3vkivld7QoHbYedzOyv7e2yjqPXz8RsiPam" + "orlZ6594zKqNiIZkDFpAc7Hdo3dae3HXnqFO97oezJCSFK3P2364nmYHVSGINtD7GQqOuC1ZOm46MfC8GQJUidFV24q2QhMfF828dl2zKF9tEk8GBKwdiqifMAFcJUEy" + "VZnliKQjFqHpxQZULef7YAsvB3uE56wnJEeeVNw3dBuGONKqaAZFhQkPm2pG3KG8U4Kky6s2IDBQPcggjzlIRtLEomkKH0sOFlVuPbEYc9AdMdiZJRaLewbogM4P183i" + "y4n32YdEldNOO2CWe3Zaw23QUIGxz6wfizTRVoInMzZwXUSu3G2OkJLdXaOBQCGaYmp0EOBXpOac2DSZBjdoT6H44VC2kxGJZPVXwjWkieGMUAAOiO5F8L8KevFQAxc1" + "yM9hjcKxfjjpRDeeK4puGeCCCSNweiX6mJxYMeT1TkDuuiJoVGgRgFZbD8C50dDGCkANlYCOtxOkbHJQE3bTNkkpoDEPG8MlaTKxEDdaDFcvojeBDT14WdB7bW0UOUnx" + "cUD1w0KvSGhN0q2CceXEiyURH7jlHHVeitZuDC6XwSCb7foDPah6IV3jiCxEJAydBhoIZUkQU0S09f94uG5eourmWN7ryDSlCQD8L7CbXV9E43dRyLMAFuqxSG6ANcDF" + "HfbRXKCJB20iD7oqjxEmCJ77L5uh6rhhOG9uA7W5WyiFHcpDdYGukcEl2A07gfiJEtuiWY4PgC4sBRx6XBH3u2ZSrXRtK7CJV5WCIhGc7TPyGPtAPXHtAkLvI1CaiNeO" + "s5XKBOushli18MMe9Mr5gO6V7yREIUdqUHaCywUbbai3R9mnGTfdKYpxvCiKx2ED375eHgmJZeKmeMwgZ70IP2xC2sxLEaRNX9Rbg0UxKaeDoZMlAK0HkBbboWdyChcX" + "LQK4vPcFUfnjUEokfuxRTqobNaXz1rdtN7OqlG2Q2xGkluIdtoyCORlnMinowepbDg7UjNyrs0SIP8XKLvWeGdNH2UJyc7kd0SVQRUUq0C29lmGawQ2WwybPzWqODKOm" + "pK7AFG4kny9nBNEADJxMcERvhKANAFldaBylGnuFT4NpaSmIlB9Wd4dFL2OcMNYivVkiDhZ6XZSgilmuA8020XOiDxiuAJqBGIXIYpRlDKea8DVaSaaxo1OA0KdCh8OK" + "Pcnu1Qw0qnYsCc2nrHQYmCQGLBjFnlrCCXgTJVvaJDh4Fp5A3Gp7ksrQgJZDyOU5KG3DmXaiebYB1LbcmaQ83XeBjxcdG5HXCpuL4xz0z9ln4uQ09RYD51OxtjVqujQG" + "rugxudP6uCi1NHxLzFj8Oev9GOGaVu185oRGv2dgifnxlBnI5aUgc1URLgEMkBxk6XeNwfIXkgSAX1iGPhirkUFhXGxUkxZ8nnGBD2Vvil1qvoi7RqNYN62K4t2qAEUt" + "to4HywGBYLmDWUb88jfm202KrPMu7wQFSTzx2RsYQ5tIlF8A1y9LOtELFfCwfyXSzGkFGAOXGCnpvnkL8lkbbwlMGnN9q4uAATdRBDWODIShjAQm1CbAyMhmSj3bImEV" + "Gz0n5HYCHdQNi4NGdCk8mMhU7jl2n09yWtHs4ja8j8HBD8oGg1rmOdQrsiyAStlzf5sK0XQr8I7sOkzQrXAiwyJ4dtcSnWWlTxVH2rMRnI9fGGrB5qEaYIgUcYXbaot2" + "T4yXMDlRbKka9coPyjBpsB0hkPLsuiwxyZPuehwyhbMmLTh38ugoxvCVUkuexNJgg9SJemdZCWAeO8kqPmPAhsKasUVx9KBEknh8Te7dgfCDkbZcl3dv6EBBfcf52Y7i" + "L7W92OarW4c9XDadAWCybRdLnbVAWE1bf8Yt9HDLEsHqnFMbF6mGzAn5D2LE94Nbl9CW5hYPMgpEsmfwrSHFQe5aCIMeLxoPWOnA5jL4Ej43XCq0HONY4XSyUbHw7Ldj" + "tTFfDVFknKXfS7kOea8BYe1IqrdHMoqnogFbXbJmDCScTV0sYskWQfdNP9XsQ5tg6ZuYKThhq98NHaDh5nZj7hXrmZ1Qld0Myx3gxQ7EzGVVr4TNZlu6nua3bHEBSReR" + "sz4SNKSPpAvqPmGEcIvmiOn1EOKWHwVxQe5MgtoWd4MsfWuLMRcjUezmWO7tNXvJUD6e9SMC1cVbRxIkQCDB2SDgfKq1ppC63vBFaRlXgBSM5yWuoItV6gsjstC513zN" + "45QSfTWnxklurxOyG5IUdp3qTJYOOuE26kxWAHSombL9vJhDKYVg1nDht0k1BYgrw2HDr9cFg9YrCyJEux48RuTWNRqhH3Jg5xqB3M3kgo089OqCWvVMJjFrBHMycp6W" + "qTb3MNUfWRX1xf2C1LtnCCy7iI0HXR5jxVjh5pNN6QTyP8wCRuPMemJ3q0f9n3iKaSkcuc6ea59NJn01DCESqzHYBVmqXKulz51Pg2zLU4gSN8czxSKbU6YEKr7oTFcG" + "UUipsdlmiSissjuDIPAJrWJWIbIn5CMLXpkxKw52R8UqPcTsqZMzQg83bw9E5WyBfA7hFsWQbIpkrniFAIS3LlHrdllbZGNweJdgD0EbmdAtXwWlN9gDtOLhlAn4M7FD" + "z47TpfEsbS5EfJSrnkrXldhVaAk9giJZO2V2jBbkT86HmJXKGMcVLMv8xyzpXghaGA9M62jRO668CUslQxQ2webUbU0r01GUdekbQuiVU8ogYwZKYIm0y0FegYeUGIWW" + "4FaFYaQMLQPwsxPdY8qTURMWZk0CiFOqMTJtEk7yK4rMadtr0DsC7npjUX2IKWh6SqSAAOnt9HAVf99vWoMDRVrNcdYQTUxE4bYBorVDRk551IGAB0bkSIQ6wjaygebb" + "UAGs8AEMZKN219QN8MWEAIlO4jceX4DrsjcjEjhorNnCOtVntniOGIWPjJ6wtKrMsw2LvyPesXWPZHXLDvEw0ZT3ET01WsqUxYug1tjiDmhUshfHSAdeuqGAfOvdLFbn" + "M6YWrlbMKea6BoswAiqIwMebilCWYgCzMI5L2stawVspDuNAwH6RiycHT72roCZ8aA3WrCGkKIk3Nddmgm3wZGmFIX83QzJ1Ginj1pT8Ogb9xc5lFsJ6sjXoCsd5zhdK" + "7kOrr9NCY20iB0gtyONYmVTOYSzlAf60OJCn7UMtY6eyoFJt8oNecvie9rQQPa5VX0eBBMcqFQFctMulc2OjPbILteMenOq0K0yB9pfpxDJ0pSbAByAyIb3xljNdOmpF" + "Ft5oT5XHFZATYYD7e6Nl7kq3eCcrxfkVxBD2fi303ap2V9CzpnxpSQZhYppRx3xIyEYzK9jKXby4lc0JQirfPz9C5j8W0OkUEt6iRHMZDW2921qf1JmnS6vEze5aTDco" + "peP03B3kmMNWOjiVO7sS6lVbPJgTVfvgKaWJoG9YqUwDwQhUMuX6Zj0V5s5Ofc8ICcu4B6EcxCJDUdQ4cqrb7aplLA4PenTQQYMth0hBqzYADblowrAZYqxhUyV157Km" + "DPS7CyARf17LdXDzNbcaqaleILwp8vhOGN4pBdFIElpmiN4eEmEWgisUb4KelBuRaFQYo1zv8d1sOE9pOaTx0R412nh1g7KZS2Zu2HTEQfV4cp3WQgluSMAiDWZAuS5r" + "w1Cfcfl1cQ0zr8LN2ih356LGAeozxiCzoifO7t5CFzrTzrvoLWz8OxvPygOzstn6xWseCF0dUvUxwcYsWWtBwxhgTI8kP2Ysjm5cpmFNpMqJipZqJhY0XoZE3LOoht1o" + "grnqy5xgaJywZdqy3FhlzFcEhhSn4u4aar9bakmfR3dG2gtLnbdQFQKkNK32NhCw1FBRE1SYVRQQsVVeM33qK7LHwlKu5Uy3AnSPrJUXgIg5JWNMX5mRNrm4HHDLJUYP" + "rHqy6VFcESv64llnQLrDArSv9Sept77x1tGDisESVpELudujZEQqhQtYOheSd42c8lD1KmDrxLQwNultwAsbfAnYgPQyDgeMLtvRWBO3KI5bGf6pt7G6WOEbvaawfwtr" + "HQBGbn9O9JvOf8MCgURM5YiuBXxxntyw3VGMyCDNWa1ya74mz4qu03mFqswcRUdRf4AhHeVotWPgj8GNAC9aDDyzGoOuXTiknip8GfLPdXqQfZhnExnZPhUQEx7mcTzj" + "4r0NFFXkD4yvuF91rehW5X6d0dLBCNO5vQCyG6ddb2GeLolmoklb6ubsTuF6vc44RaxtyNy93tG10HTc9mL4AGODkKlYKeZTwOUdeXZPth3Heq9A4DmXs9pOTw0VUXMH" + "TCvLwJB4RAvQltvgTnA7BTL2sVf7n1mhCQYiyN9rthU7NsIPDMwuTpAhy4ihfBu9RCYUVKwfkQy9PZlM97MylnBn1YLBQPyX26L6aLwYyWAt2A7CYhgrf0XilTcG5Gs3" + "wpfwWi7TBJYqiwPypz20F4Lf0BwaHnyQkIa55yduqWljAeNw17TeIVAbKLKyvZNDLB68rGf5wm6HUlsVQo03ZS5hgUhMTomm5uN37rFCQQ7lfmc7itwXszk0X4ZebpaB" + "2BPjP3K9uNpcYWlqaaKcIcXSOEc1eOBSI6tYML9Dv7jY8wvrxd9orJLAbaxS7REJ2WbF2gzPdg5vEnnsTXIGaczv9XRGteQZTM2uv1yo9S4une3RqRdVT4i2nsdYMIIC" + "izA6spPr4gdRbd4ufNyX73r9xHFHx5rBBv6CbjeolHYM1cNYFcTAC7xeaDYZ5FcFDGQgQSG8RNOt2WvXw2Od3f9BrLDTkh7k8HbTQlyd06naj6hQHKewfhC14HWFvNc6" + "udsUxYZoKbdUh5ymOTTunJpAB7urduDW903cJHmMCMbCStr7hk915ydnbGXYGxDy0fWSwFOyTsXLaULk7l0VV3bN8I4rYQxBwaNEFHRjvBlop664rE3a6zivJ1XoEwlD" + "1WcuddEVCBf9czIBmqtS3zQiNDO5wYbvCSyc3yx1L1sDdhzUNrIdVETTWL0oPoLotwbfRNiDZ3yVWboAqjvQSdtV89CVMpe7UKOpQCZPpGfjS8jlzpA8SwTuuWhQe2U2" + "yB4uvkfv7RolUiQXRUg2gOveX2dpP5mvPb8Ag75ZoNzurWIwVqQciN4H535FB0dAhKnBsRYd7H3RGtcO3pjpcQtB8xsSg0N9p3DpX4NrwETXmxF7YXSkvb6cqoHU0O6p" + "66qGEcGO2B2Y12OmXN9SgdEREUg7JUd0P6JqwWQrJRhAa7t6zIesPaE1lb9XvV0CfSMIE2FxoVIcegHVjxywhAQFQPvLi8mP0XN54kYTMREhvaIqNBMToe8dWr1qainu" + "OKIpfCPdRH7rVAi3ZfHXdMwbi7cmSMBNnRarc3BsryMmXrYy5xkweImGfWHUzoqHfArX1GyTKMrRDMcSesRbPfh0rjEtAKztLT3vMQUA4YIPQ3hFqDfaOauMII4TLWFi" + "1ZcMEWTh6IWxmlt4e5NAqXeUGZvZkEo4QptdhRAyfpg4BadQ0f7ZxI7ug77f0zRwCTo30dWaRaG41bclXqAWyxGorNmmAjU6HK6WqeIWilcGIFks4MmwGeXYUTrMRrYy" + "otltSBMFOV2wBZaJCcyZMBXUWjt6zadMPL6x7OqS6Rjw8pVp30Q0pS6p82AzFRp8w0rI8VyFil3xnSsbVVtjlr1BtwP62JY9DiBCfuIgXMgQwbccWgCUWPz8jimKViEM" + "QG4fYwPMyIB5Xrt2TTJQK1UFXJpR1mzQRVYfXDP4Oi25usQmnGZ53rrjFMVZsjos42eBvq5PpOLGO2Pvnyl7WTCljAdkme9QtjvDSlg9hWmUh7vDSSsJmPDjjVMJUjAr" + "GrQS1iq7ZJDrgkShtjneiaacHhrfI7QdOQYVR8W4g45DrfCSgDYeHHeW6i2BiHcgQrqqDaLkTo7AlFbls1CWG7AWRmTBitkb0vwfbHP2qytYPSP9KtjiKe0XL4qarIst" + "AaV7zODDa3XNdfk1f8CuSZW3jYqGWWfIUjZkZgk07oiPh4p4IiS8kKpv67anIfHj8KZx02DWS123g5VSqmezlbWtLVwz35Th1Pq7X9hV8Eu4Zzp19o6sV4Pky41pDvQR" + "OTwEwsPdHHwJnVJ87ed5oQVKWTzmDxt5fbJj9ENU8nZIdyukZhAA8Px6U248Jrih13Ooih6xy7iytmxa2a7BSBzpEu0IQUX4zAlT4egcvuHCDnk9sqRYYIe6kyhy089X" + "w3AgFyO7bQ4KCZMTtznkXKiapgAi804MVA3cqke1At5tAZqwjfhkq1Y8SMHTrmXE8kYjlwh27ikWGTvjPOLPXJL6SdIRcUo6ZXLOREYiaBiZXLxMTNFXGZEDEorLuAhG" + "hPSaUOilwgF5yUiOKPEdvIC0tbLY5DqJUGmV1AeYWhkAJ6f0nxJM8RjgPwzBhXbp28V18y9BqvqVOEG4yv9MsZjkeSGX6ngsqQkQJwBplUzMMun7qFW7gpNxXglbQ4XG" + "6GW5kEHlqVHUAqBy7l5ajtcPjcUqOjy731YYw6Wec3ETe41xct19yutbtHulPQGbVee70vlnsJ7Tk4TikpsseB323OjFLr9dTy06kwVL5Iao2zApJTAC2YM383iImpW8" + "KfYY0ZabJgJw8cWaWMTkAd2ZN0K8FBVJeVFlPztxAxmM5BKpms92VgPYF7L0bW5RCussRqkZCAoO24RTqMeojRDe12HJu7UVd16kuaspP03KqlNCoFm2hhYCZfiFM8Yr" + "iGoyTYByoQDXbGaRSpKPS0QSNHanRqfWQWiHnaq2gKrZE2LUzhT56pPhCpr7HwHCI2NoXaPWjktBceG6V5pfdGHm8DYGf2Xsxk01MdgUjfCSi3x9Gd0USDAW8DnZZfUq" + "NRMWIZujrgwSmFT4EwcRbgPUdoGvztB4WyvRsT0cyYJ11cSHvP7ArqeupitXkrAWY12ON634f9Ms0MAsCPlOaNQJvg1lqQnsnG7bV3t4ubawEUPbyrOjWkXQ8ofJyM0E" + "sgZCsOk7gi9IlE7h8pMCeIGEKGSXgYlrlepJ73uK24lP9gRwZjWR4139Y3SKUG6zsx4hnJAD3oHgz3poriDwmOYaRNlBQhF67RhSa37bGmGQOk5un58tLrO2pNrRwHnE" + "65WpxGYq8YEZsDdtCpN43gSqtZoIo66Z02JThpHPx7v6X7NB8YNsNZh1pGq7vUnW2EfumaHqPQE1n45A76P2TVTih1FIeInvGU6GQn1eq2oHJ49e08JMJltgNkrYA6tJ" + "UaGsapUGI1zx3UtOh6MolE9Qe8KvYnRtci6vLjYzH2MTtUsGWZ8eex5Et4TXqCd2W2WVDm38hHANk9wNTKn6uaZWHesGNQDZFeb66K4oWB7vM7KklRr7QbMVX1CfhGp5" + "ZSMzWxO8r4ktIpeBkOfUQcilqLe1kGJEWoFbPNItXHJPpgq5rBYxwUhpZstrwZPu8ReDbDVasmJZ337P5nZ9fVkYe9BvalKCeTEp4GLPcByxT86hpd9qKvji3V9byhHo" + "6hsKqdFjSBzLOpaYDbjXfkahQQxmX6V3ZOgipd3YlQRfC172xoejwPsx2FVutdoyTNvnjzu02SFsqmHm5cZNryXD8llPH23qpcsyJV486ICgKOOTJjPKQrCadj5OxCtH" + "9yBmGax4VyC1p9lYBqdgLy3CKn8Krz0J14c9ihn7PUVjk5mWvh4Z7qIs4BdiQIxEUGn5TAJKiHsBKMDbul94kWw5HQh5Gzu9jNnWUJ70mxEfKJFN2ci0QA9tu1EsCeqb" + "ZXWzf7A1F8eppynqKdPzNx3O5BgN8nYmHwgTSEjIdBEhPLNvhWNxQ3xzCQ0mZMq3f8VRONnGYBjFFAREGj3GyW2iWo6tmWAkCYBw8LAEgHYiyXiNP8fMZILBxyowFHit" + "4DANjLmc3rzhsLWYQy3RbvAukqUCeyk6VVtTrfIRVeYBL1V1Ih81nDHzMiYdZA6CRkrHRJoPijXjQWZtt5LAdNIWIuvTvKGd2ciS78AoLy7mHk90b1HhEJPsaaQA0VI4" + "QJJ2PbBSzfJa0Ke7RgGSmg7jbGn1kaNR7PwpLgwuME749nR0e1tp7TlSUzzigOqqTQDG9CMhFsm1FqnqRk14AvplNYFIpqvVG90bHc1XVpY5HKVG38LpPr1jMKwZdZnk" + "pAWQEvEzyeH9Q6gmu65kGXG9zprRdzdDtFFuvMUXZuxUxj2iFHRpY1nns9Xwk67lZYUf2IUJvGayvxpLIcFqAn4Ciz1DtPGtJZshJGAyqDIQrhNFAXGPlCmFKLQqivu1" + "wNsufmefA8aaUJ2PXys0vZohY3D9LAamJTH9mhrsQbhjg24uYaZEAOWitVq1SUWxt3vP1YaAgMPqefFtQNUMcsWS6yNf2wLYhiZlidLW4AovPw5cnM70uRutNMR7XpK6" + "bdZ2fkCt8BwduEXdRDhsq5BNd26MJrOqhi6F1MAFO24muDZ7nkh5ukvrwplfRy4VJWoDjAc6sNeVT6ait3MFsu1yJc6ZSq0c5cWWDlh9RDFCCpbKNsvL57PkaKTN9SYl" + "DHhrUJY77LFH3tpqEm3awwasnO1CGEenVsT4NZCfsHVojH0fH9RHu2EB7WG6L9Au9IvkDz0q0c2xRWlj1HGW4fvVnHCyaTQT6UXuTANOhrH21RE8e64GVXZLMX4ulEoC" + "jfvwv3fxalXAcak93J2435X5V2ot1z4asI1C4VYRpfwSGtznNB0B5mLgFjMnprjS8ahmKoFg4LeLHEVfQPcwisAKmu4anz7ERX53AGwHHq8sxF3LoovOnt0eq6eQUpT3" + "BrWCIf8vGnOHiJDXQz1avD8hawgn7SdGQ4qnwyeA7VOMxezFfK3XUrQoKCBaLh9LzWEcYoBWJi487mOd4qcfEeiXtcNK0mA4q9rqcPBcGFwvQF5S9uNP0Q2XbezbW5yJ" + "lC9BcZVOTeFAkEE45HyUe4Sah9Q0Q1wdBuxcfqWGftpy5cgxo4s6xbc4rHow6XizsMVUjepzgNvNMo55kWP7imBxyLRopFq9kgzN1qNmAyE43EJyGKSyG0fT8uu6qcKQ" + "A2JUAtMTdwkHpMaFvGuC0QbQ0jYMwDWKfz5bekjHGTwWMvTiBmePBgpbNxKknwTWwVTYnFaLafUuoXfspph6FZ0Tw35dXMRcVK65QoyDaEDMqhRXiQbUdFDA2W7urCit" + "7XFk0f6OCjEd4uXzX1EfRZdh29VW3pX9pgtnzBDQmFN4dadWcNIs6QEtYRceMSPiR1j2YAmGnTJVnQV4kywgi8l2LxVIdQytltz0FWi94Q090UXKtZBOKbgH4fLTiZBm" + "B4KrpBtqwgypppeHp5TR4BFJOfMPMfU8nUzVcyYOHDDDOMJIgyBY7ldin0wLhE55m9KPBu6rRiWZmAnLbR3mjjNnN5Lfi6iWZwNlI3O5yAZYKK5AJRixE11Smgd3OZ6g" + "dWbo9FOgKPnRFu50dAzBpskGBaBgENxsOGjX30Op9Tn69HekRwzgQ8XAW5KpWk6Xc7aA2qN4sWVlPniFX0wcRf10iXbDHl6RvGcEVc4Ieo643DSBp8Y2Q6W3bdKRiXQS" + "sikad35thyJBhWwrKmlK7xX9LkDNQnTrOnvFwp1epZoEWpEYyMk2CdkpLIv9eC7P2Lcdqi55y3AvnugfUmDz3AdRwsW4gW2hv2dU7Rzo2xRuvgPVoqCFawdJ9tMYFOSq" + "GFmgYHuIqTgElYhOeaq31CBtA75ywQQYIvGTF05u4rwlMbm5dQCu5zDQlTUeWn3SZQkSeWmdPNMqKJ96KyMhS9kV0PfPNqFWzAhLDyNh8QQH5WwV2FsXkXo8muzFPfn2" + "yTCbpDvYf2NbFUw7IIrLj9ujEtKX3AtBJmWrmlKwsoxCiL9KBqqn9pWzHYgqkr4b6z4R75AI0hW1AIX3ZdqX4bRQrP1pppKqQdfIwuRFVJVOdUJ64DTYacoewHOiZqF0" + "TN4kYfXJK14PJPukZnGUB4QsxUcAQIiB8z5kmT4u7y48IqmqYoFBeyec46ebmALHgH3WxUYAs9VkfaZxv3X6dm8SJiQy9RZimlChTi6ktXre24PIySvzzNCrXNMbMt2y" + "z8F18Crd6zhVqn8QbM4k0c40JuXJhVmpqnCHObKqJWdhW8KGpBxH1u4a0SE98WikDg21Pli8qkReTAZTbibSP5EiXMkmGiom56p10IjCZYKjWtbETmnCvGhEqvOEBwzJ" + "RZECwhR6t5z0uJCTGRqba0bWGbYyhODgqyIdLU43OZwcbTa2LZJI116uOlIXX4b5qOTG2OGOSRPtkYDA5IP90QXEcDMvBWXrpLAF76jHwxoklgFTq8aLi32aZ7Jk49G4" + "pNHVRW8vgG8AzZwtw9NsAqlBhg96dxyhA0SbTLHT8sfvZKeiT3uGDnjYnsFYXJOFyifnspuO4nMCRstVdRet4NUY5pFC4j9vizSCfiooaV1y6TjzYkoaXtvFp5v6M8Yh" + "vD59wmvNfN8FdQp0YhzMuewhifqwnJtiDHuyxuCcPjW2XrzIerWRJLQ3QHMopK81oMA47kW2GPkUD8lmLUK8kW6kqhbcYfKi8Xnj9gmhyweIHvjpDInoWXc4qp6IUa4g" + "xU0VWRJ6OpgKw57HUuQelTZoh49b23UOP4aenu2UrKq2QUQtKNz117It8di6SUz4uw6YgvgwNtP5ozbkvi2WlxcYwnLFW2u5vxvXbPS7uznTrJNZloRePsThUszZgXjQ" + "nCtaSXsChfj3KHYG1tNgvfsDB9ppI8Uy5Z8t0WmDj1SSKvcyRZWyDysDhcZ7Vt6JjbZHHL7iNSuSauMaHqNnEGj7p6M0Pwp1ExGOAE3fGFWyDgqlN9iWeQwqd0CdFCie" + "HcShJjrqsiy8BzlrynsOxlP2hzcfFZL9IQxWFlBqCVWZA5lyNoRY3bHNmHQWxxDYRZvVD0TNYsP0nDUJFB7up8Q8fqKDyKjF0aosm8qsfBhMx5GQbn8UgWXYg2zzrxS4" + "ntaqKmha1kxt6Rfx47788V8EGyhn8CcRZ7fPbQGhWaZWkF0fq4R3zklSts0IgITXjQihdsMRM9DM3y0ikSsr53qLVNciupUnEe77U6u6soFkeIP1zdceopnfaUp8VrEh" + "tBf0C2gJ36Qj9SmDNwhKoImuXSydeF6J3oQyZZuwQvoCfX3MyM5TOwZ4NDGuKAhGTGzDzoxz1LMqGxY0ioNVSsiQBa7u82fWg0QI3SVi8fdTHZbB5Lt2poun3RpRgk2z" + "KTeeOdJIoizow7EclBtWujO3qYecFFywkm4FwIcyvL1jXUPCVTGTs5qszA2MD6o6fHSx0gOWdKDAaLCAhQY1sUIR2Xeig2hXL8ge438Nzs0bkPRCDo2uOPp8USP1j9yS" + "vrRiLf7XKlSLuzPHuEAabOQuefUfMX8cWZYE1bphkHsBRtYMNTkgL2sPZEf9pv7AutFAgoTU4CRm9wkXqMmSKA8o4K1u4yN5CIGfboTGpovRYXGudBM0tmewTjTQlUgd" + "Or0f5OkXsz72BriRmK1Zg67yTi54Ec6n0fsFCRxeVXg1U8YrfN5eZzYrifIJWtRslVlBUK2JlOjiLaRVhIghM4IwzqYxqtKxivyeRO9upbFv7FPdAFjwTVrHkVtp6qRl" + "Bg7xjE5xE5hWp79SX152herINFnu85M8KKEQP8p9SKzK3l3D4N1m2t5Ur9uUXZEbLkudWHPhsvyCO3AvzTFKw53sC6fhF7LiotfuKaxz0gyo24UCcBwqIGTugnrLVwd8" + "xWoLisWhYcUVf1M1oEuT3AoY6qAmAhcjjBagAMmn0eG5YLl7dKCPfZyjhV2TDcPPXCIB59ADflTPGMNxX2XE0xDuGSV9yxUINaTGKjj6B6XE2j9NMQHaeqkXIacEIGgd" + "5R1pKKulhu59OwLYQkYkNQ4XpSH53AjZfCCLQcP5BPmlNVWQj4BldenBDSxOMyVEjJMyuRnZUsaJS8t9OzD0xDT9fBXjW7MVUS5tqYQZvqABvLjPiYtEjuXfsnmGpHVu" + "b3DXxzwqRvgvqSvLupHEuJ7QKykRVopiBAfC0eoJaifDP5eOQQVCXZTX9kxKNj1liNwMucE118hnarFFdAyJw4pZElOMadlXZgWJrJyINy8UU0jqmVfIwzJnYK40LEnq" + "qQC1z8ZHBPhsw2iGOHU9OK0ZxKcu12R7RBvCiD3xd9x3IfhAWF15ahuHucCYrub3MJqIL3qOPAeQ7bALAhzNrzzaDSl8BGnTKWCzJW0gR8B3kgZ9EzQOanptjrKFVaPF" + "sQXDy8h6dfJH0J79xTaQ669p9TxC44YWIGrShIEypceyvQYNBdl4Wm841dki4U2IdRxLNoHZgTh6W6UYHCSuhUXyUzrJfQLEIS1B58FcyyLH28TH3dt48bDsucEuTpHV" + "guRXXA4psiikXZyWwng20SUvddLEPv7uv1GJrZx07ZQbhWbAO6HaTsvrQ0t9zxyxJKnbtPOcmyztcEwHSwZ5640L5fHbhq6E2N9KxvzLerqSs9mGClIEdctSmCvnrPHZ" + "cSqsEzhEYTxyvuImyelXyYRXw8iudliM5nRirQkq0FsWU1CCNXaZDXp89aa4rHCvPW82yPQuE1TFy0OWFVQqQYq2coJxoeixVhNvF0ysToo9P1pGmT7lav9pAbGb0ZeP" + "YxkgPOJBatYPrbzjg2cpFOgBLjNF4gvXm9kkthr0vBGqaHQiP3k0prEXOVgDcbd1EZHY3RLCWfN1r8wI33MAl0Tttp1m4pxY87uC0uRFsGpo5E1L75XdXWPdzwn6Ptoi" + "QjTvaaNCIXv1JFovgwtFSjkAyOl2IuYLI7waVjc2Y9P9iL50aaxHTn4ylIPbCXYUR1sxINNkyxvbKFLXLnPenDySSB6Kaqpyva2lF4dgK9IEgdCp3HEalQNM4AoI5WUF" + "waz8dtnRV9ENW7VmFjNaQkWALDEoMnF7dJhku7vzQxpdnkECgZboXs5K5jYteKFZTKLf0dp0C9Ixp3HxeiKIMjpydcg8jjVam8v7ddikxj5cjRyGaxKUTMyfwCOSKQlh" + "MSTMKdTISawIMQ5dKslUzVvNZ5Mj3fuWzWYsQfQGp0af7Yjpv2tqAYdvUtcKzWB7N4sRII1HzmxzuydzURtT7CNr3UgPvuRQBwZ6zyQqQ1gmWMbCco6co5SdZw6zLeEI" + "J6AxwoGs7tuuCmrvzb6oG5CFQWlnqMWVgywMEkRL3INMY11CMWSvJvfqUnlk70JdPLUNVGRw2SaFvchUDLI4Z6BnjeT0TYfMQ8WvER52P0EKmM3iwNhfb1JKWXFAoWrJ" + "fAjtH0nqMuZ9EYU2B9pgoEDNE2f9mT4S1qLTaykXz7nJAIEHakHtJXmgxdJeTj5GqnMm3vNBr03rWSQWZJuRejUKOAZJpuUrhSxVZQnKoB1UnD9efLgw9yzn2MgzAvxP" + "lYgu65Xtbb8b84JqANEG9geYlhgmXOWb2Wr2hwEW9PFONw9d304jdcMM0fRXVAkSL81am4XsruoixnoiEAi5FhhoNviGpMV0OzvmjeIGaLUhLUKgwVr9SzWrsMLiISRk" + "L4J2dJvYxG2BKWS0IBkcdESLnfycrMc3cBfoNblnxwEv95RNcwrBDKYW4Xc1cS1htk6a7UFlkWJCHmyQvEfQrQvAhO4Kx6QiaT9mdcgerxv1KsF8yxa8g2i293yLhf9s" + "CSPae0dTUaax9kgxICExaQN1TqDBNQydumeyJrZMuyakrPfg7DSefF49H5FeWt28Z5HqKYRxS2W5hMB336yu1H3lDkooaae0aYYwUk1bFQRigVNrS2XICnBzkFpyofRb" + "4slNQ1sYxB2eqPi86utVpnGGJOMicJyost7TwZCmmPUXpIcSrEqCket54HsrdwzeXULji9XdAQaplheEzmheU0588wnrNEArl5KdNjXcZm3e5rhOheavD7LRvhYw1OZZ" + "uyzmjObnQ4EwONmeX2iTDVWX9FHbVTj4ACQIw8LLPxhom97gUdzWOWazMgEiuDhwUiKlMc3iDAJaOinVHser5U1pFDt5Z2rnrkYqH7xDlHNc0Aca0jIc0A995hiyqZ05" + "dhbkk94yekXwrDd0wEwvafP6M0PPgzZvdSe4caF675vmDpDnJG3aTOQFVzLkpeJHvaLZX0gxLS7eyTtttBMcaCpRnjIYAamsPpgYc5TM8hxSBZEHirAXC9Lsx0zaOQVk" + "ZMUr45ufrXE2Eegjm7uFWtqdGPbC5kWE4klsRwfQvDCRyP7VFw8oLhOEECLKcyfC2B1KrL4gE4dilo3gIO7QRcxpB6hB7LJ9ugQOEQl5HFa50RvG6dcrM34NQWQSnrvJ" + "PRzNNPcgve9Ws7IjkO48zchcYMfH4meq118ZVLy0C4lYMPfmPOnhpHkeX8IVh6tFoLBvV1UDjxUAVkVQtooKEwSguex5jl5mBAc1M5VArTI6F3Df5rdg245BNfg6ZyMR" + "cHYb6hfAIUhIoXDPmocIEb9TxYFdtA2EhWRqUlE5OWuvoQPPulyqJHHvpUSu93YinkxVw317uCs07NtYpvumlD7Yt3yUw4Xtb3eUsqs3coSp1EVJLS50b7UXlBQO36Kg" + "cGotiG4DXEGP496RNYESTN2NQRQNMQ7va9zb8N7IR2nXq4kaWAmdcg2jbuilyDzHYIU4o5qVpnGjO1sMjwQSaGWzErDFp2TdES6IZwhL3VNMdzjAQPAo1TuadvmtPHXs" + "aQmmtWh9lV4oYFeCYU7dv87WE8CsGW1h1hPr7C7kUYDUo6ub4FZ8AryEsxnNwfvRAa6Buh0VZicI1TGoPPHn9e1jaUrUxmSGAx9N1S5jtHpLSwXF8Hc9vCQVE5jIRNaD" + "ADuqCT9jkSwdishoonjyGtiRIYvcybZL7LZfoQPcITvkiqeNsAGOK472jMBBDKMfIkmKiLIILfKtafpRXIvUJI1NWRkogJS83R5VT2NPIMXARJ3c3cr6wcQFS9SxKsMG" + "l9baQqWu6AypCerzRxoENtF9Q0qOgAsWp3LEmyvJs7AMKZvCaXYaWAyBwXj3FSTTbHn7ed7mDJZIzUaeqeKFZhPDvZtK3ZWuzI7BL1rXf26YNSA1EJpRSEIudmDWgSCH" + "ucXfUaRL9cO2DIl0YIpiEdj6tP6Jnuoh95pSvKi4AyPlLaRmFLue1WAaRNPAgZd0Av5mEb74i0lNTUbsHNEx0QblQ1vNsjazsXtp4jilpsxPdW2l1pcMlUjLGcppwwyl" + "EZCM0yA3la0x7Zefz52zcpjIaWi7oDWSF2D7oJ2Oj5mUWNjwPZfr6rjWDIDsGW7DfSDEPJ3NaniPpJ3jgDNWAOxgL9sqoweocHxZ4fXmvobzWHftf1rWQzKd1UkEdYsz" + "v6zryHG72DSr2a94k8BingSHpof731IRgZdkdKe3bLDjJENQmwJDxWsqOwAuQqhgvZRfUYhPMPyiH0uP4PsJqdzR5ETx1DwxhLtHAieabptDF2WXuveLQiICfsVOmZiF" + "ghTaT3x0uAc3tlgj9LON6gSznLX3yNuHK9OXahGlgE8JXbsPfsNQqZHX8CT4HlhhWDyS9dKExXHiZpPUdfNTzQUyB4hihkRyDumQfGh1QOaXyxZdB2b486TjFMWop83I" + "4ROn54RivIIaoNlt3IYothpkvIw98MUuid98aNxL991IsmQYBirpPatGLuIDwU3B6wQRmhV6gpSBgGeyb2QNIFLHN5yGC8UEtv19hBrAe9K9hmH6Oa40J5sf32M4n5aU" + "usvBeI63UaLCcRWIR3okJuZnN9VUTxVeCEYHj0HQpgdK7JhhjqTeqJ1IHIlBjzqda5sMPgtEpRpGjkujgr7yrPDJrCPPBOYDT4O78mMmZingaMukU4XOtVk2TM44JrRR" + "WxyLYvmHyXkHmlPr8ndqtOR2nCEmAGvxixCShUhpsjXdV5Kyho6u7eFiS32HnpMgMF6LXNBwtzMEfbYW7k2GrhcLVgSXyavdVQ84JplceoWIohxuxRZPQEMU2NISAasb" + "hJEIXGutTDwAOYFwRFip2vx3frvrR8i0dqfws0vBa6u02wEs83uTNfySIHnFaMDcgTGMJoX0brQdm3X8QlVZob3O2E4CLZsypb9uBOzRF0FxAzGqJHaK8zGEM6xzg9NO" + "jrmQJmUtnq22Oamjk1JxAv3AYo9hWqf8mRioL2jktOMzQBWWvGLVzmSSkfGkv2viASCw85MD6LJtOQEhRSvIqkxU5eXJkarik7gSp54DoWFlkB6777ppO9FGO1LsiQdj" + "PAHEe4ugQCChaFoNKpcK6M69TEEly8OstpsaRCrS8hAdrmsf2A81GU8CPVbCQAOCm8ZBk37JtbOhlCVHCPYMXfq5oXdMIQELuXuJgSOTX7bxYgXIogQKgBpRgRuIXwij" + "AqxJcLmmEwxATfdlJkgCkwE4dvvmskmAqdBt6IOMoFcgmVKQ98p8BzzOAH4TD1mCrBUrXgSJvm1WBtmgFRjtQB3dPNRsGSIssEZzK5jm54V4GMvdIFlfDjvhc3nfByiK" + "oD9p1QN3sooRVGlrB9005zAtk3yk4ww8bvzF6cNo8oLMBpfCc5X0Z2IaCzIISSc1hpvpQzWAkLSAYDsMqd93iBi3IOcWMSYJaY7aMG72Wtt5JVdvaFx8UCEZein0aoI6" + "G02QnWSwIFCFgZtX2Uu3cCxAuOH7GVE0FUejBhlj5QWxKKykCnAh39lie3em8OU7hXyT9EYs2bKNSLtRUE2ZOW2df1dmuLAh2p7yNBix2cbg0qBgB8M6DMDyEuYato6v" + "Bg8h57gfxVKL3vZqvBXcxWeHsVv3ybHb5cugbmqOnmIY9LZY2qt0c0ZGWB4Uj6wNWPYUx2d26MK1Uw8yMyVvfa1CeIqQya4zDlFw4WQSGBD6kr2JDAOCyYfMELu4NYIk" + "Im1LOusY5GkIzcJ0ukSVppL96G6JkUJiX921QAyoYzZwsxyNP7hx6lgHHFKOoGS7oqkJES4Pc8xa2jp8sPmz5EU6O3022qM99Td974AMrREgCH36TzeNhT30evp0nXf1" + "lVeSTRN5T69XRhdsl2a8SeypEOI4mMCV4T2LC82xTbPuDqU0qhiVdpLsGbHjT3T8eJxCnAuwqYh8cv4umxB2ImB00r70S3F3Y1Fxv2tK7esiX3QxMgtE2B69okJCnfg1" + "Xj5bliKiUdKvdPSCpdWFE2X0I3acJ641mRv4J16XY2Eel8bcEIC2iV56gJ65Zuhm5qsXwWkwfIPNjbqSI8O5DKPSZQo7mYZI8FE6VtPkISou8PTAywx1BtW7vWAO2Kxl" + "iSiNWWRoMm9JOc1vCw2zcrhDJTwDMykNoGZDJuNsRpUBOtu4f2T5H2Gu4lPexy8NqjabTF2PX9ciFau3tMwIXLQ4EJkBO9RXt2rcyR7RC8NPVRMLg1BeljYLnk5oLCoP" + "z31StebrfkclR0Qw77Qf7351VIKHhruGE18XJyaeele4WydrWyGKM3uLQlwtwPX6aVOy9LOs1IR8thrFybL4HE18UE1MMZU54NEIc7csDbC5aRGT1iMoHZ24PGSJWKiH" + "aITR7TzgHMk9eBuGaTgEQcbSC40M7aX67gKufy46QK9e3K0VPOykggTpPTodlqlWpohav5H2ut0Gk0CJhiDgkQywTbHkRiuY3kIiiJR6w0coWepgQjrFNPOOPv7sMu9X" + "0bYmSwMAb9ptmMtguGgpRCzMCNfl8XAiIfTphNcRbB73rwCGdqvWMDKIRCzH0KFmSnu6SNJloKVwKRGzPqeFHJxJPLY6Aunsp4ok1M5z6s3lZiRdDFuoMzbSbLyxH2gK" + "eJg0NUmN3XRZAdhdllM6LBgYoKrh8rVT6MrblmvEa2ZfZ5Nf11sd5cRF5JFx6L95spjKPseRItiBObhy1ej3JPQGyN2TPjKmEml0EaeCS7Q6pjAq9hoPFESYdDXYsHti" + "CcO45m6SnswJOagwussDoCR8OyaqNMWN6wcqhKJ3wTWqkhN6pKnS3ivdjATA87IGo6H3EFJ37wDujeJYvB4b8VYCUXm1NrAqRfEtIcIqK2ikPYsCPj1eDYD6jfitAIEs" + "DIqZSdgfy4Fc5L9bh9fc6mnhBgKmkraLs7bG4tuV2IyP9GF81c0l0tAKvMb890BkkTOOHIIylSVDMyOeWzMowIM99lx7vJowTSEi9idXZMfsIWdTJvBRlzTHNioA9Azc" + "09r2bIfcSIJE8cNeMdfIgG5NxZ1RB4rt5Q2g2HqegJA7GRKUiS1aQKx7LNhVeUrwgughToe32KjnT1WnRc7sfdTxftLepWKSS6ShDlEH0D1JPh6oQcN2YMkE4Ldrb350" + "dUbtFInWXmtzOI2alCfQcy9PcBb6CuhuZuIVpbokV7tMJTCkXbLi1jZ2EIG3kzTlk6jBFCFcDRvKv9wWOV0XTU2A5ATFK9GoGAiy734Zi3wJ2ycomnSvWmQcs4TQ9uUj" + "WneWDGv5IiEqPxsHdTXK7DxSCzdbpuBAbkqzDKJa4KugIz4cSY5kGV3gUJLi6RpjamZXJvsW3BqKfC1BiQSLCey0ezVZfqsBswXyMc9Jah3kIgUp3Swv7EfS7DGrESvF" + "pOqSvhSyZIJEwTeqKC6o9caKIuaPHd4eBnCYTgo7EaYojz8g5WWnm2bDVrFKjoLwj9SjhGyRDKQDPD2p21hM5GrNfLMEC6qm7x0uhO62WMyuN8pd058nHkm7YHOXH3pJ" + "HtWhcACLHpLrWIbIVvep5PSn5KLUoZI5R1QjTX1kkrEm9iYKmaRWxbkiD8rLpyQJWsYA112nlacWSszCHf9zk4e2sjENqtPBZA7Qhe49qU5J7W3jaUUK3GNioDjscDS3" + "OZlAUNszMJDM2DYcZZRYM0f3MdQ5ltYdwMifZ362yqPdI872deOKl2lVXd1biT0zvR4JHY7ptlgPnk6XeDjJkaIFP1sVXrCmugACqCNbvTihHKE6X7Khtk5yE9AnLEy7" + "7xXY443yWwpiaznk4LCdfNoAZ3Pt5f2dSYwHobmHGGCHFsJhtUzeHUzFRMH4QOZbG9nyqVA209G6p1UCsGoHFuFPOmKJSi6GuAMbV3T51qGIWpslKIJaVUmgfT02fRzg" + "z7bcJokfJ9gRIGoJuK1npfPJGHAJYPSE19cKb5WUNWb5sQaKPO77xkzHs4xxCsfYiNMirHo9whRX35ffvOBx07ECq7jEYas1CpJL4f1DEdWogilu02LsrdVNlk3SPe5T" + "IJhp8VPBbvHzwx5IRJ8Fk1D0PnkyN8SLz6E3G7ysq1JSfbE4Kibc7GngjnCh6EAAHYhV4FV9ta9NU8YmbcBnsE5uujhRYoaefOJeLqWDiueoXrwzju24WfkvhjclmqIH" + "qxr83uY1orBxUaG13PGwFos81KuSGsK3cLgca32uwecM0qZBlGoh0ew1opSdutFO5sFgVkJMMvTL3DMa7iInRDngCPmtskUiWf7k6NkzaRgHbxn3ArFULRLZHK1Pf8AR" + "41p5IzPV0dqL9Uyslbj7Lu8jRepBYxZgRtbmrZTvRVz2Hf7eEiqqr4iYFAVR1NjBVtlX0eurB8pN0mkIiHOzYEkPG11zUaXbGsUASrnfnad5vdkBOO7V6hH6Ppmk5nRb" + "l5qRtVWQVHBA6Tl4052QnfM0ZoCArWhzaW7R8pCJz82txSAmVmjjO9uURvMBXz5B7xUxXMuMpw3gRb8Xb4izMjmPLGj3lXjC6rGqiR1ncQT7wUszJwY8ULA1gyHmorYL" + "46biaWPLIGcyZXrxXRaAWSt0eOLzXRO8oEIyaMcKqcEI3LrIUDslHJBbyBn9OxM4k0FHIiSXWco6TXMGDl7O9movxhXoMe6EfuJbwID7ozjZcBDdl6Yd1UloBsow2KgH" + "PXHcMTOtCgxNuPyubaoEfZvptc5VN66YX6ZXiudbRs48oMkAWy6wnxFi6a5FbUbvevXyx5PLfZzHDNRz3KD28oZ9mFSNw02zo3ntIP0VOygW54YBy12hjbJrqB8jBuLL" + "NBxqwLvZE6aBpqzMEO5ha557R2NfEsx7ZOiXPlczR0clEo9I9IHWfgkRZNXElAz1zU4gquUe6IjhLALmlHA8xr4rRSX8gQ3rVcS7xt64N9SNpvtzc0saJcfPINksdtIz" + "bcs1xgXNAUbjMCNJk7iPYxBvDemLmbbNHBikDXmfO0f9dhmIULNpXJA92tMJhgysLSC6pFNNFhqKRYaQ4hZcZp8oGnsc5Gi3IRKitF8Z8emloPcN7B5pI3MSvUSXTD5O" + "rJhZlv2BqtLUr2MwupOFVvLhXGbBQDpXrGJqjsvO11OROdDaV9IPEmOqhPT9T5POmy4i7G03mEcYWBQUHQdLnhZCpzonFitpT6nyjIVkqv4e71ZYUD7d0yW1s7bceXey" + "px1ieWtA1jMZBNCigGOi6zrn4yaMg448oKWdokqlmHbyPjtmFv15xicZrZoOWg44d9ZEvOvmo72S8PcQIgzX0Ru9G94HppiF6seGlIzxS5OYwgEHyPAmU0NPaoXKY6PI" + "PB3UD43x7W7TCABn5tv3peX7aeislqW7JbfGoJXbLbjEjd18UTQCXRelhsztDlTfBcDurtOA0R1PZ9UN5FEVPYnLTc8kMqSLb9hZn87gRsFYieSRw2MaKJquyapwwspv" + "5pPASLq0SUyf3jcsUxdYNo7JXfENRDGGS6ggN8ozFOcrcjZv5bUEaA8IhVmqGprp3rxAbBiQOniNgTUMUTeymDuinUcqlqduTDovSYGn2Z0DAorlQB93mXrDwhaQOdqQ" + "w7CfTOVfawz6861qWY0p4RaRUOTZ1vSIffJ2YPA9u1TGr8u7LipVBh7V0M403obLX9hbqW2Vr0nPeTobxQAtfJyjKCAQlJbinaKnESYumf8wbC46jV8Gj9nmekFCSacQ" + "1L5hxRoJYYDPWaHz9RuKX3qwguC9ug5vSVSgJpBLwalHKyQ2NdSnMZmLG9HKHkoOrpaTkXUnlHkmOOEKXuECg5cConzSTaYL7QKGgRSnS60JABDWULFJRpVZiYI4RUDU" + "NeQopH4oirBeSlmtkTw0dIU6ke9CpgMyWt76kAlMWdL8zXhMrA8hVrNscUakFMaigl89nzQoCULxF1OE8yMwCWelDYF6ZH5ozVk7DDf1v9iIfwaayWxTbtuzn02oBtPl" + "c38YmPEn5wD5GNuMJW5iHtrCsl3rMTFOh4GsuhOe2KF5MYd6i7sia5leRta5W2ZC9z1cVNFX8FYUe1YdDAUVd9Y8z6UbdRogOqLF1t9pLNG8PhXZf5Q18a83CNBcqVit" + "s6PdggM1aVp9YZzJmlba3CPBjOjS1FqBnf8E25bAhB2SzyGJ0tU36dYJKsLHPq6o521MtcPdu3dLujg1rEjAJ3JM4LQfBZH2Ohir6c2Erf8aroN36wVnZQhgCBiScUv3" + "cWUekhM6kAXh7Vho11ztcZNXwi67lzOkdqvAo9Hsj8ry8bVYRimaTxRNnReSjGEET1FQRMy0yMHPVavXxEkJJx9dDixwsC0FgpjB4bJpERSUOFuyDaUDnUfmyRVsySdZ" + "Irhs6rxhVNTNsPqww4jaa1N0GhVqp1ZejeRL7WVlrSYiP8CgGEVLXm12o8u0ceJ4XG2W5gEMFvoC9W3zgH4GcZ20AWfbFqL5x5TlIFxUdxJI0lLkntAzf5JytscNSQle" + "hR2gmN8gKHEbXMYmS7vX2JWIKyfENjilpBZpBHwuxdkhqCNZYtcHdQwkDzDz4agcZ7moLkZPEB9jf6HIhETmQ8CtuF24TEtwAG82zitBl9jQaAL7zeNb3ZffQWKAsT5z" + "ytI6keFnF7bxtHsbuIcQHLdbsMBUiYPJk8nWUKFATeMB3CrssazFohsRcCeHbNfP8Xw6HiyuRYE2xz2Bi8TaThrWZZz4HFYFgGZzUeDWLDvwiAlz96cLd879pN2rtBAv" + "YOZGtbSpzXI9o1uzujump3A6ejUNLYZnf5q1ZkJmQhc8P6NucMMidGdwHJDDfPmqz2wHivN6FYJTkjklDFog5rjK1PXELoN5EYIAau4fkzXSQ5h4zG2e3CF6AH4mOIxV" + "G05aJ01KnbEci8bLTcfkNniWuTWsIZvtkb7OSNTkhTm3M81uF5zoDgLflPFASQBJySXxkNTnhzOZdq7us2wrUDZkreGG3h78W5weBRjtqRMNOdQT5JRfqeqL4wtwubVN" + "Rdx36ER6KEEXFodal6wLoCZsfgD3ywpMCUVfrFpVts3t53ADnMX7jUOqbmdG3SCth9fmgB9oMjLErKm263O0F7HB0Ic2tXbamQShfaeHsAJHwtrNj0YLJhApbYzODnfm" + "JCsK3dfPQ6g0Ar00R5a2eZ21k6jb1pf8qj4DUyToexKpARA0v8nKxux1CFP83cE1mDjSu9DJgqM5lei12ugaoSEDOHE5UX3MCrx0OOlHTH2m6j2Nt6vLBbc5lmSRBMUy" + "YzLsbzu5dKRJa5t01QKn6Rjj3PW4W2krViPJyJ2RbMzecpVV9UFULPoDv82mwgTnZvh6iyu0PuTwzffc9iir2KTRGStTeM3YF8wBpEYy9ymtrm6ka1ML6STUR8T5Vhru" + "38ZsmAqZgqVdNFm6E9njDlsTUsaTkVtGZX5Dy0OVHI0PjOlvwxzmn93frhx3aSG2qI4JtJhcOiAnFneEDNgSHtDv6EFHbvX6Z0WgKwnOHVO0DOpiXzuzDXMY8ZsTnBkL" + "AADtITRAZYmyeCz8sZScUHoE3lrcgqomTKqlPyR93SSp6yc8gY7EfF6f63UwhLf5lnKCGmhnYArYFnGhJToNQ1xVyBZlhnTW1kWhEqWHqqNtNKKjiNU39EOJ5U0av8bo" + "lnHXYSJXsESVdn27bDF1nn0UzqJsxmJEXEgSmihVHdaQ9VUPZk5hFw5Grvfj6v0fFj1bdb2B4Pnukq1jCFuovFQGuCfqHeSW8FPA88H6DXqoG3pljEXOsMrpqAL5qMLI" + "958cijCHNz2zUVHamaMfmelU2F12vvqS2B5IeDOGHh18EL8hxTAArCOpb5QnvFA3JliwsR7fpAnBV1L9FCxXebWGTJnNUgHUkgsZCXBj4UASvNCcQv66PhnBK1y408qv" + "vUaYSMdZjRYpvz74RH9rcbyS6NXWSGo1vEty6OBHyElKNQT0r5SqxOp54J6tNqq8fqCs7f9WQkctJGklNpBLT8anIepqhyTWi671ZR25GcftKr3NJSdGRvNWtxh8tuff" + "75Cd4YkUdZxaLiMWLwlR30tFCifvgmsgmLiMEDSj3UgkaGgngcam9gKKum0kQ5kez6tEcfCgCF0S3BPqPbTKffStXFBqAedKWzGxXjy6OExhYCS2MQVUIF22I8H0uSnn" + "yZ3XpB10aUFBzNtgcLAszFG5QN6QjMrptv5lN34B8ZaJblrzNCCpnhOUSoqvOp5a6jXWPQDW667GMl8uwfwjc2bVTwo4BVye8OqztpxxPLoBS4fu6FBVVbWLJ3qnQ7Py" + "RhwVbVfxIVXhHhIZr7jBsiTkEAm5k98vAh5JBjQ4QUZ8vkmv4t04Csmx6GCxP174eSLKpjvnMZtpfcrb1JZeK9qKIVqroyS2J4XTwznOZlDgFpePdEgaEkt3qjDOxBjq" + "hEazZBLj1Z17B6sVBukQKdAgMBBRHR53cbALMTc3ybSLSZFJ256Gue2cFhVdcI8McwoafISqxtrtcWkydIunvGRTkFMax4uJTpGuaFz3p5SjEg11TZi9zbBLXZvTa0Rd" + "eLGmDfMGnVBOzgD2HicAserdFV520iHlmyhDQ0YrvGwoAH3fNd3xTR8TGU3EEHugMS5bxB3l1fnmGBjjOunSIqGCEDnxeGk37Dp6ySD7u32r2xiMSocELEOhtJbWRPuN" + "N9hjQO9BJOhsftTN5mMiQKuj99h2YJ3uYXO7GI5knSfx3qT6ORAeMYb1MBpEQWnn9xYaJaqccNhrTj2RE7DJw4keietODBWZpZntRKHjBja18F91sOo5HxKIHtnenWHM" + "MDDhxqJHLcepg12UmJsKS5SBjpLYFpHR4o7U3b6UbFbwTzBV1KgJAEEuNrdUyuTaBmkPH7UDlrNT6gTxvE8oKzlEwZp0L4mp4HSnJfWwYd8LvQDe1jmzdGWS3LSDjCwO" + "KS6wrCm2cKBAWAuj6dtVSPn4MGqv6NqKM9dTzzTC97HzDV8LVxdhh6NxkkGvJb455CWSy5wrCzwzo1ht2iTgC7yOw3fP22EhLQfJjXBKaJv4i1P0WQfh5P31YV0KlBnJ" + "LEIl0ShW4xDPx610ionaFtDQxQH5wi7P41nn7Hj9i4TGXmGiDRK14n9Fzc8FxHKzTJEHMv5MuTiIyLMywjd4sZmNVXSkTpzWgHdIdCS3Qk2Gbn3GvzxgX6r7iXBs0VVj" + "7wAkW3U4ye7RIORjx4c874hSTg3gRqXXQzuc20cZ9rwFSQRDT7kMpBZVHw029Qe2VtJNb0skNybIRddC6JmhKxkINoyFmAATXF1Wkv2Di3wEH0N5PEPVlKnHggxbfDFz" + "Ge7xaec1mlwFLnmvp7ovABi1nq5AIdBEWam9osM7pWFNjzCmKe6L9VbFlnrtoj1xOmsf814ODblG7jRWPsr5Q2ii7fsrQEEJDgGpUWFax2IohTSGprZV5BPVO1Q6HTlF" + "HnMzAtSycPJGMPgkeG708Tch4sckTczpZ3JAkIxVIUpENwMq1YnnyEjsnQyf8D0Uq7tJv8uLCIOp0eH3Vwb8NkFDYxEgq9Ze7qVMuq3a4Svr5J0ks8l6KQ3a74YCQf2m" + "oO76ZXyy5SJDEAmhnFg0E3oGGwF2WOfRksekLvzV0S6G0nSdZqb8CoLprApDuSs10Ig70dDxDahcRU7tuk31WKAs0NIvmyjuhtQodBxr3L99PHoVrrRyD937q3i2B6eO" + "DRoZLUX45t572VPYLc7EOQdW3U4CuVLMw23N2vjTlvB6lovNT7wfmS21t3k8e0zwlrT5PrYsMOvgyx5NcKuEWG9vLkKd54istexwoCXcfCxcAFC6Id2lcAvt1jH2PzkB" + "a1DcioPpyOlar1ECiigcNJpfunZ0CiPjEnfzAS8d5JEISQZc2Bh589ESkDGjAtz35t9X9BRWWSwsFQoUYk6ZYZasylAVCpAO5xNPtnmim0r8tjVOsw5EsSZOVJJ5oxoX" + "uKdPUzwr9LRa0bxomJ6NDPknDQjvMFQLsNc3xsHfx6kSM92ZjL8XuyJLfiUZ8vKIzVT25OHh72S5MCckKqduYyixSO33lQWglIETBi89ETYdMAu8CzUaVLS2VoIgmmzl" + "vqWGqNfxqNemLmTSqPnHMrGprE9XFZKUHPt0ILe6zoW9ktJYR9CVzEEfI4RTeZYi3SZxxUi3NNYElsU3xuX6nAGEKYORm3pl4U9n6GPWEILLSXJlEtKllUk60o6C36EB" + "863eXxwjTfYCeSDJbzoe3BWr13Q0bfb9aPoVcDF9XGpfMAIyIiI4XS9VSoYOckdgStHbMvIIvXkFEjbUsvqOWWLSKgtYUEwLMfxIu2RKhhvB6lg3BEPturG6nM6Kd9UK" + "980kkAVrFCEd6m996GxflGV5Elx9KVKIOulUahISXPNhAcM7WVBing0kmFt036sVbDHxdKNpJQWJlgcd5t6Ke2GD2D36j2wtD00hZ7mfYzlv08ulEddFRzcOcJ72WpdV" + "T5Rq0OthDAKflKg18qaqDHXtq6r85aMbylzcpFBVVcxrVptBdbeNHFRfpB32mZKT8C9vnupEgAZpCa1tkdgwYJf2zEacAyAlavONqjYIPXFTGdwyn7DWlRDJZju0amfn" + "he7uesmjLQlR5mzvOgSTaP4eSR9xkYL1n98EY19BryiaAJWGw9vcClMhZovGijmfifsWkRTo90tDDUg0TWSLmJj3tl3aId5VDcEe3bc0VzAsXXKHpakuMsJ16VUnRwlb" + "W4EBrBJtjURUaBRY1smH4YdmfFqCbbrGYVoOXCdbtrUkbOXBed9VYiaCnZSlGIQ3FMXQy9Yl7eJnkXzvKq1aGZiEJW4pqpYUcZDrsikpBKzDYTMKn3o8jT1yPfxiCj1Q" + "8vZXCV1jpnipET3ILZWc5knPxy8SewihydbfnhnwBWNEv9a5wXOcHAxsYiBgWDWTdczoMw3h8ux1u7jWPP84sDrJOjs9l2edYz7fEvcvcObdtZgeNVjCHYVwMhbegSiu" + "YHgqTrmlJqqEyeI7jm1Tp7KeWRJAgT0MgBq3Qy34IzePwKE195454MJoAA13s50chd0y5wGtc7S1hDImPc1XWqtnvjk5SVR4RfQzk8avFUhKlp1PgFoYcp9i381CVeUc" + "zQY1YQBDOEuL310sZ9xNtlqvYSyHzCFY8wIA5Iw2RIgGpcUYeFCTwJkD85xpqr0hQz3PAtvYJT2Uf3zd4yHDii7ZmekFVEb6fdHbZrhA2gYzPVVxkoHtC0YSyhif1v8w" + "h3ZISaDtt783c9XIbdO0jv1YdNjn5DcBlCqHYqfzw3F1KOZI4dppOQhH3k0XVsGfMgyDeaK6wwMdRFYOfqHZ4r6J99rL6es8lm7xbgiQN0PjTXamuWCXwBLZlTbW6E2D" + "HvEIyPh9ob1XkgRjMbsNtDCM1UzwJn87S7Yh6PFshTh3EBk9Zidu1Om4dFcO2TqkpINYIYcOYrbcsXUxoE3QTzyvyXrRQs3tVli0UYPcXJY3OJMZztfP3XWozJmlalEj" + "gfNDXx53F844LRzl4BbYYnjHhp0r7sHWQ2etUxGIHMUWak4uWVCXXnsy5IpFEK4uQlcIExvXH6AGth24iBGBLK05Vub7xUvJEKTDvAc63PBGZrFS3kiCciTQxhz1PNHX" + "NYIu1ClyDpgSnZLF6wFjyMepBr9Ka9BPVc5Sg7iLXxSV48etBMdupaJtzMUpizuBHWUR5puXJzo1pS9rddycs56bHWXK1n9R98QWZYV4z0u1qEIVYvsOUMp0xDMlcG5L" + "gCWocCGpS0KPbSQ7xCezkkeogZJmMfxJOTWbrE4jM9cEMS0l0D76uCDBG5nKKtjAE3EizLyHAi1saKiy6gf1GgJsMqiMRkQKQ9JJ70jVfPSPh34W99JywEvtnf5Lto4m" + "7vt1GPwhfKAIEIOtf9dKsKxahtGiADuspZXOJrsyWxoH5apF6PpUHWloZiYaeqzsRPZlIhHbPACpcDPRj76vdu0qMEbOUCOXF3wEvfHPGbr3g5JMyWSR72zIjxJJNCKf" + "0iyXpFuwNgj6rpkjBE4IHCtKzObwx8ij4Ht4KjHGiRx1S224IU2X2Uwb8waHBqDHvD4TOiRnWOKCf0zSWCrjrMYWbao7kQRAChUbNMROWP0XenB3LcfWwkbaXyjb97LN" + "CXZad7fprCZ63qcvGheyVpZham826nHaoKsMP2FX4x3B6IUbvyD8nX05s7BAyj5gZun6GfjtkVsbYmjtegkRJjtcTSeFl0IVSIAGBYaewt1qaQ4ucjBnYX0yXQv0yzWe" + "IgPxbG9WtwGGuJjKrGMOawrZ9W2S8qWac05pOhoXNWRrX5cwfPR9J0ORqBpPIdeBvS6yLIoyvWV96ZGwiXIWtf8118F5DOAr2ni3B1MFTH6EsesN4hRsTopQ0WtcMzq0" + "XKn1NiOwtOEJpg7xmeh3GwLaBJesdELkqAfyVT0sWMVW5PGny3X4VWm6yVpUBkeSKqn53aTGJOKTBcFvxmFZY0P9nDdIi97JkuPa97CiR4HOpC0DySeuoEJFqFOSkOBF" + "An5JcKp8RreGGMYcdzW5WGrzS5s8nSoMI5NEjeY0fhnThIZCojsq4hZCZMm2GruVHe82NIUuTyZ8Ve4jzMmwVRqjsn8e1QU6TyTnCChciN55Yt5blIVSfbxopdEu8Xvy" + "ryLhav0FFGNeJAX5Kbv1UkR9X2jvJaVE7RXWI5ebBB3pV0cRNyOnEjZPRXsjJ6D5tw3qTXblVyzso7z85K9kCkmeyfwqGVxlnnp126i6V5LPJftBQTspOrpxqoXZfV9U" + "U82F479Q6XMNLtkOYFJcrTFptfE5t56mVT93ikv6U23FDDkVXAQyIJjycCfPU8gPb3lAOs4TU9wqpT2MqYg7VuHHCoDjrDXKx6oF5TOFOOm7L4IVKuonbLk7hQe8CoBi" + "bh8KN1Gig8umrfYq7i91i3NfqJ5YO9vtFgPAMoOptX5r8UhDvKrPvIaL5AcSAUeA3A0YmIPE7qMGQtIS6R46087c0qcQZOs8AHRoP48nQ828uvXqG8p8uBKTyS1Hhz8e" + "3C7Z1Cqldsx51ZBrXBXHcqiUrMEnPabXGONJbGm0SUEmfapBfErMiIGks1emQrOZ2foELARxXjBK9YTBc9DMqveVRn6u1xSHlxPDIPQE0GDHddMjMG6UkjDiM7Fa3PLu" + "3cdYezBDAFJfiPFOBbaUwTaAnT4zXZUj9Hjdu86rNUefQVLpL7o6qNm4jfG4MtZ6dqdWCkF26idiWk6L0rnfypoBkQuyDkt92ylsGPTraw5yIDdooNU2zLYa1KX0nFCT" + "oRwtHmgggsM7da3PbIpNYocRPqiPIf7kxS8AA1XAcc7kSmMbKGqxx4J2eGbZSY3YHgWjPOzqBbrxp5s5NUS0M72SogDGttT9DTnVV1gcuoZcDqJogyN5yw7sugVRN6bp" + "INTWN8tshBwPAiZAGv5Cc1ubphjxi6zkLBdBVKm2ZO9dTlYlUnPCOFYuOZPQLaZmZTwauDYwHWtX28HRfSzmXyVPO4CkDdE8cxD7doI93cglJ970bGBumHAU8E4UW9mD" + "20MXtb8TOTI2N2hYvCd3e8J2nPOCZ075mgGzQ2UWxvZOfTpV5H9pBwbfgkpQAEBXqcbB1pjTGZfDQCfrj5s1dGSF75uBjZEhryl3ze8TX763AGbJ0s3F4g7SDrGKfWXw" + "SrfMoMiScgpEBZO9FmjG6rbUCaN35hqsAFttUDSUhzt3YXhVW7JOOeWn7v8omY094t5cBRKiwMQCd1ozm8XzTOAJSAHiUAHvXVGE9btcR7wxeUsNdeGIAncrkwazcOun" + "YGBtMhfNAU3ZLRBbJfxwb1Bqe4hy7lWmRTZGL5D5UaxtJWGLoxTJfRSbkXdyL0hVu1BWMBWvTwCdOdxdxSPuGK8LhHJA8a7xuB5yjJiek7n52OIEr6Un6mknvEGMMwPJ" + "z0cOrxUd1iikOy6Pa7PR1dkXTS3xka87St7HAs5DeeM6lNY024gp9PYWCHD1BQ6OpUoKB5BpzO293lJLLNB3a1uhqUfg56OhTdv9mzQHejr0jrvuSa05qk1hBLcMgarp" + "hbyMNgDX7mANgEz4lls1rqTQXD6oSx9iFmCZCAp4YcluCfvMnV8FPdTVdKHWDdiJ5uFfaC2knUQmrhXix5jWTJZXH5lsCbWF84NMotvIzBJX85YAc1IrEhLlhWpa9wgC" + "ogCGmOAzRuIeZLiZVcfxDSq7g8FXjIfODCNciK8ABsvTVF5XcjjGqVH2QSnV9VEcp0OtDPsPn7F43CwlCHMes3rtcPxkZ99II6JKNwQ3qerK4MKQHIHb5U6BDl29l1jl" + "lR8m1AerOxahmYRfXjwIgWIX51x8nX6z7rE3FMpCiexpy90exwixTp4XSViNosAwv5tsR9UZO13rSP51ufU5imXHYRsAGo6tD2zhsmI7YPxkaQdykzTIrsBZjKBCk4YK" + "ZMZObu9bUZsLcX4IMp9KM3s0Q671DROI4RQk9xDZ8adXfpGmmrbWG7TPf1llGXnJaIeFZJ3UFSiYJKCJ1jRpONliKeTNI35yAIBwceYpRsLaIiiINvqRNnfePbWevQw7" + "H1EN0cglTAwhc3mvzRgTuSZ1TiJneuRJylI34LzWTyiY5tCWN5JBNoHml0XbodE7iq5OuMQYk72mfcRqeeaMlWsQs6A8h5fMhVN4iARYgUayVqkcllvtlCP8nK2UTBkQ" + "oYRBQu3LAoq0si9AU0WsumS6oS5qku9drUhXGvjYC1kbKz1pKvcckAXS6n38cyvXNNY7P65vjAQevbPpSxrftWAXxEq2kvC84QeMdH2XJsbjh5OqFtUojCM28gNF0w19" + "8C3RG9EL76YVtADiDWblRdj8rspNk8t3bPahHM8sfiPtEMRazJ6iiJe3IdEEoQfiGRvcEOADZB3Xycu02YYvas92NOiGZr0aKNfvah01C1sWQvX1mIKeKcsw5IYCL20n" + "OxWlRqL03SPlVIXPCqCibbkVtzqGznRBaLjLu1K5sa2B2RQh1x4jE4pTYDm6JG4flctWeuxLbOmvgti3fJU8bj0EblvQqlow0mRBwlxxAzRBbpdP2NQyM0bucJDTBtAd" + "EYDJFx5sDE5mUh03KYrwwZ3JDiO5tmLDkvkh6AeEMcCrBPY4LBWe7JKfVB88RHvFL3wMdbiWqv7dX2IbGb0vU3SeQEUvZcDQFolmnyGoDzW5OVMhi3NJ56sWFIK0iL35" + "aaGZCg9KgHH44BqOSV8gcx8IWfnV0D89AsmEkEz5fQrYR9x8Ymp9afp4ZIwX2I6y2q5xbKkPwepqtiM93iX0CUFni7ycDvU76aerYOYS41UJRvhaN3gnkkYfQI1G4y6S" + "e02kGJRyh7O6TTjf2jyDqnHB3IiZ4CuAMsJTZlGkBP5NZSNT6Ltty2YhnBo8fga8qeHBdXvAR1XTaZ7L00L0SuMIInf0hna0IkETo1XetIM23y2ZuOXg5yFVa0AQjLe6" + "jVdPHRHk92DY1GusLJITBs9B58xDcrZ3sDJO8dFCKamvoNj06ERxR4TzCRySudNOEdKyjxWf0cfxxl9OctYpzaMEwvvxBKOEqldMCxjht6lnG74gWk7dkX9mBL7tvwm8" + "CjEKzlTBdLgL8GhUTX95wLcVK7wiTUazcOSmNW35bVYFiBnQuDBlrJGjEtE27hwafGPVlCRK2ZY8x1JH1LDvk9uWFVF5u5YaJeQoLpCL2FI4SZP2pADdNBKNqUQBA8E5" + "hdQ8JT8ZeOLNGFQpzyDvoCIT8CihhvAgaLnzWbmNdaoZtjMQZwoIbksfodHUXNoinjdOMdZHzs6TmL0x5sOOaxPR28Hp8YpsBChXkPVMggI8unKJVJ0lNxPk6LUGNJYQ" + "ixeR3qDCL3EdqHjK2IhL4Gc4ErbRtK5ImD8d7TITHIFIFWLv2fOKg3SkjaoodMHlbD0an2r5un4oX5HmfWDwSvUMStUriK61yt70EBCjC8e8nGaQYPav5ndPjY4DUNam" + "9BJbzZ2Vfeh4IBsJf2pwrm6LGwyLNpmed6IAddEQYHlZwlg2G0UgxjFtJGGpJhAUiSSnf8CzE61uXzbGuXP3g6HC0ApbKggjcRbFhFEjNkdamo3w5ySWo1FQ2cXzYvwv" + "mCHuBReBBDPVmV0eKbObuRXFQgaBwc8tAjtd7WDNcKIYuvtmAmQtPInx261sfNPNr4xbQ2T4ZJuiBIpErOAjWkLQfCYSF90EVU3BGOt9m1UDxsppz3uNPHB6mjEK15x1" + "1Mh67pSBAED8z5Jc86HK2Y5zVzHa2WMgMA0Q0MY7k3yAG8BecJ7gLri5uizxV4JUP3Z5ffKLEHDYYJcVh5Xu1Ngpgf9sGZhYPHL47995GnxjAJIFTKTT2n1NJbDF6fK2" + "HO1soH2yE5WUoeHq3CqumoGvB6xJwiXjOltr5RrUWQVAFpPjoANNF1XIFsFSyRQ3fOIN1dvxyozzwsj64eXoJSnMcwc92fCeMZjC8lnQz5mvpBv3AtBWUtHcBQEucgra" + "A7f0qTmYAK3HCwMkjl4CNEqR7m3H2Ai7fxRicNwCCKuHi75O67mPzvDxSTYZViG0G6MTyUuYg31EdxpEtOUVtnKMQSaU91iT0WagNpQ6ePb18wvoasWohmxa2KjU6vLU" + "mXETP5eRMtev7MCOC0GC9iUqpC7I6PO9mMKd9uUx7JINSiqSIV0OkMBFGq0t1FaYeYZ2gyWAGyGbdXjABhQLUAfl1H2Q4zWeYoiacOfFemB4q24ZFQKLCXtwlCFePcCq" + "UnGu0rMdl1ixStZxjYzgS8rziGQBSeBqai2I97gZfyK4Vi0m9g6WOPjVji6J54m7ZlSWJhKO7wkL6XKWJ2NmEgbvFwesrBV2nO3oxkSDigmILLVE7F4u8rPaieNIuLu0" + "JDcgZIHYM8wWXiVjpcr78OvuA514njH62CKsfQJj6gqvpHotAL3NPPFeYIoHNuUNM9M0GOLt1YtFiFe5alrKYtCxilcwA1mzQ9cw8Y1trYRXXeoW36faLf5crZmNTU6Q" + "BFwB11V2JTlUVM1h1aobE7SvRFUD4WsOmm2CTUUFR2VpjnfTMKx5DGMXDq1fxftbSpB0rrqfP5hBvVVmVX3WHN3wy4LSEjWZZgmNCkhIXBT7WSY7DMatzQy0VlfATHTK" + "ajAs56tQFOqxlcE0CrDCyimjpD1V2ni2arxHfvEnTcQSKCPVAnOr3HtiHAqhZ4M2O07KNEqycu2OKyHdjqqfQpx6Rms2REufMKkZVtMNeOqvlmoosqwpHQQqzILw6tPL" + "KSIbVL8pt74rJCFdPtvoofGRbOb1viRJAmcPlDRqK7l1fxQATpM43hLLE7zTkzn6wuHjojKYqBJPb1aRQBWNCgkPhXkn3NHmU1sklEQyyvrvDYmNoIvlJ4WOIpD9aOBH" + "7UAQKjznFEZQTwlm0plvmn8U0aC8R1e00PqDutERmgyiBQjViT3JaSN3S4ee8Wec4OErbWwxDllkg8AxLQSHy1xNwuozeGNgs5SPjsy9tY9oPbKF23kdFf38NARgD073" + "Ni20ebKrwS5P2x3ZiDBatmOQYcFis4RLESzP7RsyQy0yZxGjTUbOVCuPCbbKv6xyZrK20drbpxJNptBhAgtuXiKmVH12fZc4j5WRw7ZCoAEd7WxCoLft0mF7iexxBoHw" + "1m4oNzc3GKpeVgtfBkaAF2dQHAnK0ANcvvCAZ6EKhOUZOUa0fbwyDkD5ebuBmqas7Emf7sbnw1GWemgAoc1LtTN9eRjrmoYMer9bt9ooHXuFNgf3ilqqRkoxdRw2bGQN" + "kpcppdpQJ8hqPj5UindN2VMnSRj083g2EWXiUI3gfRh4BjDAg2UAOd260K1tv8c8RK1AYGNNNqi46stWXVezo7JKM7EbDxlApZHCv2iKXnOInqVw94HqTcM9RF9VdKkH" + "0NeKKHBupDp06QLRUfhrzIcNH23z6NgZ2RosX36JhhVki221Oiq2V7RRrqeS4RmaLR0EhvNyXziCdVhn6xZqCiI25TghneXL37ShY8VaABMAL7CAdV2RjotwNA3KSVaP" + "ypCtX0swSmritB8JrpY20jB0fDmCspd5hb2jvQ7ezQgQXh3SVbQRplCIGyIfqRMVVK4gP0FLdy0OquaWwRw7dPAEIsGHScKl3NLMpOxr9l8N5qcnusKIpdr7f7A2csaM" + "v9tW9jLPetRSgU5IvIztR4v3mSTJnEACDRGgiuqxB50pT00Xyr8M1LQ4OxNoBHbnDMsnCq5blcudGG2BMa4WWiI42CUzLlHEfV4A3hUA6ycEp0AimhOWWMArIYtYJmCr" + "EyUcmXKRYmM20Yp5Q12MoqeoRnF2wbN8jolJcbuFQPjAYoCUQICtzLkHtZ4SPZ45zopnOrrMcGLN3XwALVvzQtUHxSGmAR5tNLcUCK7siEr0DyIiGnk8sg78xEz8QrY9" + "LnVNgg2chPB3GgsroAfKGOxKZzQGUreeAzjhJTpMOXubRUzuSC8g5CLpSBwaW5gQW6jK30FZpj8GSeRZnTFvkOG0XhLKzSQcYK0xy8mIeNXtqqlwmIJjwsZgH2jBFxeM" + "7gcpguK2dAjuTqPHES2TuWr9qbqcuOfWtAHm9WCSbZdYr45xAmlQoR2WG0PSzkPmuqnZPYkJDbaE7l6WXFyWPSp8ifoMaboGJZKtOG18kUYtWisHTqqtUm60oH8pNzow" + "ecqVy7j7TbSwGcxF8d6Qb1DolS9d1rUAzsTkeBZckmwF53GhanArFfh6Beu0Maa03JMhr0wZ2yKjDGQC0aFXmhXQpeun5dXERRVEpm4GlKVyUUSzUiPxEM7krylQ0qOx" + "SN5MFuusezHsakPHpJEWOQDWpraeR1McGkNIRp362CfyR4gCgkZKZTETmSRSknmY3rdUAXOkWEYbGZgGrWNQsdGQrUBwKnYqaUSBlUa2xE1is95H95eUeyX18J6JUdEO" + "QkWjlNe5rpJpaEnXDfqnzKjAjD41YvgTNBhh6OFp2gzSxp3WTModvlaLnUTSWs0nYSs9yYOCH4wRGlbm06YyxW72nnqUjQTyweMl9bO59mLjCbwpUEj89JPTlDpOmhxj" + "kvgXib8Ds0QSfXRP00A2vSRVPvuwuKrUly10uE0kIWZU8I79cYfmOfxvaJcqsgqBUnof7SEGF7qpRjqnEqSZvC9aj7qgL41ha3S5Xtfnc31KQTLOCRiOzMUzN5nPTZBV" + "FuJi0vyi3Tr2V91Ln9rs6qAxaANylvS4XvWzuW43Jf5VqlePq5jrgtrpj0xlwrS1tgGlUn8S9ZPLDXidPRJ7vO4GpPz8l3UtdaFzyQt14crsMV8BZRORtCIzwASaitss" + "trhuzemnfzQroo9hhfNCpETpRq0ztuBd4IsXo0ig9YVQmAcNWfWx2xFYpbRta1uAcA8Hpv85T73ccoF5Bh1ks2qUOVIQAQ2we30eUThACqT0KED4YL8srHKTLCvjK8NN" + "sA3vJB1a27PMdUrwWxnnuJenMBmkGVyWackl5H5pvVUCJ77gvc3B8bLo0rd7E9pUc6K82ltxpyVrjQiffCuUdpoKM1QhgAe0qjUn58X3Bu8tH6bQUZCt2LziMmkBxYfP" + "q5DZfy6cBhfztr32lIuJQLJEMCoPcGZxZltBFvspylhF3cp2IpmwUY8xSY9f7pJCvB7OXGbz7VHJslrkdwq2MF73ecshUVnVQGvDo4Mq5LbaQtfxdEgH2CbeuP2bK0fW" + "fhdqDiZg5hU2DqTBH1msD6SFaeK4Fmsbo0VWGR6erx43nYymKhJRj70WIcoe5HTgQRIU049xHCmZsJ9fxQBT0DePEyKWaugpWXEJmeFtb2qyYLVl9hOCsgFBYrbDkVpn" + "ePTavCKiCu7NSqGzrhvnRHAtzfyCdUafKAm2yiMWQrKRuOEwdQWcEiWLRffzyPV6wFi14swEzbyw3cb5HHbrEOo0FsxPEjJUu7gEFaFVCouXodp7yenAoAS06kqRdciY" + "D7vzmNNiSGsPl2BhTJQGmpJC8jC7gFdcAXzo1OYyjtzYtH835mSRHml8d3YnEnsib16gzIpI1WKGJaMmqgIJ4w6UG5gV4FlngQyFsCajyp9qmL0iGYa8xW2CuOgRRXUg" + "Sm9645NmY3NGM2q9u6HKLBIVKYb7TUMRaUOXqnXURPzTUy8wz9IjYclZFh26I82rIbIo0FpUJynoVpXEiLKUScJJiNtUFVRCyOE2DAUPWgQ013A4RcIbcbA5J6fIaXVL" + "gahqrdDNUX5VSLNeQJakc1xlCogWkqd4ROiSesuHFgexWdZCn2NItk3uqxa2DdR4lAZEaT6PMFsaA0206YgI604Cwz6SsrIdqzjHlmW86I7LhqFSAE36pJUQJzGYXzoe" + "dpZwyHMATQgSonvONx2gVH87ifeAvDmtIIUr0jixaB2LHbJZ9GaVD7F5a6dkb1vndmzesNwrKU9uaaBzzJVJQCvKfRGwHQP2hiw09fbCr0hGmt7OrqCbZxbtMXMXiYTc" + "AsbYHIc6Ct3n1fduK5pB5Aa1wKPoU6sNe6DM9qxgQn05Y2UjsVTuo1KCWVOz2qccHOnyNzOyyAJmvQe8OPKYEhZjJbkarTYAnHN439tg6H3dzAgpXYSKb1mpuZ46dM2a" + "Hp2QCe6wtvOCnjJN0iHtBkhmbivCVU5ki3r0h4NxHx79GYzhztGD3tTLXIRVUv7unXFDTzik1MAICX7HR0GtjsECso10pqpFEXrGM2XHVKfyoRmLxTgftF8RBDb0w3YK" + "Ut19orlyPmZAKIEzs2Co2rgWx4KKIBL74YFby3OiYBjkZ3NBS17NoQZwZPzRsiSt3DtIFoUoyT1eBIZBPVjCVo5SMBDtZfHivSdNRBXaPZdR4NqQFcKxFTd73m0qTwUm" + "ibvSPTG2qLhRKHSPxLIhcSb67cEDX8Iw9jlxOk9L1Sjl7dBmVXKaBrxGCBAhffLyUdg0kGUpzOAPCNobQXfdybYYVd7xqu6itIDburLPXeNY49787H4ZpaHcPCqFL8JE" + "9iosWeknyQq1zDmxte9Try9xDliTJ95MRCCJcPLhE90czVDdGM7Ceix3kqkrrMoM16IVotiOb3299uEV2qioyNTN5XUhhiLK6gWsv6n23StSaggynPpLPo0e3PJcGLyi" + "ufRCsGGOH9H4XjH5ZLj2llZjWUEBxok7CmLV8VylrSw6UdPMztvqzYdQa2QhEepRGcJepQ98GejBrjmNVqn9RcOTQb0B0qF6jfXNkHqK2GcboQe2vMO56hgdtDSo3DsR" + "6jQJOTwqNFqJrxEsZZC9L8Fo96cxr4bXyrTtRn6joblxORZwtCn3vwRQvM5oyDZfdtxr04EzvKtftxQUgM3TRAReZi4ucmQ0fJF9zIKA6oefNleXySOn3FDtZSXuVpsM" + "Gxa27zETEiwSF0mEVtmafHq44Xeo3PgHu2mbBYz4laEYgNe04f72MGyvhXH8sRyLEGvSZCsmtg0iVW3iRoGvlxnnNVhpyDydTtwy7ifAoo7B6eI6YhKGe5ju9kQlCjSx" + "sdaJXCxLjcf66vNL2wevmXYghY05Cl3f60yn6ihEZJkj19SPqUB2olYScF7EbEy1ZifogTamkyeDovLrm21UMBi1phJpe96hnjiHyNGgKSje4K3G1AJvzzMCn37CPeHj" + "jxD0EPKOuTNg8bonMXfZByezGKruOsNDHKFBI6rvOaczDIf8FiKvTnwktnhMAcj50nOOt5IFK8olEoYwwBnzyOIoxiyCkuhXqj4GDtvXOmXeKGmjjJ7oGWNe55lPedxd" + "38c3Kq1emQwygQuMhLYxBMSsbhaVm9yv06duKAs3A7aPjPjYieKhtisNw3JFyzrKOq04u7kI0Ve34XCUU5WHcMlYk82llTYULeC5YG0MAzcJICQLX66h2f6R1j6CDij0" + "uRWArg4Gm89UXxhLKDy3oUUK5X7jInO3Z4l2Gbx0MhvuFktHGm7jRvuJwYTae3vAYbBraBMMUEY9SPmlZZHpu1Hkf6HBF8cE3rCqerUrPnUCA1H0Ie0sQBlJ9Y9y4Pur" + "U4vxcJto6zQTAeAkDsnufCwNyuHc6kOtnzlq859s3NeljjwTBe1PfATvogKY0UyE1CIysCT54JJY57DR5PhawPJq3iOC5y7Qpk6zeJyGlztXpVgFb5SJ9r5QtJHnbkE9" + "vP1fMKzemdVLSjA82ZFtGfS4DCe4YTIY9xmtSQZeHDSeV2sd2C7QwP3xkCF8FIDwVmFfzEEcgl7hA3hkPuXvT5mbQ0kbSJ9d8HUxkTeQiE37gpHRiV1mwYphX0vzxSF9" + "bMuPmGcDUEVYWavPMaAtjdAIndRn1T5Ex8jBIGSDxVU926YfZAEDAaFCQ7tbJ8RTjFfDmzzsmwNnJSDe3UinmcDznRnUkHLapuKp1GPRC1NsRxft37ujX3u8CcQVDBn4" + "qskD4n1JwOfH9yTFTF3jGzA0I77TV1JRSyCX4J3q1FRxTgdpo3hur2oLHYOuhxvDR1ZMqr8qHYkFSW9X1fUdCY1BuuPMMCPfupO1ZBNaQYsuSjKbXpJTBjhaZBqIj0MX" + "Ijetz3M0NlSZ8VO1ODDi0tn28gdptfhhiP1zLVlJNElz8gc38GvF0q9j5lCxiUWaX17qoPGHVZH6oxiTa9nN49UXvDsiaSJgbCZoZnzZpBvsXKhGemNBXCwdiuMxSL5x" + "FXVet4TQYEED9i9ptgCuXtdcBbcpRbe05HLew1I1pEcDVC5Wc3IQHgURrdYuvbKTqhR905apXUzBc79qrn8Z5Le8ASVlysA9uQqLFwVU9vHBtps2N9SwTNQph7l4WNSy" + "CtD3fW3nzP7vsgUb2SlTXeu9UXQHkjtEqBso1BfZwTpo20xWByqF9sJ568vl4i7qj6Y8oTsq5RHhTY9sscDllyl0O5Ef2lkbk1iEQRarLwCyhO39DSnRQlYVdNo1iGBu" + "Qq12tnScBaE7yEnfH1udHEjoGQrNaBqRUUIjH3VtgxIJQMWrWbdIYPpCm5gDzpsIQgBSdfHj7BwGf7tXQh841mKGFDguDUMP2OnJkkeqdOVt8pGk5Zw9SY4USccOimwb" + "EIauwjisqr1FRDeLOTGqHKkzbpu5e4rcMnoUgcHgfXZd0iIMStxWm42Rc4eM5kzBbj2bRtSeDa0cw6WfBfLswJtkdhUUdAKUCksZ5IXUAMZPYeXfZ5iwKKeetB0nRSW6" + "R64Bif5JUwdCTXslEIQc1WdIlGrhxGwPsHQhfmr3dfAtcMBA1e4bXqH6EfOGhC7YjjxCysadI7e5SUSHKlYmtqjHd3Ji1usEKZIYwEoIgu1Q8sx109yina3BWqkZrAtX" + "vJfE5NP3A8lT4MxzSanGbGpk0WekmDEqTY0A5hWrYdIbmlBsHoLHN5OyXEnRRr5tOhHYfXzQDjmp7zw9hTuyqxPPOwvwVwoCEF54pAOt9D3DEesQSfSOmxoxDJpZ4O68" + "C423Pk1TdJp4clgCrfV5Url4LxlHhXLJNCEzPd6usSoqHssNHMmEclNdu0wyrCzFzN9T7Dz42g6hO1afu84mvAks23zf5fFhZSRl0comoHZGJcbmQ1Kkc4tEuCUO5oWR" + "JfohwnmXGcPoj98AUNwRLUzsyD05z49QVY5pwmAsws82G3rkmWHd2g3JLrbFmX1oeVOEnx3lb16ywoNPqp2t2LfhVqq1mUq3MT9QYOoaCn6wVgzGbAGTAFLYerq0KF4N" + "6uLwMZEmrLPZF8yGBlE0N3rk4Uc6Imcq6gjBsyyn4ufRlrIwHrVXFY5omNg0y0TMhavkHSiu2Ku7oSmqt4CZTXt3amxhg3EHIvE419M7DMmxdxEWEHwvULkrxyJxJaPc" + "Myzo7I3MlgaHxmdjdygiUWzIE2ewa7hhweJomeZTumRWRK3I38fkGPUP5dSWwYWMatKFbZWhDAyiCY1bXkLNKzRmj4zusnMcjmzcItQpZx3Rpsn9J7l5o7tA4XuQzie0" + "O6IrM65C0x37fFSJpKVdgkArwhRLly17zOhTLtytb8gzGWeOoc7C9ErR3aPOMqsYaZzn5V4MfmfmcwzozRzh5NcRUvCYie5naS53ruUvEUVV3MqkQzCmYH2XIpuMT7gc" + "rCIuJISrcbmzH4OKr3073eT15Vz9VcyYhebWh7jbgzXDjbaNKyUzi6fGao7NZDTYSuIdCE3Nr8YHKXtmWTcFNkv9t85hzhHA9nXJ1snpPBMepwFn9b1VYNVGi0itNo2g" + "CE9EiDoApbNlVvS8KO8SCWWn0QwPNQtBwZgCjAqHNFETzshr16njxIKvYC5mgsUZduPMIJlFOWaab1yKZH2pj5Um41Gg956uZKwH3ZayWPWE5cv7QSIosGQRDEijeljL" + "nBSVMxTvOfMLXrjIe2bPiyqgjwsfFedwEEyiHWEYsSwks2p0jADqfrLGbe8w3o1TCI4CnKB46PmDeAFY3brHIt4QpGG1Sc8ia1znFxU5XqXygbUQjeWsKsXG7LTrL0Jk" + "5tOgiMLO6xrgJb5BuGsslBeVp21jBhOJdco2BGNKYEkHKwPekPn7rlfVzmSRMSPVdvXNfvrmxjoM6CFETNtI2zeTuX0J1m5F7FjrP3M9cVhcFMg3xQNddmOegl6UX8p5" + "xdSBSHVSqircHhbkhckd8Cy6VExxrKnMjXP7kkeX4Z7OLbousZXVGnu2IbR8E4pklhwcZHIaDO6OUFyHD05UZUjzCKoXEu81OleXVZTgAkYVJyzr1O0DUQblYA1ij2AM" + "zNPY5GncsBKfMBsvHxuzeBVurtLrvfeAYryuTRdwSgD9MW61AXC746rSILF7aMRnJJ8ho5qzZZbBVRx5IcF2GpdpSQqDrGBQUTAarAluIwoPCYyb6aJrbbtjpe6UmH0t" + "RXdfLoFkYu6OD7jynmGCPb4A24g8rqpRXlFUVcxcN8dm37bctHhHcIrFY6UKrZqABFaQHPQzReOkHiABOxXba8x1mKyDQPWUYe3iPFXuAIejPAW0wM7MiRkODFsr0nnP" + "wQZn6YrNNLlmq04y9x9aBZNIhGX1XlhQ96aYObh1DFipCcaQznB9qgE12YVHl8jcvRZjSCMQdEgBArhLP6UomJUr6n8rMXGVK8O7zSNQzgWzCg6KYolPgREmmCATOlmx" + "x1GkzazPBvuviO3PGEoANE3OJOia2hpmBZoL8gPVZ4XASSosAfw0f4lMWoB7nwY7t1v3uqix9Rp6gOxsXP7aK12vnCZRg2COeazeg7MUgfRJVMdTyIOja2gD1RVKH8EQ" + "YM4MevSXTpPzbRHRgys9P1Ni5xNlNJYq3Ic8MV1HygpfMHkwy9Q99fLm8zVJqencwvSsXjKLhJ0yZjgWILAMuv4ihuVQ9OzPJeENydu46bS6xqyl7n6gGQJSP6VSN9jS" + "mUULi0gMnWyBT3xCHHabqpeSqPIVCR25TcdMdHWyGBor3WJfhMw3HN05uPiuo6aWvYdpolxfVFpaE5gT3pDVz8KQKd6TrS5FQ4ljd3mbSWVVGCMjq84Y1BZ6qbg0zDIo" + "oNfZdOWQ2MGzosleFO68dQ5x2EuauEQcM7i3OQbIKrgmpQQQLgAu30QeZeX7M1IBV9P7ceLVcKaSfjUQEpMqYGGP7hhmewFfZOXuDfHMoEpXykoPXDnbiiatbz5JqPKt" + "ZHAuMXT64VqUMCYob50Vg53nRdOt5ciUrjR6pqqDBCOgEwc51fr3R0SaHo2wYBYxzHBz5PxAbr4OqUHjyN1JmOd3ZqZWbK1dTBrtgTXHDEQMFohndqQRYtS5QzYfvmVd" + "S0fh4h1hwIlf88RGxeEdO41lQOVP35P0ybM6kmRJUUMFbvv15d7yptFKIVNrJ7litI0tWvDVBPeEJmwbCPwADuK5P6cz2WEcCI48R8liEQwjBKCeo5aC89UzSh8CrtAI" + "WpraFWZ0okBBwyZ9dQUXXtdbqUIBiKiL436vFmE59aIUegtPeKTJZVHeLocXeA8VFPVhdNvKknFJdU6jzk9WQJDCjhusVcJcQrHmd16vLgpkT0D6WKIEPldIxmBrPzrH" + "b3OFB7ZmLnDsp1WMXVrNXivh11woH0AIQAfvlOvu3hbqZRWG2UDitie36JPFqtMFDRnJMkyB59RTeUBAS1iqi8Zun1KU728EZgjqJcxnX5nKXOfDWo0YOukYEou2f32A" + "vC2fKPqSgHgf4WuKmRjn0cENkZBsxQrJusN7BH37Q4dVUGijIS5I3UihZhIuNAEhlrcYAm1IXfHN1RccBpd8AQ5hBytUvJxQsmfkGILlB6K1V5J590IARfXajP9h3zew" + "FHqV8XUwo61wQSp3qB0ULBhf3ajIOix6N05cfSOOb8HhkBbmmiQ3pqs0J9tE5Ionr5O0lXlR6Af3PcqnSHi2kJB5XADd8BpjyZmjjQx73nrnI95Tx7OgFXjPRoMByXQx" + "BRBrhkVBwFY06gco2b1pbPJVIJBP00yetN896LETdAIE0AXHPLeW9AIHx4HoVd3chMLfP4dS4Lip8qVRiPKKsTYCleDmx2T5IbFLeJnsaw5Yr220opVdcPkg7M5ALBVm" + "iLxBwLXddkP8wb5utSSItKqBFofOCkTZzoRhO1fWA9BvBt0UCoZlS79xCD8wSjMjDSUW3pVXO0zBbBchXsvUzLu8yv2m8izrfkNwprxpBrjFQFsaQXhEmVPPTFJZazzf" + "awzGKqjHXguwC73xg8Y5Lf4wFPqMmTnuxvMBHSVR7AgyMjrqeIpO4GcCHedPWC52RCKMSJWwfngsC2hO7FnsCopF0PL1taPNkROuBg1tD42WZT5G32mSbNfyioI2xTE4" + "ErSQ2jcDwHkSwwFdLuYc4CIHO6zuBHBA4mc6cMOHuobBY8nJwhL3J526hzx9gzUNTU7vgUdW5zUNNAjcxkN3jF9XgnBShzeU8zg1ZLdGfLrsNjL8Z27vbMG3KphyxzCX" + "kgnNFB7hptipG721qwytqDsx7pMi10UFPWoWKMqfpM8wgpb16TXfvy2HH7VDwjNESzaQkmwjeu3NCdVnNajrGW1rIX0H0KhV4PgKG9eptvrzQ2pbwPzD6XBEh7MD8SlF" + "Ptv0aIezSdAPIckcKaFj5LjHdWZx86WFuVcKYsIajlazquF9rCeCC1bfZbKrjMeiJnp8RHheNUy0NnNNB3ClGekyFWtQCzYUCo96xB1o21u5bP7CFoqHevARgjApes4d" + "gM9WgZuHOD7qNWfYux7GorTTotOhg9hWrvRfbHrcD0Niu7iVvWgHVn3v26Rvu6B05jUGTXxfcxGLpyC2ZSKS8lQFpD7T6gmH2z9ysbbptfAdiwUZe6T3dtCaTSI8YKJE" + "HabEWVTYta5yji9Hqm8oWAT7f5W8T8HOZ80FEZWixkSVeYSKBJbEBTQEueR30G6dsLOqDLIDWqkq5BqCyr84hT0S2h5scY56j8LsWtQXHZ25BmtefeHZaUaq5jUQLXvs" + "2DGzQQpwQRiosJtRBFjOiu5wFbfcDo5mZtDboDYanJYOgyaVAKb9Sh7whEt0Ss4FVBHaUxiE9wHc2TO8Z7L1TZRUqDeUBEegIzikCHqJNEgpNuSIHD6LwQmkKPrO9M47" + "MHXdCMxuSajkGt37YGlS9WuKtv7p8gFOnImv3GXzmyleLo9fe1IiTaiGGhKU1NWNLN3pOeUzC9E63iwsHkQEPWM0CIOYyjgE3GTR6mHknc24lo0a0p8gjtDHedk8vZ5p" + "4BNHCX7Ea7paxayYDZbIUx2hA3TUXkxYivIJVCFF7drV2XMmAYSa5H88JsLA6YRZCsltAS9iXYdaonAlN4WGaz1E0m4afC06JsV0uOzfiVKsSCFSf2IXyQQlWTEqnrSu" + "nqlMMuSx8Lq4GDVWAsTATI8316L7lKV5RW9CJB1GvVM5vwrHq5L20WHs8Bvpx3wXQkK8FFwF5L8iXG1MOfS3tbXGBld8EfzF8kB9E6kk4Ehzml15oDnDQceTWxOkjKIo" + "OD83EnQKmuHvul57p1bsT8n61enTKYucBytlzBmi7mRsQ1PbjZDMvg9TjaFTMcnvKBOjq9sv6o50FXmrhQIdHkTUSxAZ0YszWEhxawDWetiT2Hnpl2DkG6AiEC23N8bc" + "YPCV4jdGnVKm7lkHpkU9gzNqNACF57y8vlsWRRZtop18ymQwxNUB0TfFo8IX01YjYWWGGPdZ4d9aR3V3FL43a40XWTf5EGYetVxENlvgmptILWp4WSShdJHFuhtSBqFZ" + "NG9LL5E6eaTcJUy5CetUFwF1ZLBIaM7NqnC1HzDyfbUE6YwZoE3SotSaIwjVfTtcQibn62LodQtVfqkwtnbiTP4ZDDUXrsFakLKKdQtDyl97KJbcgIy7AzatcJbOtv6k" + "TaB3RWYoEa0hVJ9vFbXjTNOWZfQYQffbHvSsIk5wvD7aPJH4GQjII2sB4nMTnPcG8u59HuzITQZkXdOqXOiWK5JGHUVcezyr1pCNQSfCSzjixkbb0q93KYjevgU4URQx" + "40sWaAZ0zItTAhIXMp9I1ZBXXyCiIMbZB8yW62xOXEXeHdoab1J4W98a6TxabU3oYCZcHbpFPPgUYlOOpsfXtLTbmbny7JOm5WAdn9oNB0PslpU6PZ0afwFVUw6QnBAI" + "7ugkUPAg05Q6k6Eo2vKbUceHyTctDxTb8jCAqjnQAjcsdjFdyIrXfxRuPoYlW3hbKNY7owubNosFBnkFTREPWDSaLrUc8mq9XdqRYxkMLAMSur6aal6yCK6BeeNPHdeo" + "KK8CoLNDyioVAhp2hsL5bfjBHKDQCN3ItoK3TD7ALfZlQgF2cQLnvkQdaHqVwMtwkGpIOEUZw0zcDmOVrLcYx5UmptfwkMCbQxrpFlhju1Nry2Ci9bKuRoMeuOeneV1O" + "oKsrFPXQiUcpuhOhNhlvHbnvgzvwxk4tt9TvBzw0fL9vXbDnq9uM6YtKeTW7rRrrs23cwjPJzz3vEhZxC6NkdM71Rc4nOwbrpDdxGwkZ3jSY3Jylo9Lbf0pviXcuJ3g4" + "1rzmp6gP6NEyp2rscQFE4nCr6CaQVIgjhVlsQOFOtjsvQb34oUDIBwGxDtXLPNh0ErUUoIAKxVmcJTLD2XKtme1h38QFqWEfMASip8mWjoSpwhWUT1P89iEoSeNCA6iU" + "X1Onx5LE43j9HoPlZKvYoFA0vErGTMaiNik4pmxurReM6dZIH0THe7boPk4nyih1Jss45PZ0dE5iabinKnwZjVWbyNMFWfrfYPqSbH0No1zeh1uaS2lnCiEAlyGRftQs" + "NPPSFAhPBLFeSpJYiqKeQzE1opvSoiocKki7YY3QrKCddLObN6lO4W7s1mIRF1GYJcrEY9GchljXqlJznDmZ0udaNJCUfAUcynENT8KplC4dLGUPskVd4T9MC4Y5wJ9r" + "ogZz0tPeCqnkC1gEXGSJQsMoqeI5Aa2Z0rsczLr1acHWNjahd6hwmRLcmnKycJAQlqsPUdrylLZaFMcYa6YXUh6kqYG5GEHXd7UAGwNrnm1zeHAUvcHrYOSotYDpV9dC" + "32BgvAPXdjs2G9apgOjZYC8ykWYZtXWuh0YwTpd6HBgqgeCD4MmveJqyLybFQFzdKjl4h8Gzp2m7jpmEgC3TsEEycTWwGdQf8zXEjZ3GKRLzfjrmGlOUmQuMGHmedZxB" + "pPOeCLVaVkzqnz1jP40lEjHzAqhKqKXw4qcv2l5vMAvJ0MJ7PhySWs21rQO6kZcg0uGVSa3Ykl9OQmdiUmO40n5UEx3Gc2WgfVb5nsGae7nkzpFLgrRgo6rZ0ER3bY0R" + "zCjgQx5cJTsZyuw5HasdBKjq6W3t8EYIuDNs9VWoTillo1u8qQca5hztc0oTGv5mTAcxp5mHHn2WKMK8S21OPPBaZRYFMfHj2gfU8omqVLxDzmRPWg3xZLzEqsh1QMGg" + "XQBYlP6BYpl3Yf5Y0cyBJyaz59m8OfL7tySHrJULklZkil5tMAHXWU5H1XMFmC3PvMy1qpRxDZVRvO0oHnIw58Y9pwiLInYnbOCU9400wjr6DeN5mgqzIbUqAkGfHEx5" + "Sy1qz1ZdPIyevxJyRFBqEC6Bi72ICXJh8anlXX253KJv3DQ4QTzBSqSvtmU6VYpduhhXET9x949txeo0IAXcZnHirY5ngkCIygciuFRoTTUHkEKS9kf4fS2nEyaDG09J" + "6K1r7QZCXpoVR8MttHqCPPN03dmvQSBeAMhmwIEDPM38iIqN3qC0hkCMZDrZhDiX7j5fvhoJ9kLjUPIMcu8tKpnJKB3Lv8STTSUdIULglxuDVNl0vg6m8IFUZbHXtl2D" + "J1DFEvqE73HIWFZoeJzVggpWmrDCMuhACkbnGqNGL0lUL3TUvoGBs5gVG57E2LyQZUnRPZYAo0RiOtjR1mR7CLvPphvjOVwkKNj6vibfQOhEjBG5nbkATzHomZocpL5Y" + "cHBZpNeFHh7nP48ahKBzQn49I6HJEaSPWfcfsTX6VF6lKTwQ1Z4dIKLl6lFmwwRMoaOHz0W9QaKuYNOxzD0ntW9xn8mN5sosSFZ7cMdalkIbxS12A02OteW0I7u6ZKGC" + "tmDYAk6d7juw7JdNfJiXpGQYEUkDK1373amxtzWmEu9ftz433ZtFUknfzTVuACgAenpfcVRx6oyLJoclkAk0LLKgE9tyrXjJZzZJWw70YXxn8t3D0jPx88aLYhEA8TfL" + "joAZvqcxBQ8e03pbggZFLqZAIw5ewXQhBgUDgjqsWtKELUeO9pvq6XJJhsZp2eiCahKpZKqxpU0sYBSciW1Co3eCmVLX7NvQYgNZ8epoj4OplLPqYO2oBUXdxTLXpZeG" + "gpcw1f0gsVUgreThXaZjFYcEaF3cTlqRPwayG7d2niO7l4fJ9tYLBPoOhlu0hpwwNMFf5nCccgVa10DCxFUXtG5D6jhBr5DYZcJRwZf3hdI8sFnbmQXigy1pdsmh2sUU" + "R3V09UrxqUwqO31GT7v6wCoRAqmAeNS7WeL8sYztP51JuwOewATVSNNUYU92xVpWgi1gc7zcUSnheJ4lOdTM41Q6Cl47aiTE4WhmbuW1Fgwr7p8SDOYJefMG7qFX9EBZ" + "N4L7tra75NO5qNNAj1l5RqUHvLwfP67vKkAmWdqEQOV2yofT13Hrw3bJrqB9Sh17AJD3jvSOst3sKj4UNkSJRVGJLky6t6XJyPfu9SgCqcybD8GH3A4tcotpKmUyeWaV" + "3Yb49gsjlVWNr118iAFYAtu8LuYIpv0tCDlNmoGnaOr7MYg6AhACoQGhC61W5fh0Rl8EmUYeQwr5n6FsRGANqCUlTzABzvhJeZU8R4yPzK1EKYpsNbuB0OAfjWQ4QfXo" + "3PYVwMTyxFiMTEPF7JExpQ0A0kiHocZCzBZNee4Wbcosp0V9cqlB5X2Mizv87MVqN5qeAlNv8bF3tkUTT9ny25jiWz3piarNZZriHt2Jjwr6drhDmCyAMtOzfTpWgbIo" + "4O2QBp55VldQdVAIfZaruZfYsfT0m3klv0ijQQrSXfmEonDPr9LSi5ex1k0Pt2I42sW7tO3PfHI2ess8NhKTFwaeKbhhJH4cjYWuxETfAT1hcFDp9gFVNv61hZIpPC7B" + "6Re48nr4bugJvEqjP8MgmDwCuP4wPv8whciQ8xnbDANV9atiIp6nILVVPHy6iy8aeNJ5cU8vGHKEfglaW1PF3BhNBoDtbhi2ygT7nRWncGwlpXnrPsnwBuEk1XC1d9wE" + "XSkcpTLsb3jUCod6ahdBtMBWg30E0TmP7daELf3o8SWrOXUHoCad3y9q1El10fCCDfzw7uamJ00EXaLHDNaIOckLowZUttmM1TXTyrk8V330gmOIs3pS6zF2fMgbXfdk" + "Rd7m3ocqHWKxCINsXgZlt7a5LsoulsUktOELiWnNvLJ86qo3BmYk0Ok0gGqnU4WHvVMsiCU3gEmxOxenUWlHM6JSzrk6iW0PYDjEwQvMlT4Msom4GSLDQV2oxpbvWpIg" + "xLQjt8WS6pKhDMdSMXMYproxprmKdxHdsTooFhDuIHtNZQCbqhSvAZLZ7YaVo0JMWVZHMi76eAAjPI6h1BJOmMcuywnhIlMPcSScLtu5NvwQI854m3BUSmnxwlTPrknt" + "AVoDe7cEI45aGbgaXW7ZPPSJ0lYc2OvyaTKJob2OBh8zdnA3ERIBgRXC4cNImoSvxzMBiO7BDf5zroBqceOXxtaEc7y4D7GrekVAEwIjH7DMkg7AXdBrAXKdr97HIj"; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/Source/TestImpactTestProcessLargeText.h b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/Source/TestImpactTestProcessLargeText.h new file mode 100644 index 0000000000..faaabae2d3 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/Source/TestImpactTestProcessLargeText.h @@ -0,0 +1,19 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +namespace TestImpact +{ + // Large text blob in the form of a string literal so the app does not require the Az FileIO and Application environment + extern const char* const LongText; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/Source/TestImpactTestProcessMain.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/Source/TestImpactTestProcessMain.cpp new file mode 100644 index 0000000000..5c46ee525e --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/Source/TestImpactTestProcessMain.cpp @@ -0,0 +1,19 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include "TestImpactTestProcess.h" + +int main(int argc, char* argv[]) +{ + TestImpact::TestProcess process(argc, argv); + return process.MainFunc(); +} diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/testimpactframework_testprocess_files.cmake b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/testimpactframework_testprocess_files.cmake new file mode 100644 index 0000000000..a06f1fa706 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/testimpactframework_testprocess_files.cmake @@ -0,0 +1,18 @@ +# +# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +# its licensors. +# +# For complete copyright and license terms please see the LICENSE at the root of this +# distribution (the "License"). All use of this software is governed by the License, +# or, if provided, by the license below or the license accompanying this file. Do not +# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# + +set(FILES + Source/TestImpactTestProcessMain.cpp + Source/TestImpactTestProcess.cpp + Source/TestImpactTestProcess.h + Source/TestImpactTestProcessLargeText.cpp + Source/TestImpactTestProcessLargeText.h +) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetA/CMakeLists.txt b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetA/CMakeLists.txt new file mode 100644 index 0000000000..8298bb7123 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetA/CMakeLists.txt @@ -0,0 +1,12 @@ +# +# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +# its licensors. +# +# For complete copyright and license terms please see the LICENSE at the root of this +# distribution (the "License"). All use of this software is governed by the License, +# or, if provided, by the license below or the license accompanying this file. Do not +# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# + +add_subdirectory(Code) \ No newline at end of file diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetA/Code/CMakeLists.txt b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetA/Code/CMakeLists.txt new file mode 100644 index 0000000000..7748581284 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetA/Code/CMakeLists.txt @@ -0,0 +1,28 @@ +# +# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +# its licensors. +# +# For complete copyright and license terms please see the LICENSE at the root of this +# distribution (the "License"). All use of this software is governed by the License, +# or, if provided, by the license below or the license accompanying this file. Do not +# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# + +################################################################################ +# Tests +################################################################################ + +ly_add_target( + NAME TestImpact.TestTargetA.Tests ${PAL_TRAIT_TEST_TARGET_TYPE} + NAMESPACE AZ + FILES_CMAKE + testimpactframework_testtargeta_tests_files.cmake + INCLUDE_DIRECTORIES + PRIVATE + Tests + BUILD_DEPENDENCIES + PRIVATE + AZ::AzTestShared + AZ::AzTest +) \ No newline at end of file diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetA/Code/Tests/TestImpactTestTargetA.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetA/Code/Tests/TestImpactTestTargetA.cpp new file mode 100644 index 0000000000..9598b88b98 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetA/Code/Tests/TestImpactTestTargetA.cpp @@ -0,0 +1,73 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include + +namespace UnitTest +{ + class TestFixture + : public AllocatorsTestFixture + { + }; + + TEST(TestCase, Test1_WillPass) + { + SUCCEED(); + } + + TEST(TestCase, Test2_WillPass) + { + SUCCEED(); + } + + TEST(TestCase, Test3_WillPass) + { + SUCCEED(); + } + + TEST(TestCase, Test4_WillPass) + { + SUCCEED(); + } + + TEST(TestCase, Test5_WillPass) + { + SUCCEED(); + } + + TEST(TestCase, Test6_WillPass) + { + SUCCEED(); + } + + TEST(TestCase, Test7_WillFail) + { + FAIL(); + } + + TEST_F(TestFixture, Test1_WillPass) + { + SUCCEED(); + } + + TEST_F(TestFixture, Test2_WillPass) + { + SUCCEED(); + } + + TEST_F(TestFixture, Test3_WillPass) + { + SUCCEED(); + } +} // namespace UnitTest + +AZ_UNIT_TEST_HOOK(DEFAULT_UNIT_TEST_ENV); diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetA/Code/testimpactframework_testtargeta_tests_files.cmake b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetA/Code/testimpactframework_testtargeta_tests_files.cmake new file mode 100644 index 0000000000..32c8746c9d --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetA/Code/testimpactframework_testtargeta_tests_files.cmake @@ -0,0 +1,14 @@ +# +# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +# its licensors. +# +# For complete copyright and license terms please see the LICENSE at the root of this +# distribution (the "License"). All use of this software is governed by the License, +# or, if provided, by the license below or the license accompanying this file. Do not +# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# + +set(FILES + Tests/TestImpactTestTargetA.cpp +) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetB/CMakeLists.txt b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetB/CMakeLists.txt new file mode 100644 index 0000000000..8298bb7123 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetB/CMakeLists.txt @@ -0,0 +1,12 @@ +# +# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +# its licensors. +# +# For complete copyright and license terms please see the LICENSE at the root of this +# distribution (the "License"). All use of this software is governed by the License, +# or, if provided, by the license below or the license accompanying this file. Do not +# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# + +add_subdirectory(Code) \ No newline at end of file diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetB/Code/CMakeLists.txt b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetB/Code/CMakeLists.txt new file mode 100644 index 0000000000..57b55fb9f4 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetB/Code/CMakeLists.txt @@ -0,0 +1,29 @@ +# +# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +# its licensors. +# +# For complete copyright and license terms please see the LICENSE at the root of this +# distribution (the "License"). All use of this software is governed by the License, +# or, if provided, by the license below or the license accompanying this file. Do not +# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# + +################################################################################ +# Tests +################################################################################ + +ly_add_target( + NAME TestImpact.TestTargetB.Tests ${PAL_TRAIT_TEST_TARGET_TYPE} + NAMESPACE AZ + FILES_CMAKE + testimpactframework_testtargetb_tests_files.cmake + INCLUDE_DIRECTORIES + PRIVATE + Tests + BUILD_DEPENDENCIES + PRIVATE + AZ::AzCore + AZ::AzTestShared + AZ::AzTest +) \ No newline at end of file diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetB/Code/Tests/TestImpactTestTargetB.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetB/Code/Tests/TestImpactTestTargetB.cpp new file mode 100644 index 0000000000..be58b4fa17 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetB/Code/Tests/TestImpactTestTargetB.cpp @@ -0,0 +1,78 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include + +namespace UnitTest +{ + class TestFixture + : public AllocatorsTestFixture + { + }; + + class TestFixtureWithParams + : public AllocatorsTestFixture + , public ::testing::WithParamInterface> + { + }; + + TEST(TestCase, Test1_WillPass) + { + SUCCEED(); + } + + TEST(TestCase, Test2_WillPass) + { + SUCCEED(); + } + + TEST(TestCase, Test3_WillPass) + { + SUCCEED(); + } + + TEST_F(TestFixture, Test1_WillPass) + { + SUCCEED(); + } + + TEST_P(TestFixtureWithParams, Test1_WillPass) + { + SUCCEED(); + } + + TEST_P(TestFixtureWithParams, Test2_WillPass) + { + SUCCEED(); + } + + INSTANTIATE_TEST_CASE_P( + PermutationA, + TestFixtureWithParams, + ::testing::Combine( + ::testing::Values(1, 2, 4), + ::testing::Values(3, 5, 7), + ::testing::Values(-0.0f, 0.0f, 1.0f) + )); + + INSTANTIATE_TEST_CASE_P( + , + TestFixtureWithParams, + ::testing::Combine( + ::testing::Values(8, 16, 32), + ::testing::Values(9, 13, 17), + ::testing::Values(-10.0f, 0.05f, 10.0f) + )); +} // namespace UnitTest + +AZ_UNIT_TEST_HOOK(DEFAULT_UNIT_TEST_ENV); diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetB/Code/testimpactframework_testtargetb_tests_files.cmake b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetB/Code/testimpactframework_testtargetb_tests_files.cmake new file mode 100644 index 0000000000..949074a5af --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetB/Code/testimpactframework_testtargetb_tests_files.cmake @@ -0,0 +1,14 @@ +# +# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +# its licensors. +# +# For complete copyright and license terms please see the LICENSE at the root of this +# distribution (the "License"). All use of this software is governed by the License, +# or, if provided, by the license below or the license accompanying this file. Do not +# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# + +set(FILES + Tests/TestImpactTestTargetB.cpp +) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetC/CMakeLists.txt b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetC/CMakeLists.txt new file mode 100644 index 0000000000..8298bb7123 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetC/CMakeLists.txt @@ -0,0 +1,12 @@ +# +# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +# its licensors. +# +# For complete copyright and license terms please see the LICENSE at the root of this +# distribution (the "License"). All use of this software is governed by the License, +# or, if provided, by the license below or the license accompanying this file. Do not +# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# + +add_subdirectory(Code) \ No newline at end of file diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetC/Code/CMakeLists.txt b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetC/Code/CMakeLists.txt new file mode 100644 index 0000000000..47862f80df --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetC/Code/CMakeLists.txt @@ -0,0 +1,29 @@ +# +# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +# its licensors. +# +# For complete copyright and license terms please see the LICENSE at the root of this +# distribution (the "License"). All use of this software is governed by the License, +# or, if provided, by the license below or the license accompanying this file. Do not +# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# + +################################################################################ +# Tests +################################################################################ + +ly_add_target( + NAME TestImpact.TestTargetC.Tests ${PAL_TRAIT_TEST_TARGET_TYPE} + NAMESPACE AZ + FILES_CMAKE + testimpactframework_testtargetc_tests_files.cmake + INCLUDE_DIRECTORIES + PRIVATE + Tests + BUILD_DEPENDENCIES + PRIVATE + AZ::AzCore + AZ::AzTestShared + AZ::AzTest +) \ No newline at end of file diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetC/Code/Tests/TestImpactTestTargetC.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetC/Code/Tests/TestImpactTestTargetC.cpp new file mode 100644 index 0000000000..48f0ffe9dc --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetC/Code/Tests/TestImpactTestTargetC.cpp @@ -0,0 +1,63 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include + +namespace UnitTest +{ + class TestFixture + : public AllocatorsTestFixture + { + }; + + template + class TestFixtureWithTypes + : public AllocatorsTestFixture + { + }; + + using TestTypes = testing::Types; + TYPED_TEST_CASE(TestFixtureWithTypes, TestTypes); + + TEST_F(TestFixture, Test1_WillPass) + { + SUCCEED(); + } + + TEST_F(TestFixture, Test2_WillPass) + { + SUCCEED(); + } + + TYPED_TEST(TestFixtureWithTypes, Test1_WillPass) + { + SUCCEED(); + } + + TYPED_TEST(TestFixtureWithTypes, Test2_WillPass) + { + SUCCEED(); + } + + TYPED_TEST(TestFixtureWithTypes, Test3_WillPass) + { + SUCCEED(); + } + + TYPED_TEST(TestFixtureWithTypes, Test4_WillPass) + { + SUCCEED(); + } +} // namespace UnitTest + +AZ_UNIT_TEST_HOOK(DEFAULT_UNIT_TEST_ENV); diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetC/Code/testimpactframework_testtargetc_tests_files.cmake b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetC/Code/testimpactframework_testtargetc_tests_files.cmake new file mode 100644 index 0000000000..fa5b4f6e9a --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetC/Code/testimpactframework_testtargetc_tests_files.cmake @@ -0,0 +1,14 @@ +# +# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +# its licensors. +# +# For complete copyright and license terms please see the LICENSE at the root of this +# distribution (the "License"). All use of this software is governed by the License, +# or, if provided, by the license below or the license accompanying this file. Do not +# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# + +set(FILES + Tests/TestImpactTestTargetC.cpp +) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetD/CMakeLists.txt b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetD/CMakeLists.txt new file mode 100644 index 0000000000..8298bb7123 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetD/CMakeLists.txt @@ -0,0 +1,12 @@ +# +# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +# its licensors. +# +# For complete copyright and license terms please see the LICENSE at the root of this +# distribution (the "License"). All use of this software is governed by the License, +# or, if provided, by the license below or the license accompanying this file. Do not +# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# + +add_subdirectory(Code) \ No newline at end of file diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetD/Code/CMakeLists.txt b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetD/Code/CMakeLists.txt new file mode 100644 index 0000000000..bf821ac0d6 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetD/Code/CMakeLists.txt @@ -0,0 +1,29 @@ +# +# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +# its licensors. +# +# For complete copyright and license terms please see the LICENSE at the root of this +# distribution (the "License"). All use of this software is governed by the License, +# or, if provided, by the license below or the license accompanying this file. Do not +# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# + +################################################################################ +# Tests +################################################################################ + +ly_add_target( + NAME TestImpact.TestTargetD.Tests ${PAL_TRAIT_TEST_TARGET_TYPE} + NAMESPACE AZ + FILES_CMAKE + testimpactframework_testtargetd_tests_files.cmake + INCLUDE_DIRECTORIES + PRIVATE + Tests + BUILD_DEPENDENCIES + PRIVATE + AZ::AzCore + AZ::AzTestShared + AZ::AzTest +) \ No newline at end of file diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetD/Code/Tests/TestImpactTestTargetD.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetD/Code/Tests/TestImpactTestTargetD.cpp new file mode 100644 index 0000000000..bd27b51fd9 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetD/Code/Tests/TestImpactTestTargetD.cpp @@ -0,0 +1,188 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include + +namespace UnitTest +{ + class TestFixture1 + : public AllocatorsTestFixture + { + }; + + class DISABLED_TestFixture2 + : public AllocatorsTestFixture + { + }; + + template + class TestFixtureWithTypes1 + : public AllocatorsTestFixture + { + }; + + template + class DISABLED_TestFixtureWithTypes2 + : public AllocatorsTestFixture + { + }; + + class TestFixtureWithParams1 + : public AllocatorsTestFixture + , public ::testing::WithParamInterface> + { + }; + + class DISABLED_TestFixtureWithParams2 + : public AllocatorsTestFixture + , public ::testing::WithParamInterface> + { + }; + + using TestTypes = testing::Types; + TYPED_TEST_CASE(TestFixtureWithTypes1, TestTypes); + TYPED_TEST_CASE(DISABLED_TestFixtureWithTypes2, TestTypes); + + TEST(TestCase, Test1_WillPass) + { + SUCCEED(); + } + + TEST(TestCase, DISABLED_Test2_WillPass) + { + SUCCEED(); + } + + TEST(TestCase, Test3_WillPass) + { + SUCCEED(); + } + + TEST(TestCase, Test4_WillPass) + { + SUCCEED(); + } + + TEST(TestCase, Test5_WillPass) + { + SUCCEED(); + } + + TEST_F(TestFixture1, Test1_WillPass) + { + SUCCEED(); + } + + TEST_F(TestFixture1, Test2_WillPass) + { + SUCCEED(); + } + + TEST_F(DISABLED_TestFixture2, Test1_WillPass) + { + SUCCEED(); + } + + TEST_F(DISABLED_TestFixture2, Test2_WillPass) + { + SUCCEED(); + } + + TEST_P(TestFixtureWithParams1, Test1_WillPass) + { + SUCCEED(); + } + + TEST_P(TestFixtureWithParams1, DISABLED_Test2_WillPass) + { + SUCCEED(); + } + + TEST_P(DISABLED_TestFixtureWithParams2, Test1_WillPass) + { + SUCCEED(); + } + + TEST_P(DISABLED_TestFixtureWithParams2, DISABLED_Test2_WillPass) + { + SUCCEED(); + } + + INSTANTIATE_TEST_CASE_P( + PermutationA, + TestFixtureWithParams1, + ::testing::Combine( + ::testing::Values(1, 2, 4), + ::testing::Values(3, 5, 7), + ::testing::Values(-0.0f, 0.0f, 1.0f) + )); + + INSTANTIATE_TEST_CASE_P( + , + TestFixtureWithParams1, + ::testing::Combine( + ::testing::Values(8, 16, 32), + ::testing::Values(9, 13, 17), + ::testing::Values(-10.0f, 0.05f, 10.0f) + )); + + INSTANTIATE_TEST_CASE_P( + PermutationA, + DISABLED_TestFixtureWithParams2, + ::testing::Combine( + ::testing::Values(1, 2, 4), + ::testing::Values(3, 5, 7), + ::testing::Values(-0.0f, 0.0f, 1.0f) + )); + + INSTANTIATE_TEST_CASE_P( + , + DISABLED_TestFixtureWithParams2, + ::testing::Combine( + ::testing::Values(8, 16, 32), + ::testing::Values(9, 13, 17), + ::testing::Values(-10.0f, 0.05f, 10.0f) + )); + + TYPED_TEST(TestFixtureWithTypes1, Test1_WillPass) + { + SUCCEED(); + } + + TYPED_TEST(TestFixtureWithTypes1, DISABLED_Test2_WillPass) + { + SUCCEED(); + } + + TYPED_TEST(TestFixtureWithTypes1, Test3_WillPass) + { + SUCCEED(); + } + + TYPED_TEST(DISABLED_TestFixtureWithTypes2, Test1_WillPass) + { + SUCCEED(); + } + + TYPED_TEST(DISABLED_TestFixtureWithTypes2, DISABLED_Test2_WillPass) + { + SUCCEED(); + } + + TYPED_TEST(DISABLED_TestFixtureWithTypes2, Test3_WillPass) + { + SUCCEED(); + } +} // namespace UnitTest + +AZ_UNIT_TEST_HOOK(DEFAULT_UNIT_TEST_ENV); diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetD/Code/testimpactframework_testtargetd_tests_files.cmake b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetD/Code/testimpactframework_testtargetd_tests_files.cmake new file mode 100644 index 0000000000..ebb5a422c7 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetD/Code/testimpactframework_testtargetd_tests_files.cmake @@ -0,0 +1,14 @@ +# +# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +# its licensors. +# +# For complete copyright and license terms please see the LICENSE at the root of this +# distribution (the "License"). All use of this software is governed by the License, +# or, if provided, by the license below or the license accompanying this file. Do not +# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# + +set(FILES + Tests/TestImpactTestTargetD.cpp +) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_files.cmake b/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_files.cmake index 74c7c84dcc..fd65a16e39 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_files.cmake +++ b/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_files.cmake @@ -10,5 +10,20 @@ # set(FILES - Source/Dummy.cpp + Include/TestImpactFramework/TestImpactException.h + Include/TestImpactFramework/TestImpactFrameworkPath.h + Include/TestImpactFramework/TestImpactCallback.h + Source/TestImpactException.cpp + Source/TestImpactFrameworkPath.cpp + Source/Process/TestImpactProcess.cpp + Source/Process/TestImpactProcess.h + Source/Process/TestImpactProcessException.h + Source/Process/TestImpactProcessInfo.cpp + Source/Process/TestImpactProcessInfo.h + Source/Process/TestImpactProcessLauncher.h + Source/Process/JobRunner/TestImpactProcessJob.h + Source/Process/JobRunner/TestImpactProcessJobInfo.h + Source/Process/JobRunner/TestImpactProcessJobRunner.h + Source/Process/Scheduler/TestImpactProcessScheduler.cpp + Source/Process/Scheduler/TestImpactProcessScheduler.h ) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_tests_files.cmake b/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_tests_files.cmake new file mode 100644 index 0000000000..61059dc543 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_tests_files.cmake @@ -0,0 +1,22 @@ +# +# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +# its licensors. +# +# For complete copyright and license terms please see the LICENSE at the root of this +# distribution (the "License"). All use of this software is governed by the License, +# or, if provided, by the license below or the license accompanying this file. Do not +# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# + +set(FILES + Tests/Process/TestImpactProcessSchedulerTest.cpp + Tests/Process/TestImpactProcessTest.cpp + Tests/TestImpactExceptionTest.cpp + Tests/TestImpactFrameworkPathTest.cpp + Tests/TestImpactProcessSchedulerTest.cpp + Tests/TestImpactProcessTest.cpp + Tests/TestImpactProcessTestShared.cpp + Tests/TestImpactProcessTestShared.h + Tests/TestImpactTestMain.cpp +) From db630794c8badee93b3bb2549defa733ce7d48b6 Mon Sep 17 00:00:00 2001 From: jonawals Date: Wed, 28 Apr 2021 14:18:40 +0100 Subject: [PATCH 002/102] Update TIAF artifact file formats and test target enumeration. --- cmake/TestImpactFramework/EnumeratedTests.in | 48 ++++++++------- .../LYTestImpactFramework.cmake | 58 +++++++++---------- .../SourceToTargetMapping.in | 27 +++++---- 3 files changed, 72 insertions(+), 61 deletions(-) diff --git a/cmake/TestImpactFramework/EnumeratedTests.in b/cmake/TestImpactFramework/EnumeratedTests.in index acd33fb661..02eb648f5f 100644 --- a/cmake/TestImpactFramework/EnumeratedTests.in +++ b/cmake/TestImpactFramework/EnumeratedTests.in @@ -1,23 +1,31 @@ -#Lumberyard enumerated tests -[google] -[google.test] -tests = [ -${google_tests} -] -[google.benchmark] -tests = [ +{ + "google": { + "benchmark": { + "tests": [ ${google_benchmarks} -] -[python] -[python.test] -tests = [ -${python_tests} -] -[python.editor] -tests = [ + ] + }, + "test": { + "tests": [ +${google_tests} + ] + } + }, + "python": { + "editor": { + "tests": [ ${python_editor_tests} -] -[unknown] -tests = [ + ] + }, + "test": { + "tests": [ +${python_tests} + ] + } + }, + "unknown": { + "tests": [ ${unknown_tests} -] \ No newline at end of file + ] + } +} \ No newline at end of file diff --git a/cmake/TestImpactFramework/LYTestImpactFramework.cmake b/cmake/TestImpactFramework/LYTestImpactFramework.cmake index c10c5bf637..c08c64db80 100644 --- a/cmake/TestImpactFramework/LYTestImpactFramework.cmake +++ b/cmake/TestImpactFramework/LYTestImpactFramework.cmake @@ -25,7 +25,7 @@ set(LY_TEST_IMPACT_WORKING_DIR "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/TestImpactFram set(LY_TEST_IMPACT_TEMP_DIR "${LY_TEST_IMPACT_WORKING_DIR}/Temp") # Directory for static artifacts produced as part of the build system generation process -set(LY_TEST_IMPACT_ARTIFACT_DIR "${LY_TEST_IMPACT_WORKING_DIR}/Artefact") +set(LY_TEST_IMPACT_ARTIFACT_DIR "${LY_TEST_IMPACT_WORKING_DIR}/Artifact") # Directory for source to build target mappings set(LY_TEST_IMPACT_SOURCE_TARGET_MAPPING_DIR "${LY_TEST_IMPACT_ARTIFACT_DIR}/Mapping") @@ -77,28 +77,28 @@ function(ly_test_impact_rebase_files_to_repo_root INPUT_FILES OUTPUT_FILES RELAT set(${OUTPUT_FILES} ${rebased_files} PARENT_SCOPE) endfunction() -#! ly_test_impact_get_target_type_string: gets the target type string (either executable, dynalib or unknown) for the specified target. +#! ly_test_impact_get_test_launch_method: gets the launch method (either standalone or testrunner) for the specified target. # # \arg:TARGET_NAME name of the target -# \arg:TARGET_TYPE the type string for the specified target -function(ly_test_impact_get_target_type_string TARGET_NAME TARGET_TYPE) - # Get the test impact framework-friendly target type string +# \arg:LAUNCH_METHOD the type string for the specified target +function(ly_test_impact_get_test_launch_method TARGET_NAME LAUNCH_METHOD) + # Get the test impact framework-friendly launch method string get_target_property(target_type ${TARGET_NAME} TYPE) if("${target_type}" STREQUAL "SHARED_LIBRARY" OR "${target_type}" STREQUAL "MODULE_LIBRARY") - set(${TARGET_TYPE} "dynlib" PARENT_SCOPE) + set(${LAUNCH_METHOD} "test_runner" PARENT_SCOPE) elseif("${target_type}" STREQUAL "EXECUTABLE") - set(${TARGET_TYPE} "executable" PARENT_SCOPE) + set(${LAUNCH_METHOD} "stand_alone" PARENT_SCOPE) else() - set(${TARGET_TYPE} "unknown" PARENT_SCOPE) + message(FATAL_ERROR "Cannot deduce test target launch method for the target type ${target_type}") endif() endfunction() #! ly_test_impact_extract_google_test: explodes a composite google test string into namespace, test and suite components. # # \arg:COMPOSITE_TEST test in the form 'namespace::test' -# \arg:TEST_QUALIFER qualifier for the test (namespace) +# \arg:TEST_NAMESPACE namespace for the test # \arg:TEST_NAME name of test -function(ly_test_impact_extract_google_test COMPOSITE_TEST TEST_QUALIFER TEST_NAME) +function(ly_test_impact_extract_google_test COMPOSITE_TEST TEST_NAMESPACE TEST_NAME) get_property(test_components GLOBAL PROPERTY LY_ALL_TESTS_${COMPOSITE_TEST}_TEST_NAME) # Namespace and test are mandetiry string(REPLACE "::" ";" test_components ${test_components}) @@ -107,19 +107,19 @@ function(ly_test_impact_extract_google_test COMPOSITE_TEST TEST_QUALIFER TEST_NA message(FATAL_ERROR "The test ${test_components} appears to have been specified without a namespace, i.e.:\ly_add_googletest/benchmark(NAME ${test_components})\nInstead of (perhaps):\ly_add_googletest/benchmark(NAME Gem::${test_components})\nPlease add the missing namespace before proceeding.") endif() - list(GET test_components 0 test_qualifier) + list(GET test_components 0 test_namespace) list(GET test_components 1 test_name) - set(${TEST_QUALIFER} ${test_qualifier} PARENT_SCOPE) + set(${TEST_NAMESPACE} ${test_namespace} PARENT_SCOPE) set(${TEST_NAME} ${test_name} PARENT_SCOPE) endfunction() #! ly_test_impact_extract_python_test: explodes a composite python test string into filename, namespace, test and suite components. # # \arg:COMPOSITE_TEST test in form 'namespace::test' or 'test' -# \arg:TEST_QUALIFER qualifier for the test (optional) +# \arg:TEST_NAMESPACE namespace for the test (optional) # \arg:TEST_NAME name of test # \arg:TEST_FILE the Python script path for this test -function(ly_test_impact_extract_python_test COMPOSITE_TEST TEST_QUALIFER TEST_NAME TEST_FILE) +function(ly_test_impact_extract_python_test COMPOSITE_TEST TEST_NAMESPACE TEST_NAME TEST_FILE) get_property(test_components GLOBAL PROPERTY LY_ALL_TESTS_${COMPOSITE_TEST}_TEST_NAME) get_property(test_file GLOBAL PROPERTY LY_ALL_TESTS_${COMPOSITE_TEST}_SCRIPT_PATH) @@ -127,10 +127,10 @@ function(ly_test_impact_extract_python_test COMPOSITE_TEST TEST_QUALIFER TEST_NA string(REPLACE "::" ";" test_components ${test_components}) list(LENGTH test_components num_test_components) if(num_test_components GREATER 1) - list(GET test_components 0 test_qualifier) + list(GET test_components 0 test_namespace) list(GET test_components 1 test_name) else() - set(test_qualifier "") + set(test_namespace "") set(test_name ${test_components}) endif() @@ -141,7 +141,7 @@ function(ly_test_impact_extract_python_test COMPOSITE_TEST TEST_QUALIFER TEST_NA ${LY_ROOT_FOLDER} ) - set(${TEST_QUALIFER} ${test_qualifier} PARENT_SCOPE) + set(${TEST_NAMESPACE} ${test_namespace} PARENT_SCOPE) set(${TEST_NAME} ${test_name} PARENT_SCOPE) set(${TEST_FILE} ${test_file} PARENT_SCOPE) endfunction() @@ -166,24 +166,24 @@ function(ly_test_impact_write_test_enumeration_file TEST_ENUMERATION_TEMPLATE_FI get_property(test_suite GLOBAL PROPERTY LY_ALL_TESTS_${test}_TEST_SUITE) if("${test_type}" STREQUAL "pytest") # Python tests - ly_test_impact_extract_python_test(${test} test_qualifier test_name test_file) - list(APPEND python_tests "{ name = \"${test_name}\", qualifier = \"${test_qualifier}\", suite = \"${test_suite}\", path = \"${test_file}\" }") + ly_test_impact_extract_python_test(${test} test_namespace test_name test_file) + list(APPEND python_tests " { \"name\": \"${test_name}\", \"namespace\": \"${test_namespace}\", \"suite\": \"${test_suite}\", \"path\": \"${test_file}\" }") elseif("${test_type}" STREQUAL "pytest_editor") # Python editor tests - ly_test_impact_extract_python_test(${test} test_qualifier test_name test_file) - list(APPEND python_editor_tests "{ name = \"${test_name}\", qualifier = \"${test_qualifier}\", suite = \"${test_suite}\", path = \"${test_file}\" }") + ly_test_impact_extract_python_test(${test} test_namespace test_name test_file) + list(APPEND python_editor_tests " { \"name\": \"${test_name}\", \"namespace\": \"${test_namespace}\", \"suite\": \"${test_suite}\", \"path\": \"${test_file}\" }") elseif("${test_type}" STREQUAL "googletest") # Google tests - ly_test_impact_extract_google_test(${test} test_qualifier test_name) - ly_test_impact_get_target_type_string(${test_name} target_type) - list(APPEND google_tests "{ name = \"${test_name}\", qualifier = \"${test_qualifier}\", suite = \"${test_suite}\", build_type = \"${target_type}\" }") + ly_test_impact_extract_google_test(${test} test_namespace test_name) + ly_test_impact_get_test_launch_method(${test_name} launch_method) + list(APPEND google_tests " { \"name\": \"${test_name}\", \"namespace\": \"${test_namespace}\", \"suite\": \"${test_suite}\", \"launch_method\": \"${launch_method}\" }") elseif("${test_type}" STREQUAL "googlebenchmark") # Google benchmarks - ly_test_impact_extract_google_test(${test} test_qualifier test_name) - list(APPEND google_benchmarks "{ name = \"${test_name}\", qualifier = \"${test_qualifier}\", suite = \"${test_suite}\" }") + ly_test_impact_extract_google_test(${test} test_namespace test_name) + list(APPEND google_benchmarks " { \"name\": \"${test_name}\", \"namespace\": \"${test_namespace}\", \"suite\": \"${test_suite}\" }") else() message("${test} is of unknown type (TEST_LIBRARY property is empty)") - list(APPEND unknown_tests "{ name = \"${test}\" }") + list(APPEND unknown_tests " { \"name\": \"${test}\" }") endif() endforeach() @@ -279,7 +279,7 @@ function(ly_test_impact_write_config_file CONFIG_TEMPLATE_FILE PERSISTENT_DATA_D if(NOT LY_TEST_IMPACT_INSTRUMENTATION_BIN) message(FATAL_ERROR "No test impact framework instrumentation binary was specified, please provide the path with option LY_TEST_IMPACT_INSTRUMENTATION_BIN") endif() - set(instrumentation_bin ${LY_TEST_IMPACT_INSTRUMENTATION_BIN}) + file(TO_CMAKE_PATH ${LY_TEST_IMPACT_INSTRUMENTATION_BIN} instrumentation_bin) # test impact framework working dir ly_test_impact_rebase_file_to_repo_root( @@ -323,7 +323,7 @@ function(ly_test_impact_write_config_file CONFIG_TEMPLATE_FILE PERSISTENT_DATA_D ${LY_ROOT_FOLDER} ) - # Bild dependency artifact dir + # Build dependency artifact dir ly_test_impact_rebase_file_to_repo_root( "${LY_TEST_IMPACT_TARGET_DEPENDENCY_DIR}" target_dependency_dir diff --git a/cmake/TestImpactFramework/SourceToTargetMapping.in b/cmake/TestImpactFramework/SourceToTargetMapping.in index c94c52a145..da837ed398 100644 --- a/cmake/TestImpactFramework/SourceToTargetMapping.in +++ b/cmake/TestImpactFramework/SourceToTargetMapping.in @@ -1,15 +1,18 @@ -#Lumberyard source to target mapping -[target] -name = "${target_name}" -output_name = "${target_output_name}" -path = "${target_path}" -[sources] -input = [ +{ + "sources": { + "input": [ ${autogen_input_files} -] -output = [ + ], + "output": [ ${autogen_output_files} -] -static = [ + ], + "static": [ ${static_sources} -] \ No newline at end of file + ] + }, + "target": { + "name": "${target_name}", + "output_name": "${target_output_name}", + "path": "${target_path}" + } +} \ No newline at end of file From b9dc7639b4182cbdace284ed4226c0c179ffec52 Mon Sep 17 00:00:00 2001 From: jonawals Date: Wed, 28 Apr 2021 17:15:48 +0100 Subject: [PATCH 003/102] Add dynamic dependency map. --- .../Frontend/Console/Code/CMakeLists.txt | 2 +- .../Artifact/Dynamic/TestImpactCoverage.h | 9 +- .../TestImpactChangeDependencyList.cpp | 41 ++ .../TestImpactChangeDependencyList.h | 41 ++ .../TestImpactDependencyException.h | 26 ++ .../TestImpactDynamicDependencyMap.cpp | 414 ++++++++++++++++++ .../TestImpactDynamicDependencyMap.h | 120 +++++ .../TestImpactSourceCoveringTestsList.cpp | 58 +++ .../TestImpactSourceCoveringTestsList.h | 53 +++ .../Test/Run/TestImpactTestCoverage.cpp | 4 +- .../Source/Test/Run/TestImpactTestCoverage.h | 4 +- .../Tests/Test/TestImpactTestCoverageTest.cpp | 12 +- .../testimpactframework_runtime_files.cmake | 68 ++- ...timpactframework_runtime_tests_files.cmake | 24 +- 14 files changed, 853 insertions(+), 23 deletions(-) create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactChangeDependencyList.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactChangeDependencyList.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDependencyException.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsList.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsList.h diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Code/CMakeLists.txt b/Code/Tools/TestImpactFramework/Frontend/Console/Code/CMakeLists.txt index 7a043a30ca..da2c707cb8 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Code/CMakeLists.txt +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Code/CMakeLists.txt @@ -20,4 +20,4 @@ ly_add_target( BUILD_DEPENDENCIES PRIVATE AZ::TestImpact.Runtime.Static -) +) \ No newline at end of file diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Dynamic/TestImpactCoverage.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Dynamic/TestImpactCoverage.h index 3c5b5353a7..54b78d682e 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Dynamic/TestImpactCoverage.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Dynamic/TestImpactCoverage.h @@ -12,9 +12,8 @@ #pragma once -#include +#include #include -#include namespace TestImpact { @@ -28,14 +27,14 @@ namespace TestImpact //! Coverage information about a particular source file. struct SourceCoverage { - AZ::IO::Path m_path; //!< Source file path. - AZStd::optional> m_coverage; //!< Source file line coverage (empty if source level coverage only). + AZStd::string m_path; //!< Source file path. + AZStd::vector m_coverage; //!< Source file line coverage (empty if source level coverage only). }; //! Coverage information about a particular module (executable, shared library). struct ModuleCoverage { - AZ::IO::Path m_path; //!< Module path. + AZStd::string m_path; //!< Module path. AZStd::vector m_sources; //!< Sources of this module that are covered. }; } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactChangeDependencyList.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactChangeDependencyList.cpp new file mode 100644 index 0000000000..53e71d3a30 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactChangeDependencyList.cpp @@ -0,0 +1,41 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include + +namespace TestImpact +{ + ChangeDependencyList::ChangeDependencyList( + AZStd::vector&& createSourceDependencies, + AZStd::vector&& updateSourceDependencies, + AZStd::vector&& deleteSourceDependencies) + : m_createSourceDependencies(AZStd::move(createSourceDependencies)) + , m_updateSourceDependencies(AZStd::move(updateSourceDependencies)) + , m_deleteSourceDependencies(AZStd::move(deleteSourceDependencies)) + { + } + + const AZStd::vector& ChangeDependencyList::GetCreateSourceDependencies() const + { + return m_createSourceDependencies; + } + + const AZStd::vector& ChangeDependencyList::GetUpdateSourceDependencies() const + { + return m_updateSourceDependencies; + } + + const AZStd::vector& ChangeDependencyList::GetDeleteSourceDependencies() const + { + return m_deleteSourceDependencies; + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactChangeDependencyList.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactChangeDependencyList.h new file mode 100644 index 0000000000..bf8b0f558d --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactChangeDependencyList.h @@ -0,0 +1,41 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +namespace TestImpact +{ + //! Representation of a change list where all CRUD sources have been resolved to source dependencies from the dynamic dependency map. + class ChangeDependencyList + { + public: + ChangeDependencyList( + AZStd::vector&& createSourceDependencies, + AZStd::vector&& updateSourceDependencies, + AZStd::vector&& deleteSourceDependencies); + + //! Gets the sources dependencies of the created source files from the change list. + const AZStd::vector& GetCreateSourceDependencies() const; + + //! Gets the sources dependencies of the updated source files from the change list. + const AZStd::vector& GetUpdateSourceDependencies() const; + + //! Gets the sources dependencies of the deleted source files from the change list. + const AZStd::vector& GetDeleteSourceDependencies() const; + private: + AZStd::vector m_createSourceDependencies; + AZStd::vector m_updateSourceDependencies; + AZStd::vector m_deleteSourceDependencies; + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDependencyException.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDependencyException.h new file mode 100644 index 0000000000..e256fa3ccc --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDependencyException.h @@ -0,0 +1,26 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +namespace TestImpact +{ + //! Exception for dependency related operations. + class DependencyException + : public Exception + { + public: + using Exception::Exception; + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp new file mode 100644 index 0000000000..f3edbe2507 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp @@ -0,0 +1,414 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include + +namespace TestImpact +{ + DynamicDependencyMap::DynamicDependencyMap( + AZStd::vector&& productionTargetDescriptors, + AZStd::vector&& testTargetDescriptors) + : m_productionTargets(AZStd::move(productionTargetDescriptors)) + , m_testTargets(AZStd::move(testTargetDescriptors)) + { + const auto mapBuildTargetSources = [this](const auto* target) + { + for (const auto& source : target->GetSources().m_staticSources) + { + if (auto mapping = m_sourceDependencyMap.find(source); mapping != m_sourceDependencyMap.end()) + { + // This is an existing entry in the dependency map so update the parent build targets with this target + mapping->second.m_parentTargets.insert(target); + } + else + { + // This is a new entry on the dependency map so create an entry with this parent target and no covering targets + m_sourceDependencyMap.emplace(source, DependencyData{ {target}, {} }); + } + } + + // Populate the autogen input to output mapping with any autogen sources + for (const auto& autogen : target->GetSources().m_autogenSources) + { + for (const auto& output : autogen.m_outputs) + { + m_autogenInputToOutputMap[autogen.m_input].push_back(output); + } + } + }; + + for (const auto& target : m_productionTargets.GetTargets()) + { + mapBuildTargetSources(&target); + } + + for (const auto& target : m_testTargets.GetTargets()) + { + mapBuildTargetSources(&target); + } + } + + size_t DynamicDependencyMap::GetNumTargets() const + { + return m_productionTargets.GetNumTargets() + m_testTargets.GetNumTargets(); + } + + size_t DynamicDependencyMap::GetNumSources() const + { + return m_sourceDependencyMap.size(); + } + + const BuildTarget* DynamicDependencyMap::GetBuildTarget(const AZStd::string& name) const + { + const BuildTarget* buildTarget = nullptr; + AZStd::visit([&buildTarget](auto&& target) + { + if constexpr (IsProductionTarget || IsTestTarget) + { + buildTarget = target; + } + + }, GetTarget(name)); + + return buildTarget; + } + + const BuildTarget* DynamicDependencyMap::GetBuildTargetOrThrow(const AZStd::string& name) const + { + const BuildTarget* buildTarget = nullptr; + AZStd::visit([&buildTarget](auto&& target) + { + if constexpr (IsProductionTarget || IsTestTarget) + { + buildTarget = target; + } + }, GetTargetOrThrow(name)); + + return buildTarget; + } + + AZStd::variant DynamicDependencyMap::GetTarget(const AZStd::string& name) const + { + if (auto testTarget = m_testTargets.GetTarget(name); testTarget != nullptr) + { + return testTarget; + } + else if (auto productionTarget = m_productionTargets.GetTarget(name); productionTarget != nullptr) + { + return productionTarget; + } + + return AZStd::monostate{}; + } + + AZStd::variant DynamicDependencyMap::GetTargetOrThrow(const AZStd::string& name) const + { + AZStd::variant buildTarget; + AZStd::visit([&buildTarget, &name](auto&& target) + { + if constexpr (IsProductionTarget || IsTestTarget) + { + buildTarget = target; + } + else + { + throw(TargetException(AZStd::string::format("Couldn't find target %s", name.c_str()).c_str())); + } + }, GetTarget(name)); + + return buildTarget; + } + + void DynamicDependencyMap::ReplaceSourceCoverage(const SourceCoveringTestsList& sourceCoverageDelta) + { + for (const auto& sourceCoverage : sourceCoverageDelta.GetCoverage()) + { + // Autogen input files are not compiled sources and thus supplying coverage data for them makes no sense + AZ_TestImpact_Eval( + m_autogenInputToOutputMap.find(sourceCoverage.GetPath()) == m_autogenInputToOutputMap.end(), + DependencyException, AZStd::string::format("Couldn't replace source coverage for %s, source file is an autogen input file", + sourceCoverage.GetPath().c_str()).c_str()); + + auto [it, inserted] = m_sourceDependencyMap.insert(sourceCoverage.GetPath()); + auto& [key, sourceDependency] = *it; + + // Clear any existing coverage for the delta + sourceDependency.m_coveringTestTargets.clear(); + + // Update the dependency with any new coverage data + for (const auto& unresolvedTestTarget : sourceCoverage.GetCoveringTestTargets()) + { + const TestTarget* testTarget = m_testTargets.GetTarget(unresolvedTestTarget); + if (testTarget) + { + // Source to covering test target mapping + sourceDependency.m_coveringTestTargets.insert(testTarget); + + // Build target to covering test target mapping + for (const auto& parentTarget : sourceDependency.m_parentTargets) + { + m_buildTargetCoverage[parentTarget.GetBuildTarget()].insert(testTarget); + } + } + else + { + AZ_Warning("ReplaceSourceCoverage", false, AZStd::string::format("Test target %s exists in the coverage data " + "but has since been removed from the build system", unresolvedTestTarget.c_str()).c_str()); + } + } + + // If the new coverage data results in a parentless and coverageless entry, consider it a dead entry and remove accordingly + if (sourceDependency.m_coveringTestTargets.empty() && sourceDependency.m_parentTargets.empty()) + { + m_sourceDependencyMap.erase(it); + } + } + } + + void DynamicDependencyMap::ClearSourceCoverage(const AZStd::vector& paths) + { + for (const auto& path : paths) + { + if (const auto outputSources = m_autogenInputToOutputMap.find(path); outputSources != m_autogenInputToOutputMap.end()) + { + // Clearing the coverage data of an autogen input source instead clears the coverage data of its output sources + for (const auto& outputSource : outputSources->second) + { + ReplaceSourceCoverage(SourceCoveringTestsList({ SourceCoveringTests(outputSource, { }) })); + } + } + else + { + ReplaceSourceCoverage(SourceCoveringTestsList({ SourceCoveringTests(path, { }) })); + } + } + } + + const ProductionTargetList& DynamicDependencyMap::GetProductionTargetList() const + { + return m_productionTargets; + } + + const TestTargetList& DynamicDependencyMap::GetTestTargetList() const + { + return m_testTargets; + } + + AZStd::vector DynamicDependencyMap::GetCoveringTestTargetsForProductionTarget(const ProductionTarget& productionTarget) const + { + AZStd::vector coveringTestTargets; + if (const auto coverage = m_buildTargetCoverage.find(&productionTarget); coverage != m_buildTargetCoverage.end()) + { + coveringTestTargets.reserve(coverage->second.size()); + AZStd::copy(coverage->second.begin(), coverage->second.end(), AZStd::back_inserter(coveringTestTargets)); + } + + return coveringTestTargets; + } + + AZStd::optional DynamicDependencyMap::GetSourceDependency(const AZStd::string& path) const + { + AZStd::unordered_set parentTargets; + AZStd::unordered_set coveringTestTargets; + + const auto getSourceDependency = [&parentTargets, &coveringTestTargets, this](const AZStd::string& path) + { + const auto sourceDependency = m_sourceDependencyMap.find(path); + if (sourceDependency != m_sourceDependencyMap.end()) + { + for (const auto& parentTarget : sourceDependency->second.m_parentTargets) + { + parentTargets.insert(parentTarget); + } + + for (const auto& testTarget : sourceDependency->second.m_coveringTestTargets) + { + coveringTestTargets.insert(testTarget); + } + } + }; + + if (const auto outputSources = m_autogenInputToOutputMap.find(path); outputSources != m_autogenInputToOutputMap.end()) + { + // Consolidate the parentage and coverage of each of the autogen input file's generated output files + for (const auto& outputSource : outputSources->second) + { + getSourceDependency(outputSource); + } + } + else + { + getSourceDependency(path); + } + + if (!parentTargets.empty() || !coveringTestTargets.empty()) + { + return SourceDependency(path, DependencyData{ AZStd::move(parentTargets), AZStd::move(coveringTestTargets) }); + } + + return AZStd::nullopt; + } + + SourceDependency DynamicDependencyMap::GetSourceDependencyOrThrow(const AZStd::string& path) const + { + auto sourceDependency = GetSourceDependency(path); + AZ_TestImpact_Eval(sourceDependency.has_value(), DependencyException, AZStd::string::format("Couldn't find source %s", path.c_str()).c_str()); + return sourceDependency.value(); + } + + SourceCoveringTestsList DynamicDependencyMap::ExportSourceCoverage() const + { + AZStd::vector coverage; + for (const auto& [path, dependency] : m_sourceDependencyMap) + { + AZStd::vector souceCoveringTests; + for (const auto& testTarget : dependency.m_coveringTestTargets) + { + souceCoveringTests.push_back(testTarget->GetName()); + } + + coverage.push_back(SourceCoveringTests(path, AZStd::move(souceCoveringTests))); + } + + return coverage; + } + + AZStd::vector DynamicDependencyMap::GetOrphanSourceFiles() const + { + AZStd::vector orphans; + for (const auto& [source, dependency] : m_sourceDependencyMap) + { + if (dependency.m_parentTargets.empty()) + { + orphans.push_back(source); + } + } + + return orphans; + } + + ChangeDependencyList DynamicDependencyMap::ApplyAndResoveChangeList(const ChangeList& changeList) + { + AZStd::vector createDependencies; + AZStd::vector updateDependencies; + AZStd::vector deleteDependencies; + + // Keep track of the coverage to delete as a post step rather than deleting it in situ so that erroneous change lists + // do not corrupt the dynamic dependency map + AZStd::vector coverageToDelete; + + // Create operations + for (const auto& createdFile : changeList.m_createdFiles) + { + auto sourceDependency = GetSourceDependency(createdFile); + if (sourceDependency.has_value()) + { + if (sourceDependency->GetNumParentTargets()) + { + if (sourceDependency->GetNumCoveringTestTargets()) + { + const AZStd::string msg = AZStd::string::format("The newly-created file %s belongs to a build target yet " + "still has coverage data in the source covering test list implying that a delete CRUD operation has been " + "missed, thus the integrity of the source covering test list has been compromised", createdFile.c_str()); + AZ_Error("File Creation", false, msg.c_str()); + throw DependencyException(msg); + } + else + { + createDependencies.emplace_back(AZStd::move(*sourceDependency)); + } + } + else + { + if (sourceDependency->GetNumCoveringTestTargets()) + { + const AZStd::string msg = AZStd::string::format("The newly-created file %s does not belong to a build target " + "yet still has coverage data in the source covering test list, implying that a delete CRUD operation has been " + "missed, thus the integrity of the source covering test list has been compromised", createdFile.c_str()); + AZ_Error("File Creation", false, msg.c_str()); + throw DependencyException(msg); + } + } + } + } + + // Update operations + for (const auto& updatedFile : changeList.m_updatedFiles) + { + auto sourceDependency = GetSourceDependency(updatedFile); + if (sourceDependency.has_value()) + { + if (sourceDependency->GetNumParentTargets()) + { + updateDependencies.emplace_back(AZStd::move(*sourceDependency)); + } + else + { + if (sourceDependency->GetNumCoveringTestTargets()) + { + AZ_Warning( + "File Update", false, AZStd::string::format("Source file %s is potentially an orphan (used by build targets " + "without explicitly being added to the build system, e.g. an include directive pulling in a header from the " + "repository). Running the covering tests for this file with instrumentation will confirm whether or nor this " + "is the case", updatedFile.c_str()).c_str()); + + updateDependencies.emplace_back(AZStd::move(*sourceDependency)); + coverageToDelete.push_back(updatedFile); + } + } + } + } + + // Delete operations + for (const auto& deletedFile : changeList.m_deletedFiles) + { + auto sourceDependency = GetSourceDependency(deletedFile); + if (sourceDependency.has_value()) + { + if (sourceDependency->GetNumParentTargets()) + { + if (sourceDependency->GetNumCoveringTestTargets()) + { + const AZStd::string msg = AZStd::string::format("The deleted file %s still belongs to a build target and still " + "has coverage data in the source covering test list, implying that the integrity of both the source to target " + "mappings and the source covering test list has been compromised", deletedFile.c_str()); + AZ_Error("File Delete", false, msg.c_str()); + throw DependencyException(msg); + } + else + { + const AZStd::string msg = AZStd::string::format("The deleted file %s still belongs to a build target implying " + "that the integrity of the source to target mappings has been compromised", deletedFile.c_str()); + AZ_Error("File Delete", false, msg.c_str()); + throw DependencyException(msg); + } + } + else + { + if (sourceDependency->GetNumCoveringTestTargets()) + { + deleteDependencies.emplace_back(AZStd::move(*sourceDependency)); + coverageToDelete.push_back(deletedFile); + } + } + } + } + + if (!coverageToDelete.empty()) + { + ClearSourceCoverage(coverageToDelete); + } + + return ChangeDependencyList(AZStd::move(createDependencies), AZStd::move(updateDependencies), AZStd::move(deleteDependencies)); + } +} diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.h new file mode 100644 index 0000000000..0057fdc264 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.h @@ -0,0 +1,120 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace TestImpact +{ + //! Representation of the repository source tree and its relation to the build targets and coverage data. + class DynamicDependencyMap + { + public: + //! Constructs the dependency map with entries for each build target's source files with empty test coverage data. + DynamicDependencyMap( + AZStd::vector&& productionTargetDescriptors, + AZStd::vector&& testTargetDescriptors); + + //! Gets the total number of production and test targets in the repository. + size_t GetNumTargets() const; + + //! Gets the total number of unique source files in the repository. + //! @note This includes autogen output sources. + size_t GetNumSources() const; + + //! Attempts to get the specified build target. + //! @param name The name of the build target to get. + //! @returns If found, the pointer to the specified build target, otherwise nullptr. + const BuildTarget* GetBuildTarget(const AZStd::string& name) const; + + //! Attempts to get the specified build target or throw TargetException. + //! @param name The name of the build target to get. + const BuildTarget* GetBuildTargetOrThrow(const AZStd::string& name) const; + + //! Attempts to get the specified target's specialized type. + //! @param name The name of the target to get. + //! @returns If found, the pointer to the specialized target, otherwise AZStd::monostate. + AZStd::variant GetTarget(const AZStd::string& name) const; + + //! Attempts to get the specified target's specialized type or throw TargetException. + //! @param name The name of the target to get. + AZStd::variant GetTargetOrThrow(const AZStd::string& name) const; + + //! Get the list of production targets in the repository. + const ProductionTargetList& GetProductionTargetList() const; + + //! Get the list of test targets in the repository. + const TestTargetList& GetTestTargetList() const; + + //! Gets the test targets covering the specified production target. + //! @param productionTarget The production target to retrieve the covering tests for. + AZStd::vector GetCoveringTestTargetsForProductionTarget(const ProductionTarget& productionTarget) const; + + //! Gets the source dependency for the specified source file. + //! @note Autogen input source dependencies are the consolidated source dependencies of all of their generated output sources. + //! @returns If found, the source dependency information for the specified source file, otherwise empty. + AZStd::optional GetSourceDependency(const AZStd::string& path) const; + + //! Gets the source dependency for the specified source file or throw DependencyException. + SourceDependency GetSourceDependencyOrThrow(const AZStd::string& path) const; + + //! Replaces the source coverage of the specified sources with the specified source coverage. + //! @note The covering targets for the parent test target(s) will not be pruned if those covering targets are removed. + //! @param sourceCoverageDelta The source coverage delta to replace in the dependency map. + void ReplaceSourceCoverage(const SourceCoveringTestsList& sourceCoverageDelta); + + //! Exports the coverage of all sources in the dependency map. + SourceCoveringTestsList ExportSourceCoverage() const; + + //! Gets the list of orphaned source files in the dependency map that have coverage data but belong to no parent build targets. + AZStd::vector GetOrphanSourceFiles() const; + + //! Applies the specified change list to the dynamic dependency map and resolves the change list to a change dependency list + //! containing the updated source dependencies for each source file in the change list. + //! @param changeList The change list to apply and resolve. + //! @returns The change list as resolved to the appropriate source dependencies. + [[nodiscard]] ChangeDependencyList ApplyAndResoveChangeList(const ChangeList& changeList); + + private: + //! Clears the source coverage of the specified sources. + //! @note The covering targets for the parent test target(s) will not be pruned if those covering targets are removed. + void ClearSourceCoverage(const AZStd::vector& paths); + + //! The sorted list of unique production targets in the repository. + ProductionTargetList m_productionTargets; + + //! The sorted list of unique test targets in the repository. + TestTargetList m_testTargets; + + //! The dependency map of sources to their parent build targets and covering test targets. + AZStd::unordered_map m_sourceDependencyMap; + + //! The map of build targets and their covering test targets. + AZStd::unordered_map> m_buildTargetCoverage; + + //! Mapping of autogen input sources to their generated output sources. + AZStd::unordered_map> m_autogenInputToOutputMap; + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsList.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsList.cpp new file mode 100644 index 0000000000..57a11e9f69 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsList.cpp @@ -0,0 +1,58 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include + +#include + +namespace TestImpact +{ + SourceCoveringTests::SourceCoveringTests(const AZStd::string& path, AZStd::vector&& coveringTestTargets) + : m_path(path) + , m_coveringTestTargets(AZStd::move(coveringTestTargets)) + { + } + + const AZStd::string& SourceCoveringTests::GetPath() const + { + return m_path; + } + + size_t SourceCoveringTests::GetNumCoveringTestTargets() const + { + return m_coveringTestTargets.size(); + } + + const AZStd::vector& SourceCoveringTests::GetCoveringTestTargets() const + { + return m_coveringTestTargets; + } + + SourceCoveringTestsList::SourceCoveringTestsList(AZStd::vector&& sourceCoveringTests) + : m_coverage(AZStd::move(sourceCoveringTests)) + { + AZStd::sort(m_coverage.begin(), m_coverage.end(), [](const SourceCoveringTests& lhs, const SourceCoveringTests& rhs) + { + return lhs.GetPath() < rhs.GetPath(); + }); + } + + size_t SourceCoveringTestsList::GetNumSources() const + { + return m_coverage.size(); + } + + const AZStd::vector& SourceCoveringTestsList::GetCoverage() const + { + return m_coverage; + } +} diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsList.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsList.h new file mode 100644 index 0000000000..68b09d9e5a --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsList.h @@ -0,0 +1,53 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include +#include + +namespace TestImpact +{ + //! Represents the unresolved test target coverage for a given source file. + class SourceCoveringTests + { + public: + SourceCoveringTests(const AZStd::string& path, AZStd::vector&& coveringTestTargets); + + //! Returns the path of this source file. + const AZStd::string& GetPath() const; + + //! Returns the number of unresolved test targets covering this source file. + size_t GetNumCoveringTestTargets() const; + + //! Returns the unresolved test targets covering this source file. + const AZStd::vector& GetCoveringTestTargets() const; + private: + AZStd::string m_path; //!< The path of this source file. + AZStd::vector m_coveringTestTargets; //!< The unresolved test targets that cover this source file. + }; + + //! Sorted collection of source file test coverage. + class SourceCoveringTestsList + { + public: + SourceCoveringTestsList(AZStd::vector&& sourceCoveringTests); + + //! Returns the number of source files in the collection. + size_t GetNumSources() const; + + //! Returns the source file coverages. + const AZStd::vector& GetCoverage() const; + private: + AZStd::vector m_coverage; //!< The collection of source file coverages. + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestCoverage.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestCoverage.cpp index 3a6f5d0b79..0d50dbbf9a 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestCoverage.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestCoverage.cpp @@ -25,7 +25,7 @@ namespace TestImpact for (const auto& sourceCovered : moduleCovered.m_sources) { m_sourcesCovered.emplace_back(sourceCovered.m_path); - if (sourceCovered.m_coverage.has_value()) + if (!sourceCovered.m_coverage.empty()) { m_coverageLevel = CoverageLevel::Line; } @@ -51,7 +51,7 @@ namespace TestImpact return m_modules.size(); } - const AZStd::vector& TestCoverage::GetSourcesCovered() const + const AZStd::vector& TestCoverage::GetSourcesCovered() const { return m_sourcesCovered; } diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestCoverage.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestCoverage.h index ab92b1bd80..7253a29df0 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestCoverage.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestCoverage.h @@ -38,7 +38,7 @@ namespace TestImpact size_t GetNumModulesCovered() const; //! Returns the sorted set of unique sources covered (empty if no coverage). - const AZStd::vector& GetSourcesCovered() const; + const AZStd::vector& GetSourcesCovered() const; //! Returns the modules covered (empty if no coverage). const AZStd::vector& GetModuleCoverages() const; @@ -48,7 +48,7 @@ namespace TestImpact private: AZStd::vector m_modules; - AZStd::vector m_sourcesCovered; + AZStd::vector m_sourcesCovered; AZStd::optional m_coverageLevel; }; } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestCoverageTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestCoverageTest.cpp index 538097d11e..c1d7ba3e4b 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestCoverageTest.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestCoverageTest.cpp @@ -19,12 +19,12 @@ namespace UnitTest { namespace { - AZ::IO::Path GenerateSourcePath(AZ::u32 index) + AZStd::string GenerateSourcePath(AZ::u32 index) { return AZStd::string::format("SourceFile%u", index); } - AZ::IO::Path GenerateModulePath(AZ::u32 index) + AZStd::string GenerateModulePath(AZ::u32 index) { return AZStd::string::format("Module%u", index); } @@ -47,7 +47,7 @@ namespace UnitTest sourceCoverage.m_path = GenerateSourcePath(index); if (coverageLevel == TestImpact::CoverageLevel::Line) { - sourceCoverage.m_coverage.emplace(GenerateLineCoverages(index + 1)); + sourceCoverage.m_coverage = GenerateLineCoverages(index + 1); } return sourceCoverage; @@ -152,9 +152,9 @@ namespace UnitTest if (m_coverageLevel == TestImpact::CoverageLevel::Line) { // Expect there to actually be line coverage data if this coverage was procedurally generated with line data - EXPECT_TRUE(sourceCoverage.m_coverage.has_value()); + EXPECT_FALSE(sourceCoverage.m_coverage.empty()); - const AZStd::vector& lineCoverages = sourceCoverage.m_coverage.value(); + const AZStd::vector& lineCoverages = sourceCoverage.m_coverage; // Expect the source's number of lines to match that of the corresponding procedurally generated source EXPECT_EQ(lineCoverages.size(), sourceIndex + 1); @@ -171,7 +171,7 @@ namespace UnitTest else { // Do not expect there to actually be line coverage data if this coverage was not procedurally generated with line data - EXPECT_FALSE(sourceCoverage.m_coverage.has_value()); + EXPECT_TRUE(sourceCoverage.m_coverage.empty()); } } } diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_files.cmake b/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_files.cmake index fd65a16e39..f4128f7002 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_files.cmake +++ b/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_files.cmake @@ -10,11 +10,37 @@ # set(FILES + Include/TestImpactFramework/TestImpactBitwise.h + Include/TestImpactFramework/TestImpactCallback.h Include/TestImpactFramework/TestImpactException.h Include/TestImpactFramework/TestImpactFrameworkPath.h - Include/TestImpactFramework/TestImpactCallback.h - Source/TestImpactException.cpp - Source/TestImpactFrameworkPath.cpp + Source/Artifact/TestImpactArtifactException.h + Source/Artifact/Factory/TestImpactBuildTargetDescriptorFactory.cpp + Source/Artifact/Factory/TestImpactBuildTargetDescriptorFactory.h + Source/Artifact/Factory/TestImpactChangeListFactory.cpp + Source/Artifact/Factory/TestImpactChangeListFactory.h + Source/Artifact/Factory/TestImpactTestEnumerationSuiteFactory.cpp + Source/Artifact/Factory/TestImpactTestEnumerationSuiteFactory.h + Source/Artifact/Factory/TestImpactTestRunSuiteFactory.cpp + Source/Artifact/Factory/TestImpactTestRunSuiteFactory.h + Source/Artifact/Factory/TestImpactTestTargetMetaMapFactory.cpp + Source/Artifact/Factory/TestImpactTestTargetMetaMapFactory.h + Source/Artifact/Factory/TestImpactModuleCoverageFactory.cpp + Source/Artifact/Factory/TestImpactModuleCoverageFactory.h + Source/Artifact/Static/TestImpactBuildTargetDescriptor.cpp + Source/Artifact/Static/TestImpactBuildTargetDescriptor.h + Source/Artifact/Static/TestImpactTargetDescriptorCompiler.cpp + Source/Artifact/Static/TestImpactTargetDescriptorCompiler.h + Source/Artifact/Static/TestImpactProductionTargetDescriptor.cpp + Source/Artifact/Static/TestImpactProductionTargetDescriptor.h + Source/Artifact/Static/TestImpactTestTargetMeta.h + Source/Artifact/Static/TestImpactTestTargetDescriptor.cpp + Source/Artifact/Static/TestImpactTestTargetDescriptor.h + Source/Artifact/Dynamic/TestImpactChangelist.h + Source/Artifact/Dynamic/TestImpactTestEnumerationSuite.h + Source/Artifact/Dynamic/TestImpactTestRunSuite.h + Source/Artifact/Dynamic/TestImpactTestSuite.h + Source/Artifact/Dynamic/TestImpactCoverage.h Source/Process/TestImpactProcess.cpp Source/Process/TestImpactProcess.h Source/Process/TestImpactProcessException.h @@ -26,4 +52,40 @@ set(FILES Source/Process/JobRunner/TestImpactProcessJobRunner.h Source/Process/Scheduler/TestImpactProcessScheduler.cpp Source/Process/Scheduler/TestImpactProcessScheduler.h + + Source/Target/TestImpactBuildTarget.cpp + Source/Target/TestImpactBuildTarget.h + Source/Target/TestImpactBuildTargetList.h + Source/Target/TestImpactProductionTarget.cpp + Source/Target/TestImpactProductionTarget.h + Source/Target/TestImpactProductionTargetList.h + Source/Target/TestImpactTargetException.h + Source/Target/TestImpactTestTarget.cpp + Source/Target/TestImpactTestTarget.h + Source/Target/TestImpactTestTargetList.h + Source/Test/Enumeration/TestImpactTestEnumeration.h + Source/Test/Enumeration/TestImpactTestEnumerationException.h + Source/Test/Enumeration/TestImpactTestEnumerationSerializer.cpp + Source/Test/Enumeration/TestImpactTestEnumerationSerializer.h + Source/Test/Enumeration/TestImpactTestEnumerator.cpp + Source/Test/Enumeration/TestImpactTestEnumerator.h + Source/Test/Run/TestImpactTestRunSerializer.cpp + Source/Test/Run/TestImpactTestRunSerializer.h + Source/Test/Run/TestImpactTestRunner.cpp + Source/Test/Run/TestImpactTestRunner.h + Source/Test/Run/TestImpactInstrumentedTestRunner.cpp + Source/Test/Run/TestImpactInstrumentedTestRunner.h + Source/Test/Run/TestImpactTestRun.cpp + Source/Test/Run/TestImpactTestRun.h + Source/Test/Run/TestImpactTestRunJobData.cpp + Source/Test/Run/TestImpactTestRunJobData.h + Source/Test/Run/TestImpactTestCoverage.cpp + Source/Test/Run/TestImpactTestCoverage.h + Source/Test/Run/TestImpactTestRunException.h + Source/Test/Job/TestImpactTestJobRunner.h + Source/Test/Job/TestImpactTestJobException.h + Source/Test/Job/TestImpactTestJobCommon.h + Source/Test/TestImpactTestSuiteContainer.h + Source/TestImpactException.cpp + Source/TestImpactFrameworkPath.cpp ) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_tests_files.cmake b/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_tests_files.cmake index 61059dc543..13f788c37b 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_tests_files.cmake +++ b/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_tests_files.cmake @@ -10,13 +10,29 @@ # set(FILES + Tests/Artifact/TestImpactTargetDescriptorCompilerTest.cpp + Tests/Artifact/TestImpactBuildTargetDescriptorFactoryTest.cpp + Tests/Artifact/TestImpactModuleCoverageFactoryTest.cpp + Tests/Artifact/TestImpactChangeListFactoryTest.cpp + Tests/Artifact/TestImpactTestEnumerationSuiteFactoryTest.cpp + Tests/Artifact/TestImpactTestRunSuiteFactoryTest.cpp + Tests/Artifact/TestImpactTestTargetMetaMapFactoryTest.cpp + Tests/Process/TestImpactProcessSchedulerTest.cpp + Tests/Process/TestImpactProcessTest.cpp + Tests/Target/TestImpactBuildTargetTest.cpp Tests/TestImpactExceptionTest.cpp Tests/TestImpactFrameworkPathTest.cpp - Tests/TestImpactProcessSchedulerTest.cpp - Tests/TestImpactProcessTest.cpp - Tests/TestImpactProcessTestShared.cpp - Tests/TestImpactProcessTestShared.h + Tests/Test/TestImpactTestEnumeratorTest.cpp + Tests/Test/TestImpactTestEumerationSerializerTest.cpp + Tests/Test/TestImpactTestRunSerializerTest.cpp + Tests/Test/TestImpactTestRunnerTest.cpp + Tests/Test/TestImpactInstrumentedTestRunnerTest.cpp + Tests/Test/TestImpactTestCoverageTest.cpp + Tests/TestImpactTestJobRunnerCommon.h Tests/TestImpactTestMain.cpp + Tests/TestImpactTestUtils.cpp + Tests/TestImpactTestUtils.h + ) From 692dd21b892dfc647053734230e26bda3bbfa9b4 Mon Sep 17 00:00:00 2001 From: jonawals Date: Wed, 28 Apr 2021 17:26:28 +0100 Subject: [PATCH 004/102] Add missing SourceDependency. --- .../Dependency/TestImpactSourceDependency.cpp | 82 ++++++++++++++++ .../Dependency/TestImpactSourceDependency.h | 96 +++++++++++++++++++ .../testimpactframework_runtime_files.cmake | 10 +- 3 files changed, 187 insertions(+), 1 deletion(-) create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceDependency.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceDependency.h diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceDependency.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceDependency.cpp new file mode 100644 index 0000000000..49d82e341a --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceDependency.cpp @@ -0,0 +1,82 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include +#include +#include +#include + +namespace TestImpact +{ + ParentTarget::ParentTarget(const TestTarget* target) + : m_buildTarget(target) + , m_target(target) + { + } + + ParentTarget::ParentTarget(const ProductionTarget* target) + : m_buildTarget(target) + , m_target(target) + { + } + + bool ParentTarget::operator==(const ParentTarget& other) const + { + return m_buildTarget == other.m_buildTarget; + } + + const BuildTarget* ParentTarget::GetBuildTarget() const + { + return m_buildTarget; + } + + const AZStd::variant& ParentTarget::GetTarget() const + { + return m_target; + } + + SourceDependency::SourceDependency( + const AZStd::string& path, + DependencyData&& dependencyData) + : m_path(path) + , m_dependencyData(AZStd::move(dependencyData)) + { + } + + const AZStd::string& SourceDependency::GetPath() const + { + return m_path; + } + + size_t SourceDependency::GetNumParentTargets() const + { + return m_dependencyData.m_parentTargets.size(); + } + + size_t SourceDependency::GetNumCoveringTestTargets() const + { + return m_dependencyData.m_coveringTestTargets.size(); + } + + const AZStd::unordered_set& SourceDependency::GetParentTargets() const + { + return m_dependencyData.m_parentTargets; + } + + const AZStd::unordered_set& SourceDependency::GetCoveringTestTargets() const + { + return m_dependencyData.m_coveringTestTargets; + } + +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceDependency.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceDependency.h new file mode 100644 index 0000000000..e6cd89df9d --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceDependency.h @@ -0,0 +1,96 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include +#include +#include +#include +#include + +namespace TestImpact +{ + class BuildTarget; + class ProductionTarget; + class TestTarget; + + //! Representation of a source dependency's parent target. + class ParentTarget + { + public: + //! Constructor overload for test target types. + ParentTarget(const TestTarget* target); + + //! Constructor overload for production target types. + ParentTarget(const ProductionTarget* target); + + //! Returns the base build target pointer for this parent. + const BuildTarget* GetBuildTarget() const; + + //! Returns the specialized target pointer for this parent. + const AZStd::variant& GetTarget() const; + + bool operator==(const ParentTarget& other) const; + private: + const BuildTarget* m_buildTarget; //! The base built target pointer for this parent. + AZStd::variant m_target; //! The specialized target pointer for this parent. + }; +} + +namespace AZStd +{ + //! Hash function for ParentTarget types for use in maps and sets + template<> struct hash + { + size_t operator()(const TestImpact::ParentTarget& parentTarget) const noexcept + { + return reinterpret_cast(parentTarget.GetBuildTarget()); + } + }; +} + +namespace TestImpact +{ + struct DependencyData + { + AZStd::unordered_set m_parentTargets; + AZStd::unordered_set m_coveringTestTargets; + }; + + //! Test target coverage and build target dependency information for a given source file in the dynamic dependency map. + class SourceDependency + { + public: + SourceDependency( + const AZStd::string& path, + DependencyData&& dependencyData); + + //! Returns the path of this source file. + const AZStd::string& GetPath() const; + + //! Returns the number of parent build targets this source belongs to. + size_t GetNumParentTargets() const; + + //! Returns the number of test targets covering this source file. + size_t GetNumCoveringTestTargets() const; + + //! Returns the parent targets that this source file belongs to. + const AZStd::unordered_set& GetParentTargets() const; + + //! Returns the test targets covering this source file. + const AZStd::unordered_set& GetCoveringTestTargets() const; + private: + AZStd::string m_path; //!< The path of this source file. + DependencyData m_dependencyData; //!< + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_files.cmake b/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_files.cmake index f4128f7002..19f3ec23f7 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_files.cmake +++ b/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_files.cmake @@ -52,7 +52,15 @@ set(FILES Source/Process/JobRunner/TestImpactProcessJobRunner.h Source/Process/Scheduler/TestImpactProcessScheduler.cpp Source/Process/Scheduler/TestImpactProcessScheduler.h - + Source/Dependency/TestImpactDynamicDependencyMap.cpp + Source/Dependency/TestImpactDynamicDependencyMap.h + Source/Dependency/TestImpactChangeDependencyList.cpp + Source/Dependency/TestImpactChangeDependencyList.h + Source/Dependency/TestImpactDependencyException.h + Source/Dependency/TestImpactSourceDependency.h + Source/Dependency/TestImpactSourceDependency.cpp + Source/Dependency/TestImpactSourceCoveringTestsList.h + Source/Dependency/TestImpactSourceCoveringTestsList.cpp Source/Target/TestImpactBuildTarget.cpp Source/Target/TestImpactBuildTarget.h Source/Target/TestImpactBuildTargetList.h From 2b21b70635ef0d78b3af55adcf3c3018d491b840 Mon Sep 17 00:00:00 2001 From: jonawals Date: Thu, 29 Apr 2021 08:28:42 +0100 Subject: [PATCH 005/102] Add test selector and prioritizor. --- .../TestImpactModuleCoverageFactory.cpp | 13 +- .../Static/TestImpactDependencyGraphData.h | 26 ++ .../TestImpactTestSelectorAndPrioritizer.cpp | 224 ++++++++++++++++++ .../TestImpactTestSelectorAndPrioritizer.h | 77 ++++++ .../testimpactframework_runtime_files.cmake | 3 + 5 files changed, 333 insertions(+), 10 deletions(-) create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactDependencyGraphData.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactTestSelectorAndPrioritizer.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactTestSelectorAndPrioritizer.h diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactModuleCoverageFactory.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactModuleCoverageFactory.cpp index 54ba65c762..889f91c600 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactModuleCoverageFactory.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactModuleCoverageFactory.cpp @@ -97,7 +97,7 @@ namespace TestImpact { // Module ModuleCoverage moduleCoverage; - moduleCoverage.m_path = AZ::IO::Path(package_node->first_attribute(Keys[NameKey])->value()); + moduleCoverage.m_path = package_node->first_attribute(Keys[NameKey])->value(); const auto classes_node = package_node->first_node(Keys[ClassesKey]); if (classes_node) @@ -107,13 +107,11 @@ namespace TestImpact { // Source SourceCoverage sourceCoverage; - sourceCoverage.m_path = AZ::IO::Path(pathRoot + class_node->first_attribute(Keys[FileNameKey])->value()); + sourceCoverage.m_path = pathRoot + class_node->first_attribute(Keys[FileNameKey])->value(); const auto lines_node = class_node->first_node(Keys[LinesKey]); if (lines_node) { - AZStd::vector lineCoverage; - // Lines for (auto line_node = lines_node->first_node(); line_node; line_node = line_node->next_sibling()) { @@ -121,12 +119,7 @@ namespace TestImpact const size_t number = AZStd::stol(AZStd::string(line_node->first_attribute(Keys[NumberKey])->value())); const size_t hits = AZStd::stol(AZStd::string(line_node->first_attribute(Keys[HitsKey])->value())); - lineCoverage.emplace_back(LineCoverage{number, hits}); - } - - if (!lineCoverage.empty()) - { - sourceCoverage.m_coverage.emplace(AZStd::move(lineCoverage)); + sourceCoverage.m_coverage.emplace_back(LineCoverage{number, hits}); } } diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactDependencyGraphData.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactDependencyGraphData.h new file mode 100644 index 0000000000..ee8ca722b4 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactDependencyGraphData.h @@ -0,0 +1,26 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include +#include + +namespace TestImpact +{ + struct DependencyGraphData + { + AZStd::string m_root; + AZStd::vector m_vertexes; + AZStd::vector> m_edges; + }; +} diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactTestSelectorAndPrioritizer.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactTestSelectorAndPrioritizer.cpp new file mode 100644 index 0000000000..f8213738e5 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactTestSelectorAndPrioritizer.cpp @@ -0,0 +1,224 @@ +/* +* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +* its licensors. +* +* For complete copyright and license terms please see the LICENSE at the root of this +* distribution (the "License"). All use of this software is governed by the License, +* or, if provided, by the license below or the license accompanying this file. Do not +* remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* +*/ + +#include +#include +#include +#include + +namespace TestImpact +{ + TestSelectorAndPrioritizer::TestSelectorAndPrioritizer(const DynamicDependencyMap* dynamicDependencyMap, DependencyGraphDataMap&& dependencyGraphDataMap) + : m_dynamicDependencyMap(dynamicDependencyMap) + , m_dependencyGraphDataMap(AZStd::move(dependencyGraphDataMap)) + { + } + + AZStd::vector TestSelectorAndPrioritizer::SelectTestTargets(const ChangeDependencyList& changeDependencyList, TestSelectionStrategy testSelectionStrategy) + { + const auto selectedTestTargetAndDependerMap = SelectTestTargets(changeDependencyList); + const auto prioritizedSelectedTests = PrioritizeSelectedTestTargets(selectedTestTargetAndDependerMap, testSelectionStrategy); + return prioritizedSelectedTests; + } + + TestSelectorAndPrioritizer::SelectedTestTargetAndDependerMap TestSelectorAndPrioritizer::SelectTestTargets( + const ChangeDependencyList& changeDependencyList) + { + SelectedTestTargetAndDependerMap selectedTestTargetMap; + + // Create operations + for (const auto& sourceDependency : changeDependencyList.GetCreateSourceDependencies()) + { + for (const auto& parentTarget : sourceDependency.GetParentTargets()) + { + AZStd::visit([&selectedTestTargetMap, this](auto&& target) + { + if constexpr (IsProductionTarget) + { + // Parent Targets: Yes + // Coverage Data : No + // Source Type : Production + // + // Scenario + // 1. The file has been newly created + // 2. This file exists in one or more source to production target mapping artifacts + // 3. There exists no coverage data for this file in the source covering test list + // + // Action + // 1. Select all test targets covering the parent production targets + const auto coverage = m_dynamicDependencyMap->GetCoveringTestTargetsForProductionTarget(*target); + for (const auto* testTarget : coverage) + { + selectedTestTargetMap[testTarget].insert(target); + } + } + else + { + // Parent Targets: Yes + // Coverage Data : No + // Source Type : Test + // + // Scenario + // 1. The file has been newly created + // 2. This file exists in one or more source to test target mapping artifacts + // 3. There exists no coverage data for this file in the source covering test list + // + // Action + // 1. Select all parent test targets + selectedTestTargetMap.insert(target); + } + }, parentTarget.GetTarget()); + } + } + + // Update operations + for (const auto& sourceDependency : changeDependencyList.GetUpdateSourceDependencies()) + { + if (sourceDependency.GetNumParentTargets()) + { + if (sourceDependency.GetNumCoveringTestTargets()) + { + for (const auto& parentTarget : sourceDependency.GetParentTargets()) + { + AZStd::visit([&selectedTestTargetMap, &sourceDependency, this](auto&& target) + { + if constexpr (IsProductionTarget) + { + // Parent Targets: Yes + // Coverage Data : Yes + // Source Type : Production + // + // Scenario + // 1. The existing file has been modified + // 2. This file exists in one or more source to production target mapping artifacts + // 3. There exists coverage data for this file in the source covering test list + // + // Action + // 1. Select all test targets covering this file + for (const auto* testTarget : sourceDependency.GetCoveringTestTargets()) + { + selectedTestTargetMap[testTarget].insert(target); + } + } + else + { + // Parent Targets: Yes + // Coverage Data : Yes + // Source Type : Test + // + // Scenario + // 1. The existing file has been modified + // 2. This file exists in one or more source to test target mapping artifacts + // 3. There exists coverage data for this file in the source covering test list + // + // Action + // 1. Select the parent test targets for this file + selectedTestTargetMap.insert(target); + } + }, parentTarget.GetTarget()); + } + } + else + { + for (const auto& parentTarget : sourceDependency.GetParentTargets()) + { + AZStd::visit([&selectedTestTargetMap, &sourceDependency, this](auto&& target) + { + if constexpr (IsTestTarget) + { + // Parent Targets: Yes + // Coverage Data : No + // Source Type : Test + // + // Scenario + // 1. The existing file has been modified + // 2. This file exists in one or more source to test target mapping artifacts + // 3. There exists no coverage data for this file in the source covering test list + // + // Action + // 1. Select the parent test targets for this file + selectedTestTargetMap.insert(target); + } + }, parentTarget.GetTarget()); + } + } + } + else + { + // Parent Targets: No + // Coverage Data : Yes + // Source Type : Indeterminate + // + // Scenario + // 1. The existing file has been modified + // 2. Either: + // a) This file previously existed in one or more source to target mapping artifacts + // b) This file no longer exists in any source to target mapping artifacts + // c) The coverage data for this file was has yet to be deleted from the source covering test list + // 3. Or: + // a) The file is being used by build targets but has erroneously not been explicitly added to the build + // system (e.g. include directive pulling in a header from the repository that has not been added to + // any build targets due to an oversight) + // + // Action + // 1. Log potential orphaned source file warning + // 2. Select all test targets covering this file + // 3. Delete the existing coverage data from the source covering test list + + for (const auto* testTarget : sourceDependency.GetCoveringTestTargets()) + { + selectedTestTargetMap.insert(testTarget); + } + } + } + + // Delete operations + for (const auto& sourceDependency : changeDependencyList.GetDeleteSourceDependencies()) + { + // Parent Targets: No + // Coverage Data : Yes + // Source Type : Indeterminate + // + // Scenario + // 1. The existing file has been deleted + // 2. This file previously existed in one or more source to target mapping artifacts + // 2. This file does not exist in any source to target mapping artifacts + // 4. The coverage data for this file was has yet to be deleted from the source covering test list + // + // Action + // 1. Select all test targets covering this file + // 2. Delete the existing coverage data from the source covering test list + for (const auto* testTarget : sourceDependency.GetCoveringTestTargets()) + { + selectedTestTargetMap.insert(testTarget); + } + } + + return selectedTestTargetMap; + } + + AZStd::vector TestSelectorAndPrioritizer::PrioritizeSelectedTestTargets( + const SelectedTestTargetAndDependerMap& selectedTestTargetAndDependerMap, + [[maybe_unused]]TestSelectionStrategy testSelectionStrategy) + { + AZStd::vector selectedTestTargets; + + // Prioritization disabled for now + // https://jira.agscollab.com/browse/SPEC-6563 + for (const auto& [testTarget, dependerTargets] : selectedTestTargetAndDependerMap) + { + selectedTestTargets.push_back(testTarget); + } + + return selectedTestTargets; + } +} diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactTestSelectorAndPrioritizer.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactTestSelectorAndPrioritizer.h new file mode 100644 index 0000000000..261e469ee2 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactTestSelectorAndPrioritizer.h @@ -0,0 +1,77 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include +#include + +#include +#include +#include + +namespace TestImpact +{ + class DynamicDependencyMap; + class BuildTarget; + class TestTarget; + + //! Strategy for selecting tests given a set of source changes. + enum class TestSelectionStrategy : bool + { + SelectOnly, //!< Select tests only, do not attempt prioritization of those selected tests. + SelectAndPriotitize //!< Select tests and prioritize according to dependency graph locality of coverer and coveree. + }; + + //! Map of build targets and their dependency graph data. + //! For test targets, the dependency graph data is that of the build targets which the test target depends on. + //! For production targets, the dependency graph is that of the build targets that depend on it (dependers). + //! @note No dependency graph data is not an error, it simple means that the target cannot be prioritized. + using DependencyGraphDataMap = AZStd::unordered_map; + + //! Selects the test targets that cover a given set of changes based on the CRUD rules and optionally prioritizes the test + //! selection according to their locality of their covering production targets in the their dependency graphs. + //! @note the CRUD rules for how tests are selected can be found in the MicroRepo header file. + class TestSelectorAndPrioritizer + { + public: + //! Constructs the test selector and prioritizer for the given dynamic dependency map. + //! @param dynamicDependencyMap The dynamic dependency map representing the repository source tree. + //! @param dependencyGraphDataMap The map of build targets and their dependency graph data for use in test prioritization. + TestSelectorAndPrioritizer(const DynamicDependencyMap* dynamicDependencyMap, DependencyGraphDataMap&& dependencyGraphDataMap); + + //! Select the covering test targets for the given set of source changes and optionally prioritizes said test selection. + //! @param changeDependencyList The resolved list of source dependencies for the CRUD source changes. + //! @param testSelectionStrategy The test selection and prioritization strategy to apply to the given CRUD source changes. + AZStd::vector SelectTestTargets(const ChangeDependencyList& changeDependencyList, TestSelectionStrategy testSelectionStrategy); + + private: + //! Map of selected test targets and the production targets they cover for the given set of source changes. + using SelectedTestTargetAndDependerMap = AZStd::unordered_map>; + + //! Selects the test targets covering the set of source changes in the change dependency list. + //! @param changeDependencyList The change dependency list containing the CRUD source changes to select tests for. + //! @returns The selected tests and their covering production targets for the given set of source changes. + SelectedTestTargetAndDependerMap SelectTestTargets(const ChangeDependencyList& changeDependencyList); + + //! Prioritizes the selected tests according to the specified test selection strategy, + //! @note If no dependency graph data exists for a given test target then that test target still be selected albeit not prioritized. + //! @param selectedTestTargetAndDependerMap The selected tests to prioritize. + //! @param testSelectionStrategy The test selection strategy to prioritize the selected tests. + //! @returns The selected tests either in either arbitrary order or in prioritized with highest priority first. + AZStd::vector PrioritizeSelectedTestTargets( + const SelectedTestTargetAndDependerMap& selectedTestTargetAndDependerMap, TestSelectionStrategy testSelectionStrategy); + + const DynamicDependencyMap* m_dynamicDependencyMap; + DependencyGraphDataMap m_dependencyGraphDataMap; + }; +} diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_files.cmake b/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_files.cmake index 19f3ec23f7..da3693433e 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_files.cmake +++ b/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_files.cmake @@ -36,6 +36,7 @@ set(FILES Source/Artifact/Static/TestImpactTestTargetMeta.h Source/Artifact/Static/TestImpactTestTargetDescriptor.cpp Source/Artifact/Static/TestImpactTestTargetDescriptor.h + Source/Artifact/Static/TestImpactDependencyGraphData.h Source/Artifact/Dynamic/TestImpactChangelist.h Source/Artifact/Dynamic/TestImpactTestEnumerationSuite.h Source/Artifact/Dynamic/TestImpactTestRunSuite.h @@ -59,6 +60,8 @@ set(FILES Source/Dependency/TestImpactDependencyException.h Source/Dependency/TestImpactSourceDependency.h Source/Dependency/TestImpactSourceDependency.cpp + Source/Dependency/TestImpactTestSelectorAndPrioritizer.h + Source/Dependency/TestImpactTestSelectorAndPrioritizer.cpp Source/Dependency/TestImpactSourceCoveringTestsList.h Source/Dependency/TestImpactSourceCoveringTestsList.cpp Source/Target/TestImpactBuildTarget.cpp From eb5dd7ee47ee3ce0d6c4ab3fdc86fd923fff503e Mon Sep 17 00:00:00 2001 From: jonawals Date: Thu, 29 Apr 2021 14:51:51 +0100 Subject: [PATCH 006/102] Address PR comments. --- .../Source/Artifact/Static/TestImpactDependencyGraphData.h | 2 +- .../Dependency/TestImpactTestSelectorAndPrioritizer.cpp | 4 ++-- .../Source/Dependency/TestImpactTestSelectorAndPrioritizer.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactDependencyGraphData.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactDependencyGraphData.h index ee8ca722b4..8dae41c579 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactDependencyGraphData.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactDependencyGraphData.h @@ -23,4 +23,4 @@ namespace TestImpact AZStd::vector m_vertexes; AZStd::vector> m_edges; }; -} +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactTestSelectorAndPrioritizer.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactTestSelectorAndPrioritizer.cpp index f8213738e5..2962e91381 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactTestSelectorAndPrioritizer.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactTestSelectorAndPrioritizer.cpp @@ -213,7 +213,7 @@ namespace TestImpact AZStd::vector selectedTestTargets; // Prioritization disabled for now - // https://jira.agscollab.com/browse/SPEC-6563 + // SPEC-6563 for (const auto& [testTarget, dependerTargets] : selectedTestTargetAndDependerMap) { selectedTestTargets.push_back(testTarget); @@ -221,4 +221,4 @@ namespace TestImpact return selectedTestTargets; } -} +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactTestSelectorAndPrioritizer.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactTestSelectorAndPrioritizer.h index 261e469ee2..c742176073 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactTestSelectorAndPrioritizer.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactTestSelectorAndPrioritizer.h @@ -74,4 +74,4 @@ namespace TestImpact const DynamicDependencyMap* m_dynamicDependencyMap; DependencyGraphDataMap m_dependencyGraphDataMap; }; -} +} // namespace TestImpact From 20243549e6a36bd59e5b12f829b691cf88eca8ce Mon Sep 17 00:00:00 2001 From: jonawals Date: Fri, 30 Apr 2021 12:51:26 +0100 Subject: [PATCH 007/102] Address PR comments. --- .../Source/Artifact/Static/TestImpactDependencyGraphData.h | 7 ++++--- .../Dependency/TestImpactTestSelectorAndPrioritizer.cpp | 6 ++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactDependencyGraphData.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactDependencyGraphData.h index 8dae41c579..3c9f455254 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactDependencyGraphData.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactDependencyGraphData.h @@ -17,10 +17,11 @@ namespace TestImpact { + //! Raw representation of the dependency graph for a given build target. struct DependencyGraphData { - AZStd::string m_root; - AZStd::vector m_vertexes; - AZStd::vector> m_edges; + AZStd::string m_root; //!< The build target this dependency graph is for. + AZStd::vector m_vertices; //!< The depender/depending built targets in this graph. + AZStd::vector> m_edges; //!< The dependency connectivity of the build targets in this graph. }; } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactTestSelectorAndPrioritizer.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactTestSelectorAndPrioritizer.cpp index 2962e91381..549ef0c108 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactTestSelectorAndPrioritizer.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactTestSelectorAndPrioritizer.cpp @@ -17,13 +17,15 @@ namespace TestImpact { - TestSelectorAndPrioritizer::TestSelectorAndPrioritizer(const DynamicDependencyMap* dynamicDependencyMap, DependencyGraphDataMap&& dependencyGraphDataMap) + TestSelectorAndPrioritizer::TestSelectorAndPrioritizer( + const DynamicDependencyMap* dynamicDependencyMap, DependencyGraphDataMap&& dependencyGraphDataMap) : m_dynamicDependencyMap(dynamicDependencyMap) , m_dependencyGraphDataMap(AZStd::move(dependencyGraphDataMap)) { } - AZStd::vector TestSelectorAndPrioritizer::SelectTestTargets(const ChangeDependencyList& changeDependencyList, TestSelectionStrategy testSelectionStrategy) + AZStd::vector TestSelectorAndPrioritizer::SelectTestTargets( + const ChangeDependencyList& changeDependencyList, TestSelectionStrategy testSelectionStrategy) { const auto selectedTestTargetAndDependerMap = SelectTestTargets(changeDependencyList); const auto prioritizedSelectedTests = PrioritizeSelectedTestTargets(selectedTestTargetAndDependerMap, testSelectionStrategy); From 39ba583ad7ae1761e13d356bd50b02ae29e3f6b4 Mon Sep 17 00:00:00 2001 From: jonawals Date: Fri, 30 Apr 2021 12:58:48 +0100 Subject: [PATCH 008/102] Address PR comments. --- .../TestImpactDynamicDependencyMap.cpp | 110 +++++++++--------- .../TestImpactDynamicDependencyMap.h | 5 +- .../TestImpactSourceCoveringTestsList.cpp | 7 +- .../TestImpactSourceCoveringTestsList.h | 3 +- .../Dependency/TestImpactSourceDependency.cpp | 19 +-- .../Dependency/TestImpactSourceDependency.h | 11 +- .../Source/Target/TestImpactBuildTarget.h | 10 ++ 7 files changed, 89 insertions(+), 76 deletions(-) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp index f3edbe2507..f99300ad4e 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp @@ -25,7 +25,8 @@ namespace TestImpact { for (const auto& source : target->GetSources().m_staticSources) { - if (auto mapping = m_sourceDependencyMap.find(source); mapping != m_sourceDependencyMap.end()) + if (auto mapping = m_sourceDependencyMap.find(source); + mapping != m_sourceDependencyMap.end()) { // This is an existing entry in the dependency map so update the parent build targets with this target mapping->second.m_parentTargets.insert(target); @@ -97,13 +98,15 @@ namespace TestImpact return buildTarget; } - AZStd::variant DynamicDependencyMap::GetTarget(const AZStd::string& name) const + OptionalTarget DynamicDependencyMap::GetTarget(const AZStd::string& name) const { - if (auto testTarget = m_testTargets.GetTarget(name); testTarget != nullptr) + if (const auto testTarget = m_testTargets.GetTarget(name); + testTarget != nullptr) { return testTarget; } - else if (auto productionTarget = m_productionTargets.GetTarget(name); productionTarget != nullptr) + else if (auto productionTarget = m_productionTargets.GetTarget(name); + productionTarget != nullptr) { return productionTarget; } @@ -111,9 +114,9 @@ namespace TestImpact return AZStd::monostate{}; } - AZStd::variant DynamicDependencyMap::GetTargetOrThrow(const AZStd::string& name) const + Target DynamicDependencyMap::GetTargetOrThrow(const AZStd::string& name) const { - AZStd::variant buildTarget; + Target buildTarget; AZStd::visit([&buildTarget, &name](auto&& target) { if constexpr (IsProductionTarget || IsTestTarget) @@ -148,8 +151,8 @@ namespace TestImpact // Update the dependency with any new coverage data for (const auto& unresolvedTestTarget : sourceCoverage.GetCoveringTestTargets()) { - const TestTarget* testTarget = m_testTargets.GetTarget(unresolvedTestTarget); - if (testTarget) + if (const TestTarget* testTarget = m_testTargets.GetTarget(unresolvedTestTarget); + testTarget) { // Source to covering test target mapping sourceDependency.m_coveringTestTargets.insert(testTarget); @@ -179,12 +182,13 @@ namespace TestImpact { for (const auto& path : paths) { - if (const auto outputSources = m_autogenInputToOutputMap.find(path); outputSources != m_autogenInputToOutputMap.end()) + if (const auto outputSources = m_autogenInputToOutputMap.find(path); + outputSources != m_autogenInputToOutputMap.end()) { // Clearing the coverage data of an autogen input source instead clears the coverage data of its output sources for (const auto& outputSource : outputSources->second) { - ReplaceSourceCoverage(SourceCoveringTestsList({ SourceCoveringTests(outputSource, { }) })); + ReplaceSourceCoverage(SourceCoveringTestsList({ SourceCoveringTests(outputSource) })); } } else @@ -207,7 +211,8 @@ namespace TestImpact AZStd::vector DynamicDependencyMap::GetCoveringTestTargetsForProductionTarget(const ProductionTarget& productionTarget) const { AZStd::vector coveringTestTargets; - if (const auto coverage = m_buildTargetCoverage.find(&productionTarget); coverage != m_buildTargetCoverage.end()) + if (const auto coverage = m_buildTargetCoverage.find(&productionTarget); + coverage != m_buildTargetCoverage.end()) { coveringTestTargets.reserve(coverage->second.size()); AZStd::copy(coverage->second.begin(), coverage->second.end(), AZStd::back_inserter(coveringTestTargets)); @@ -280,7 +285,7 @@ namespace TestImpact coverage.push_back(SourceCoveringTests(path, AZStd::move(souceCoveringTests))); } - return coverage; + return SourceCoveringTestsList(AZStd::move(coverage)); } AZStd::vector DynamicDependencyMap::GetOrphanSourceFiles() const @@ -313,31 +318,18 @@ namespace TestImpact auto sourceDependency = GetSourceDependency(createdFile); if (sourceDependency.has_value()) { + if (sourceDependency->GetNumCoveringTestTargets()) + { + const AZStd::string msg = AZStd::string::format("The newly-created file %s belongs to a build target yet " + "still has coverage data in the source covering test list implying that a delete CRUD operation has been " + "missed, thus the integrity of the source covering test list has been compromised", createdFile.c_str()); + AZ_Error("File Creation", false, msg.c_str()); + throw DependencyException(msg); + } + if (sourceDependency->GetNumParentTargets()) { - if (sourceDependency->GetNumCoveringTestTargets()) - { - const AZStd::string msg = AZStd::string::format("The newly-created file %s belongs to a build target yet " - "still has coverage data in the source covering test list implying that a delete CRUD operation has been " - "missed, thus the integrity of the source covering test list has been compromised", createdFile.c_str()); - AZ_Error("File Creation", false, msg.c_str()); - throw DependencyException(msg); - } - else - { - createDependencies.emplace_back(AZStd::move(*sourceDependency)); - } - } - else - { - if (sourceDependency->GetNumCoveringTestTargets()) - { - const AZStd::string msg = AZStd::string::format("The newly-created file %s does not belong to a build target " - "yet still has coverage data in the source covering test list, implying that a delete CRUD operation has been " - "missed, thus the integrity of the source covering test list has been compromised", createdFile.c_str()); - AZ_Error("File Creation", false, msg.c_str()); - throw DependencyException(msg); - } + createDependencies.emplace_back(AZStd::move(*sourceDependency)); } } } @@ -373,33 +365,35 @@ namespace TestImpact for (const auto& deletedFile : changeList.m_deletedFiles) { auto sourceDependency = GetSourceDependency(deletedFile); - if (sourceDependency.has_value()) + if (!sourceDependency.has_value()) { - if (sourceDependency->GetNumParentTargets()) + continue; + } + + if (sourceDependency->GetNumParentTargets()) + { + if (sourceDependency->GetNumCoveringTestTargets()) { - if (sourceDependency->GetNumCoveringTestTargets()) - { - const AZStd::string msg = AZStd::string::format("The deleted file %s still belongs to a build target and still " - "has coverage data in the source covering test list, implying that the integrity of both the source to target " - "mappings and the source covering test list has been compromised", deletedFile.c_str()); - AZ_Error("File Delete", false, msg.c_str()); - throw DependencyException(msg); - } - else - { - const AZStd::string msg = AZStd::string::format("The deleted file %s still belongs to a build target implying " - "that the integrity of the source to target mappings has been compromised", deletedFile.c_str()); - AZ_Error("File Delete", false, msg.c_str()); - throw DependencyException(msg); - } + const AZStd::string msg = AZStd::string::format("The deleted file %s still belongs to a build target and still " + "has coverage data in the source covering test list, implying that the integrity of both the source to target " + "mappings and the source covering test list has been compromised", deletedFile.c_str()); + AZ_Error("File Delete", false, msg.c_str()); + throw DependencyException(msg); } else { - if (sourceDependency->GetNumCoveringTestTargets()) - { - deleteDependencies.emplace_back(AZStd::move(*sourceDependency)); - coverageToDelete.push_back(deletedFile); - } + const AZStd::string msg = AZStd::string::format("The deleted file %s still belongs to a build target implying " + "that the integrity of the source to target mappings has been compromised", deletedFile.c_str()); + AZ_Error("File Delete", false, msg.c_str()); + throw DependencyException(msg); + } + } + else + { + if (sourceDependency->GetNumCoveringTestTargets()) + { + deleteDependencies.emplace_back(AZStd::move(*sourceDependency)); + coverageToDelete.push_back(deletedFile); } } } @@ -411,4 +405,4 @@ namespace TestImpact return ChangeDependencyList(AZStd::move(createDependencies), AZStd::move(updateDependencies), AZStd::move(deleteDependencies)); } -} +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.h index 0057fdc264..527a6555c6 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.h @@ -24,7 +24,6 @@ #include #include #include -#include namespace TestImpact { @@ -56,11 +55,11 @@ namespace TestImpact //! Attempts to get the specified target's specialized type. //! @param name The name of the target to get. //! @returns If found, the pointer to the specialized target, otherwise AZStd::monostate. - AZStd::variant GetTarget(const AZStd::string& name) const; + OptionalTarget GetTarget(const AZStd::string& name) const; //! Attempts to get the specified target's specialized type or throw TargetException. //! @param name The name of the target to get. - AZStd::variant GetTargetOrThrow(const AZStd::string& name) const; + Target GetTargetOrThrow(const AZStd::string& name) const; //! Get the list of production targets in the repository. const ProductionTargetList& GetProductionTargetList() const; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsList.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsList.cpp index 57a11e9f69..792eb1f0b2 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsList.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsList.cpp @@ -16,6 +16,11 @@ namespace TestImpact { + SourceCoveringTests::SourceCoveringTests(const AZStd::string& path) + : m_path(path) + { + } + SourceCoveringTests::SourceCoveringTests(const AZStd::string& path, AZStd::vector&& coveringTestTargets) : m_path(path) , m_coveringTestTargets(AZStd::move(coveringTestTargets)) @@ -55,4 +60,4 @@ namespace TestImpact { return m_coverage; } -} +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsList.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsList.h index 68b09d9e5a..52f3a7c3f6 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsList.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsList.h @@ -21,6 +21,7 @@ namespace TestImpact class SourceCoveringTests { public: + explicit SourceCoveringTests(const AZStd::string& path); SourceCoveringTests(const AZStd::string& path, AZStd::vector&& coveringTestTargets); //! Returns the path of this source file. @@ -40,7 +41,7 @@ namespace TestImpact class SourceCoveringTestsList { public: - SourceCoveringTestsList(AZStd::vector&& sourceCoveringTests); + explicit SourceCoveringTestsList(AZStd::vector&& sourceCoveringTests); //! Returns the number of source files in the collection. size_t GetNumSources() const; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceDependency.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceDependency.cpp index 49d82e341a..39193d4dd9 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceDependency.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceDependency.cpp @@ -20,28 +20,33 @@ namespace TestImpact { ParentTarget::ParentTarget(const TestTarget* target) - : m_buildTarget(target) - , m_target(target) + : m_target(target) { } ParentTarget::ParentTarget(const ProductionTarget* target) - : m_buildTarget(target) - , m_target(target) + : m_target(target) { } bool ParentTarget::operator==(const ParentTarget& other) const { - return m_buildTarget == other.m_buildTarget; + return GetBuildTarget() == other.GetBuildTarget(); } const BuildTarget* ParentTarget::GetBuildTarget() const { - return m_buildTarget; + const BuildTarget* buildTarget; + AZStd::visit([&buildTarget](auto&& target) + { + buildTarget = target; + + }, m_target); + + return buildTarget; } - const AZStd::variant& ParentTarget::GetTarget() const + const Target& ParentTarget::GetTarget() const { return m_target; } diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceDependency.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceDependency.h index e6cd89df9d..a3b6783d13 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceDependency.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceDependency.h @@ -12,15 +12,15 @@ #pragma once +#include + #include #include #include #include -#include namespace TestImpact { - class BuildTarget; class ProductionTarget; class TestTarget; @@ -38,12 +38,11 @@ namespace TestImpact const BuildTarget* GetBuildTarget() const; //! Returns the specialized target pointer for this parent. - const AZStd::variant& GetTarget() const; + const Target& GetTarget() const; bool operator==(const ParentTarget& other) const; private: - const BuildTarget* m_buildTarget; //! The base built target pointer for this parent. - AZStd::variant m_target; //! The specialized target pointer for this parent. + Target m_target; //! The specialized target pointer for this parent. }; } @@ -91,6 +90,6 @@ namespace TestImpact const AZStd::unordered_set& GetCoveringTestTargets() const; private: AZStd::string m_path; //!< The path of this source file. - DependencyData m_dependencyData; //!< + DependencyData m_dependencyData; //!< The dependency data for this source file. }; } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactBuildTarget.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactBuildTarget.h index d4346c15ef..0cd6369472 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactBuildTarget.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactBuildTarget.h @@ -14,10 +14,20 @@ #include +#include #include namespace TestImpact { + class TestTarget; + class ProductionTarget; + + //! Holder for specializations of BuildTarget. + using Target = AZStd::variant; + + //! Optional holder for specializations of BuildTarget. + using OptionalTarget = AZStd::variant; + //! Type id for querying specialized derived target types from base pointer/reference. enum class TargetType : bool { From d880aa5d2257528dfd050c32d0c52def4a106504 Mon Sep 17 00:00:00 2001 From: jonawals Date: Thu, 6 May 2021 15:33:02 +0100 Subject: [PATCH 009/102] Fix anonymous namespaces for unity builds. --- ...TestImpactBuildTargetDescriptorFactory.cpp | 54 +++++++-------- .../TestImpactModuleCoverageFactory.cpp | 65 +++++++++---------- .../TestImpactTestEnumerationSuiteFactory.cpp | 37 +++++------ .../Factory/TestImpactTestRunSuiteFactory.cpp | 57 ++++++++-------- .../TestImpactTestTargetMetaMapFactory.cpp | 9 ++- .../TestImpactTestEnumerationSerializer.cpp | 22 +++---- .../Test/Run/TestImpactTestRunSerializer.cpp | 52 +++++++-------- 7 files changed, 141 insertions(+), 155 deletions(-) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactBuildTargetDescriptorFactory.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactBuildTargetDescriptorFactory.cpp index 9e82170dff..5e1f2bc890 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactBuildTargetDescriptorFactory.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactBuildTargetDescriptorFactory.cpp @@ -19,35 +19,6 @@ namespace TestImpact { - namespace - { - // Keys for pertinent JSON node and attribute names - constexpr const char* Keys[] = - { - "target", - "name", - "output_name", - "path", - "sources", - "static", - "input", - "output" - }; - - enum - { - TargetKey, - NameKey, - OutputNameKey, - PathKey, - SourcesKey, - StaticKey, - InputKey, - OutputKey - }; - - } // namespace - AutogenSources PairAutogenSources( const AZStd::vector& inputSources, const AZStd::vector& outputSources, @@ -98,6 +69,31 @@ namespace TestImpact const AZStd::vector& autogenInputExtensionExcludes, const AZStd::string& autogenMatcher) { + // Keys for pertinent JSON node and attribute names + constexpr const char* Keys[] = + { + "target", + "name", + "output_name", + "path", + "sources", + "static", + "input", + "output" + }; + + enum + { + TargetKey, + NameKey, + OutputNameKey, + PathKey, + SourcesKey, + StaticKey, + InputKey, + OutputKey + }; + AZ_TestImpact_Eval(!autogenMatcher.empty(), ArtifactException, "Autogen matcher cannot be empty"); BuildTargetDescriptor buildTargetDescriptor; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactModuleCoverageFactory.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactModuleCoverageFactory.cpp index 889f91c600..68eee409e7 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactModuleCoverageFactory.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactModuleCoverageFactory.cpp @@ -18,46 +18,43 @@ namespace TestImpact { - namespace - { - // Keys for pertinent XML node and attribute names - constexpr const char* Keys[] = - { - "packages", - "name", - "filename", - "coverage", - "classes", - "lines", - "line", - "number", - "hits", - "sources", - "source" - }; - - enum - { - PackagesKey, - NameKey, - FileNameKey, - CoverageKey, - ClassesKey, - LinesKey, - LineKey, - NumberKey, - HitsKey, - SourcesKey, - SourceKey - }; - } // namespace - namespace Cobertura { // Note: OpenCppCoverage appears to have a very liberal interpretation of the Cobertura coverage file format so consider // this implementation to be provisional and coupled to the Windows platform and OpenCppCoverage tool AZStd::vector ModuleCoveragesFactory(const AZStd::string& coverageData) { + // Keys for pertinent XML node and attribute names + constexpr const char* Keys[] = + { + "packages", + "name", + "filename", + "coverage", + "classes", + "lines", + "line", + "number", + "hits", + "sources", + "source" + }; + + enum + { + PackagesKey, + NameKey, + FileNameKey, + CoverageKey, + ClassesKey, + LinesKey, + LineKey, + NumberKey, + HitsKey, + SourcesKey, + SourceKey + }; + AZ_TestImpact_Eval(!coverageData.empty(), ArtifactException, "Cannot parse coverage, string is empty"); AZStd::vector modules; AZStd::vector rawData(coverageData.begin(), coverageData.end()); diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestEnumerationSuiteFactory.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestEnumerationSuiteFactory.cpp index bf8a19c812..8162256aaf 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestEnumerationSuiteFactory.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestEnumerationSuiteFactory.cpp @@ -18,30 +18,27 @@ namespace TestImpact { - namespace - { - // Keys for pertinent XML node and attribute names - constexpr const char* Keys[] = - { - "testsuites", - "testsuite", - "name", - "testcase" - }; - - enum - { - TestSuitesKey, - TestSuiteKey, - NameKey, - TestCaseKey - }; - } // namespace - namespace GTest { AZStd::vector TestEnumerationSuitesFactory(const AZStd::string& testEnumerationData) { + // Keys for pertinent XML node and attribute names + constexpr const char* Keys[] = + { + "testsuites", + "testsuite", + "name", + "testcase" + }; + + enum + { + TestSuitesKey, + TestSuiteKey, + NameKey, + TestCaseKey + }; + AZ_TestImpact_Eval(!testEnumerationData.empty(), ArtifactException, "Cannot parse enumeration, string is empty"); AZStd::vector testSuites; AZStd::vector rawData(testEnumerationData.begin(), testEnumerationData.end()); diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestRunSuiteFactory.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestRunSuiteFactory.cpp index 506e058824..ba5c398188 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestRunSuiteFactory.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestRunSuiteFactory.cpp @@ -19,38 +19,35 @@ namespace TestImpact { - namespace - { - // Keys for pertinent XML node and attribute names - constexpr const char* Keys[] = - { - "testsuites", - "testsuite", - "name", - "testcase", - "status", - "run", - "notrun", - "time" - }; - - enum - { - TestSuitesKey, - TestSuiteKey, - NameKey, - TestCaseKey, - StatusKey, - RunKey, - NotRunKey, - DurationKey - }; - } // namespace - namespace GTest { AZStd::vector TestRunSuitesFactory(const AZStd::string& testEnumerationData) { + // Keys for pertinent XML node and attribute names + constexpr const char* Keys[] = + { + "testsuites", + "testsuite", + "name", + "testcase", + "status", + "run", + "notrun", + "time" + }; + + enum + { + TestSuitesKey, + TestSuiteKey, + NameKey, + TestCaseKey, + StatusKey, + RunKey, + NotRunKey, + DurationKey + }; + AZ_TestImpact_Eval(!testEnumerationData.empty(), ArtifactException, "Cannot parse test run, string is empty"); AZStd::vector testSuites; AZStd::vector rawData(testEnumerationData.begin(), testEnumerationData.end()); @@ -71,7 +68,7 @@ namespace TestImpact return !name.starts_with("DISABLED_") && name.find("/DISABLED_") == AZStd::string::npos; }; - const auto getDuration = [](const AZ::rapidxml::xml_node<>* node) + const auto getDuration = [&Keys](const AZ::rapidxml::xml_node<>* node) { const AZStd::string duration = node->first_attribute(Keys[DurationKey])->value(); return AZStd::chrono::milliseconds(AZStd::stof(duration) * 1000.f); @@ -85,7 +82,7 @@ namespace TestImpact for (auto testcase_node = testsuite_node->first_node(Keys[TestCaseKey]); testcase_node; testcase_node = testcase_node->next_sibling()) { - const auto getStatus = [](const AZ::rapidxml::xml_node<>* node) + const auto getStatus = [&Keys](const AZ::rapidxml::xml_node<>* node) { const AZStd::string status = node->first_attribute(Keys[StatusKey])->value(); if (status == Keys[RunKey]) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestTargetMetaMapFactory.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestTargetMetaMapFactory.cpp index 9135a3c683..db4d5f8563 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestTargetMetaMapFactory.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestTargetMetaMapFactory.cpp @@ -19,7 +19,7 @@ namespace TestImpact { - namespace + TestTargetMetaMap TestTargetMetaMapFactory(const AZStd::string& masterTestListData) { // Keys for pertinent JSON node and attribute names constexpr const char* Keys[] = @@ -45,16 +45,15 @@ namespace TestImpact StandAloneKey, NameKey }; - } // namespace - TestTargetMetaMap TestTargetMetaMapFactory(const AZStd::string& masterTestListData) - { + AZ_TestImpact_Eval(!masterTestListData.empty(), ArtifactException, "test meta-data cannot be empty"); + TestTargetMetaMap testMetas; rapidjson::Document masterTestList; if (masterTestList.Parse(masterTestListData.c_str()).HasParseError()) { - throw TestImpact::ArtifactException("Could not parse test meta-data file"); + throw TestImpact::ArtifactException("Could not parse test meta-data"); } const auto tests = masterTestList[Keys[GoogleKey]][Keys[TestKey]][Keys[TestsKey]].GetArray(); diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerationSerializer.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerationSerializer.cpp index 0b171ba404..efed04970d 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerationSerializer.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerationSerializer.cpp @@ -20,7 +20,7 @@ namespace TestImpact { - namespace + namespace TestEnumFields { // Keys for pertinent JSON node and attribute names constexpr const char* Keys[] = @@ -46,23 +46,23 @@ namespace TestImpact rapidjson::PrettyWriter writer(stringBuffer); writer.StartObject(); - writer.Key(Keys[SuitesKey]); + writer.Key(TestEnumFields::Keys[TestEnumFields::SuitesKey]); writer.StartArray(); for (const auto& suite : testEnum.GetTestSuites()) { writer.StartObject(); - writer.Key(Keys[NameKey]); + writer.Key(TestEnumFields::Keys[TestEnumFields::NameKey]); writer.String(suite.m_name.c_str()); - writer.Key(Keys[EnabledKey]); + writer.Key(TestEnumFields::Keys[TestEnumFields::EnabledKey]); writer.Bool(suite.m_enabled); - writer.Key(Keys[TestsKey]); + writer.Key(TestEnumFields::Keys[TestEnumFields::TestsKey]); writer.StartArray(); for (const auto& test : suite.m_tests) { writer.StartObject(); - writer.Key(Keys[NameKey]); + writer.Key(TestEnumFields::Keys[TestEnumFields::NameKey]); writer.String(test.m_name.c_str()); - writer.Key(Keys[EnabledKey]); + writer.Key(TestEnumFields::Keys[TestEnumFields::EnabledKey]); writer.Bool(test.m_enabled); writer.EndObject(); } @@ -85,13 +85,13 @@ namespace TestImpact throw TestEnumerationException("Could not parse enumeration data"); } - for (const auto& suite : doc[Keys[SuitesKey]].GetArray()) + for (const auto& suite : doc[TestEnumFields::Keys[TestEnumFields::SuitesKey]].GetArray()) { - testSuites.emplace_back(TestEnumerationSuite{suite[Keys[NameKey]].GetString(), suite[Keys[EnabledKey]].GetBool(), {}}); - for (const auto& test : suite[Keys[TestsKey]].GetArray()) + testSuites.emplace_back(TestEnumerationSuite{suite[TestEnumFields::Keys[TestEnumFields::NameKey]].GetString(), suite[TestEnumFields::Keys[TestEnumFields::EnabledKey]].GetBool(), {}}); + for (const auto& test : suite[TestEnumFields::Keys[TestEnumFields::TestsKey]].GetArray()) { testSuites.back().m_tests.emplace_back( - TestEnumerationCase{test[Keys[NameKey]].GetString(), test[Keys[EnabledKey]].GetBool()}); + TestEnumerationCase{test[TestEnumFields::Keys[TestEnumFields::NameKey]].GetString(), test[TestEnumFields::Keys[TestEnumFields::EnabledKey]].GetBool()}); } } diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunSerializer.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunSerializer.cpp index 15a135ad25..7f29dd77a9 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunSerializer.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunSerializer.cpp @@ -20,7 +20,7 @@ namespace TestImpact { - namespace + namespace TestRunFields { // Keys for pertinent JSON node and attribute names constexpr const char* Keys[] = @@ -55,11 +55,11 @@ namespace TestImpact writer.StartObject(); // Run duration - writer.Key(Keys[DurationKey]); + writer.Key(TestRunFields::Keys[TestRunFields::DurationKey]); writer.Uint(testRun.GetDuration().count()); // Suites - writer.Key(Keys[SuitesKey]); + writer.Key(TestRunFields::Keys[TestRunFields::SuitesKey]); writer.StartArray(); for (const auto& suite : testRun.GetTestSuites()) @@ -68,19 +68,19 @@ namespace TestImpact writer.StartObject(); // Suite name - writer.Key(Keys[NameKey]); + writer.Key(TestRunFields::Keys[TestRunFields::NameKey]); writer.String(suite.m_name.c_str()); // Suite duration - writer.Key(Keys[DurationKey]); + writer.Key(TestRunFields::Keys[TestRunFields::DurationKey]); writer.Uint(suite.m_duration.count()); // Suite enabled - writer.Key(Keys[EnabledKey]); + writer.Key(TestRunFields::Keys[TestRunFields::EnabledKey]); writer.Bool(suite.m_enabled); // Suite tests - writer.Key(Keys[TestsKey]); + writer.Key(TestRunFields::Keys[TestRunFields::TestsKey]); writer.StartArray(); for (const auto& test : suite.m_tests) { @@ -88,30 +88,30 @@ namespace TestImpact writer.StartObject(); // Test name - writer.Key(Keys[NameKey]); + writer.Key(TestRunFields::Keys[TestRunFields::NameKey]); writer.String(test.m_name.c_str()); // Test enabled - writer.Key(Keys[EnabledKey]); + writer.Key(TestRunFields::Keys[TestRunFields::EnabledKey]); writer.Bool(test.m_enabled); // Test duration - writer.Key(Keys[DurationKey]); + writer.Key(TestRunFields::Keys[TestRunFields::DurationKey]); writer.Uint(test.m_duration.count()); // Test status - writer.Key(Keys[StatusKey]); + writer.Key(TestRunFields::Keys[TestRunFields::StatusKey]); writer.Bool(static_cast(test.m_status)); // Test result if (test.m_status == TestRunStatus::Run) { - writer.Key(Keys[ResultKey]); + writer.Key(TestRunFields::Keys[TestRunFields::ResultKey]); writer.Bool(static_cast(test.m_result.value())); } else { - writer.Key(Keys[ResultKey]); + writer.Key(TestRunFields::Keys[TestRunFields::ResultKey]); writer.Null(); } @@ -146,38 +146,38 @@ namespace TestImpact } // Run duration - const AZStd::chrono::milliseconds runDuration = AZStd::chrono::milliseconds{doc[Keys[DurationKey]].GetUint()}; + const AZStd::chrono::milliseconds runDuration = AZStd::chrono::milliseconds{doc[TestRunFields::Keys[TestRunFields::DurationKey]].GetUint()}; // Suites - for (const auto& suite : doc[Keys[SuitesKey]].GetArray()) + for (const auto& suite : doc[TestRunFields::Keys[TestRunFields::SuitesKey]].GetArray()) { // Suite name - const AZStd::string name = suite[Keys[NameKey]].GetString(); + const AZStd::string name = suite[TestRunFields::Keys[TestRunFields::NameKey]].GetString(); // Suite duration - const AZStd::chrono::milliseconds suiteDuration = AZStd::chrono::milliseconds{suite[Keys[DurationKey]].GetUint()}; + const AZStd::chrono::milliseconds suiteDuration = AZStd::chrono::milliseconds{suite[TestRunFields::Keys[TestRunFields::DurationKey]].GetUint()}; // Suite enabled - const bool enabled = suite[Keys[EnabledKey]].GetBool(); + const bool enabled = suite[TestRunFields::Keys[TestRunFields::EnabledKey]].GetBool(); testSuites.emplace_back(TestRunSuite{ - suite[Keys[NameKey]].GetString(), - suite[Keys[EnabledKey]].GetBool(), + suite[TestRunFields::Keys[TestRunFields::NameKey]].GetString(), + suite[TestRunFields::Keys[TestRunFields::EnabledKey]].GetBool(), {}, - AZStd::chrono::milliseconds{suite[Keys[DurationKey]].GetUint()}}); + AZStd::chrono::milliseconds{suite[TestRunFields::Keys[TestRunFields::DurationKey]].GetUint()}}); // Suite tests - for (const auto& test : suite[Keys[TestsKey]].GetArray()) + for (const auto& test : suite[TestRunFields::Keys[TestRunFields::TestsKey]].GetArray()) { AZStd::optional result; - TestRunStatus status = static_cast(test[Keys[StatusKey]].GetBool()); + TestRunStatus status = static_cast(test[TestRunFields::Keys[TestRunFields::StatusKey]].GetBool()); if (status == TestRunStatus::Run) { - result = static_cast(test[Keys[ResultKey]].GetBool()); + result = static_cast(test[TestRunFields::Keys[TestRunFields::ResultKey]].GetBool()); } - const AZStd::chrono::milliseconds testDuration = AZStd::chrono::milliseconds{test[Keys[DurationKey]].GetUint()}; + const AZStd::chrono::milliseconds testDuration = AZStd::chrono::milliseconds{test[TestRunFields::Keys[TestRunFields::DurationKey]].GetUint()}; testSuites.back().m_tests.emplace_back( - TestRunCase{test[Keys[NameKey]].GetString(), test[Keys[EnabledKey]].GetBool(), result, testDuration, status}); + TestRunCase{test[TestRunFields::Keys[TestRunFields::NameKey]].GetString(), test[TestRunFields::Keys[TestRunFields::EnabledKey]].GetBool(), result, testDuration, status}); } } From f47f712bd07a7af10cd2e6ab48902c36e10f4e7f Mon Sep 17 00:00:00 2001 From: jonawals Date: Thu, 6 May 2021 15:53:54 +0100 Subject: [PATCH 010/102] Fix missing pragma directive. --- .../Runtime/Code/Source/Test/Job/TestImpactTestJobCommon.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Job/TestImpactTestJobCommon.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Job/TestImpactTestJobCommon.h index ee245f1ff3..03bbd91454 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Job/TestImpactTestJobCommon.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Job/TestImpactTestJobCommon.h @@ -17,6 +17,8 @@ #include #include +#pragma once + namespace TestImpact { template From 006a9ce85f833b5c53b88a8e97d5d778e93abdb2 Mon Sep 17 00:00:00 2001 From: jonawals Date: Thu, 6 May 2021 16:33:40 +0100 Subject: [PATCH 011/102] Fix unused variable warning. --- .../Code/Tests/Process/TestImpactProcessSchedulerTest.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Process/TestImpactProcessSchedulerTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Process/TestImpactProcessSchedulerTest.cpp index c64b05e431..2b32067688 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Process/TestImpactProcessSchedulerTest.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Process/TestImpactProcessSchedulerTest.cpp @@ -227,8 +227,6 @@ namespace UnitTest // Expects the process to have exited gracefully of its own accord (i.e. not terminated for any reason by the scheduler) void ProcessSchedulerTestFixtureWithParams::ExpectGracefulExit(TestImpact::ProcessId pid) { - const auto& [launchResult, exitStatus, createTime, exitTime, duration, returnCode, std] = m_processResults[pid]; - ExpectSuccessfulLaunch(pid); ExpectExitCondition(pid, TestImpact::ExitCondition::Gracefull); } @@ -236,8 +234,6 @@ namespace UnitTest // Expects the process to have been terminated by the client or scheduler void ProcessSchedulerTestFixtureWithParams::ExpectTerminatedProcess(TestImpact::ProcessId pid) { - const auto& [launchResult, exitStatus, createTime, exitTime, duration, returnCode, std] = m_processResults[pid]; - ExpectSuccessfulLaunch(pid); ExpectExitCondition(pid, TestImpact::ExitCondition::Terminated); } @@ -245,8 +241,6 @@ namespace UnitTest // Expects the process to have been terminated by the scheduler due to the process or scheduler timing out void ProcessSchedulerTestFixtureWithParams::ExpectTimeoutProcess(TestImpact::ProcessId pid) { - const auto& [launchResult, exitStatus, createTime, exitTime, duration, returnCode, std] = m_processResults[pid]; - ExpectSuccessfulLaunch(pid); ExpectExitCondition(pid, TestImpact::ExitCondition::Timeout); } From 31312606b198bf02c52617b0d74056bb4960d6aa Mon Sep 17 00:00:00 2001 From: jonawals Date: Thu, 6 May 2021 16:50:52 +0100 Subject: [PATCH 012/102] Fix unused variable warning. --- .../Runtime/Code/Tests/Target/TestImpactBuildTargetTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Target/TestImpactBuildTargetTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Target/TestImpactBuildTargetTest.cpp index e416d3836e..e7ebec17d1 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Target/TestImpactBuildTargetTest.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Target/TestImpactBuildTargetTest.cpp @@ -332,7 +332,7 @@ namespace UnitTest try { // When attempting to find a target that does not exist - auto target = targetList.GetTargetOrThrow(GenerateBuildTargetName(i + targetList.GetNumTargets())); + targetList.GetTargetOrThrow(GenerateBuildTargetName(i + targetList.GetNumTargets())); // Do not expect the target list construction to succeed FAIL(); From fee010f6021c3f1f7462e80f1f0166a78de3a24e Mon Sep 17 00:00:00 2001 From: jonawals Date: Thu, 6 May 2021 17:04:22 +0100 Subject: [PATCH 013/102] Fix test utils. --- .../Runtime/Code/Tests/TestImpactTestUtils.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactTestUtils.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactTestUtils.cpp index 90f384de85..6b2fa72310 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactTestUtils.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactTestUtils.cpp @@ -1142,7 +1142,7 @@ namespace UnitTest "C:\\Lumberyard\\Code\\Tools\\TestImpactFramework\\Runtime\\Code\\Tests\\TestTargetA\\Code\\Tests\\TestImpactTestTargetA.cpp"; sourceCoverage.m_coverage = AZStd::vector(); - auto& lines = sourceCoverage.m_coverage.value(); + auto& lines = sourceCoverage.m_coverage; lines.push_back(TestImpact::LineCoverage{22, 1}); lines.push_back(TestImpact::LineCoverage{23, 1}); lines.push_back(TestImpact::LineCoverage{24, 1}); @@ -1202,7 +1202,7 @@ namespace UnitTest "C:\\Lumberyard\\Code\\Tools\\TestImpactFramework\\Runtime\\Code\\Tests\\TestTargetB\\Code\\Tests\\TestImpactTestTargetB.cpp"; sourceCoverage.m_coverage = AZStd::vector(); - auto& lines = sourceCoverage.m_coverage.value(); + auto& lines = sourceCoverage.m_coverage; lines.push_back(TestImpact::LineCoverage{29, 1}); lines.push_back(TestImpact::LineCoverage{30, 1}); lines.push_back(TestImpact::LineCoverage{31, 1}); @@ -1250,7 +1250,7 @@ namespace UnitTest "C:\\Lumberyard\\Code\\Tools\\TestImpactFramework\\Runtime\\Code\\Tests\\TestTargetC\\Code\\Tests\\TestImpactTestTargetC.cpp"; sourceCoverage.m_coverage = AZStd::vector(); - auto& lines = sourceCoverage.m_coverage.value(); + auto& lines = sourceCoverage.m_coverage; lines.push_back(TestImpact::LineCoverage{32, 1}); lines.push_back(TestImpact::LineCoverage{33, 1}); lines.push_back(TestImpact::LineCoverage{34, 1}); @@ -1294,7 +1294,7 @@ namespace UnitTest "C:\\Lumberyard\\Code\\Tools\\TestImpactFramework\\Runtime\\Code\\Tests\\TestTargetD\\Code\\Tests\\TestImpactTestTargetD.cpp"; sourceCoverage.m_coverage = AZStd::vector(); - auto& lines = sourceCoverage.m_coverage.value(); + auto& lines = sourceCoverage.m_coverage; lines.push_back(TestImpact::LineCoverage{56, 1}); lines.push_back(TestImpact::LineCoverage{57, 1}); lines.push_back(TestImpact::LineCoverage{58, 1}); @@ -1762,18 +1762,18 @@ namespace UnitTest return false; } - if (lhs.m_coverage.has_value() != rhs.m_coverage.has_value()) + if (lhs.m_coverage.empty() != rhs.m_coverage.empty()) { AZ_Error( - "LineCoverage ==", false, "lhs.m_coverage.has_value(): %u, rhs.m_coverage.has_value(): %u", lhs.m_coverage.has_value(), - rhs.m_coverage.has_value()); + "LineCoverage ==", false, "lhs.m_coverage.empty(): %u, rhs.m_coverage.empty(): %u", lhs.m_coverage.empty(), + rhs.m_coverage.empty()); return false; } - if (lhs.m_coverage.has_value()) + if (!lhs.m_coverage.empty()) { return AZStd::equal( - lhs.m_coverage.value().begin(), lhs.m_coverage.value().end(), rhs.m_coverage.value().begin(), + lhs.m_coverage.begin(), lhs.m_coverage.end(), rhs.m_coverage.begin(), [](const TestImpact::LineCoverage& left, const TestImpact::LineCoverage& right) { return left == right; }); From 884022eb6411f5543c103c060c18d4bb9576f4a9 Mon Sep 17 00:00:00 2001 From: jonawals Date: Wed, 19 May 2021 13:31:47 +0100 Subject: [PATCH 014/102] Add additional meta-data for static artifacts --- cmake/LYTestWrappers.cmake | 29 +- .../ConsoleFrontendConfig.in | 295 +++++++++++------- .../LYTestImpactFramework.cmake | 181 ++++++----- 3 files changed, 299 insertions(+), 206 deletions(-) diff --git a/cmake/LYTestWrappers.cmake b/cmake/LYTestWrappers.cmake index 2a65929cf6..40669f7426 100644 --- a/cmake/LYTestWrappers.cmake +++ b/cmake/LYTestWrappers.cmake @@ -151,12 +151,6 @@ function(ly_add_test) set(LY_ADDED_TEST_NAME ${qualified_test_run_name_with_suite}::TEST_RUN) set(LY_ADDED_TEST_NAME ${LY_ADDED_TEST_NAME} PARENT_SCOPE) - # Store the test so we can walk through all of them in LYTestImpactFramework.cmake - set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS ${LY_ADDED_TEST_NAME}) - set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${LY_ADDED_TEST_NAME}_TEST_NAME ${ly_add_test_NAME}) - set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${LY_ADDED_TEST_NAME}_TEST_SUITE ${ly_add_test_TEST_SUITE}) - set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${LY_ADDED_TEST_NAME}_TEST_LIBRARY ${ly_add_test_TEST_LIBRARY}) - set(final_labels SUITE_${ly_add_test_TEST_SUITE}) if (ly_add_test_TEST_REQUIRES) @@ -247,6 +241,12 @@ function(ly_add_test) endif() + # Store the test so we can walk through all of them in LYTestImpactFramework.cmake + set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS ${ly_add_test_NAME}) + set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_test_NAME}_TEST_SUITE ${ly_add_test_TEST_SUITE}) + set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_test_NAME}_TEST_LIBRARY ${ly_add_test_TEST_LIBRARY}) + set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_test_NAME}_TEST_TIMEOUT ${ly_add_test_TIMEOUT}) + endfunction() #! ly_add_pytest: registers target PyTest-based test with CTest @@ -298,8 +298,8 @@ function(ly_add_pytest) ${ly_add_pytest_UNPARSED_ARGUMENTS} ) + set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_pytest_NAME}_SCRIPT_PATH ${ly_add_pytest_PATH}) set_tests_properties(${LY_ADDED_TEST_NAME} PROPERTIES RUN_SERIAL "${ly_add_pytest_TEST_SERIAL}") - set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${LY_ADDED_TEST_NAME}_SCRIPT_PATH ${ly_add_pytest_PATH}) endfunction() #! ly_add_editor_python_test: registers target Editor Python Bindings test with CTest @@ -363,8 +363,8 @@ function(ly_add_editor_python_test) COMPONENT ${ly_add_editor_python_test_COMPONENT} ) + set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_editor_python_test_NAME}_SCRIPT_PATH ${ly_add_editor_python_test_PATH}) set_tests_properties(${LY_ADDED_TEST_NAME} PROPERTIES RUN_SERIAL "${ly_add_editor_python_test_TEST_SERIAL}") - set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${LY_ADDED_TEST_NAME}_SCRIPT_PATH ${ly_add_editor_python_test_PATH}) endfunction() #! ly_add_googletest: Adds a new RUN_TEST using for the specified target using the supplied command or fallback to running @@ -424,8 +424,14 @@ function(ly_add_googletest) set(full_test_command $ $ AzRunUnitTests) # Add AzTestRunner as a build dependency ly_add_dependencies(${build_target} AZ::AzTestRunner) + # Ideally, we would populate the full command procedurally but the generator expressions won't be expanded by the time we need this data + set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_googletest_NAME}_TEST_COMMAND "AzRunUnitTests") else() set(full_test_command ${ly_add_googletest_TEST_COMMAND}) + # Remove the generator expressions so we are left with the argument(s) required to run unit tests for executable targets + string(REPLACE ";" "" stripped_test_command ${full_test_command}) + string(GENEX_STRIP ${stripped_test_command} stripped_test_command) + set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_googletest_NAME}_TEST_COMMAND ${stripped_test_command}) endif() string(REPLACE "::" "_" report_directory "${GTEST_XML_OUTPUT_DIR}/${ly_add_googletest_NAME}.xml") @@ -507,8 +513,14 @@ function(ly_add_googlebenchmark) # If command is not supplied attempts, uses the AzTestRunner to run googlebenchmarks on the supplied TARGET set(full_test_command $ $ AzRunBenchmarks ${output_format_args}) + # Ideally, we would populate the full command procedurally but the generator expressions won't be expanded by the time we need this data + set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_googlebenchmark_NAME}_TEST_COMMAND "AzRunUnitTests") else() set(full_test_command ${ly_add_googlebenchmark_TEST_COMMAND}) + # Remove the generator expressions so we are left with the argument(s) required to run unit tests for executable targets + string(REPLACE ";" "" stripped_test_command ${full_test_command}) + string(GENEX_STRIP ${stripped_test_command} stripped_test_command) + set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_googletest_NAME}_TEST_COMMAND ${stripped_test_command}) endif() ly_add_test( @@ -524,6 +536,5 @@ function(ly_add_googlebenchmark) AZ::AzTestRunner COMPONENT ${ly_add_googlebenchmark_COMPONENT} ) - endfunction() diff --git a/cmake/TestImpactFramework/ConsoleFrontendConfig.in b/cmake/TestImpactFramework/ConsoleFrontendConfig.in index 5f3908671e..73e90ea0ce 100644 --- a/cmake/TestImpactFramework/ConsoleFrontendConfig.in +++ b/cmake/TestImpactFramework/ConsoleFrontendConfig.in @@ -1,111 +1,184 @@ -[path.configuration] -repo_dir = "${repo_dir}" -working_dir = "${working_dir}" -bin_dir = "${runtime_bin_dir}" -tests_dir = "${tests_dir}" -temp_dir = "${temp_dir}" -target_mappings_dir = "${source_target_mapping_dir}" -test_type_dir = "${test_type_dir}" -dependencies_dir = "${target_dependency_dir}" - -[sourcetree.configuration.filters.autogen] -# E.g. matches input /Foo/{Bar}.FooBar.xml with output /Baz/{Bar}.BazBar.cpp -input_output_pairer = "(.*)\\..*" -[sourcetree.configuration.filters.autogen.input] -exclude_filter = [".jinja"] -[sourcetree.configuration.filters.source] -exclude_filter = [".cmake"] -[sourcetree.configuration.testtype.enumerated] -file = "All.tests" -# The table to read from the test enumeration file that contains the test targets -target_table = "google.test" -[sourcetree.configuration.dependency] -# E.g. matches WhiteBox.Editor.Static\n(Gem::WhiteBox.Editor.Static) or WhiteBox.Editor.Static -target_dependency_file_matcher = "target\\.(.*)\\.(dependers)?" -# E.g. matches target.WhiteBox.Editor.Static (for dependency) target.WhiteBox.Editor.Static.dependers (for dependers) -target_vertex_matcher = "(?:(.*)\\n|(.*)" - -[spartia.configuration] -test_impact_Data_file = "TestImpactData.spartia" -test_run_coverage_file = "{test_dir}\\{test_target}.coverage.xml" -test_run_results_file = "{test_dir}\\{test_target}.results.xml" -test_enumeration_file = "{temp_dir}\\{test_target}.enum" -test_shard_selection_file = "{temp_dir}\\{test_target}.filter.{shard_id}" -exclude_filter = [ -{ target = "AssetBundler.Tests", tests = ["*"] }, -{ target = "AssetProcessor.Tests", tests = ["*"] }, -{ target = "CryRenderD3D11.Tests", tests = ["*"] }, -{ target = "CryRenderD3D12.Tests", tests = ["*"] }, -{ target = "LyzardApplicationDescriptors.Tests", tests = ["*"] }, -{ target = "EMotionFX.Editor.Tests", tests = ["UIFixture.*", "SimulatedObjectModelTestsFixture.*", "TestParametersFixture.*", "CanSeeJointsFixture.*", "LODSkinnedMeshFixtureTests/LODSkinnedMeshFixture.CheckLODLevels/*"] }, -{ target = "EMotionFX.Tests", tests = ["UIFixture.*", "SimulatedObjectModelTestsFixture.*", "TestParametersFixture.*", "CanSeeJointsFixture.*"] }, -{ target = "AzCore.Tests", tests = ["AllocatorsTestFixtureLeakDetectionDeathTest_SKIPCODECOVERAGE.AllocatorLeak"] }, -] -[spartia.configuration.shard] -# Long tests that will be sharded -include_filter = [ -{ target = "AzCore.Tests", policy = "fixture_contiguous" }, -{ target = "AzToolsFramework.Tests", policy = "fixture_contiguous" }, -{ target = "Framework.Tests", policy = "test_interleaved" }, -{ target = "LmbrCentral.Editor.Tests", policy = "test_interleaved" }, -{ target = "EditorLib.Tests", policy = "test_interleaved" }, -{ target = "PhysX.Tests", policy = "test_interleaved" }, -{ target = "ImageProcessing.Tests", policy = "test_interleaved" }, -{ target = "Atom_RPI.Tests", policy = "test_interleaved" }, -{ target = "Atom_RHI.Tests", policy = "test_interleaved" }, -{ target = "AzManipulatorFramework.Tests", policy = "test_interleaved" }, -{ target = "WhiteBox.Editor.Tests", policy = "test_interleaved" }, -{ target = "AzManipulatorTestFramework.Tests", policy = "test_interleaved" }, -{ target = "AtomCore.Tests", policy = "test_interleaved" }, -{ target = "ImageProcessingAtom.Editor.Tests", policy = "test_interleaved" }, -{ target = "EditorPythonBindings.Tests", policy = "test_interleaved" }, -{ target = "Atom_Utils.Tests", policy = "test_interleaved" }, -{ target = "AudioEngineWwise.Editor.Tests", policy = "test_interleaved" }, -{ target = "Multiplayer.Tests", policy = "test_interleaved" }, -{ target = "LmbrCentral.Tests", policy = "test_interleaved" }, -{ target = "LyMetricsShared.Tests", policy = "fixture_contiguous" }, -{ target = "PhysX.Editor.Tests", policy = "test_interleaved" }, -{ target = "ComponentEntityEditorPlugin.Tests", policy = "test_interleaved" }, -{ target = "DeltaCataloger.Tests", policy = "test_interleaved" }, -{ target = "GradientSignal.Tests", policy = "test_interleaved" }, -{ target = "LyShine.Tests", policy = "test_interleaved" }, -{ target = "EMotionFX.Editor.Tests", policy = "test_interleaved" }, -{ target = "EMotionFX.Tests", policy = "test_interleaved" }, -{ target = "CrySystem.Tests", policy = "test_interleaved" }, -] -[spartia.configuration.instrumentation] -abs_bin = "${instrumentation_bin}" -[spartia.configuration.instrumentation.errors] -# AzCppCoverage error codes -incorrect_args = -1618178468 -[spartia.configuration.instrumentation.test_coverage] -args = "--export_type cobertura:\"{test_run_coverage_file}\"" -[spartia.configuration.instrumentation.test_selection] -args = "--gtest_filter={test_selection}" -[spartia.configuration.instrumentation.test_enumeration] -args = "--gtest_list_tests" -[spartia.configuration.instrumentation.test_results] -args = "--gtest_output=xml:\"{test_run_results_file}\"" -[spartia.configuration.instrumentation.test_results.errors] -test_success = 0 -test_failures = 1 -[spartia.configuration.instrumentation.binary_type.dynlib] -abs_bin = "{bin_dir}\\AzTestRunner.exe" -args = "\"{bin_dir}\\{test_target}.dll\" AzRunUnitTests" -[spartia.configuration.instrumentation.binary_type.dynlib.test_enumeration] -args = "--stdout_to_file \"{test_enumeration_file}\" {test_enumeration}" -[spartia.configuration.instrumentation.binary_type.dynlib.test_shard_selection] -args = "--args_from_file \"{test_shard_selection_file}\"" -[spartia.configuration.instrumentation.binary_type.dynlib.errors] -# AzTestRunner error codes -failed_to_find_target_bin = 102 -incorrect_args = 101 -known_errors = [ 103, 104] -[spartia.configuration.instrumentation.binary_type.executable] -abs_bin = "{bin_dir}\\{test_target}.exe" -[spartia.configuration.instrumentation.binary_type.executable.test_enumeration] -args = "--stdout_to_file \"{test_enumeration_file}\" {test_enumeration}" -[spartia.configuration.instrumentation.binary_type.executable.test_shard_selection] -args = "--args_from_file \"{test_shard_selection_file}\"" -[spartia.configuration.test_run.seed] -instrumentation_args = "--modules \"{bin_dir}\" --excluded_modules \"{binary_type.dynlib.abs_bin}\" --sources \"{repo_dir}\" --no_breakpoints {test_coverage} -- " \ No newline at end of file +{ + "meta": { + "platform": "${platform}", + "timestamp": "${timestamp}" + }, + "repo": { + "root": "${repo_dir}" + }, + "workspace": { + "temp": { + "root": "${temp_dir}", + "artifact_dir": "RuntimeArtifact" + }, + "persistent": { + "root": "${persistent_dir}", + "test_impact_data_file": "TestImpactData.spartia", + "enumeration_cache_dir": "EnumerationCache" + } + }, + "artifacts": { + "static": { + "build_target_descriptor": { + "dir": "${source_target_mapping_dir}", + "target_sources": { + "static": { + "include_filters": [ + ".h", ".hpp", ".hxx", ".inl", ".c", ".cpp", ".cxx" + ] + }, + "autogen": { + "input_output_pairer": "(.*)\\..*", + "input": { + "include_filters": [ + ".xml" + ] + } + } + } + }, + "dependency_graph_data": { + "dir": "${target_dependency_dir}", + "matchers": { + "target_dependency_file": "target\\.(.*)\\.(dependers)?", + "target_vertex": "(?:(.*)\\n|(.*)" + } + }, + "test_target_meta": { + "file": "${test_target_type_file}" + } + } + }, + "test_engine": { + "test_runner": { + "bin": "${test_runner_bin}" + }, + "instrumentation": { + "bin": "${instrumentation_bin}" + } + }, + "target": { + "dir": "${bin_dir}", + "exclude": [ + + ], + "shard": [ + { + "policy": "fixture_contiguous", + "target": "AzCore.Tests" + }, + { + "policy": "fixture_contiguous", + "target": "AzToolsFramework.Tests" + }, + { + "policy": "test_interleaved", + "target": "Framework.Tests" + }, + { + "policy": "test_interleaved", + "target": "LmbrCentral.Editor.Tests" + }, + { + "policy": "test_interleaved", + "target": "EditorLib.Tests" + }, + { + "policy": "test_interleaved", + "target": "PhysX.Tests" + }, + { + "policy": "test_interleaved", + "target": "ImageProcessing.Tests" + }, + { + "policy": "test_interleaved", + "target": "Atom_RPI.Tests" + }, + { + "policy": "test_interleaved", + "target": "Atom_RHI.Tests" + }, + { + "policy": "test_interleaved", + "target": "AzManipulatorFramework.Tests" + }, + { + "policy": "test_interleaved", + "target": "WhiteBox.Editor.Tests" + }, + { + "policy": "test_interleaved", + "target": "ImageProcessing.Tests" + }, + { + "policy": "test_interleaved", + "target": "AzManipulatorTestFramework.Tests" + }, + { + "policy": "test_interleaved", + "target": "AtomCore.Tests" + }, + { + "policy": "test_interleaved", + "target": "ImageProcessingAtom.Editor.Tests" + }, + { + "policy": "test_interleaved", + "target": "EditorPythonBindings.Tests" + }, + { + "policy": "test_interleaved", + "target": "Atom_Utils.Tests" + }, + { + "policy": "test_interleaved", + "target": "AudioEngineWwise.Editor.Tests" + }, + { + "policy": "test_interleaved", + "target": "Multiplayer.Tests" + }, + { + "policy": "test_interleaved", + "target": "LmbrCentral.Tests" + }, + { + "policy": "fixture_contiguous", + "target": "LyMetricsShared.Tests" + }, + { + "policy": "test_interleaved", + "target": "PhysX.Editor.Tests" + }, + { + "policy": "test_interleaved", + "target": "ComponentEntityEditorPlugin.Tests" + }, + { + "policy": "test_interleaved", + "target": "DeltaCataloger.Tests" + }, + { + "policy": "test_interleaved", + "target": "GradientSignal.Tests" + }, + { + "policy": "test_interleaved", + "target": "LyShine.Tests" + }, + { + "policy": "test_interleaved", + "target": "EMotionFX.Editor.Tests" + }, + { + "policy": "test_interleaved", + "target": "EMotionFX.Tests" + }, + { + "policy": "test_interleaved", + "target": "CrySystem.Tests" + } + ] + } +} diff --git a/cmake/TestImpactFramework/LYTestImpactFramework.cmake b/cmake/TestImpactFramework/LYTestImpactFramework.cmake index c08c64db80..606ab0df4b 100644 --- a/cmake/TestImpactFramework/LYTestImpactFramework.cmake +++ b/cmake/TestImpactFramework/LYTestImpactFramework.cmake @@ -15,6 +15,9 @@ option(LY_TEST_IMPACT_ACTIVE "Enable test impact framework" OFF) # Path to test instrumentation binary option(LY_TEST_IMPACT_INSTRUMENTATION_BIN "Path to test impact framework instrumentation binary" OFF) +# Name of test impact framework console static library target +set(LY_TEST_IMPACT_CONSOLE_STATIC_TARGET "TestImpact.Frontend.Console.Static") + # Name of test impact framework console target set(LY_TEST_IMPACT_CONSOLE_TARGET "TestImpact.Frontend.Console") @@ -33,11 +36,8 @@ set(LY_TEST_IMPACT_SOURCE_TARGET_MAPPING_DIR "${LY_TEST_IMPACT_ARTIFACT_DIR}/Map # Directory for build target dependency/depender graphs set(LY_TEST_IMPACT_TARGET_DEPENDENCY_DIR "${LY_TEST_IMPACT_ARTIFACT_DIR}/Dependency") -# Directory for test type enumeration files -set(LY_TEST_IMPACT_TEST_TYPE_DIR "${LY_TEST_IMPACT_ARTIFACT_DIR}/TestType") - # Master test enumeration file for all test types -set(LY_TEST_IMPACT_TEST_TYPE_FILE "${LY_TEST_IMPACT_TEST_TYPE_DIR}/All.tests") +set(LY_TEST_IMPACT_TEST_TYPE_FILE "${LY_TEST_IMPACT_ARTIFACT_DIR}/TestType/All.tests") #! ly_test_impact_rebase_file_to_repo_root: rebases the relative and/or absolute path to be relative to repo root directory and places the resulting path in quotes. # @@ -93,14 +93,13 @@ function(ly_test_impact_get_test_launch_method TARGET_NAME LAUNCH_METHOD) endif() endfunction() -#! ly_test_impact_extract_google_test: explodes a composite google test string into namespace, test and suite components. +#! ly_test_impact_extract_google_test_name: extracts the google test name from the composite 'namespace::test_name' string # # \arg:COMPOSITE_TEST test in the form 'namespace::test' -# \arg:TEST_NAMESPACE namespace for the test # \arg:TEST_NAME name of test function(ly_test_impact_extract_google_test COMPOSITE_TEST TEST_NAMESPACE TEST_NAME) get_property(test_components GLOBAL PROPERTY LY_ALL_TESTS_${COMPOSITE_TEST}_TEST_NAME) - # Namespace and test are mandetiry + # Namespace and test are mandetory string(REPLACE "::" ";" test_components ${test_components}) list(LENGTH test_components num_test_components) if(num_test_components LESS 2) @@ -113,37 +112,74 @@ function(ly_test_impact_extract_google_test COMPOSITE_TEST TEST_NAMESPACE TEST_N set(${TEST_NAME} ${test_name} PARENT_SCOPE) endfunction() -#! ly_test_impact_extract_python_test: explodes a composite python test string into filename, namespace, test and suite components. +#! ly_test_impact_extract_python_test_name: extracts the python test name from the composite 'namespace::test_name' string # # \arg:COMPOSITE_TEST test in form 'namespace::test' or 'test' -# \arg:TEST_NAMESPACE namespace for the test (optional) # \arg:TEST_NAME name of test -# \arg:TEST_FILE the Python script path for this test -function(ly_test_impact_extract_python_test COMPOSITE_TEST TEST_NAMESPACE TEST_NAME TEST_FILE) +function(ly_test_impact_extract_python_test COMPOSITE_TEST TEST_NAME) get_property(test_components GLOBAL PROPERTY LY_ALL_TESTS_${COMPOSITE_TEST}_TEST_NAME) - get_property(test_file GLOBAL PROPERTY LY_ALL_TESTS_${COMPOSITE_TEST}_SCRIPT_PATH) # namespace is optional, in which case this component will be simply the test name string(REPLACE "::" ";" test_components ${test_components}) list(LENGTH test_components num_test_components) if(num_test_components GREATER 1) - list(GET test_components 0 test_namespace) list(GET test_components 1 test_name) else() - set(test_namespace "") + set(test_name ${test_components}) + endif() + + set(${TEST_NAME} ${test_name} PARENT_SCOPE) +endfunction() + +#! ly_test_impact_extract_google_test_params: extracts the google test name and command parameters. +# +# \arg:COMPOSITE_TEST test in the form 'namespace::test' +# \arg:TEST_NAMESPACE namespace for the test +# \arg:TEST_NAME name of test +# \arg:TEST_NAME optional command arguments to run the test +# \arg:TEST_NAME test timeout value +function(ly_test_impact_extract_google_test_params COMPOSITE_TEST TEST_NAME TEST_COMMAND) + get_property(test_command GLOBAL PROPERTY LY_ALL_TESTS_${COMPOSITE_TEST}_TEST_COMMAND) + # Namespace and test are mandetory + string(REPLACE "::" ";" test_components ${COMPOSITE_TEST}) + list(LENGTH test_components num_test_components) + if(num_test_components LESS 2) + message(FATAL_ERROR "The test ${test_components} appears to have been specified without a namespace, i.e.:\ly_add_googletest/benchmark(NAME ${test_components})\nInstead of (perhaps):\ly_add_googletest/benchmark(NAME Gem::${test_components})\nPlease add the missing namespace before proceeding.") + endif() + + list(GET test_components 0 test_namespace) + list(GET test_components 1 test_name) + set(${TEST_NAMESPACE} ${test_namespace} PARENT_SCOPE) + set(${TEST_NAME} ${test_name} PARENT_SCOPE) + set(${TEST_COMMAND} ${test_command} PARENT_SCOPE) +endfunction() + +#! ly_test_impact_extract_python_test_params: extracts the python test name and relative script path parameters. +# +# \arg:COMPOSITE_TEST test in form 'namespace::test' or 'test' +# \arg:TEST_NAME name of test +# \arg:SCRIPT_PATH name of test +function(ly_test_impact_extract_python_test_params COMPOSITE_TEST TEST_NAME SCRIPT_PATH) + get_property(script_path GLOBAL PROPERTY LY_ALL_TESTS_${COMPOSITE_TEST}_SCRIPT_PATH) + + # namespace is optional, in which case this component will be simply the test name + string(REPLACE "::" ";" test_components ${COMPOSITE_TEST}) + list(LENGTH test_components num_test_components) + if(num_test_components GREATER 1) + list(GET test_components 1 test_name) + else() set(test_name ${test_components}) endif() # Get python script path relative to repo root ly_test_impact_rebase_file_to_repo_root( - ${test_file} - test_file + ${script_path} + script_path ${LY_ROOT_FOLDER} ) - set(${TEST_NAMESPACE} ${test_namespace} PARENT_SCOPE) set(${TEST_NAME} ${test_name} PARENT_SCOPE) - set(${TEST_FILE} ${test_file} PARENT_SCOPE) + set(${SCRIPT_PATH} ${script_path} PARENT_SCOPE) endfunction() #! ly_test_impact_write_test_enumeration_file: exports the master test lists to file. @@ -164,25 +200,26 @@ function(ly_test_impact_write_test_enumeration_file TEST_ENUMERATION_TEMPLATE_FI message(TRACE "Parsing ${test}") get_property(test_type GLOBAL PROPERTY LY_ALL_TESTS_${test}_TEST_LIBRARY) get_property(test_suite GLOBAL PROPERTY LY_ALL_TESTS_${test}_TEST_SUITE) + get_property(test_timeout GLOBAL PROPERTY LY_ALL_TESTS_${test}_TEST_TIMEOUT) if("${test_type}" STREQUAL "pytest") # Python tests - ly_test_impact_extract_python_test(${test} test_namespace test_name test_file) - list(APPEND python_tests " { \"name\": \"${test_name}\", \"namespace\": \"${test_namespace}\", \"suite\": \"${test_suite}\", \"path\": \"${test_file}\" }") + ly_test_impact_extract_python_test_params(${test} test_name script_path) + list(APPEND python_tests " { \"name\": \"${test_name}\", \"suite\": \"${test_suite}\", \"script\": \"${script_path}\", \"timeout\":${test_timeout} }") elseif("${test_type}" STREQUAL "pytest_editor") # Python editor tests - ly_test_impact_extract_python_test(${test} test_namespace test_name test_file) - list(APPEND python_editor_tests " { \"name\": \"${test_name}\", \"namespace\": \"${test_namespace}\", \"suite\": \"${test_suite}\", \"path\": \"${test_file}\" }") + ly_test_impact_extract_python_test_params(${test} test_name script_path) + list(APPEND python_editor_tests " { \"name\": \"${test_name}\", \"suite\": \"${test_suite}\", \"script\": \"${script_path}\", \"timeout\":${test_timeout} }") elseif("${test_type}" STREQUAL "googletest") # Google tests - ly_test_impact_extract_google_test(${test} test_namespace test_name) + ly_test_impact_extract_google_test_params(${test} test_name test_command) ly_test_impact_get_test_launch_method(${test_name} launch_method) - list(APPEND google_tests " { \"name\": \"${test_name}\", \"namespace\": \"${test_namespace}\", \"suite\": \"${test_suite}\", \"launch_method\": \"${launch_method}\" }") + list(APPEND google_tests " { \"name\": \"${test_name}\", \"suite\": \"${test_suite}\", \"command\": \"${test_command}\", \"timeout\":${test_timeout}, \"launch_method\": \"${launch_method}\" }") elseif("${test_type}" STREQUAL "googlebenchmark") # Google benchmarks - ly_test_impact_extract_google_test(${test} test_namespace test_name) - list(APPEND google_benchmarks " { \"name\": \"${test_name}\", \"namespace\": \"${test_namespace}\", \"suite\": \"${test_suite}\" }") + ly_test_impact_extract_google_test_params(${test} test_name test_command) + list(APPEND google_benchmarks " { \"name\": \"${test_name}\", \"suite\": \"${test_suite}\", \"command\": \"${test_command}\", \"timeout\":${test_timeout} }") else() - message("${test} is of unknown type (TEST_LIBRARY property is empty)") + message("${test_name} is of unknown type (TEST_LIBRARY property is empty)") list(APPEND unknown_tests " { \"name\": \"${test}\" }") endif() endforeach() @@ -242,12 +279,7 @@ function(ly_test_impact_export_source_target_mappings MAPPING_TEMPLATE_FILE) endif() # Static source file mappings - get_target_property(target_type ${target} TYPE) - if("${target_type}" STREQUAL "INTERFACE_LIBRARY") - get_target_property(static_sources ${target}_HEADERS SOURCES) - else() - get_target_property(static_sources ${target} SOURCES) - endif() + get_target_property(static_sources ${target} SOURCES) # Rebase static source files to repo root ly_test_impact_rebase_files_to_repo_root( @@ -271,75 +303,52 @@ endfunction() # # \arg:CONFIG_TEMPLATE_FILE path to the runtime configuration template file # \arg:PERSISTENT_DATA_DIR path to the test impact framework persistent data directory -# \arg:RUNTIME_BIN_DIR path to repo binary ourput directory -function(ly_test_impact_write_config_file CONFIG_TEMPLATE_FILE PERSISTENT_DATA_DIR RUNTIME_BIN_DIR) - set(repo_dir ${LY_ROOT_FOLDER}) +# \arg:BIN_DIR path to repo binary output directory +function(ly_test_impact_write_config_file CONFIG_TEMPLATE_FILE PERSISTENT_DATA_DIR BIN_DIR) + # Platform this config file is being generated for + set(platform ${PAL_PLATFORM_NAME}) - # SparTIA instrumentation binary + # Timestamp this config file was generated at + string(TIMESTAMP timestamp "%Y-%m-%d %H:%M:%S") + + # Instrumentation binary if(NOT LY_TEST_IMPACT_INSTRUMENTATION_BIN) message(FATAL_ERROR "No test impact framework instrumentation binary was specified, please provide the path with option LY_TEST_IMPACT_INSTRUMENTATION_BIN") endif() file(TO_CMAKE_PATH ${LY_TEST_IMPACT_INSTRUMENTATION_BIN} instrumentation_bin) - - # test impact framework working dir - ly_test_impact_rebase_file_to_repo_root( - ${LY_TEST_IMPACT_WORKING_DIR} - working_dir - ${LY_ROOT_FOLDER} - ) - - # test impact framework console binary dir - ly_test_impact_rebase_file_to_repo_root( - ${RUNTIME_BIN_DIR} - runtime_bin_dir - ${LY_ROOT_FOLDER} - ) - # Test dir - ly_test_impact_rebase_file_to_repo_root( - "${PERSISTENT_DATA_DIR}/Tests" - tests_dir - ${LY_ROOT_FOLDER} - ) + # Testrunner binary + set(test_runner_bin $) + + # Repository root + set(repo_dir ${LY_ROOT_FOLDER}) + + # Test impact framework output binary dir + set(bin_dir ${BIN_DIR}) # Temp dir - ly_test_impact_rebase_file_to_repo_root( - "${LY_TEST_IMPACT_TEMP_DIR}" - temp_dir - ${LY_ROOT_FOLDER} - ) - + set(temp_dir "${LY_TEST_IMPACT_TEMP_DIR}") + + # Persistent dir + set(persistent_dir "${PERSISTENT_DATA_DIR}") + # Source to target mappings dir - ly_test_impact_rebase_file_to_repo_root( - "${LY_TEST_IMPACT_SOURCE_TARGET_MAPPING_DIR}" - source_target_mapping_dir - ${LY_ROOT_FOLDER} - ) + set(source_target_mapping_dir "${LY_TEST_IMPACT_SOURCE_TARGET_MAPPING_DIR}") - # Test type artifact dir - ly_test_impact_rebase_file_to_repo_root( - "${LY_TEST_IMPACT_TEST_TYPE_DIR}" - test_type_dir - ${LY_ROOT_FOLDER} - ) + # Test type artifact file + set(test_target_type_file "${LY_TEST_IMPACT_TEST_TYPE_FILE}") # Build dependency artifact dir - ly_test_impact_rebase_file_to_repo_root( - "${LY_TEST_IMPACT_TARGET_DEPENDENCY_DIR}" - target_dependency_dir - ${LY_ROOT_FOLDER} - ) + set(target_dependency_dir "${LY_TEST_IMPACT_TARGET_DEPENDENCY_DIR}") # Substitute config file template with above vars file(READ "${CONFIG_TEMPLATE_FILE}" config_file) string(CONFIGURE ${config_file} config_file) # Write out entire config contents to a file in the build directory of the test impact framework console target - string(TIMESTAMP timestamp "%Y-%m-%d %H:%M:%S") - set(header "# Test Impact Framework configuration file for Lumberyard\n# Platform: ${CMAKE_SYSTEM_NAME}\n# Build: $\n# ${timestamp}") file(GENERATE - OUTPUT "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/$/$.$.cfg" - CONTENT "${header}\n\n${config_file}" + OUTPUT "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/$/$.$.json" + CONTENT ${config_file} ) endfunction() @@ -353,7 +362,7 @@ function(ly_test_impact_post_step) set(persistent_data_dir "${LY_ROOT_FOLDER}/Tests/test_impact_framework/${CMAKE_SYSTEM_NAME}/$") # Directory for binaries built for this profile - set(runtime_bin_dir "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/$") + set(bin_dir "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/$") # Erase any existing non-persistent data to avoid getting test impact framework out of sync with current repo state file(REMOVE_RECURSE "${LY_TEST_IMPACT_WORKING_DIR}") @@ -372,7 +381,7 @@ function(ly_test_impact_post_step) ly_test_impact_write_config_file( "cmake/TestImpactFramework/ConsoleFrontendConfig.in" ${persistent_data_dir} - ${runtime_bin_dir} + ${bin_dir} ) # Copy over the graphviz options file for the build dependency graphs @@ -380,6 +389,6 @@ function(ly_test_impact_post_step) file(COPY "cmake/TestImpactFramework/CMakeGraphVizOptions.cmake" DESTINATION ${CMAKE_BINARY_DIR}) # Set the above config file as the default config file to use for the test impact framework console target - target_compile_definitions(${LY_TEST_IMPACT_CONSOLE_TARGET} PRIVATE "LY_TEST_IMPACT_DEFAULT_CONFIG_FILE=\"$.$.cfg\"") + target_compile_definitions(${LY_TEST_IMPACT_CONSOLE_STATIC_TARGET} PUBLIC "LY_TEST_IMPACT_DEFAULT_CONFIG_FILE=\"${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/$/$.$.json\"") message(DEBUG "Test impact framework post steps complete") endfunction() \ No newline at end of file From 0647b3186f557d25b47b8756289ad15b39f1968a Mon Sep 17 00:00:00 2001 From: jonawals Date: Wed, 19 May 2021 13:40:49 +0100 Subject: [PATCH 015/102] Add missing new line --- cmake/TestImpactFramework/LYTestImpactFramework.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/TestImpactFramework/LYTestImpactFramework.cmake b/cmake/TestImpactFramework/LYTestImpactFramework.cmake index 606ab0df4b..fc3ba0af44 100644 --- a/cmake/TestImpactFramework/LYTestImpactFramework.cmake +++ b/cmake/TestImpactFramework/LYTestImpactFramework.cmake @@ -391,4 +391,4 @@ function(ly_test_impact_post_step) # Set the above config file as the default config file to use for the test impact framework console target target_compile_definitions(${LY_TEST_IMPACT_CONSOLE_STATIC_TARGET} PUBLIC "LY_TEST_IMPACT_DEFAULT_CONFIG_FILE=\"${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/$/$.$.json\"") message(DEBUG "Test impact framework post steps complete") -endfunction() \ No newline at end of file +endfunction() From abe35aad75fd4cdb6484e7bca1e2b831c9195c2c Mon Sep 17 00:00:00 2001 From: jonawals Date: Wed, 19 May 2021 13:45:26 +0100 Subject: [PATCH 016/102] Add missing args to PythonBindingsExample.Tests --- Code/Tools/PythonBindingsExample/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Code/Tools/PythonBindingsExample/CMakeLists.txt b/Code/Tools/PythonBindingsExample/CMakeLists.txt index 3abd24b5fc..90cb718ce4 100644 --- a/Code/Tools/PythonBindingsExample/CMakeLists.txt +++ b/Code/Tools/PythonBindingsExample/CMakeLists.txt @@ -114,6 +114,6 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED) ly_add_googletest( NAME AZ::PythonBindingsExample.Tests - TEST_COMMAND $ + TEST_COMMAND $ --unittest ) endif() From b00906ef36a9d6d5540f814d8b1e1159ae8c928a Mon Sep 17 00:00:00 2001 From: jonawals Date: Wed, 19 May 2021 13:49:35 +0100 Subject: [PATCH 017/102] Delete moved and obselete runtime files --- .../TestImpactFramework/TestImpactBitwise.h | 35 - .../TestImpactFramework/TestImpactCallback.h | 23 - .../TestImpactFrameworkPath.h | 41 - .../Artifact/Dynamic/TestImpactChangeList.h | 27 - .../Factory/TestImpactChangeListFactory.cpp | 175 ---- .../Factory/TestImpactChangeListFactory.h | 29 - .../Code/Source/TestImpactFrameworkPath.cpp | 38 - .../TestImpactChangeListFactoryTest.cpp | 288 ------ .../TestImpactInstrumentedTestRunnerTest.cpp | 822 ---------------- .../Tests/Test/TestImpactTestCoverageTest.cpp | 208 ---- .../Test/TestImpactTestEnumeratorTest.cpp | 890 ------------------ ...TestImpactTestEumerationSerializerTest.cpp | 99 -- .../Test/TestImpactTestRunSerializerTest.cpp | 99 -- .../Tests/Test/TestImpactTestRunnerTest.cpp | 516 ---------- .../Tests/TestImpactFrameworkPathTest.cpp | 137 --- .../Runtime/Code/Tests/TestImpactTestMain.cpp | 33 - 16 files changed, 3460 deletions(-) delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactBitwise.h delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactCallback.h delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactFrameworkPath.h delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Dynamic/TestImpactChangeList.h delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactChangeListFactory.cpp delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactChangeListFactory.h delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactFrameworkPath.cpp delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactChangeListFactoryTest.cpp delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactInstrumentedTestRunnerTest.cpp delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestCoverageTest.cpp delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestEnumeratorTest.cpp delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestEumerationSerializerTest.cpp delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestRunSerializerTest.cpp delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestRunnerTest.cpp delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactFrameworkPathTest.cpp delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactTestMain.cpp diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactBitwise.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactBitwise.h deleted file mode 100644 index 73278f6be6..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactBitwise.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#pragma once - -namespace TestImpact -{ - //! Convinience namespace for allowing bitwise operations on enum classes. - //! @note Any types declared in this namespace will have the bitwise operator overloads declared within. - namespace Bitwise - { - template - Flags operator|(Flags lhs, Flags rhs) - { - return static_cast( - static_cast::type>(lhs) | static_cast::type>(rhs)); - } - - template - bool IsFlagSet(Flags flags, Flags flag) - { - return static_cast( - static_cast::type>(flags) & static_cast::type>(flag)); - } - } // namespace Bitwise -} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactCallback.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactCallback.h deleted file mode 100644 index a8c9542b47..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactCallback.h +++ /dev/null @@ -1,23 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#pragma once - -namespace TestImpact -{ - //! Generic callback result used by test impact systems. - enum class CallbackResult : bool - { - Continue, - Abort - }; -} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactFrameworkPath.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactFrameworkPath.h deleted file mode 100644 index affab6daa7..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactFrameworkPath.h +++ /dev/null @@ -1,41 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#pragma once - -#include - -namespace TestImpact -{ - //! Wrapper for OS paths relative to a specified parent path. - //! @note Mimics path semantics only, makes no guarantees about validity. - class FrameworkPath - { - public: - FrameworkPath() = default; - //! Creates a path with no parent. - explicit FrameworkPath(const AZ::IO::Path& absolutePath); - //! Creates a path with an absolute path and path relative to the specified parent path. - explicit FrameworkPath(const AZ::IO::Path& absolutePath, const FrameworkPath& relativeTo); - - //! Retrieves the absolute path. - const AZ::IO::Path& Absolute() const; - //! Retrieves the path relative to the specified parent path. - const AZ::IO::Path& Relative() const; - - private: - //! The absolute path value. - AZ::IO::Path m_absolutePath; - //! The path value relative to the specified parent path. - AZ::IO::Path m_relativePath; - }; -} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Dynamic/TestImpactChangeList.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Dynamic/TestImpactChangeList.h deleted file mode 100644 index 6d966385ac..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Dynamic/TestImpactChangeList.h +++ /dev/null @@ -1,27 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#pragma once - -#include -#include - -namespace TestImpact -{ - //! Artifact produced by the unified diff parsing process representing the file CRUD operations of a given diff. - struct ChangeList - { - AZStd::vector m_createdFiles; - AZStd::vector m_updatedFiles; - AZStd::vector m_deletedFiles; - }; -} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactChangeListFactory.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactChangeListFactory.cpp deleted file mode 100644 index 4b2a33355b..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactChangeListFactory.cpp +++ /dev/null @@ -1,175 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include -#include - -#include - -namespace TestImpact -{ - namespace Utils - { - template - auto split(C&& str, const AZStd::string& delimiter) - { - AZStd::vector strings; - - for (auto p = str.data(), end = p + str.length(); p != end; p += ((p == end) ? 0 : delimiter.length())) - { - const auto pre = p; - p = AZStd::search(pre, end, delimiter.cbegin(), delimiter.cend()); - - if (p != pre) - { - strings.emplace_back(pre, p - pre); - } - } - - return strings; - } - } // namespace Utils - - namespace UnifiedDiff - { - class UnifiedDiffParser - { - public: - ChangeList Parse(const AZStd::string& unifiedDiff); - - private: - AZStd::optional GetTargetFile(const AZStd::string_view& targetFile); - ChangeList GenerateChangelist( - const AZStd::vector>& src, - const AZStd::vector>& dst); - - const AZStd::string m_srcFilePrefix = "--- "; - const AZStd::string m_dstFilePrefix = "+++ "; - const AZStd::string m_gitTargetPrefix = "b/"; - const AZStd::string m_perforceTargetPrefix = "/b/"; - const AZStd::string m_renameFromPrefix = "rename from "; - const AZStd::string m_renameToPrefix = "rename to "; - const AZStd::string m_nullFile = "/dev/null"; - bool m_hasGitHeader = false; - }; - - AZStd::optional UnifiedDiffParser::GetTargetFile(const AZStd::string_view& targetFile) - { - size_t startIndex = 0; - - if (targetFile.starts_with(m_renameFromPrefix)) - { - startIndex = m_renameFromPrefix.length(); - } - else if (targetFile.starts_with(m_renameToPrefix)) - { - startIndex = m_renameToPrefix.length(); - } - else if (targetFile.find(m_nullFile) != AZStd::string::npos) - { - return AZStd::nullopt; - } - else - { - startIndex = m_hasGitHeader ? m_gitTargetPrefix.size() + m_dstFilePrefix.size() - : m_perforceTargetPrefix.size() + m_dstFilePrefix.size(); - } - - const auto endIndex = targetFile.find('\t'); - - if (endIndex != AZStd::string::npos) - { - return targetFile.substr(startIndex, endIndex - startIndex); - } - - return targetFile.substr(startIndex); - } - - ChangeList UnifiedDiffParser::GenerateChangelist( - const AZStd::vector>& src, const AZStd::vector>& dst) - { - AZ_TestImpact_Eval(src.size() == dst.size(), ArtifactException, "Change list source and destination file count mismatch"); - ChangeList changelist; - - for (size_t i = 0; i < src.size(); i++) - { - if (!src[i].has_value()) - { - changelist.m_createdFiles.emplace_back(dst[i].value()); - } - else if (!dst[i].has_value()) - { - changelist.m_deletedFiles.emplace_back(src[i].value()); - } - else if (src[i] != dst[i]) - { - changelist.m_deletedFiles.emplace_back(src[i].value()); - changelist.m_createdFiles.emplace_back(dst[i].value()); - } - else - { - changelist.m_updatedFiles.emplace_back(src[i].value()); - } - } - - return changelist; - } - - ChangeList UnifiedDiffParser::Parse(const AZStd::string& unifiedDiff) - { - const AZStd::string GitHeader = "diff --git"; - const auto lines = Utils::split(unifiedDiff, "\n"); - - AZStd::vector> src; - AZStd::vector> dst; - - for (const auto& line : lines) - { - if (line.starts_with(GitHeader)) - { - m_hasGitHeader = true; - } - else if (line.starts_with(m_srcFilePrefix)) - { - src.emplace_back(GetTargetFile(line)); - } - else if (line.starts_with(m_dstFilePrefix)) - { - dst.emplace_back(GetTargetFile(line)); - } - else if (line.starts_with(m_renameFromPrefix)) - { - src.emplace_back(GetTargetFile(line)); - } - else if (line.starts_with(m_renameToPrefix)) - { - dst.emplace_back(GetTargetFile(line)); - } - } - - return GenerateChangelist(src, dst); - } - - ChangeList ChangeListFactory(const AZStd::string& unifiedDiff) - { - AZ_TestImpact_Eval(!unifiedDiff.empty(), ArtifactException, "Unified diff is empty"); - UnifiedDiffParser diff; - ChangeList changeList = diff.Parse(unifiedDiff); - AZ_TestImpact_Eval( - !changeList.m_createdFiles.empty() || - !changeList.m_updatedFiles.empty() || - !changeList.m_deletedFiles.empty(), - ArtifactException, "The unified diff contained no changes"); - return changeList; - } - } // namespace UnifiedDiff -} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactChangeListFactory.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactChangeListFactory.h deleted file mode 100644 index 82074a66ef..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactChangeListFactory.h +++ /dev/null @@ -1,29 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#pragma once - -#include - -#include -#include - -namespace TestImpact -{ - namespace UnifiedDiff - { - //! Constructs a change list artifact from the specified unified diff data. - //! @param unifiedDiffData The raw change list data in unified diff format. - //! @return The constructed change list artifact. - ChangeList ChangeListFactory(const AZStd::string& unifiedDiffData); - } // namespace UnifiedDiff -} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactFrameworkPath.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactFrameworkPath.cpp deleted file mode 100644 index 6b5d671776..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactFrameworkPath.cpp +++ /dev/null @@ -1,38 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include - -namespace TestImpact -{ - FrameworkPath::FrameworkPath(const AZ::IO::Path& absolutePath) - { - m_absolutePath = AZ::IO::Path(absolutePath).MakePreferred(); - m_relativePath = m_absolutePath.LexicallyRelative(m_absolutePath); - } - - FrameworkPath::FrameworkPath(const AZ::IO::Path& absolutePath, const FrameworkPath& relativeTo) - { - m_absolutePath = AZ::IO::Path(absolutePath).MakePreferred(); - m_relativePath = m_absolutePath.LexicallyRelative(relativeTo.Absolute()); - } - - const AZ::IO::Path& FrameworkPath::Absolute() const - { - return m_absolutePath; - } - - const AZ::IO::Path& FrameworkPath::Relative() const - { - return m_relativePath; - } -} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactChangeListFactoryTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactChangeListFactoryTest.cpp deleted file mode 100644 index aeb8f118a4..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactChangeListFactoryTest.cpp +++ /dev/null @@ -1,288 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include -#include - -#include -#include - -namespace UnitTest -{ - TEST(ChangeListFactoryTest, NoRawData_ExpectArtifactException) - { - // Given an empty unified diff string - const AZStd::string unifiedDiff; - - try - { - // When attempting to construct the change list - const TestImpact::ChangeList changeList = TestImpact::UnifiedDiff::ChangeListFactory(unifiedDiff); - - // Do not expect this statement to be reachable - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::ArtifactException& e) - { - // Expect an artifact exception - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST(ChangeListFactoryTest, NoChanges_ExpectArtifactException) - { - // Given a unified diff string with no changes - const AZStd::string unifiedDiff = "On this day in 1738 absolutely nothing happened"; - - try - { - // When attempting to construct the change list - const TestImpact::ChangeList changeList = TestImpact::UnifiedDiff::ChangeListFactory(unifiedDiff); - - // Do not expect this statement to be reachable - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::ArtifactException& e) - { - // Expect an artifact exception - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST(ChangeListFactoryTest, CreateOnly_ExpectValidChangeListWithFileCreateOperations) - { - // Given a unified diff with only one file creation and no file updates or deletions - const AZStd::string unifiedDiff = - "From f642a2f698452fc18484758b0046132415f09467 Mon Sep 17 00:00:00 2001\n" - "From: user \n" - "Date: Sat, 13 Mar 2021 22:58:07 +0000\n" - "Subject: Test\n" - "\n" - "---\n" - " New.txt | 1 +\n" - " create mode 100644 New.txt\n" - "diff --git a/New.txt b/New.txt\n" - "new file mode 100644\n" - "index 0000000..30d74d2\n" - "--- /dev/null\n" - "+++ b/New.txt\n" - "@@ -0,0 +1 @@\n" - "+test\n" - "\\ No newline at end of file\n" - "-- \n" - "2.30.0.windows.2\n" - "\n" - "\n"; - - // When attempting to construct the change list - const TestImpact::ChangeList changeList = TestImpact::UnifiedDiff::ChangeListFactory(unifiedDiff); - - // Expect the change list to contain the 1 created file - EXPECT_EQ(changeList.m_createdFiles.size(), 1); - EXPECT_TRUE( - AZStd::find(changeList.m_createdFiles.begin(), changeList.m_createdFiles.end(), "New.txt") != changeList.m_createdFiles.end()); - - // Expect the change list to contain no updated files - EXPECT_TRUE(changeList.m_updatedFiles.empty()); - - // Expect the change list to contain no deleted files - EXPECT_TRUE(changeList.m_deletedFiles.empty()); - } - - TEST(ChangeListFactoryTest, UpdateOnly_ExpectValidChangeListWithFileUpdateOperations) - { - // Given a unified diff with only one file update and no file creations or deletions - const AZStd::string unifiedDiff = - "From f642a2f698452fc18484758b0046132415f09467 Mon Sep 17 00:00:00 2001\n" - "From: user \n" - "Date: Sat, 13 Mar 2021 22:58:07 +0000\n" - "Subject: Test\n" - "\n" - "---\n" - " A.txt | 2 +-\n" - "diff --git a/A.txt b/A.txt\n" - "index 7c4a013..e132db2 100644\n" - "--- a/A.txt\n" - "+++ b/A.txt\n" - "@@ -1 +1 @@\n" - "-aaa\n" - "\\ No newline at end of file\n" - "+zzz\n" - "\\ No newline at end of file\n" - "-- \n" - "2.30.0.windows.2\n" - "\n" - "\n"; - - // When attempting to construct the change list - const TestImpact::ChangeList changeList = TestImpact::UnifiedDiff::ChangeListFactory(unifiedDiff); - - // Expect the change list to contain no created files - EXPECT_TRUE(changeList.m_createdFiles.empty()); - - // Expect the change list to contain one updated file - EXPECT_EQ(changeList.m_updatedFiles.size(), 1); - EXPECT_TRUE( - AZStd::find(changeList.m_updatedFiles.begin(), changeList.m_updatedFiles.end(), "A.txt") != changeList.m_updatedFiles.end()); - - // Expect the change list to contain no deleted files - EXPECT_TRUE(changeList.m_deletedFiles.empty()); - } - - TEST(ChangeListFactoryTest, DeleteOnly_ExpectValidChangeListWithFileDeleteOperations) - { - // Given a unified diff with only one file deletion and no file creations or updates - const AZStd::string unifiedDiff = - "From f642a2f698452fc18484758b0046132415f09467 Mon Sep 17 00:00:00 2001\n" - "From: user \n" - "Date: Sat, 13 Mar 2021 22:58:07 +0000\n" - "Subject: Test\n" - "\n" - "---\n" - " B.txt | 1 -\n" - " delete mode 100644 B.txt\n" - "diff --git a/B.txt b/B.txt\n" - "deleted file mode 100644\n" - "index 01f02e3..0000000\n" - "--- a/B.txt\n" - "+++ /dev/null\n" - "@@ -1 +0,0 @@\n" - "-bbb\n" - "\\ No newline at end of file\n" - "-- \n" - "2.30.0.windows.2\n" - "\n" - "\n"; - - // When attempting to construct the change list - const TestImpact::ChangeList changeList = TestImpact::UnifiedDiff::ChangeListFactory(unifiedDiff); - - // Expect the change list to contain no created files - EXPECT_TRUE(changeList.m_createdFiles.empty()); - - // Expect the change list to contain no updated files - EXPECT_TRUE(changeList.m_updatedFiles.empty()); - - // Expect the change list to contain one deleted file - EXPECT_EQ(changeList.m_deletedFiles.size(), 1); - EXPECT_TRUE( - AZStd::find(changeList.m_deletedFiles.begin(), changeList.m_deletedFiles.end(), "B.txt") != changeList.m_deletedFiles.end()); - } - - TEST(ChangeListFactoryTest, ParseUnifiedDiffWithAllPossibleOperations_ExpectChangeListMatchingOperations) - { - // Given a unified diff with created files, updated files, deleted files, renamed files and moved files - const AZStd::string unifiedDiff = - "From f642a2f698452fc18484758b0046132415f09467 Mon Sep 17 00:00:00 2001\n" - "From: user \n" - "Date: Sat, 13 Mar 2021 22:58:07 +0000\n" - "Subject: Test\n" - "\n" - "---\n" - " A.txt | 2 +-\n" - " B.txt | 1 -\n" - " D.txt => Foo/D.txt | 0\n" - " E.txt => Foo/Y.txt | 0\n" - " New.txt | 1 +\n" - " C.txt => X.txt | 0\n" - " 6 files changed, 2 insertions(+), 2 deletions(-)\n" - " delete mode 100644 B.txt\n" - " rename D.txt => Foo/D.txt (100%)\n" - " rename E.txt => Foo/Y.txt (100%)\n" - " create mode 100644 New.txt\n" - " rename C.txt => X.txt (100%)\n" - "\n" - "diff --git a/A.txt b/A.txt\n" - "index 7c4a013..e132db2 100644\n" - "--- a/A.txt\n" - "+++ b/A.txt\n" - "@@ -1 +1 @@\n" - "-aaa\n" - "\\ No newline at end of file\n" - "+zzz\n" - "\\ No newline at end of file\n" - "diff --git a/B.txt b/B.txt\n" - "deleted file mode 100644\n" - "index 01f02e3..0000000\n" - "--- a/B.txt\n" - "+++ /dev/null\n" - "@@ -1 +0,0 @@\n" - "-bbb\n" - "\\ No newline at end of file\n" - "diff --git a/D.txt b/Foo/D.txt\n" - "similarity index 100%\n" - "rename from D.txt\n" - "rename to Foo/D.txt\n" - "diff --git a/E.txt b/Foo/Y.txt\n" - "similarity index 100%\n" - "rename from E.txt\n" - "rename to Foo/Y.txt\n" - "diff --git a/New.txt b/New.txt\n" - "new file mode 100644\n" - "index 0000000..30d74d2\n" - "--- /dev/null\n" - "+++ b/New.txt\n" - "@@ -0,0 +1 @@\n" - "+test\n" - "\\ No newline at end of file\n" - "diff --git a/C.txt b/X.txt\n" - "similarity index 100%\n" - "rename from C.txt\n" - "rename to X.txt\n" - "-- \n" - "2.30.0.windows.2\n" - "\n" - "\n"; - - // When attempting to construct the change list - const TestImpact::ChangeList changeList = TestImpact::UnifiedDiff::ChangeListFactory(unifiedDiff); - - // Expect the change list to contain the 4 created files - EXPECT_EQ(changeList.m_createdFiles.size(), 4); - EXPECT_TRUE( - AZStd::find(changeList.m_createdFiles.begin(), changeList.m_createdFiles.end(), "Foo/D.txt") != - changeList.m_createdFiles.end()); - EXPECT_TRUE( - AZStd::find(changeList.m_createdFiles.begin(), changeList.m_createdFiles.end(), "Foo/Y.txt") != - changeList.m_createdFiles.end()); - EXPECT_TRUE( - AZStd::find(changeList.m_createdFiles.begin(), changeList.m_createdFiles.end(), "X.txt") != changeList.m_createdFiles.end()); - EXPECT_TRUE( - AZStd::find(changeList.m_createdFiles.begin(), changeList.m_createdFiles.end(), "New.txt") != changeList.m_createdFiles.end()); - - // Expect the change list to contain the 1 updated file - EXPECT_EQ(changeList.m_updatedFiles.size(), 1); - EXPECT_TRUE( - AZStd::find(changeList.m_updatedFiles.begin(), changeList.m_updatedFiles.end(), "A.txt") != changeList.m_updatedFiles.end()); - - // Expect the change list to contain the 4 deleted files - EXPECT_EQ(changeList.m_deletedFiles.size(), 4); - EXPECT_TRUE( - AZStd::find(changeList.m_deletedFiles.begin(), changeList.m_deletedFiles.end(), "B.txt") != changeList.m_deletedFiles.end()); - EXPECT_TRUE( - AZStd::find(changeList.m_deletedFiles.begin(), changeList.m_deletedFiles.end(), "D.txt") != changeList.m_deletedFiles.end()); - EXPECT_TRUE( - AZStd::find(changeList.m_deletedFiles.begin(), changeList.m_deletedFiles.end(), "E.txt") != changeList.m_deletedFiles.end()); - EXPECT_TRUE( - AZStd::find(changeList.m_deletedFiles.begin(), changeList.m_deletedFiles.end(), "C.txt") != changeList.m_deletedFiles.end()); - } -} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactInstrumentedTestRunnerTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactInstrumentedTestRunnerTest.cpp deleted file mode 100644 index 897eda012c..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactInstrumentedTestRunnerTest.cpp +++ /dev/null @@ -1,822 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include -#include - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace UnitTest -{ - using JobExceptionPolicy = TestImpact::InstrumentedTestRunner::JobExceptionPolicy; - using CoverageExceptionPolicy = TestImpact::InstrumentedTestRunner::CoverageExceptionPolicy; - - struct TargetPaths - { - AZ::IO::Path m_targetBinary; - AZ::IO::Path m_testRunArtifact; - AZ::IO::Path m_testCoverageArtifact; - }; - - // Indices for looking up job command arguments for the different coverage levels - enum CoverageLevel : uint8_t - { - LineLevel = 0, - SourceLevel - }; - - // Get the job command for an instrumented test run - AZStd::string GetRunCommandForTarget(const TargetPaths& testTarget, CoverageLevel coverageLevel, const char* sourcesFilter) - { - AZStd::string args = AZStd::string::format( - "%s " // 1. Instrumented test runner - "--coverage_level %s " // 2. Coverage level - "--export_type cobertura:\"%s\" " // 3. Test coverage artifact path - "--modules \"%s\" " // 4. Modules path - "--excluded_modules \"%s\" " // 5. Exclude modules - "--sources \"%s\" -- " // 6. Sources path - "\"%s\" " // 7. Test runner binary - "\"%s\" " // 8. Test target bin - "AzRunUnitTests " - "--gtest_output=xml:%s", // 9. Test run result artifact - - LY_TEST_IMPACT_INSTRUMENTATION_BIN, // 1. - (coverageLevel == CoverageLevel::LineLevel ? "line" : "source"), // 2. - testTarget.m_testCoverageArtifact.c_str(), // 3. - LY_TEST_IMPACT_MODULES_DIR, // 4. - LY_TEST_IMPACT_AZ_TESTRUNNER_BIN, // 5. - sourcesFilter, // 6. - LY_TEST_IMPACT_AZ_TESTRUNNER_BIN, // 7. - testTarget.m_targetBinary.c_str(), // 8. - testTarget.m_testRunArtifact.c_str() // 9. - ); - - // OpenCppCoverage doesn't support forward slash directory separators so replace all with escaped backslashes - return AZStd::regex_replace(args, AZStd::regex("/"), "\\"); - } - - // Get the job command for an instrumented test run with valid source filters to produce coverage artifact - AZStd::string GetRunCommandForTargetWithSources(const TargetPaths& testTarget, CoverageLevel coverageLevel) - { - return GetRunCommandForTarget(testTarget, coverageLevel, LY_TEST_IMPACT_COVERAGE_SOURCES_DIR); - } - - // Get the job command for an instrumented test run without valid source filters to produce empty coverage artifact - AZStd::string GetRunCommandForTargetWithoutSources(const TargetPaths& testTarget, CoverageLevel coverageLevel) - { - return GetRunCommandForTarget(testTarget, coverageLevel, "C:\\No\\Sources\\Here\\At\\All\\Ever\\Ever\\Ever"); - } - - class InstrumentedTestRunnerFixture - : public AllocatorsTestFixture - { - public: - void SetUp() override; - void TearDown() override; - - protected: - using JobInfo = TestImpact::InstrumentedTestRunner::JobInfo; - using JobData = TestImpact::InstrumentedTestRunner::JobData; - - AZStd::vector m_jobInfos; - AZStd::unique_ptr m_testRunner; - AZStd::vector> m_testTargetJobArgs; - AZStd::vector m_testTargetPaths; - AZStd::vector m_expectedTestTargetRuns; - AZStd::vector> m_expectedTestTargetCoverages; - AZStd::vector m_expectedTestTargetResult; - size_t m_maxConcurrency = 0; - CoverageLevel m_coverageLevel = CoverageLevel::LineLevel; - inline static AZ::u32 s_uniqueTestCaseId = 0; // Unique id for each test case to be used as the suffix for the written files - }; - - void InstrumentedTestRunnerFixture::SetUp() - { - AllocatorsTestFixture::SetUp(); - - // Suffix each artifact file with the unique test case id to ensure that there are no possible race conditions between cleaning - // up the artifact files after each test case and those artifact files being written and read again in future test cases - const AZStd::string fileExtension = AZStd::string::format(".%u.xml", s_uniqueTestCaseId++); - - const AZStd::string runPath = AZStd::string(LY_TEST_IMPACT_TEST_TARGET_RESULTS_DIR) + "/%s.Run" + fileExtension; - const AZStd::string coveragePath = AZStd::string(LY_TEST_IMPACT_TEST_TARGET_COVERAGE_DIR) + "/%s.Coverage" + fileExtension; - - // TestTargetA - m_testTargetPaths.emplace_back(TargetPaths{ - LY_TEST_IMPACT_TEST_TARGET_A_BIN, AZStd::string::format(runPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_A_BASE_NAME), - AZStd::string::format(coveragePath.c_str(), LY_TEST_IMPACT_TEST_TARGET_A_BASE_NAME)}); - - // TestTargetB - m_testTargetPaths.emplace_back(TargetPaths{ - LY_TEST_IMPACT_TEST_TARGET_B_BIN, AZStd::string::format(runPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_B_BASE_NAME), - AZStd::string::format(coveragePath.c_str(), LY_TEST_IMPACT_TEST_TARGET_B_BASE_NAME)}); - - // TestTargetC - m_testTargetPaths.emplace_back(TargetPaths{ - LY_TEST_IMPACT_TEST_TARGET_C_BIN, AZStd::string::format(runPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_C_BASE_NAME), - AZStd::string::format(coveragePath.c_str(), LY_TEST_IMPACT_TEST_TARGET_C_BASE_NAME)}); - - // TestTargetD - m_testTargetPaths.emplace_back(TargetPaths{ - LY_TEST_IMPACT_TEST_TARGET_D_BIN, AZStd::string::format(runPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_D_BASE_NAME), - AZStd::string::format(coveragePath.c_str(), LY_TEST_IMPACT_TEST_TARGET_D_BASE_NAME)}); - - m_expectedTestTargetRuns.emplace_back(GetTestTargetATestRunSuites(), AZStd::chrono::milliseconds{500}); // TestTargetA - m_expectedTestTargetRuns.emplace_back(GetTestTargetBTestRunSuites(), AZStd::chrono::milliseconds{500}); // TestTargetB - m_expectedTestTargetRuns.emplace_back(GetTestTargetCTestRunSuites(), AZStd::chrono::milliseconds{500}); // TestTargetC - m_expectedTestTargetRuns.emplace_back(GetTestTargetDTestRunSuites(), AZStd::chrono::milliseconds{500}); // TestTargetD - - // TestTargetA - m_expectedTestTargetCoverages.emplace_back(AZStd::array{ - TestImpact::TestCoverage(GetTestTargetALineModuleCoverages()), - TestImpact::TestCoverage(GetTestTargetASourceModuleCoverages())}); - - // TestTargetB - m_expectedTestTargetCoverages.emplace_back(AZStd::array{ - TestImpact::TestCoverage(GetTestTargetBLineModuleCoverages()), - TestImpact::TestCoverage(GetTestTargetBSourceModuleCoverages())}); - - // TestTargetC - m_expectedTestTargetCoverages.emplace_back(AZStd::array{ - TestImpact::TestCoverage(GetTestTargetCLineModuleCoverages()), - TestImpact::TestCoverage(GetTestTargetCSourceModuleCoverages())}); - - // TestTargetD - m_expectedTestTargetCoverages.emplace_back(AZStd::array{ - TestImpact::TestCoverage(GetTestTargetDLineModuleCoverages()), - TestImpact::TestCoverage(GetTestTargetDSourceModuleCoverages())}); - - m_expectedTestTargetResult.emplace_back(TestImpact::TestRunResult::Failed); // TestTargetA - m_expectedTestTargetResult.emplace_back(TestImpact::TestRunResult::Passed); // TestTargetB - m_expectedTestTargetResult.emplace_back(TestImpact::TestRunResult::Passed); // TestTargetC - m_expectedTestTargetResult.emplace_back(TestImpact::TestRunResult::Passed); // TestTargetD - - // Generate the job command arguments for both line level and source level coverage permutations - for (const auto& testTarget : m_testTargetPaths) - { - m_testTargetJobArgs.emplace_back(AZStd::array{ - GetRunCommandForTargetWithSources(testTarget, CoverageLevel::LineLevel), - GetRunCommandForTargetWithSources(testTarget, CoverageLevel::SourceLevel)}); - } - } - - void InstrumentedTestRunnerFixture::TearDown() - { - DeleteFiles(LY_TEST_IMPACT_TEST_TARGET_COVERAGE_DIR, "*.xml"); - DeleteFiles(LY_TEST_IMPACT_TEST_TARGET_RESULTS_DIR, "*.xml"); - - AllocatorsTestFixture::TearDown(); - } - - using ConcurrencyAndCoveragePermutation = AZStd::tuple - < - size_t, // Max number of concurrent processes - CoverageLevel // Coverage level - >; - - // Fixture parameterized for different max number of concurrent jobs and coverage levels - class InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageParams - : public InstrumentedTestRunnerFixture - , public ::testing::WithParamInterface - { - public: - void SetUp() override - { - InstrumentedTestRunnerFixture::SetUp(); - auto [maxConcurrency, coverageLevel] = GetParam(); - m_maxConcurrency = maxConcurrency; - m_coverageLevel = coverageLevel; - } - }; - - using ConcurrencyAndJobExceptionPermutation = AZStd::tuple - < - size_t, // Max number of concurrent processes - CoverageLevel, // Coverage level - JobExceptionPolicy // Test job exception policy - >; - - // Fixture parameterized for different max number of concurrent jobs, coverage levels and different job exception policies - class InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndJobExceptionParams - : public InstrumentedTestRunnerFixture - , public ::testing::WithParamInterface - { - public: - void SetUp() override - { - InstrumentedTestRunnerFixture::SetUp(); - const auto& [maxConcurrency, coverageLevel, jobExceptionPolicy] = GetParam(); - m_maxConcurrency = maxConcurrency; - m_coverageLevel = coverageLevel; - m_jobExceptionPolicy = jobExceptionPolicy; - } - - protected: - JobExceptionPolicy m_jobExceptionPolicy = JobExceptionPolicy::Never; - }; - - class InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndFailedToLaunchExceptionParams - : public InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndJobExceptionParams - { - }; - - class InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndExecutedWithFailureExceptionParams - : public InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndJobExceptionParams - { - }; - - using ConcurrencyAndCoverageExceptionPermutation = AZStd::tuple - < - size_t, // Max number of concurrent processes - CoverageExceptionPolicy // Test coverage exception policy - >; - - // Fixture parameterized for different max number of concurrent jobs and different coverage exception policies - class InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndCoverageExceptionParams - : public InstrumentedTestRunnerFixture - , public ::testing::WithParamInterface - { - public: - void SetUp() override - { - InstrumentedTestRunnerFixture::SetUp(); - const auto& [maxConcurrency, coverageExceptionPolicy] = GetParam(); - m_maxConcurrency = maxConcurrency; - m_coverageExceptionPolicy = coverageExceptionPolicy; - } - - protected: - CoverageExceptionPolicy m_coverageExceptionPolicy = CoverageExceptionPolicy::Never; - }; - - namespace - { - AZStd::array MaxConcurrentRuns = {1, 2, 3, 4}; - - AZStd::array CoverageLevels = {CoverageLevel::LineLevel, CoverageLevel::SourceLevel}; - - AZStd::array FailedToLaunchExceptionPolicies = { - JobExceptionPolicy::Never, JobExceptionPolicy::OnFailedToExecute}; - - AZStd::array ExecutedWithFailureExceptionPolicies = { - JobExceptionPolicy::Never, JobExceptionPolicy::OnExecutedWithFailure}; - } // namespace - - // Validates that the specified test coverage matches the expected output - void ValidateTestTargetCoverage(const TestImpact::TestCoverage& actualResult, const TestImpact::TestCoverage& expectedResult) - { - EXPECT_TRUE(actualResult == expectedResult); - } - - // Validates that the specified test coverage is empty - void ValidateEmptyTestTargetCoverage(const TestImpact::TestCoverage& actualResult) - { - EXPECT_TRUE(actualResult.GetSourcesCovered().empty()); - EXPECT_TRUE(actualResult.GetModuleCoverages().empty()); - EXPECT_EQ(actualResult.GetNumSourcesCovered(), 0); - EXPECT_EQ(actualResult.GetNumModulesCovered(), 0); - } - - TEST_P( - InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndCoverageExceptionParams, - EmptyTestCoverages_ExpectEmptyTestCoveragesOrTestRunException) - { - // Given a test runner with no client callback or run timeout or runner timeout - m_testRunner = - AZStd::make_unique(AZStd::nullopt, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt); - - // Given a mixture of instrumented test run jobs with and without coverage sources - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - const AZStd::string args = (jobId % 2) ? GetRunCommandForTargetWithoutSources(m_testTargetPaths[jobId], m_coverageLevel) - : m_testTargetJobArgs[jobId][m_coverageLevel]; - - JobData jobData(m_testTargetPaths[jobId].m_testRunArtifact, m_testTargetPaths[jobId].m_testCoverageArtifact); - m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData))); - } - - try - { - // When the instrumented test run jobs are executed with different exception policies - const auto runnerJobs = m_testRunner->RunInstrumentedTests(m_jobInfos, m_coverageExceptionPolicy, JobExceptionPolicy::Never); - - // Expect this statement to be reachable only if no exception policy for empty coverages - EXPECT_FALSE(::IsFlagSet(m_coverageExceptionPolicy, CoverageExceptionPolicy::OnEmptyCoverage)); - - for (const auto& job : runnerJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]); - ValidateTestTargetRun(job.GetPayload().value().first, m_expectedTestTargetRuns[jobId]); - - if (jobId % 2) - { - // Expect jobs have a empty test coverages - ValidateEmptyTestTargetCoverage(job.GetPayload().value().second); - } - else - { - // Expect the jobs to successfully result in a test run and coverage that matches the expected test run data - ValidateTestTargetCoverage(job.GetPayload().value().second, m_expectedTestTargetCoverages[jobId][m_coverageLevel]); - } - } - } - catch ([[maybe_unused]] const TestImpact::TestRunException& e) - { - // Expect this statement to be reachable only if there is an exception policy for empty coverages - EXPECT_TRUE(::IsFlagSet(m_coverageExceptionPolicy, CoverageExceptionPolicy::OnEmptyCoverage)); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_P( - InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndFailedToLaunchExceptionParams, - InvalidCommandArgument_ExpectJobResulFailedToExecuteeOrTestJobException) - { - // Given a test runner with no client callback or run timeout or runner timeout - m_testRunner = - AZStd::make_unique(AZStd::nullopt, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt); - - // Given a mixture of instrumented test run jobs with valid and invalid command arguments - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - const AZStd::string args = (jobId % 2) ? InvalidProcessPath : m_testTargetJobArgs[jobId][m_coverageLevel]; - JobData jobData(m_testTargetPaths[jobId].m_testRunArtifact, m_testTargetPaths[jobId].m_testCoverageArtifact); - m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData))); - } - - try - { - // When the instrumented test run jobs are executed with different exception policies - const auto runnerJobs = m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, m_jobExceptionPolicy); - - // Expect this statement to be reachable only if no exception policy for launch failures - EXPECT_FALSE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnFailedToExecute)); - - for (const auto& job : runnerJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - if (jobId % 2) - { - // Expect invalid jobs have a job result of FailedToExecute - ValidateJobFailedToExecute(job); - } - else - { - // Expect the valid jobs to successfully result in a test run that matches the expected test run data - ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]); - ValidateTestTargetRun(job.GetPayload().value().first, m_expectedTestTargetRuns[jobId]); - ValidateTestTargetCoverage(job.GetPayload().value().second, m_expectedTestTargetCoverages[jobId][m_coverageLevel]); - } - } - } - catch ([[maybe_unused]] const TestImpact::TestJobException& e) - { - // Expect this statement to be reachable only if there is an exception policy for launch failures - EXPECT_TRUE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnFailedToExecute)); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_P( - InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndExecutedWithFailureExceptionParams, - ErroneousReturnCode_ExpectJobResultExecutedWithFailureOrTestJobException) - { - // Given a test runner with no client callback or run timeout or runner timeout - m_testRunner = - AZStd::make_unique(AZStd::nullopt, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt); - - // Given a mixture of instrumented test run jobs that execute and return either successfully or with failure - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - JobData jobData(m_testTargetPaths[jobId].m_testRunArtifact, m_testTargetPaths[jobId].m_testCoverageArtifact); - m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId][m_coverageLevel], AZStd::move(jobData))); - } - - try - { - // When the instrumented test run jobs are executed with different exception policies - const auto runnerJobs = m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, m_jobExceptionPolicy); - - // Expect this statement to be reachable only if no exception policy for jobs that return with error - EXPECT_FALSE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnExecutedWithFailure)); - - for (const auto& job : runnerJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - - // Expect the valid jobs to successfully result in a test run that matches the expected test run data - ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]); - ValidateTestTargetRun(job.GetPayload().value().first, m_expectedTestTargetRuns[jobId]); - ValidateTestTargetCoverage(job.GetPayload().value().second, m_expectedTestTargetCoverages[jobId][m_coverageLevel]); - } - } - catch ([[maybe_unused]] const TestImpact::TestJobException& e) - { - // Expect this statement to be reachable only if there is an exception policy for jobs that return with error - EXPECT_TRUE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnExecutedWithFailure)); - } - catch ([[maybe_unused]] const TestImpact::Exception& e) - { - FAIL(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(InstrumentedTestRunnerFixture, EmptyRunRawData_ExpectTestRunnerException) - { - // Given a test runner with no client callback, concurrency, run timeout or runner timeout - m_testRunner = - AZStd::make_unique(AZStd::nullopt, OneConcurrentProcess, AZStd::nullopt, AZStd::nullopt); - - // Given an test runner job that will return successfully but with an empty artifact string - JobData jobData("", m_testTargetPaths[TestTargetA].m_testCoverageArtifact); - m_jobInfos.emplace_back(JobInfo({TestTargetA}, m_testTargetJobArgs[TestTargetA][LineLevel], AZStd::move(jobData))); - - try - { - // When the test runner job is executed - const auto runnerJobs = - m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, JobExceptionPolicy::Never); - - // Do not expect this statement to be reachable - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::TestRunException& e) - { - // Expect an runner exception - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(InstrumentedTestRunnerFixture, EmptyCoverageRawData_ExpectTestRunnerException) - { - // Given a test runner with no client callback, concurrency, run timeout or runner timeout - m_testRunner = - AZStd::make_unique(AZStd::nullopt, OneConcurrentProcess, AZStd::nullopt, AZStd::nullopt); - - // Given an test runner job that will return successfully but with an empty artifact string - JobData jobData(m_testTargetPaths[TestTargetA].m_testRunArtifact, ""); - m_jobInfos.emplace_back(JobInfo({TestTargetA}, m_testTargetJobArgs[TestTargetA][LineLevel], AZStd::move(jobData))); - - try - { - // When the test runner job is executed - const auto runnerJobs = - m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, JobExceptionPolicy::Never); - - // Do not expect this statement to be reachable - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::TestRunException& e) - { - // Expect an runner exception - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(InstrumentedTestRunnerFixture, InvalidRunArtifact_ExpectArtifactException) - { - // Given a run artifact with invalid contents - WriteTextToFile("There is nothing valid here", m_testTargetPaths[TestTargetA].m_testRunArtifact); - - // Given a job command that will write the run artifact to a different location that what we will read from - TargetPaths invalidRunArtifact = m_testTargetPaths[TestTargetA]; - invalidRunArtifact.m_testRunArtifact /= ".xml"; - const AZStd::string args = GetRunCommandForTargetWithSources(invalidRunArtifact, LineLevel); - - // Given a test runner with no client callback, concurrency, run timeout or runner timeout - m_testRunner = - AZStd::make_unique(AZStd::nullopt, OneConcurrentProcess, AZStd::nullopt, AZStd::nullopt); - - // Given an test runner job that will return successfully but not produce a run artifact - JobData jobData(m_testTargetPaths[TestTargetA].m_testRunArtifact, m_testTargetPaths[TestTargetA].m_testCoverageArtifact); - m_jobInfos.emplace_back(JobInfo({TestTargetA}, args, AZStd::move(jobData))); - - try - { - // When the test runner job is executed - const auto runnerJobs = - m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, JobExceptionPolicy::Never); - - // Do not expect this statement to be reachable - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::ArtifactException& e) - { - // Expect an runner exception - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(InstrumentedTestRunnerFixture, InvalidCoverageArtifact_ExpectArtifactException) - { - // Given a coverage artifact with invalid contents - WriteTextToFile("There is nothing valid here", m_testTargetPaths[TestTargetA].m_testCoverageArtifact); - - // Given a job command that will write the coverage artifact to a different location that what we will read from - TargetPaths invalidCoverageArtifact = m_testTargetPaths[TestTargetA]; - invalidCoverageArtifact.m_testCoverageArtifact /= ".xml"; - const AZStd::string args = GetRunCommandForTargetWithSources(invalidCoverageArtifact, LineLevel); - - // Given a test runner with no client callback, concurrency, run timeout or runner timeout - m_testRunner = - AZStd::make_unique(AZStd::nullopt, OneConcurrentProcess, AZStd::nullopt, AZStd::nullopt); - - // Given an test runner job that will return successfully but not produce a run artifact - JobData jobData(m_testTargetPaths[TestTargetA].m_testRunArtifact, m_testTargetPaths[TestTargetA].m_testCoverageArtifact); - m_jobInfos.emplace_back(JobInfo({TestTargetA}, args, AZStd::move(jobData))); - - try - { - // When the test runner job is executed - const auto runnerJobs = - m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, JobExceptionPolicy::Never); - - // Do not expect this statement to be reachable - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::ArtifactException& e) - { - // Expect an runner exception - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_P(InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageParams, RunTestTargets_RunsAndCoverageMatchTestSuitesInTarget) - { - // Given a test runner with no client callback, runner timeout or runner timeout - m_testRunner = - AZStd::make_unique(AZStd::nullopt, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt); - - // Given an test runner job for each test target with no runner caching - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - JobData jobData(m_testTargetPaths[jobId].m_testRunArtifact, m_testTargetPaths[jobId].m_testCoverageArtifact); - m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId][m_coverageLevel], AZStd::move(jobData))); - } - - // When the test runner jobs are executed - const auto runnerJobs = m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, JobExceptionPolicy::Never); - - // Expect each job to successfully result in a test runner that matches the expected test runner data for that test target - for (const auto& job : runnerJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]); - ValidateTestTargetRun(job.GetPayload().value().first, m_expectedTestTargetRuns[jobId]); - ValidateTestTargetCoverage(job.GetPayload().value().second, m_expectedTestTargetCoverages[jobId][m_coverageLevel]); - } - } - - TEST_P( - InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageParams, - RunTestTargetsWithArbitraryJobIds_RunsAndCoverageMatchTestSuitesInTarget) - { - // Given a set of arbitrary job ids to be used for the test target jobs - enum - { - ArbitraryA = 36, - ArbitraryB = 890, - ArbitraryC = 19, - ArbitraryD = 1 - }; - - const AZStd::unordered_map sequentialToArbitrary = - { - {TestTargetA, ArbitraryA}, - {TestTargetB, ArbitraryB}, - {TestTargetC, ArbitraryC}, - {TestTargetD, ArbitraryD}, - }; - - const AZStd::unordered_map arbitraryToSequential = - { - {ArbitraryA, TestTargetA}, - {ArbitraryB, TestTargetB}, - {ArbitraryC, TestTargetC}, - {ArbitraryD, TestTargetD}, - }; - - // Given a test runner with no client callback, run timeout or runner timeout - m_testRunner = - AZStd::make_unique(AZStd::nullopt, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt); - - // Given an test run job for each test target - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - JobData jobData(m_testTargetPaths[jobId].m_testRunArtifact, m_testTargetPaths[jobId].m_testCoverageArtifact); - m_jobInfos.emplace_back( - JobInfo({sequentialToArbitrary.at(jobId)}, m_testTargetJobArgs[jobId][m_coverageLevel], AZStd::move(jobData))); - } - - // When the instrumented test run jobs are executed - const auto runnerJobs = m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, JobExceptionPolicy::Never); - - // Expect each job to successfully result in a test run that matches the expected test run data for that test target - for (const auto& job : runnerJobs) - { - const JobInfo::IdType jobId = arbitraryToSequential.at(job.GetJobInfo().GetId().m_value); - ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]); - ValidateTestTargetRun(job.GetPayload().value().first, m_expectedTestTargetRuns[jobId]); - ValidateTestTargetCoverage(job.GetPayload().value().second, m_expectedTestTargetCoverages[jobId][m_coverageLevel]); - } - } - - TEST_P(InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageParams, RunTestTargetsWithCallback_RunsAndCoverageMatchTestSuitesInTarget) - { - // Given a client callback function that tracks the number of successful runs - size_t numSuccesses = 0; - const auto jobCallback = - [&numSuccesses]([[maybe_unused]] const TestImpact::InstrumentedTestRunner::JobInfo& jobInfo, const TestImpact::JobMeta& meta) - { - if (meta.m_result == TestImpact::JobResult::ExecutedWithSuccess) - { - numSuccesses++; - } - }; - - // Given a test runner with no run timeout or runner timeout - m_testRunner = - AZStd::make_unique(jobCallback, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt); - - // Given an test run job for each test target - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - JobData jobData(m_testTargetPaths[jobId].m_testRunArtifact, m_testTargetPaths[jobId].m_testCoverageArtifact); - m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId][m_coverageLevel], AZStd::move(jobData))); - } - - // When the instrumented test run jobs are executed - const auto runnerJobs = m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, JobExceptionPolicy::Never); - - // Expect the number of successful runs tracked in the callback to match the number of test targets run with no failures - EXPECT_EQ( - numSuccesses, - AZStd::count_if(m_expectedTestTargetResult.begin(), m_expectedTestTargetResult.end(), [](TestImpact::TestRunResult result) { - return result == TestImpact::TestRunResult::Passed; - })); - - // Expect each job to successfully result in a test run that matches the expected test run data for that test target - for (const auto& job : runnerJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]); - ValidateTestTargetRun(job.GetPayload().value().first, m_expectedTestTargetRuns[jobId]); - ValidateTestTargetCoverage(job.GetPayload().value().second, m_expectedTestTargetCoverages[jobId][m_coverageLevel]); - } - } - - TEST_P(InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageParams, JobRunnerTimeout_InFlightJobsTimeoutAndQueuedJobsUnlaunched) - { - // Given a test runner with no client callback or runner timeout and 2 second run timeout - m_testRunner = AZStd::make_unique( - AZStd::nullopt, m_maxConcurrency, AZStd::chrono::seconds(2), AZStd::nullopt); - - // Given an test run job for each test target where half will sleep indefinitely - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - JobData jobData(m_testTargetPaths[jobId].m_testRunArtifact, m_testTargetPaths[jobId].m_testCoverageArtifact); - const AZStd::string args = (jobId % 2) - ? AZStd::string::format("%s %s", ValidProcessPath, ConstructTestProcessArgs(jobId, LongSleep).c_str()) - : m_testTargetJobArgs[jobId][m_coverageLevel]; - m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData))); - } - - // When the instrumented test run jobs are executed - const auto runnerJobs = m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, JobExceptionPolicy::Never); - - // Expect half the jobs to successfully result in a test run that matches the expected test run data for that test target - // with the other half having timed out - for (const auto& job : runnerJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - if (jobId % 2) - { - ValidateJobTimeout(job); - } - else - { - ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]); - ValidateTestTargetRun(job.GetPayload().value().first, m_expectedTestTargetRuns[jobId]); - ValidateTestTargetCoverage(job.GetPayload().value().second, m_expectedTestTargetCoverages[jobId][m_coverageLevel]); - } - } - } - - TEST_F(InstrumentedTestRunnerFixture, JobTimeout_InFlightJobTimeoutAndQueuedJobsUnlaunched) - { - // Given a test runner with no client callback or run timeout and a 5 second runner timeout - m_testRunner = AZStd::make_unique( - AZStd::nullopt, FourConcurrentProcesses, AZStd::nullopt, AZStd::chrono::seconds(5)); - - // Given an test run job for each test target where half will sleep indefinitely - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - JobData jobData(m_testTargetPaths[jobId].m_testRunArtifact, m_testTargetPaths[jobId].m_testCoverageArtifact); - const AZStd::string args = (jobId % 2) - ? AZStd::string::format("%s %s", ValidProcessPath, ConstructTestProcessArgs(jobId, LongSleep).c_str()) - : m_testTargetJobArgs[jobId][m_coverageLevel]; - m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData))); - } - - // When the instrumented test run jobs are executed - const auto runnerJobs = m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, JobExceptionPolicy::Never); - - // Expect half the jobs to successfully result in a test run that matches the expected test run data for that test target - // with the other half having timed out - for (const auto& job : runnerJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - if (jobId % 2) - { - ValidateJobTimeout(job); - } - else - { - ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]); - ValidateTestTargetRun(job.GetPayload().value().first, m_expectedTestTargetRuns[jobId]); - ValidateTestTargetCoverage(job.GetPayload().value().second, m_expectedTestTargetCoverages[jobId][m_coverageLevel]); - } - } - } - - INSTANTIATE_TEST_CASE_P( - , - InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndCoverageExceptionParams, - ::testing::Combine( - ::testing::ValuesIn(MaxConcurrentRuns), - ::testing::Values(CoverageExceptionPolicy::Never, CoverageExceptionPolicy::OnEmptyCoverage))); - - INSTANTIATE_TEST_CASE_P( - , - InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndFailedToLaunchExceptionParams, - ::testing::Combine( - ::testing::ValuesIn(MaxConcurrentRuns), ::testing::ValuesIn(CoverageLevels), - ::testing::ValuesIn(FailedToLaunchExceptionPolicies))); - - INSTANTIATE_TEST_CASE_P( - , - InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndExecutedWithFailureExceptionParams, - ::testing::Combine( - ::testing::ValuesIn(MaxConcurrentRuns), ::testing::ValuesIn(CoverageLevels), - ::testing::ValuesIn(ExecutedWithFailureExceptionPolicies))); - - INSTANTIATE_TEST_CASE_P( - , - InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageParams, - ::testing::Combine(::testing::ValuesIn(MaxConcurrentRuns), ::testing::ValuesIn(CoverageLevels))); -} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestCoverageTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestCoverageTest.cpp deleted file mode 100644 index c1d7ba3e4b..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestCoverageTest.cpp +++ /dev/null @@ -1,208 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include - -#include -#include - -namespace UnitTest -{ - namespace - { - AZStd::string GenerateSourcePath(AZ::u32 index) - { - return AZStd::string::format("SourceFile%u", index); - } - - AZStd::string GenerateModulePath(AZ::u32 index) - { - return AZStd::string::format("Module%u", index); - } - - AZStd::vector GenerateLineCoverages(AZ::u32 numLines) - { - AZStd::vector lineCoverages; - for (size_t i = 0; i < numLines; i++) - { - // Fudge some superficially different but trivially checkable line coverage data - lineCoverages.emplace_back(TestImpact::LineCoverage{i, i * 2}); - } - - return lineCoverages; - } - - TestImpact::SourceCoverage GenerateSourceCoverage(AZ::u32 index, TestImpact::CoverageLevel coverageLevel) - { - TestImpact::SourceCoverage sourceCoverage; - sourceCoverage.m_path = GenerateSourcePath(index); - if (coverageLevel == TestImpact::CoverageLevel::Line) - { - sourceCoverage.m_coverage = GenerateLineCoverages(index + 1); - } - - return sourceCoverage; - } - - AZStd::vector GenerateSourceCoverages(AZ::u32 numSources, TestImpact::CoverageLevel coverageLevel) - { - AZStd::vector sourceCoverages; - for (AZ::u32 i = 0; i < numSources; i++) - { - sourceCoverages.emplace_back(GenerateSourceCoverage(i, coverageLevel)); - } - - return sourceCoverages; - } - - TestImpact::ModuleCoverage GenerateModuleCoverage(AZ::u32 index, AZ::u32 numSources, TestImpact::CoverageLevel coverageLevel) - { - TestImpact::ModuleCoverage moduleCoverage; - moduleCoverage.m_path = GenerateModulePath(index); - moduleCoverage.m_sources = GenerateSourceCoverages(numSources, coverageLevel); - return moduleCoverage; - } - - AZStd::vector GenerateModuleCoverages(AZ::u32 numModules, TestImpact::CoverageLevel coverageLevel) - { - AZStd::vector moduleCoverages; - for (AZ::u32 i = 0; i < numModules; i++) - { - // Fudge some superficially different but trivially deducible module coverage data - moduleCoverages.emplace_back(GenerateModuleCoverage(i, i + 1, coverageLevel)); - } - - return moduleCoverages; - } - } // namespace - - using CoveragePermutation = AZStd::tuple - < - unsigned, // Number of modules covered - TestImpact::CoverageLevel // Test coverage level - >; - - // Fixture parameterized for different max number of concurrent jobs - class TestCoverageFixtureWithCoverageParams - : public AllocatorsTestFixture - , public ::testing::WithParamInterface - { - public: - void SetUp() override; - - protected: - void ValidateTestCoverage(const TestImpact::TestCoverage& testCoverage); - - size_t m_numModulesCovered; - TestImpact::CoverageLevel m_coverageLevel; - }; - - void TestCoverageFixtureWithCoverageParams::SetUp() - { - AllocatorsTestFixture::SetUp(); - const auto& [numModulesCovered, coverageLevel] = GetParam(); - m_numModulesCovered = numModulesCovered; - m_coverageLevel = coverageLevel; - } - - void TestCoverageFixtureWithCoverageParams::ValidateTestCoverage(const TestImpact::TestCoverage& testCoverage) - { - // Expect the coverage level to match that which was used to generate the module coverages generated - EXPECT_EQ(testCoverage.GetCoverageLevel(), m_coverageLevel); - - // Expect the number of modules covered to match the number of module coverages generated - EXPECT_EQ(testCoverage.GetNumModulesCovered(), m_numModulesCovered); - - // Expect the number of unique sources covered to match the number of modules coverages generated - EXPECT_EQ(testCoverage.GetNumSourcesCovered(), m_numModulesCovered); - - // Expect the unique sources covered to match the procedurally generated source paths - for (size_t sourceIndex = 0; sourceIndex < testCoverage.GetNumSourcesCovered(); sourceIndex++) - { - EXPECT_EQ(testCoverage.GetSourcesCovered()[sourceIndex], GenerateSourcePath(sourceIndex)); - } - - // Expect each module covered to match that of the corresponding procedurally generated modules - for (size_t moduleIndex = 0; moduleIndex < testCoverage.GetNumModulesCovered(); moduleIndex++) - { - const TestImpact::ModuleCoverage& moduleCoverage = testCoverage.GetModuleCoverages()[moduleIndex]; - - // Expect the module path to match that of the corresponding procedurally generated module - EXPECT_EQ(moduleCoverage.m_path, GenerateModulePath(moduleIndex)); - - // Expect the module's number of sources to match that of the corresponding procedurally generated module - EXPECT_EQ(moduleCoverage.m_sources.size(), moduleIndex + 1); - - for (size_t sourceIndex = 0; sourceIndex < moduleCoverage.m_sources.size(); sourceIndex++) - { - const TestImpact::SourceCoverage& sourceCoverage = moduleCoverage.m_sources[sourceIndex]; - - // Expect the source path to match the procedurally generated source path - EXPECT_EQ(sourceCoverage.m_path, GenerateSourcePath(sourceIndex)); - - if (m_coverageLevel == TestImpact::CoverageLevel::Line) - { - // Expect there to actually be line coverage data if this coverage was procedurally generated with line data - EXPECT_FALSE(sourceCoverage.m_coverage.empty()); - - const AZStd::vector& lineCoverages = sourceCoverage.m_coverage; - - // Expect the source's number of lines to match that of the corresponding procedurally generated source - EXPECT_EQ(lineCoverages.size(), sourceIndex + 1); - - for (size_t lineIndex = 0; lineIndex < lineCoverages.size(); lineIndex++) - { - // The expected line number and hit count are deduced as follows: - // Line number: line index - // Hit count: 2x line index - EXPECT_EQ(lineCoverages[lineIndex].m_lineNumber, lineIndex); - EXPECT_EQ(lineCoverages[lineIndex].m_hitCount, lineIndex * 2); - } - } - else - { - // Do not expect there to actually be line coverage data if this coverage was not procedurally generated with line data - EXPECT_TRUE(sourceCoverage.m_coverage.empty()); - } - } - } - } - - TEST(TestCoverage, EmptyCoverage_ExpectTestRunException) - { - // When constructing a test coverage from the empty module coverages - TestImpact::TestCoverage testCoverage(AZStd::vector{}); - - // Expect the test coverage fields to be empty - EXPECT_EQ(testCoverage.GetNumModulesCovered(), 0); - EXPECT_EQ(testCoverage.GetNumSourcesCovered(), 0); - EXPECT_TRUE(testCoverage.GetModuleCoverages().empty()); - EXPECT_TRUE(testCoverage.GetSourcesCovered().empty()); - } - - TEST_P(TestCoverageFixtureWithCoverageParams, AllCoveragePermutations_ExpectTestCoverageMetaDatasToMatchPermutations) - { - // Given a procedurally generated test coverage - const TestImpact::TestCoverage testCoverage(GenerateModuleCoverages(m_numModulesCovered, m_coverageLevel)); - - // Expect the test coverage data and meta-data to match that of the rules used to procedurally generate the coverage data - ValidateTestCoverage(testCoverage); - } - - INSTANTIATE_TEST_CASE_P( - , - TestCoverageFixtureWithCoverageParams, - ::testing::Combine( - ::testing::Range(1u, 11u), - ::testing::Values(TestImpact::CoverageLevel::Line, TestImpact::CoverageLevel::Source)) - ); -} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestEnumeratorTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestEnumeratorTest.cpp deleted file mode 100644 index 9b67eafa54..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestEnumeratorTest.cpp +++ /dev/null @@ -1,890 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include -#include - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace UnitTest -{ - using JobExceptionPolicy = TestImpact::TestEnumerator::JobExceptionPolicy; - using CacheExceptionPolicy = TestImpact::TestEnumerator::CacheExceptionPolicy; - - // Generates the command to run the given test target through AzTestRunner and get gtest to output the enumeration file - AZStd::string GetEnumerateCommandForTarget(AZStd::pair testTarget) - { - return AZStd::string::format( - "%s %s AzRunUnitTests --gtest_list_tests --gtest_output=xml:%s", - LY_TEST_IMPACT_AZ_TESTRUNNER_BIN, - testTarget.first.c_str(), // Path to test target bin - testTarget.second.c_str()); // Path to test target gtest enumeration file - } - - class TestEnumeratorFixture - : public AllocatorsTestFixture - { - public: - void SetUp() override; - - protected: - using JobInfo = TestImpact::TestEnumerator::JobInfo; - using JobData = TestImpact::TestEnumerator::JobData; - - AZStd::vector m_jobInfos; - AZStd::unique_ptr m_testEnumerator; - AZStd::vector m_testTargetJobArgs; - AZStd::vector> m_testTargetPaths; - AZStd::vector m_expectedTestTargetEnumerations; - AZStd::vector m_cacheFiles; - size_t m_maxConcurrency = 0; - }; - - void TestEnumeratorFixture::SetUp() - { - UnitTest::AllocatorsTestFixture::SetUp(); - - DeleteFiles(LY_TEST_IMPACT_TEST_TARGET_ENUMERATION_DIR, "*.cache"); - DeleteFiles(LY_TEST_IMPACT_TEST_TARGET_ENUMERATION_DIR, "*.xml"); - - // first: path to test target bin - // second: path to test target gtest enumeration file in XML format - const AZStd::string enumPath = AZStd::string(LY_TEST_IMPACT_TEST_TARGET_ENUMERATION_DIR) + "/%s.Enumeration.xml"; - m_testTargetPaths.emplace_back( - LY_TEST_IMPACT_TEST_TARGET_A_BIN, AZStd::string::format(enumPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_A_BASE_NAME)); - m_testTargetPaths.emplace_back( - LY_TEST_IMPACT_TEST_TARGET_B_BIN, AZStd::string::format(enumPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_B_BASE_NAME)); - m_testTargetPaths.emplace_back( - LY_TEST_IMPACT_TEST_TARGET_C_BIN, AZStd::string::format(enumPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_C_BASE_NAME)); - m_testTargetPaths.emplace_back( - LY_TEST_IMPACT_TEST_TARGET_D_BIN, AZStd::string::format(enumPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_D_BASE_NAME)); - - m_expectedTestTargetEnumerations.emplace_back(GetTestTargetATestEnumerationSuites()); - m_expectedTestTargetEnumerations.emplace_back(GetTestTargetBTestEnumerationSuites()); - m_expectedTestTargetEnumerations.emplace_back(GetTestTargetCTestEnumerationSuites()); - m_expectedTestTargetEnumerations.emplace_back(GetTestTargetDTestEnumerationSuites()); - - // Path to enumeration file in TIAF internal JSON format - m_cacheFiles.emplace_back( - AZStd::string::format("%s/%s.cache", LY_TEST_IMPACT_TEST_TARGET_ENUMERATION_DIR, LY_TEST_IMPACT_TEST_TARGET_A_BASE_NAME)); - m_cacheFiles.emplace_back( - AZStd::string::format("%s/%s.cache", LY_TEST_IMPACT_TEST_TARGET_ENUMERATION_DIR, LY_TEST_IMPACT_TEST_TARGET_B_BASE_NAME)); - m_cacheFiles.emplace_back( - AZStd::string::format("%s/%s.cache", LY_TEST_IMPACT_TEST_TARGET_ENUMERATION_DIR, LY_TEST_IMPACT_TEST_TARGET_C_BASE_NAME)); - m_cacheFiles.emplace_back( - AZStd::string::format("%s/%s.cache", LY_TEST_IMPACT_TEST_TARGET_ENUMERATION_DIR, LY_TEST_IMPACT_TEST_TARGET_D_BASE_NAME)); - - for (const auto& testTarget : m_testTargetPaths) - { - m_testTargetJobArgs.emplace_back(GetEnumerateCommandForTarget(testTarget)); - } - } - - // Fixture parameterized for different max number of concurrent jobs - class TestEnumeratorFixtureWithConcurrencyParams - : public TestEnumeratorFixture - , public ::testing::WithParamInterface - { - public: - void SetUp() override - { - TestEnumeratorFixture::SetUp(); - m_maxConcurrency = GetParam(); - } - }; - - using ConcurrencyAndJobExceptionPermutation = AZStd::tuple - < - size_t, // Max number of concurrent processes - JobExceptionPolicy // Test job exception policy - >; - - // Fixture parameterized for different max number of concurrent jobs and different job exception policies - class TestEnumeratorFixtureWithConcurrencyAndJobExceptionParams - : public TestEnumeratorFixture - , public ::testing::WithParamInterface - { - public: - void SetUp() override - { - TestEnumeratorFixture::SetUp(); - const auto& [maxConcurrency, jobExceptionPolicy] = GetParam(); - m_maxConcurrency = maxConcurrency; - m_jobExceptionPolicy = jobExceptionPolicy; - } - - protected: - JobExceptionPolicy m_jobExceptionPolicy = JobExceptionPolicy::Never; - }; - - using ConcurrencyAndCacheExceptionPermutation = AZStd::tuple - < - size_t, // Max number of concurrent processes - CacheExceptionPolicy // Test enumeration exception policy - >; - - // Fixture parameterized for different max number of concurrent jobs and different enumeration exception policies - class TestEnumeratorFixtureWithConcurrencyAndCacheExceptionParams - : public TestEnumeratorFixture - , public ::testing::WithParamInterface - { - public: - void SetUp() override - { - TestEnumeratorFixture::SetUp(); - const auto& [maxConcurrency, cacheExceptionPolicy] = GetParam(); - m_maxConcurrency = maxConcurrency; - m_cacheExceptionPolicy = cacheExceptionPolicy; - } - - protected: - CacheExceptionPolicy m_cacheExceptionPolicy = CacheExceptionPolicy::Never; - }; - - namespace - { - AZStd::array MaxConcurrentEnumerations = {{1, 2, 3, 4}}; - - AZStd::array JobExceptionPolicies = - { - JobExceptionPolicy::Never, - JobExceptionPolicy::OnExecutedWithFailure, - JobExceptionPolicy::OnFailedToExecute - }; - - AZStd::array CacheExceptionPolicies = - { - CacheExceptionPolicy::Never, - CacheExceptionPolicy::OnCacheNotExist, - CacheExceptionPolicy::OnCacheReadFailure, - CacheExceptionPolicy::OnCacheWriteFailure - }; - } - - // Validates that the specified job successfully read from its test enumeration cache - void ValidateJobSuccessfulCacheRead(const TestImpact::TestEnumerator::Job& job) - { - EXPECT_EQ(job.GetResult(), TestImpact::JobResult::NotExecuted); - EXPECT_EQ(job.GetStartTime(), AZStd::chrono::high_resolution_clock::time_point()); - EXPECT_EQ(job.GetEndTime(), AZStd::chrono::high_resolution_clock::time_point()); - EXPECT_EQ(job.GetDuration(), AZStd::chrono::milliseconds(0)); - EXPECT_FALSE(job.GetReturnCode().has_value()); - EXPECT_TRUE(job.GetPayload().has_value()); - } - - // Validates that the specified test enumeration matched the expected output - void ValidateTestTargetEnumeration(const TestImpact::TestEnumeration& actualResult, const TestImpact::TestEnumeration& expectedResult) - { - EXPECT_TRUE(actualResult == expectedResult); - EXPECT_EQ(actualResult.GetNumTestSuites(), CalculateNumTestSuites(expectedResult.GetTestSuites())); - EXPECT_EQ(actualResult.GetNumTests(), CalculateNumTests(expectedResult.GetTestSuites())); - EXPECT_EQ(actualResult.GetNumEnabledTests(), CalculateNumEnabledTests(expectedResult.GetTestSuites())); - EXPECT_EQ(actualResult.GetNumDisabledTests(), CalculateNumDisabledTests(expectedResult.GetTestSuites())); - } - - // Validates that the specified test enumeration cache matches the expected output - void ValidateTestEnumerationCache(const AZ::IO::Path& cacheFile, const TestImpact::TestEnumeration& expectedEnumeration) - { - // Cache file must exist - const auto fileSize = AZ::IO::SystemFile::Length(cacheFile.c_str()); - EXPECT_GT(fileSize, 0); - - // Read raw byte data from cache - AZStd::vector buffer(fileSize + 1); - buffer[fileSize] = 0; - EXPECT_TRUE(AZ::IO::SystemFile::Read(cacheFile.c_str(), buffer.data())); - - // Transform raw byte data to raw string data and attempt to construct the test enumeration - AZStd::string rawEnum(buffer.begin(), buffer.end()); - TestImpact::TestEnumeration actualEnumeration = TestImpact::DeserializeTestEnumeration(rawEnum); - - // Check that the constructed test enumeration matches the expected enumeration - ValidateTestTargetEnumeration(actualEnumeration, expectedEnumeration); - } - - // Validates that the specified cache file does not exist - void ValidateInvalidTestEnumerationCache(const AZ::IO::Path& cacheFile) - { - EXPECT_FALSE(AZ::IO::SystemFile::Exists(cacheFile.c_str())); - } - - TEST_P( - TestEnumeratorFixtureWithConcurrencyAndJobExceptionParams, InvalidCommandArgument_ExpectJobResulFailedToExecuteeOrTestJobException) - { - // Given a test enumerator with no client callback or enumeration timeout or enumerator timeout - m_testEnumerator = AZStd::make_unique( - AZStd::nullopt, - m_maxConcurrency, - AZStd::nullopt, - AZStd::nullopt); - - // Given a mixture of test enumeration jobs with valid and invalid command arguments - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - const AZStd::string args = (jobId % 2) ? InvalidProcessPath : m_testTargetJobArgs[jobId]; - JobData jobData(m_testTargetPaths[jobId].second, AZStd::nullopt); - m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData))); - } - - try - { - // When the test enumeration jobs are executed with different exception policies - const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, m_jobExceptionPolicy); - - // Expect this statement to be reachable only if no exception policy for launch failures - EXPECT_FALSE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnFailedToExecute)); - - for (const auto& job : enumerationJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - if (jobId % 2) - { - // Expect invalid jobs have a job result of FailedToExecute - ValidateJobFailedToExecute(job); - } - else - { - // Expect the valid jobs job to successfully result in a test enumeration that matches the expected test enumeration data - ValidateJobExecutedSuccessfully(job); - ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]); - } - } - } - catch ([[maybe_unused]] const TestImpact::TestJobException& e) - { - // Expect this statement to be reachable only if there is an exception policy for launch failures - EXPECT_TRUE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnFailedToExecute)); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_P( - TestEnumeratorFixtureWithConcurrencyAndJobExceptionParams, ErroneousReturnCode_ExpectJobResultExecutedWithFailureOrTestJobException) - { - // Given a test enumerator with no client callback or enumeration timeout or enumerator timeout - m_testEnumerator = AZStd::make_unique( - AZStd::nullopt, - m_maxConcurrency, - AZStd::nullopt, - AZStd::nullopt); - - // Given a mixture of test enumeration jobs that execute and return either successfully or with failure - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - JobData jobData(m_testTargetPaths[jobId].second, AZStd::nullopt); - const AZStd::string args = (jobId % 2) - ? AZStd::string::format("%s %s", ValidProcessPath, ConstructTestProcessArgs(jobId, AZStd::chrono::milliseconds(0)).c_str()) - : m_testTargetJobArgs[jobId]; - m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData))); - } - - try - { - // When the test enumeration jobs are executed with different exception policies - const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, m_jobExceptionPolicy); - - // Expect this statement to be reachable only if no exception policy for jobs that return with error - EXPECT_FALSE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnExecutedWithFailure)); - - for (const auto& job : enumerationJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - if (jobId % 2) - { - // Expect failed jobs to have job result ExecutedWithFailure and a non-zero return code - ValidateJobExecutedWithFailure(job); - } - else - { - // Expect the valid jobs job to successfully result in a test enumeration that matches the expected test enumeration data - ValidateJobExecutedSuccessfully(job); - ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]); - } - } - } - catch ([[maybe_unused]] const TestImpact::TestJobException& e) - { - // Expect this statement to be reachable only if there is an exception policy for jobs that return with error - EXPECT_TRUE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnExecutedWithFailure)); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_P(TestEnumeratorFixtureWithConcurrencyAndCacheExceptionParams, EmptyCacheRead_NoCacheDataButEnumerationsMatchTestSuitesInTarget) - { - // Given a test enumerator with no client callback, enumeration timeout or enumerator timeout - m_testEnumerator = AZStd::make_unique( - AZStd::nullopt, - m_maxConcurrency, - AZStd::nullopt, - AZStd::nullopt); - - // Given an test enumeration job for each test target that reads from an enumeration caching - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - JobData jobData(m_testTargetPaths[jobId].second, JobData::Cache{JobData::CachePolicy::Read, m_cacheFiles[jobId]}); - m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId], AZStd::move(jobData))); - } - - try - { - // When the test enumeration jobs are executed with different exception policies - const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, m_cacheExceptionPolicy, JobExceptionPolicy::Never); - - // Expect this statement to be reachable only if no exception policy for read attempts of non-existent caches - EXPECT_FALSE(::IsFlagSet(m_cacheExceptionPolicy, CacheExceptionPolicy::OnCacheNotExist)); - - // Expect each job to successfully result in a test enumeration that matches the expected test enumeration data for that test target - // even though the cache files could not be read - for (const auto& job : enumerationJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - ValidateJobExecutedSuccessfully(job); - ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]); - ValidateInvalidTestEnumerationCache(job.GetJobInfo().GetCache()->m_file); - } - } - catch ([[maybe_unused]] const TestImpact::TestEnumerationException& e) - { - // Expect this statement to be reachable only if there is an exception policy for read attempts of non-existent caches - EXPECT_TRUE(::IsFlagSet(m_cacheExceptionPolicy, CacheExceptionPolicy::OnCacheNotExist)); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - // Note: this test only cues up one test for enumeration but still runs the permutations for max concurrency so there is duplicated work - TEST_P( - TestEnumeratorFixtureWithConcurrencyAndCacheExceptionParams, - EmptyCacheDataRead_ExpectEnumerationsMatchTestSuitesInTargetOrTestEnumerationException) - { - // Given an enumeration cache for Test Target A with invalid JSON data - WriteTextToFile("", m_cacheFiles[TestTargetA]); - - // Given a test enumerator with no client callback, enumeration timeout or enumerator timeout - m_testEnumerator = AZStd::make_unique( - AZStd::nullopt, - m_maxConcurrency, - AZStd::nullopt, - AZStd::nullopt); - - // Given an test enumeration job that will attempt to read an invalid enumeration cache - JobData jobData(m_testTargetPaths[TestTargetA].second, JobData::Cache{JobData::CachePolicy::Read, m_cacheFiles[TestTargetA]}); - m_jobInfos.emplace_back(JobInfo({TestTargetA}, m_testTargetJobArgs[TestTargetA], AZStd::move(jobData))); - - try - { - // When the test enumeration jobs are executed with different exception policies - const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, m_cacheExceptionPolicy, JobExceptionPolicy::Never); - - // Expect this statement to be reachable only if no exception policy for cache reads that fail - EXPECT_FALSE(::IsFlagSet(m_cacheExceptionPolicy, CacheExceptionPolicy::OnCacheReadFailure)); - - for (const auto& job : enumerationJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - - // Expect the valid jobs job to successfully result in a test enumeration that matches the expected test enumeration data - ValidateJobExecutedSuccessfully(job); - ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]); - } - } - catch ([[maybe_unused]] const TestImpact::TestEnumerationException& e) - { - // Expect this statement to be reachable only if there is an exception policy for cache reads that fail - EXPECT_TRUE(::IsFlagSet(m_cacheExceptionPolicy, CacheExceptionPolicy::OnCacheReadFailure)); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_P( - TestEnumeratorFixtureWithConcurrencyAndCacheExceptionParams, - InvalidCacheWrite_ExpectEnumerationsMatchTestSuitesInTargetOrTestEnumerationException) - { - // Given a test enumerator with no client callback,, enumeration timeout or enumerator timeout - m_testEnumerator = AZStd::make_unique( - AZStd::nullopt, - m_maxConcurrency, - AZStd::nullopt, - AZStd::nullopt); - - // Given an test enumeration job that will attempt to write to an invalid enumeration cache - JobData jobData(m_testTargetPaths[TestTargetA].second, JobData::Cache{JobData::CachePolicy::Write, InvalidProcessPath}); - m_jobInfos.emplace_back(JobInfo({TestTargetA}, m_testTargetJobArgs[TestTargetA], AZStd::move(jobData))); - - try - { - // When the test enumeration job is executed - const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, m_cacheExceptionPolicy, JobExceptionPolicy::Never); - - // Expect this statement to be reachable only if no exception policy for cache writes that fail - EXPECT_FALSE(::IsFlagSet(m_cacheExceptionPolicy, CacheExceptionPolicy::OnCacheWriteFailure)); - - for (const auto& job : enumerationJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - - // Expect the valid jobs job to successfully result in a test enumeration that matches the expected test enumeration data - ValidateJobExecutedSuccessfully(job); - ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]); - } - } - catch ([[maybe_unused]] const TestImpact::TestEnumerationException& e) - { - // Expect this statement to be reachable only if there is an exception policy for cache writes that fail - EXPECT_TRUE(::IsFlagSet(m_cacheExceptionPolicy, CacheExceptionPolicy::OnCacheWriteFailure)); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_P(TestEnumeratorFixtureWithConcurrencyParams, ValidAndInvalidCacheRead_CachedEnumerationsMatchTestSuitesInTarget) - { - // Given the cache file written for only test target B - WriteTextToFile(TestImpact::SerializeTestEnumeration(m_expectedTestTargetEnumerations[TestTargetB]), m_cacheFiles[TestTargetB]); - - // Given a test enumerator with no client callback, enumeration timeout or enumerator timeout - m_testEnumerator = AZStd::make_unique( - AZStd::nullopt, - m_maxConcurrency, - AZStd::nullopt, - AZStd::nullopt); - - // Given test enumeration jobs test target A and D with no enumeration caching - m_jobInfos.emplace_back( - JobInfo({TestTargetA}, m_testTargetJobArgs[TestTargetA], JobData{m_testTargetPaths[TestTargetA].second, AZStd::nullopt})); - m_jobInfos.emplace_back( - JobInfo({TestTargetD}, m_testTargetJobArgs[TestTargetD], JobData{m_testTargetPaths[TestTargetD].second, AZStd::nullopt})); - - // Given test target B with enumeration cache reading and a valid cache file - m_jobInfos.emplace_back(JobInfo( - {TestTargetB}, m_testTargetJobArgs[TestTargetB], - JobData{m_testTargetPaths[TestTargetB].second, JobInfo::Cache{JobData::CachePolicy::Read, m_cacheFiles[TestTargetB]}})); - - // Given test target C with enumeration cache reading and an invalid cache file - m_jobInfos.emplace_back(JobInfo( - {TestTargetC}, m_testTargetJobArgs[TestTargetC], - JobData{m_testTargetPaths[TestTargetC].second, JobInfo::Cache{JobData::CachePolicy::Read, "nothing"}})); - - // When the test enumeration jobs are executed - const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never); - - // Expect each job to successfully result in a test enumeration that matches the expected test enumeration data for that test target - for (const auto& job : enumerationJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - - switch (jobId) - { - case TestTargetA: // No cache read - case TestTargetC: // Cache read, but invalid cache, so re-enumerate anyway - case TestTargetD: // No cache read - { - ValidateJobExecutedSuccessfully(job); - break; - } - case TestTargetB: // Cache read, successful cache read, so job not executed - { - ValidateJobSuccessfulCacheRead(job); - break; - } - default: - { - FAIL(); - } - } - - // Regardless of cache policy and cache failures all targets should still produce the expected test enumerations - ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]); - } - } - - TEST_F(TestEnumeratorFixture, InvalidCacheDataRead_TestEnumerationException) - { - // Given an enumeration cache for Test Target A with invalid JSON data - WriteTextToFile("There is no valid cache data here", m_cacheFiles[TestTargetA]); - - // Given a test enumerator with no client callback, concurrency, enumeration timeout or enumerator timeout - m_testEnumerator = AZStd::make_unique( - AZStd::nullopt, - OneConcurrentProcess, - AZStd::nullopt, - AZStd::nullopt); - - // Given an test enumeration job that will attempt to read an invalid enumeration cache - JobData jobData(m_testTargetPaths[TestTargetA].second, JobData::Cache{JobData::CachePolicy::Read, m_cacheFiles[TestTargetA]}); - m_jobInfos.emplace_back(JobInfo({TestTargetA}, m_testTargetJobArgs[TestTargetA], AZStd::move(jobData))); - - try - { - // When the test enumeration jobs are executed with different exception policies - const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never); - } - catch ([[maybe_unused]] const TestImpact::TestEnumerationException& e) - { - // Expect this statement to be reachable only if there is an exception policy for jobs that return with error - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_P(TestEnumeratorFixtureWithConcurrencyParams, ValidCacheWrite_CachedEnumerationsMatchTestSuitesInTarget) - { - // Given a test enumerator with no client callback, enumeration timeout or enumerator timeout - m_testEnumerator = AZStd::make_unique( - AZStd::nullopt, - m_maxConcurrency, - AZStd::nullopt, - AZStd::nullopt); - - // Given an test enumeration job for each test target with write enumeration caching - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - JobData jobData(m_testTargetPaths[jobId].second, JobData::Cache{JobData::CachePolicy::Write, m_cacheFiles[jobId]}); - m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId], AZStd::move(jobData))); - } - - // When the test enumeration jobs are executed - const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never); - - // Expect each job to successfully result in a test enumeration and cache that matches the expected test enumeration data for that - // test target - for (const auto& job : enumerationJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - ValidateJobExecutedSuccessfully(job); - ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]); - ValidateTestEnumerationCache(job.GetJobInfo().GetCache()->m_file, m_expectedTestTargetEnumerations[jobId]); - } - } - - TEST_F(TestEnumeratorFixture, EmptyArtifact_ExpectTestEnumerationException) - { - // Given a test enumerator with no client callback, concurrency, enumeration timeout or enumerator timeout - m_testEnumerator = AZStd::make_unique( - AZStd::nullopt, - OneConcurrentProcess, - AZStd::nullopt, - AZStd::nullopt); - - // Given an test enumeration job that will return successfully but with an empty artifact string - m_jobInfos.emplace_back(JobInfo({0}, m_testTargetJobArgs[0], JobData("", AZStd::nullopt))); - - try - { - // When the test enumeration job is executed - const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never); - - // Do not expect this statement to be reachable - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::TestEnumerationException& e) - { - // Expect an enumeration exception - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(TestEnumeratorFixture, InvalidArtifact_ExpectTestEnumerationException) - { - // Given a test enumerator with no client callback, concurrency, enumeration timeout or enumerator timeout - m_testEnumerator = AZStd::make_unique( - AZStd::nullopt, - OneConcurrentProcess, - AZStd::nullopt, - AZStd::nullopt); - - // Given an test enumeration job that will return successfully but not produce an artifact - m_jobInfos.emplace_back(JobInfo( - {0}, AZStd::string::format("%s %s", ValidProcessPath, ConstructTestProcessArgs(0, AZStd::chrono::milliseconds(0)).c_str()), - JobData("", AZStd::nullopt))); - - try - { - // When the test enumeration job is executed - const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never); - - // Do not expect this statement to be reachable - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::TestEnumerationException& e) - { - // Expect an enumeration exception - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_P(TestEnumeratorFixtureWithConcurrencyParams, EnumerateTestTargets_EnumerationsMatchTestSuitesInTarget) - { - // Given a test enumerator with no client callback, enumeration timeout or enumerator timeout - m_testEnumerator = AZStd::make_unique( - AZStd::nullopt, - m_maxConcurrency, - AZStd::nullopt, - AZStd::nullopt); - - // Given an test enumeration job for each test target with no enumeration caching - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - JobData jobData(m_testTargetPaths[jobId].second, AZStd::nullopt); - m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId], AZStd::move(jobData))); - } - - // When the test enumeration jobs are executed - const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never); - - // Expect each job to successfully result in a test enumeration that matches the expected test enumeration data for that test target - for (const auto& job : enumerationJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - ValidateJobExecutedSuccessfully(job); - ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]); - } - } - - TEST_P(TestEnumeratorFixtureWithConcurrencyParams, EnumerateTestTargetsWithArbitraryJobIds_EnumerationsMatchTestSuitesInTarget) - { - // Given a set of arbitrary job ids to be used for the test target jobs - enum - { - ArbitraryA = 36, - ArbitraryB = 890, - ArbitraryC = 19, - ArbitraryD = 1 - }; - - const AZStd::unordered_map sequentialToArbitrary = - { - {TestTargetA, ArbitraryA}, - {TestTargetB, ArbitraryB}, - {TestTargetC, ArbitraryC}, - {TestTargetD, ArbitraryD}, - }; - - const AZStd::unordered_map arbitraryToSequential = - { - {ArbitraryA, TestTargetA}, - {ArbitraryB, TestTargetB}, - {ArbitraryC, TestTargetC}, - {ArbitraryD, TestTargetD}, - }; - - // Given a test enumerator with no client callback, enumeration timeout or enumerator timeout - m_testEnumerator = AZStd::make_unique( - AZStd::nullopt, - m_maxConcurrency, - AZStd::nullopt, - AZStd::nullopt); - - // Given an test enumeration job for each test target with no enumeration caching - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - JobData jobData(m_testTargetPaths[jobId].second, AZStd::nullopt); - m_jobInfos.emplace_back(JobInfo({sequentialToArbitrary.at(jobId)}, m_testTargetJobArgs[jobId], AZStd::move(jobData))); - } - - // When the test enumeration jobs are executed - const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never); - - // Expect each job to successfully result in a test enumeration that matches the expected test enumeration data for that test target - for (const auto& job : enumerationJobs) - { - const JobInfo::IdType jobId = arbitraryToSequential.at(job.GetJobInfo().GetId().m_value); - ValidateJobExecutedSuccessfully(job); - ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]); - } - } - - TEST_P(TestEnumeratorFixtureWithConcurrencyParams, EnumerateTestTargetsWithCallback_EnumerationsMatchTestSuitesInTarget) - { - // Given a client callback function that tracks the number of successful enumerations - size_t numSuccesses = 0; - const auto jobCallback = [&numSuccesses]([[maybe_unused]] const TestImpact::TestEnumerator::JobInfo& jobInfo, const TestImpact::JobMeta& meta) - { - if (meta.m_result == TestImpact::JobResult::ExecutedWithSuccess) - { - numSuccesses++; - } - }; - - // Given a test enumerator with no enumeration timeout or enumerator timeout - m_testEnumerator = AZStd::make_unique( - jobCallback, - m_maxConcurrency, - AZStd::nullopt, - AZStd::nullopt); - - // Given an test enumeration job for each test target with no enumeration caching - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - JobData jobData(m_testTargetPaths[jobId].second, AZStd::nullopt); - m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId], AZStd::move(jobData))); - } - - // When the test enumeration jobs are executed - const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never); - - // Expect the number of successful enumerations tracked in the callback to match the number of test targets enumerated - EXPECT_EQ(numSuccesses, enumerationJobs.size()); - - // Expect each job to successfully result in a test enumeration that matches the expected test enumeration data for that test target - for (const auto& job : enumerationJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - ValidateJobExecutedSuccessfully(job); - ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]); - } - } - - TEST_P(TestEnumeratorFixtureWithConcurrencyParams, JobRunnerTimeout_InFlightJobsTimeoutAndQueuedJobsUnlaunched) - { - // Given a test enumerator with no client callback or enumerator timeout and 500ms enumeration timeout - m_testEnumerator = AZStd::make_unique( - AZStd::nullopt, - m_maxConcurrency, - AZStd::chrono::milliseconds(500), - AZStd::nullopt); - - // Given an test enumeration job for each test target with no enumeration caching where half will sleep indefinitely - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - JobData jobData(m_testTargetPaths[jobId].second, AZStd::nullopt); - const AZStd::string args = (jobId % 2) - ? AZStd::string::format("%s %s", ValidProcessPath, ConstructTestProcessArgs(jobId, LongSleep).c_str()) - : m_testTargetJobArgs[jobId]; - m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData))); - } - - // When the test enumeration jobs are executed - const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never); - - // Expect half the jobs to successfully result in a test enumeration that matches the expected test enumeration data for that test - // target with the other half having timed out - for (const auto& job : enumerationJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - if (jobId % 2) - { - ValidateJobTimeout(job); - } - else - { - ValidateJobExecutedSuccessfully(job); - ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]); - } - } - } - - TEST_F(TestEnumeratorFixture, JobTimeout_InFlightJobTimeoutAndQueuedJobsUnlaunched) - { - // Given a test enumerator with no client callback or enumeration timeout and a 5 second enumerator timeout - m_testEnumerator = AZStd::make_unique( - AZStd::nullopt, - FourConcurrentProcesses, - AZStd::nullopt, - AZStd::chrono::milliseconds(5000)); - - // Given an test enumeration job for each test target with no enumeration caching where half will sleep indefinitely - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - JobData jobData(m_testTargetPaths[jobId].second, AZStd::nullopt); - const AZStd::string args = (jobId % 2) - ? AZStd::string::format("%s %s", ValidProcessPath, ConstructTestProcessArgs(jobId, LongSleep).c_str()) - : m_testTargetJobArgs[jobId]; - m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData))); - } - - // When the test enumeration jobs are executed - const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never); - - // Expect half the jobs to successfully result in a test enumeration that matches the expected test enumeration data for that test - // target with the other half having timed out - for (const auto& job : enumerationJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - if (jobId % 2) - { - ValidateJobTimeout(job); - } - else - { - ValidateJobExecutedSuccessfully(job); - ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]); - } - } - } - - INSTANTIATE_TEST_CASE_P( - , - TestEnumeratorFixtureWithConcurrencyAndJobExceptionParams, - ::testing::Combine( - ::testing::ValuesIn(MaxConcurrentEnumerations), - ::testing::ValuesIn(JobExceptionPolicies)) - ); - - INSTANTIATE_TEST_CASE_P( - , - TestEnumeratorFixtureWithConcurrencyAndCacheExceptionParams, - ::testing::Combine( - ::testing::ValuesIn(MaxConcurrentEnumerations), - ::testing::ValuesIn(CacheExceptionPolicies)) - ); - - INSTANTIATE_TEST_CASE_P( - , - TestEnumeratorFixtureWithConcurrencyParams, - ::testing::ValuesIn(MaxConcurrentEnumerations) - ); -} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestEumerationSerializerTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestEumerationSerializerTest.cpp deleted file mode 100644 index aa94657378..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestEumerationSerializerTest.cpp +++ /dev/null @@ -1,99 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include - -#include -#include - -#include -#include - -namespace UnitTest -{ - const TestImpact::TestEnumeration EmptyTestEnum(AZStd::vector{}); - - TEST(TestEnumerationSerializer, EmptyTestEnumerationSuites_ExpectTemptyTestEnumeration) - { - // Given an empty set of test enumeration suites - // When the test enumeration is serialized and deserialized back again - const AZStd::string serializedString = TestImpact::SerializeTestEnumeration(EmptyTestEnum); - const TestImpact::TestEnumeration actualEnumeration = TestImpact::DeserializeTestEnumeration(serializedString); - - // Expect the actual test enumeration to match the expected test enumeration - EXPECT_TRUE(actualEnumeration == EmptyTestEnum); - } - - TEST(TestEnumerationSerializer, SerializeAndDeserializeSuitesForTestTargetA_ActualSuiteDataMatchesExpectedSuiteData) - { - // Given the test of test enumeration suites for Test target A - const TestImpact::TestEnumeration expectedEnumeration = TestImpact::TestEnumeration(GetTestTargetATestEnumerationSuites()); - - // When the test enumeration is serialized and deserialized back again - const AZStd::string serializedString = TestImpact::SerializeTestEnumeration(expectedEnumeration); - const TestImpact::TestEnumeration actualEnumeration = TestImpact::DeserializeTestEnumeration(serializedString); - - // Expect the actual test enumeration to match the expected test enumeration - EXPECT_TRUE(actualEnumeration == expectedEnumeration); - - // Do not expect the actual test enumeration to match the empty test enumeration - EXPECT_FALSE(actualEnumeration == EmptyTestEnum); - } - - TEST(TestEnumerationSerializer, SerializeAndDeserializeSuitesForTestTargetB_ActualSuiteDataMatchesExpectedSuiteData) - { - // Given the test of test enumeration suites for Test target B - const TestImpact::TestEnumeration expectedEnumeration = TestImpact::TestEnumeration(GetTestTargetBTestEnumerationSuites()); - - // When the test enumeration is serialized and deserialized back again - const AZStd::string serializedString = TestImpact::SerializeTestEnumeration(expectedEnumeration); - const TestImpact::TestEnumeration actualEnumeration = TestImpact::DeserializeTestEnumeration(serializedString); - - // Expect the actual test enumeration to match the expected test enumeration - EXPECT_TRUE(actualEnumeration == expectedEnumeration); - - // Do not expect the actual test enumeration to match the empty test enumeration - EXPECT_FALSE(actualEnumeration == EmptyTestEnum); - } - - TEST(TestEnumerationSerializer, SerializeAndDeserializeSuitesForTestTargetC_ActualSuiteDataMatchesExpectedSuiteData) - { - // Given the test of test enumeration suites for Test target B - const TestImpact::TestEnumeration expectedEnumeration = TestImpact::TestEnumeration(GetTestTargetCTestEnumerationSuites()); - - // When the test enumeration is serialized and deserialized back again - const AZStd::string serializedString = TestImpact::SerializeTestEnumeration(expectedEnumeration); - const TestImpact::TestEnumeration actualEnumeration = TestImpact::DeserializeTestEnumeration(serializedString); - - // Expect the actual test enumeration to match the expected test enumeration - EXPECT_TRUE(actualEnumeration == expectedEnumeration); - - // Do not expect the actual test enumeration to match the empty test enumeration - EXPECT_FALSE(actualEnumeration == EmptyTestEnum); - } - - TEST(TestEnumerationSerializer, SerializeAndDeserializeSuitesForTestTargetD_ActualSuiteDataMatchesExpectedSuiteData) - { - // Given the test of test enumeration suites for Test target B - const TestImpact::TestEnumeration expectedEnumeration = TestImpact::TestEnumeration(GetTestTargetDTestEnumerationSuites()); - - // When the test enumeration is serialized and deserialized back again - const AZStd::string serializedString = TestImpact::SerializeTestEnumeration(expectedEnumeration); - const TestImpact::TestEnumeration actualEnumeration = TestImpact::DeserializeTestEnumeration(serializedString); - - // Expect the actual test enumeration to match the expected test enumeration - EXPECT_TRUE(actualEnumeration == expectedEnumeration); - - // Do not expect the actual test enumeration to match the empty test enumeration - EXPECT_FALSE(actualEnumeration == EmptyTestEnum); - } -} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestRunSerializerTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestRunSerializerTest.cpp deleted file mode 100644 index 1fc7c62b3d..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestRunSerializerTest.cpp +++ /dev/null @@ -1,99 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include - -#include -#include - -#include -#include - -namespace UnitTest -{ - const TestImpact::TestRun EmptyTestRun(AZStd::vector{}, AZStd::chrono::milliseconds{0}); - - TEST(TestRunSerializer, EmptyTestRunSuites_ExpectTemptyTestRun) - { - // Given an empty set of test run suites - // When the test run is serialized and deserialized back again - const auto serializedString = TestImpact::SerializeTestRun(EmptyTestRun); - const auto actualRun = TestImpact::DeserializeTestRun(serializedString); - - // Expect the actual test run to match the expected test run - EXPECT_TRUE(actualRun == EmptyTestRun); - } - - TEST(TestRunSerializer, SerializeAndDeserializeSuitesForTestTargetA_ActualSuiteDataMatchesExpectedSuiteData) - { - // Given the test of test run suites for Test target A - const auto expectedRun = TestImpact::TestRun(GetTestTargetATestRunSuites(), AZStd::chrono::milliseconds{500}); - - // When the test run is serialized and deserialized back again - const auto serializedString = TestImpact::SerializeTestRun(expectedRun); - const auto actualRun = TestImpact::DeserializeTestRun(serializedString); - - // Expect the actual test run to match the expected test run - EXPECT_TRUE(actualRun == expectedRun); - - // Do not expect the actual test run to match the empty test run - EXPECT_FALSE(actualRun == EmptyTestRun); - } - - TEST(TestRunSerializer, SerializeAndDeserializeSuitesForTestTargetB_ActualSuiteDataMatchesExpectedSuiteData) - { - // Given the test of test run suites for Test target B - const auto expectedRun = TestImpact::TestRun(GetTestTargetBTestRunSuites(), AZStd::chrono::milliseconds{500}); - - // When the test run is serialized and deserialized back again - const auto serializedString = TestImpact::SerializeTestRun(expectedRun); - const auto actualRun = TestImpact::DeserializeTestRun(serializedString); - - // Expect the actual test run to match the expected test run - EXPECT_TRUE(actualRun == expectedRun); - - // Do not expect the actual test run to match the empty test run - EXPECT_FALSE(actualRun == EmptyTestRun); - } - - TEST(TestRunSerializer, SerializeAndDeserializeSuitesForTestTargetC_ActualSuiteDataMatchesExpectedSuiteData) - { - // Given the test of test run suites for Test target B - const auto expectedRun = TestImpact::TestRun(GetTestTargetCTestRunSuites(), AZStd::chrono::milliseconds{500}); - - // When the test run is serialized and deserialized back again - const auto serializedString = TestImpact::SerializeTestRun(expectedRun); - const auto actualRun = TestImpact::DeserializeTestRun(serializedString); - - // Expect the actual test run to match the expected test run - EXPECT_TRUE(actualRun == expectedRun); - - // Do not expect the actual test run to match the empty test run - EXPECT_FALSE(actualRun == EmptyTestRun); - } - - TEST(TestRunSerializer, SerializeAndDeserializeSuitesForTestTargetD_ActualSuiteDataMatchesExpectedSuiteData) - { - // Given the test of test run suites for Test target B - const auto expectedRun = TestImpact::TestRun(GetTestTargetDTestRunSuites(), AZStd::chrono::milliseconds{500}); - - // When the test run is serialized and deserialized back again - const auto serializedString = TestImpact::SerializeTestRun(expectedRun); - const auto actualRun = TestImpact::DeserializeTestRun(serializedString); - - // Expect the actual test run to match the expected test run - EXPECT_TRUE(actualRun == expectedRun); - - // Do not expect the actual test run to match the empty test run - EXPECT_FALSE(actualRun == EmptyTestRun); - } -} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestRunnerTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestRunnerTest.cpp deleted file mode 100644 index 08a0306e7f..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestRunnerTest.cpp +++ /dev/null @@ -1,516 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include -#include - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace UnitTest -{ - using JobExceptionPolicy = TestImpact::TestRunner::JobExceptionPolicy; - - AZStd::string GetRunCommandForTarget(AZStd::pair testTarget) - { - return AZStd::string::format( - "%s %s AzRunUnitTests --gtest_output=xml:%s", LY_TEST_IMPACT_AZ_TESTRUNNER_BIN, testTarget.first.c_str(), - testTarget.second.c_str()); - } - - class TestRunnerFixture - : public AllocatorsTestFixture - { - public: - void SetUp() override; - - protected: - using JobInfo = TestImpact::TestRunner::JobInfo; - using JobData = TestImpact::TestRunner::JobData; - - AZStd::vector m_jobInfos; - AZStd::unique_ptr m_testRunner; - AZStd::vector m_testTargetJobArgs; - AZStd::vector> m_testTargetPaths; - AZStd::vector m_expectedTestTargetRuns; - AZStd::vector m_expectedTestTargetResult; - size_t m_maxConcurrency = 0; - }; - - void TestRunnerFixture::SetUp() - { - UnitTest::AllocatorsTestFixture::SetUp(); - - DeleteFiles(LY_TEST_IMPACT_TEST_TARGET_RESULTS_DIR, "*.xml"); - - // first: path to test target bin - // second: path to test target gtest results file in XML format - const AZStd::string runPath = AZStd::string(LY_TEST_IMPACT_TEST_TARGET_RESULTS_DIR) + "/%s.Run.xml"; - m_testTargetPaths.emplace_back( - LY_TEST_IMPACT_TEST_TARGET_A_BIN, AZStd::string::format(runPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_A_BASE_NAME)); - m_testTargetPaths.emplace_back( - LY_TEST_IMPACT_TEST_TARGET_B_BIN, AZStd::string::format(runPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_B_BASE_NAME)); - m_testTargetPaths.emplace_back( - LY_TEST_IMPACT_TEST_TARGET_C_BIN, AZStd::string::format(runPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_C_BASE_NAME)); - m_testTargetPaths.emplace_back( - LY_TEST_IMPACT_TEST_TARGET_D_BIN, AZStd::string::format(runPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_D_BASE_NAME)); - - m_expectedTestTargetRuns.emplace_back(GetTestTargetATestRunSuites(), AZStd::chrono::milliseconds{500}); - m_expectedTestTargetRuns.emplace_back(GetTestTargetBTestRunSuites(), AZStd::chrono::milliseconds{500}); - m_expectedTestTargetRuns.emplace_back(GetTestTargetCTestRunSuites(), AZStd::chrono::milliseconds{500}); - m_expectedTestTargetRuns.emplace_back(GetTestTargetDTestRunSuites(), AZStd::chrono::milliseconds{500}); - - m_expectedTestTargetResult.emplace_back(TestImpact::TestRunResult::Failed); - m_expectedTestTargetResult.emplace_back(TestImpact::TestRunResult::Passed); - m_expectedTestTargetResult.emplace_back(TestImpact::TestRunResult::Passed); - m_expectedTestTargetResult.emplace_back(TestImpact::TestRunResult::Passed); - - for (const auto& testTarget : m_testTargetPaths) - { - m_testTargetJobArgs.emplace_back(GetRunCommandForTarget(testTarget)); - } - } - - // Fixture parameterized for different max number of concurrent jobs - class TestRunnerFixtureWithConcurrencyParams - : public TestRunnerFixture - , public ::testing::WithParamInterface - { - public: - void SetUp() override - { - TestRunnerFixture::SetUp(); - m_maxConcurrency = GetParam(); - } - }; - - using ConcurrencyAndJobExceptionPermutation = AZStd::tuple - < - size_t, // Max number of concurrent processes - JobExceptionPolicy // Test job exception policy - >; - - // Fixture parameterized for different max number of concurrent jobs and different job exception policies - class TestRunnerFixtureWithConcurrencyAndJobExceptionParams - : public TestRunnerFixture - , public ::testing::WithParamInterface - { - public: - void SetUp() override - { - TestRunnerFixture::SetUp(); - const auto& [maxConcurrency, jobExceptionPolicy] = GetParam(); - m_maxConcurrency = maxConcurrency; - m_jobExceptionPolicy = jobExceptionPolicy; - } - - protected: - JobExceptionPolicy m_jobExceptionPolicy = JobExceptionPolicy::Never; - }; - - class TestRunnerFixtureWithConcurrencyAndFailedToLaunchExceptionParams - : public TestRunnerFixtureWithConcurrencyAndJobExceptionParams - { - }; - - class TestRunnerFixtureWithConcurrencyAndExecutedWithFailureExceptionParams - : public TestRunnerFixtureWithConcurrencyAndJobExceptionParams - { - }; - - namespace - { - AZStd::array MaxConcurrentRuns = {1, 2, 3, 4}; - - AZStd::array FailedToLaunchExceptionPolicies = { - JobExceptionPolicy::Never, JobExceptionPolicy::OnFailedToExecute}; - - AZStd::array ExecutedWithFailureExceptionPolicies = { - JobExceptionPolicy::Never, JobExceptionPolicy::OnExecutedWithFailure}; - } // namespace - - TEST_P( - TestRunnerFixtureWithConcurrencyAndFailedToLaunchExceptionParams, - InvalidCommandArgument_ExpectJobResulFailedToExecuteeOrTestJobException) - { - // Given a test runner with no client callback or run timeout or runner timeout - m_testRunner = AZStd::make_unique(AZStd::nullopt, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt); - - // Given a mixture of test run jobs with valid and invalid command arguments - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - const AZStd::string args = (jobId % 2) ? InvalidProcessPath : m_testTargetJobArgs[jobId]; - JobData jobData(m_testTargetPaths[jobId].second); - m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData))); - } - - try - { - // When the test run jobs are executed with different exception policies - const auto runnerJobs = m_testRunner->RunTests(m_jobInfos, m_jobExceptionPolicy); - - // Expect this statement to be reachable only if no exception policy for launch failures - EXPECT_FALSE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnFailedToExecute)); - - for (const auto& job : runnerJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - if (jobId % 2) - { - // Expect invalid jobs have a job result of FailedToExecute - ValidateJobFailedToExecute(job); - } - else - { - // Expect the valid jobs to successfully result in a test run that matches the expected test run data - ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]); - ValidateTestTargetRun(job.GetPayload().value(), m_expectedTestTargetRuns[jobId]); - } - } - } - catch ([[maybe_unused]] const TestImpact::TestJobException& e) - { - // Expect this statement to be reachable only if there is an exception policy for launch failures - EXPECT_TRUE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnFailedToExecute)); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_P( - TestRunnerFixtureWithConcurrencyAndExecutedWithFailureExceptionParams, - ErroneousReturnCode_ExpectJobResultExecutedWithFailureOrTestJobException) - { - // Given a test runner with no client callback or run timeout or runner timeout - m_testRunner = AZStd::make_unique(AZStd::nullopt, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt); - - // Given a mixture of test run jobs that execute and return either successfully or with failure - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - JobData jobData(m_testTargetPaths[jobId].second); - m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId], AZStd::move(jobData))); - } - - try - { - // When the test run jobs are executed with different exception policies - const auto runnerJobs = m_testRunner->RunTests(m_jobInfos, m_jobExceptionPolicy); - - // Expect this statement to be reachable only if no exception policy for jobs that return with error - EXPECT_FALSE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnExecutedWithFailure)); - - for (const auto& job : runnerJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - - // Expect the valid jobs to successfully result in a test run that matches the expected test run data - ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]); - ValidateTestTargetRun(job.GetPayload().value(), m_expectedTestTargetRuns[jobId]); - } - } - catch ([[maybe_unused]] const TestImpact::TestJobException& e) - { - // Expect this statement to be reachable only if there is an exception policy for jobs that return with error - EXPECT_TRUE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnExecutedWithFailure)); - } - catch ([[maybe_unused]] const TestImpact::Exception& e) - { - FAIL(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(TestRunnerFixture, EmptyArtifact_ExpectTestRunnerException) - { - // Given a test runner with no client callback, concurrency, run timeout or runner timeout - m_testRunner = AZStd::make_unique(AZStd::nullopt, OneConcurrentProcess, AZStd::nullopt, AZStd::nullopt); - - // Given an test runner job that will return successfully but with an empty artifact string - m_jobInfos.emplace_back(JobInfo({0}, m_testTargetJobArgs[0], JobData(""))); - - try - { - // When the test runner job is executed - const auto runnerJobs = m_testRunner->RunTests(m_jobInfos, JobExceptionPolicy::Never); - - // Do not expect this statement to be reachable - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::TestRunException& e) - { - // Expect an runner exception - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(TestRunnerFixture, InvalidRunArtifact_ExpectArtifactException) - { - // Given a test run artifact with invalid contents - WriteTextToFile("There is nothing valid here", m_testTargetPaths[TestTargetA].second); - - // Given a job command that will write the test run artifact to a different location that what we will read from - auto invalidRunArtifact = m_testTargetPaths[TestTargetA]; - invalidRunArtifact.second /= ".xml"; - const AZStd::string args = GetRunCommandForTarget(invalidRunArtifact); - - // Given a test runner with no client callback, concurrency, run timeout or runner timeout - m_testRunner = AZStd::make_unique(AZStd::nullopt, OneConcurrentProcess, AZStd::nullopt, AZStd::nullopt); - - // Given an test runner job that will return successfully but not produce an artifact - JobData jobData(m_testTargetPaths[TestTargetA].second); - m_jobInfos.emplace_back(JobInfo({TestTargetA}, args, AZStd::move(jobData))); - - try - { - // When the test runner job is executed - const auto runnerJobs = m_testRunner->RunTests(m_jobInfos, JobExceptionPolicy::Never); - - // Do not expect this statement to be reachable - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::ArtifactException& e) - { - // Expect an runner exception - SUCCEED(); - } - catch (const TestImpact::Exception& e) - { - std::cout << e.what(); - FAIL(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_P(TestRunnerFixtureWithConcurrencyParams, RunTestTargets_RunsMatchTestSuitesInTarget) - { - // Given a test runner with no client callback, runner timeout or runner timeout - m_testRunner = AZStd::make_unique(AZStd::nullopt, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt); - - // Given an test runner job for each test target with no runner caching - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - JobData jobData(m_testTargetPaths[jobId].second); - m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId], AZStd::move(jobData))); - } - - // When the test runner jobs are executed - const auto runnerJobs = m_testRunner->RunTests(m_jobInfos, JobExceptionPolicy::Never); - - // Expect each job to successfully result in a test runner that matches the expected test runner data for that test target - for (const auto& job : runnerJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]); - ValidateTestTargetRun(job.GetPayload().value(), m_expectedTestTargetRuns[jobId]); - } - } - - TEST_P(TestRunnerFixtureWithConcurrencyParams, RunTestTargetsWithArbitraryJobIds_RunsMatchTestSuitesInTarget) - { - // Given a set of arbitrary job ids to be used for the test target jobs - enum - { - ArbitraryA = 36, - ArbitraryB = 890, - ArbitraryC = 19, - ArbitraryD = 1 - }; - - const AZStd::unordered_map sequentialToArbitrary = - { - {TestTargetA, ArbitraryA}, - {TestTargetB, ArbitraryB}, - {TestTargetC, ArbitraryC}, - {TestTargetD, ArbitraryD}, - }; - - const AZStd::unordered_map arbitraryToSequential = - { - {ArbitraryA, TestTargetA}, - {ArbitraryB, TestTargetB}, - {ArbitraryC, TestTargetC}, - {ArbitraryD, TestTargetD}, - }; - - // Given a test runner with no client callback, run timeout or runner timeout - m_testRunner = AZStd::make_unique(AZStd::nullopt, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt); - - // Given an test run job for each test target - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - JobData jobData(m_testTargetPaths[jobId].second); - m_jobInfos.emplace_back(JobInfo({sequentialToArbitrary.at(jobId)}, m_testTargetJobArgs[jobId], AZStd::move(jobData))); - } - - // When the test run jobs are executed - const auto runnerJobs = m_testRunner->RunTests(m_jobInfos, JobExceptionPolicy::Never); - - // Expect each job to successfully result in a test run that matches the expected test run data for that test target - for (const auto& job : runnerJobs) - { - const JobInfo::IdType jobId = arbitraryToSequential.at(job.GetJobInfo().GetId().m_value); - ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]); - ValidateTestTargetRun(job.GetPayload().value(), m_expectedTestTargetRuns[jobId]); - } - } - - TEST_P(TestRunnerFixtureWithConcurrencyParams, RunTestTargetsWithCallback_RunsMatchTestSuitesInTarget) - { - // Given a client callback function that tracks the number of successful runs - size_t numSuccesses = 0; - const auto jobCallback = - [&numSuccesses]([[maybe_unused]] const TestImpact::TestRunner::JobInfo& jobInfo, const TestImpact::JobMeta& meta) { - if (meta.m_result == TestImpact::JobResult::ExecutedWithSuccess) - { - numSuccesses++; - } - }; - - // Given a test runner with no run timeout or runner timeout - m_testRunner = AZStd::make_unique(jobCallback, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt); - - // Given an test run job for each test target - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - JobData jobData(m_testTargetPaths[jobId].second); - m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId], AZStd::move(jobData))); - } - - // When the test run jobs are executed - const auto runnerJobs = m_testRunner->RunTests(m_jobInfos, JobExceptionPolicy::Never); - - // Expect the number of successful runs tracked in the callback to match the number of test targets run with no failures - EXPECT_EQ( - numSuccesses, - AZStd::count_if(m_expectedTestTargetResult.begin(), m_expectedTestTargetResult.end(), [](TestImpact::TestRunResult result) { - return result == TestImpact::TestRunResult::Passed; - })); - - // Expect each job to successfully result in a test run that matches the expected test run data for that test target - for (const auto& job : runnerJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]); - ValidateTestTargetRun(job.GetPayload().value(), m_expectedTestTargetRuns[jobId]); - } - } - - TEST_P(TestRunnerFixtureWithConcurrencyParams, JobRunnerTimeout_InFlightJobsTimeoutAndQueuedJobsUnlaunched) - { - // Given a test runner with no client callback or runner timeout and 500ms run timeout - m_testRunner = - AZStd::make_unique(AZStd::nullopt, m_maxConcurrency, AZStd::chrono::milliseconds(500), AZStd::nullopt); - - // Given an test run job for each test target where half will sleep indefinitely - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - JobData jobData(m_testTargetPaths[jobId].second); - const AZStd::string args = (jobId % 2) - ? AZStd::string::format("%s %s", ValidProcessPath, ConstructTestProcessArgs(jobId, LongSleep).c_str()) - : m_testTargetJobArgs[jobId]; - m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData))); - } - - // When the test run jobs are executed - const auto runnerJobs = m_testRunner->RunTests(m_jobInfos, JobExceptionPolicy::Never); - - // Expect half the jobs to successfully result in a test run that matches the expected test run data for that test target - // with the other half having timed out - for (const auto& job : runnerJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - if (jobId % 2) - { - ValidateJobTimeout(job); - } - else - { - ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]); - ValidateTestTargetRun(job.GetPayload().value(), m_expectedTestTargetRuns[jobId]); - } - } - } - - TEST_F(TestRunnerFixture, JobTimeout_InFlightJobTimeoutAndQueuedJobsUnlaunched) - { - // Given a test runner with no client callback or run timeout and a 5 second runner timeout - m_testRunner = AZStd::make_unique( - AZStd::nullopt, FourConcurrentProcesses, AZStd::nullopt, AZStd::chrono::milliseconds(5000)); - - // Given an test run job for each test target where half will sleep indefinitely - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - JobData jobData(m_testTargetPaths[jobId].second); - const AZStd::string args = (jobId % 2) - ? AZStd::string::format("%s %s", ValidProcessPath, ConstructTestProcessArgs(jobId, LongSleep).c_str()) - : m_testTargetJobArgs[jobId]; - m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData))); - } - - // When the test run jobs are executed - const auto runnerJobs = m_testRunner->RunTests(m_jobInfos, JobExceptionPolicy::Never); - - // Expect half the jobs to successfully result in a test run that matches the expected test run data for that test target - // with the other half having timed out - for (const auto& job : runnerJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - if (jobId % 2) - { - ValidateJobTimeout(job); - } - else - { - ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]); - ValidateTestTargetRun(job.GetPayload().value(), m_expectedTestTargetRuns[jobId]); - } - } - } - - INSTANTIATE_TEST_CASE_P( - , - TestRunnerFixtureWithConcurrencyAndFailedToLaunchExceptionParams, - ::testing::Combine(::testing::ValuesIn(MaxConcurrentRuns), ::testing::ValuesIn(FailedToLaunchExceptionPolicies))); - - INSTANTIATE_TEST_CASE_P( - , - TestRunnerFixtureWithConcurrencyAndExecutedWithFailureExceptionParams, - ::testing::Combine(::testing::ValuesIn(MaxConcurrentRuns), ::testing::ValuesIn(ExecutedWithFailureExceptionPolicies))); - - INSTANTIATE_TEST_CASE_P(, TestRunnerFixtureWithConcurrencyParams, ::testing::ValuesIn(MaxConcurrentRuns)); -} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactFrameworkPathTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactFrameworkPathTest.cpp deleted file mode 100644 index 2982cb0551..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactFrameworkPathTest.cpp +++ /dev/null @@ -1,137 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include -#include -#include - -namespace UnitTest -{ - class FrameworkPathTestFixture - : public AllocatorsTestFixture - { - public: - void SetUp() override - { - UnitTest::AllocatorsTestFixture::SetUp(); - - m_parentPathAbs = AZStd::string(AZStd::string("parent") + AZ_TRAIT_OS_PATH_SEPARATOR + "path"); - m_childPathRel = AZStd::string(AZStd::string("child") + AZ_TRAIT_OS_PATH_SEPARATOR + "path"); - m_childPathAbs = AZStd::string(m_parentPathAbs + AZ_TRAIT_OS_PATH_SEPARATOR + m_childPathRel); - - m_pathComponentA = AZStd::string("DirA"); - m_pathComponentB = AZStd::string("DirB"); - m_pathComponentC = AZStd::string("DirC"); - - m_posixPath = m_pathComponentA + AZ::IO::PosixPathSeparator + m_pathComponentB + AZ::IO::PosixPathSeparator + m_pathComponentC; - - m_windowsPath = - m_pathComponentA + AZ::IO::WindowsPathSeparator + m_pathComponentB + AZ::IO::WindowsPathSeparator + m_pathComponentC; - - m_mixedPath = - m_pathComponentA + AZ::IO::WindowsPathSeparator + m_pathComponentB + AZ::IO::PosixPathSeparator + m_pathComponentC; - - m_referredPath = - m_pathComponentA + AZ_TRAIT_OS_PATH_SEPARATOR + m_pathComponentB + AZ_TRAIT_OS_PATH_SEPARATOR + m_pathComponentC; - } - - void TearDown() override - { - UnitTest::AllocatorsTestFixture::TearDown(); - } - - AZStd::string m_parentPathAbs; - AZStd::string m_childPathRel; - AZStd::string m_childPathAbs; - - AZStd::string m_pathComponentA; - AZStd::string m_pathComponentB; - AZStd::string m_pathComponentC; - - AZ::IO::Path m_posixPath; - AZ::IO::Path m_windowsPath; - AZ::IO::Path m_mixedPath; - AZ::IO::Path m_referredPath; - }; - - TEST_F(FrameworkPathTestFixture, DefaultConstructor_HasEmptyAbsAndRelPaths) - { - // Given an empty framework path - TestImpact::FrameworkPath path; - - // Expect the absolute path to be empty - EXPECT_TRUE(path.Absolute().empty()); - - // Expect the relative path to be empty - EXPECT_TRUE(path.Relative().empty()); - } - - TEST_F(FrameworkPathTestFixture, OrphanConstructor_HasAbsAndEmptyRelPaths) - { - // Given an orhpan framework path - TestImpact::FrameworkPath path(m_parentPathAbs); - - // Expect the absolute path to be equal to the specified path - EXPECT_EQ(path.Absolute().String(), m_parentPathAbs); - - // Expect the relative path to be current directory symbol - EXPECT_STREQ(path.Relative().c_str(), "."); - } - - TEST_F(FrameworkPathTestFixture, ParentConstructor_HasAbsAndRelPaths) - { - // Given a child framework path - TestImpact::FrameworkPath path(m_childPathAbs, TestImpact::FrameworkPath(m_parentPathAbs)); - - // Expect the absolute path to be equal to the concatenation of the parent and child path - EXPECT_EQ(path.Absolute().String(), m_childPathAbs); - - // Expect the relative path to equal to the specified path - EXPECT_EQ(path.Relative().String(), m_childPathRel); - } - - TEST_F(FrameworkPathTestFixture, PosixSeperators_HasUniformPreferredSeperators) - { - // Given an orhpan framework path with Posix separators - TestImpact::FrameworkPath path(m_posixPath); - - // Expect the absolute path to be equal to the specified path with preferred separators - EXPECT_EQ(path.Absolute(), m_referredPath); - - // Expect the relative path to be current directory symbol - EXPECT_STREQ(path.Relative().c_str(), "."); - } - - TEST_F(FrameworkPathTestFixture, WindowsSeperators_HasUniformPreferredSeperators) - { - // Given an orhpan framework path with Windows separators - TestImpact::FrameworkPath path(m_windowsPath); - - // Expect the absolute path to be equal to the specified path with preferred separators - EXPECT_EQ(path.Absolute(), m_referredPath); - - // Expect the relative path to be current directory symbol - EXPECT_STREQ(path.Relative().c_str(), "."); - } - - TEST_F(FrameworkPathTestFixture, MixedSeperators_HasUniformPreferredSeperators) - { - // Given an orhpan framework path with mixed separators - TestImpact::FrameworkPath path(m_windowsPath); - - // Expect the absolute path to be equal to the specified path with preferred separators - EXPECT_EQ(path.Absolute(), m_referredPath); - - // Expect the relative path to be current directory symbol - EXPECT_STREQ(path.Relative().c_str(), "."); - } -} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactTestMain.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactTestMain.cpp deleted file mode 100644 index 632e84ad9f..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactTestMain.cpp +++ /dev/null @@ -1,33 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include -#include -#include - -class TestImpactTestEnvironment : public AZ::Test::ITestEnvironment -{ -protected: - void SetupEnvironment() override - { - AZ::AllocatorInstance::Create(); - AZ::AllocatorInstance::Create(); - } - - void TeardownEnvironment() override - { - AZ::AllocatorInstance::Destroy(); - AZ::AllocatorInstance::Destroy(); - } -}; - -AZ_UNIT_TEST_HOOK(new TestImpactTestEnvironment); From c011c4c64c12c470c02a2abe743cf30ff97fb1f7 Mon Sep 17 00:00:00 2001 From: jonawals Date: Wed, 19 May 2021 16:59:58 +0100 Subject: [PATCH 018/102] Address PR comments --- cmake/TestImpactFramework/ConsoleFrontendConfig.in | 10 +++++++--- cmake/TestImpactFramework/LYTestImpactFramework.cmake | 8 +++----- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/cmake/TestImpactFramework/ConsoleFrontendConfig.in b/cmake/TestImpactFramework/ConsoleFrontendConfig.in index 73e90ea0ce..f6af3d801d 100644 --- a/cmake/TestImpactFramework/ConsoleFrontendConfig.in +++ b/cmake/TestImpactFramework/ConsoleFrontendConfig.in @@ -9,12 +9,16 @@ "workspace": { "temp": { "root": "${temp_dir}", - "artifact_dir": "RuntimeArtifact" + "relative_paths": { + "artifact_dir": "RuntimeArtifact" + } }, "persistent": { "root": "${persistent_dir}", - "test_impact_data_file": "TestImpactData.spartia", - "enumeration_cache_dir": "EnumerationCache" + "relative_paths": { + "test_impact_data_file": "TestImpactData.spartia", + "enumeration_cache_dir": "EnumerationCache" + } } }, "artifacts": { diff --git a/cmake/TestImpactFramework/LYTestImpactFramework.cmake b/cmake/TestImpactFramework/LYTestImpactFramework.cmake index fc3ba0af44..c85b5ff1e9 100644 --- a/cmake/TestImpactFramework/LYTestImpactFramework.cmake +++ b/cmake/TestImpactFramework/LYTestImpactFramework.cmake @@ -99,7 +99,7 @@ endfunction() # \arg:TEST_NAME name of test function(ly_test_impact_extract_google_test COMPOSITE_TEST TEST_NAMESPACE TEST_NAME) get_property(test_components GLOBAL PROPERTY LY_ALL_TESTS_${COMPOSITE_TEST}_TEST_NAME) - # Namespace and test are mandetory + # Namespace and test are mandatory string(REPLACE "::" ";" test_components ${test_components}) list(LENGTH test_components num_test_components) if(num_test_components LESS 2) @@ -134,13 +134,11 @@ endfunction() #! ly_test_impact_extract_google_test_params: extracts the google test name and command parameters. # # \arg:COMPOSITE_TEST test in the form 'namespace::test' -# \arg:TEST_NAMESPACE namespace for the test # \arg:TEST_NAME name of test -# \arg:TEST_NAME optional command arguments to run the test -# \arg:TEST_NAME test timeout value +# \arg:TEST_COMMAND optional command arguments to run the test function(ly_test_impact_extract_google_test_params COMPOSITE_TEST TEST_NAME TEST_COMMAND) get_property(test_command GLOBAL PROPERTY LY_ALL_TESTS_${COMPOSITE_TEST}_TEST_COMMAND) - # Namespace and test are mandetory + # Namespace and test are mandatory string(REPLACE "::" ";" test_components ${COMPOSITE_TEST}) list(LENGTH test_components num_test_components) if(num_test_components LESS 2) From e70d86501cb95e6ead6c1bf785a5026af8c4626d Mon Sep 17 00:00:00 2001 From: jonawals Date: Wed, 19 May 2021 20:35:13 +0100 Subject: [PATCH 019/102] Add runtime api headers --- .../TestImpactChangeList.h | 28 +++ .../TestImpactChangeListException.h | 26 +++ .../TestImpactChangeListSerializer.h | 26 +++ .../TestImpactClientFailureReport.h | 177 +++++++++++++++++ .../TestImpactClientTestRun.h | 46 +++++ .../TestImpactClientTestSelection.h | 47 +++++ .../TestImpactConfiguration.h | 137 +++++++++++++ .../TestImpactConfigurationException.h | 26 +++ .../TestImpactFramework/TestImpactRepoPath.h | 42 ++++ .../TestImpactFramework/TestImpactRuntime.h | 183 ++++++++++++++++++ .../TestImpactRuntimeException.h | 26 +++ .../TestImpactTestSequence.h | 91 +++++++++ .../TestImpactFramework/TestImpactUtils.h | 60 ++++++ 13 files changed, 915 insertions(+) create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactChangeList.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactChangeListException.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactChangeListSerializer.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientFailureReport.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientTestRun.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientTestSelection.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactConfiguration.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactConfigurationException.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRepoPath.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntimeException.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactTestSequence.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactUtils.h diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactChangeList.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactChangeList.h new file mode 100644 index 0000000000..e36b343544 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactChangeList.h @@ -0,0 +1,28 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +#include + +namespace TestImpact +{ + //! Representation of the file CRUD operations of a given set of source changes. + struct ChangeList + { + AZStd::vector m_createdFiles; //!< Files that were newly created. + AZStd::vector m_updatedFiles; //!< Files that were updated. + AZStd::vector m_deletedFiles; //!< Files that were deleted. + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactChangeListException.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactChangeListException.h new file mode 100644 index 0000000000..d9139cf47f --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactChangeListException.h @@ -0,0 +1,26 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +namespace TestImpact +{ + //! Exception for change list operations. + class ChangeListException + : public Exception + { + public: + using Exception::Exception; + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactChangeListSerializer.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactChangeListSerializer.h new file mode 100644 index 0000000000..13c41db564 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactChangeListSerializer.h @@ -0,0 +1,26 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +#include + +namespace TestImpact +{ + //! Serializes the specified change list to JSON format. + AZStd::string SerializeChangeList(const ChangeList& changeList); + + //! Deserializes a change list from the specified test run data in JSON format. + ChangeList DeserializeChangeList(const AZStd::string& changeListString); +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientFailureReport.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientFailureReport.h new file mode 100644 index 0000000000..e5cadcbbd2 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientFailureReport.h @@ -0,0 +1,177 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include +#include + +namespace TestImpact +{ + namespace Client + { + //! Represents a test target that failed, either due to failing to execute, completing in an abnormal state or completing with failing tests. + class TargetFailure + { + public: + TargetFailure(const AZStd::string& targetName); + + //! Returns the name of the test target this failure pertains to. + const AZStd::string& GetTargetName() const; + private: + AZStd::string m_targetName; + }; + + //! Represents a test target that failed to execute. + class ExecutionFailure + : public TargetFailure + { + public: + ExecutionFailure(const AZStd::string& targetName, const AZStd::string& command); + + //! Returns the command string used to execute this test target. + const AZStd::string& GetCommandString() const; + private: + AZStd::string m_commandString; + }; + + //! Represents a test target that terminated abnormally. + class LauncherFailure + : public ExecutionFailure + { + public: + LauncherFailure(const AZStd::string& targetName, const AZStd::string& command, int returnCode); + + //! The return code of the test target that terminated abnormally. + int GetReturnCode() const; + private: + int m_returnCode; + }; + + //! Represents an individual test of a test target that failed. + class TestFailure + { + public: + TestFailure(const AZStd::string& testName, const AZStd::string& errorMessage); + + //! Returns the name of the test that failed. + const AZStd::string& GetName() const; + + //! Returns the error message of the test that failed. + const AZStd::string& GetErrorMessage() const; + + private: + AZStd::string m_name; + AZStd::string m_errorMessage; + }; + + //! Represents a collection of tests that failed. + //! @note Only the failing tests are included in the collection. + class TestCaseFailure + { + public: + TestCaseFailure(const AZStd::string& testCaseName, AZStd::vector&& testFailures); + + //! Returns the name of the test case containing the failing tests. + const AZStd::string& GetName() const; + + //! Returns the collection of tests in this test case that failed. + const AZStd::vector& GetTestFailures() const; + + private: + AZStd::string m_name; + AZStd::vector m_testFailures; + }; + + //! Represents a test target that launched successfully but contains failing tests. + class TestRunFailure + : public TargetFailure + { + public: + TestRunFailure(const AZStd::string& targetName, AZStd::vector&& testFailures); + + //! Returns the total number of failing tests in this run. + size_t GetNumTestFailures() const; + + //! Returns the test cases in this run containing failing tests. + const AZStd::vector& GetTestCaseFailures() const; + + private: + AZStd::vector m_testCaseFailures; + }; + + //! Base class for reporting failing test sequences. + class SequenceFailure + { + public: + SequenceFailure( + AZStd::vector&& executionFailures, + AZStd::vector&& launcherFailures, + AZStd::vector&& unexecutionTests); + + //! Returns the test targets in this sequence that failed to execute. + const AZStd::vector& GetExecutionFailures() const; + + //! Returns the test targets in this sequence that terminated abnormally. + const AZStd::vector& GetLauncherFailures() const; + + //! Returns the test targets in this sequence that were not executed due to the sequence terminating prematurely. + const AZStd::vector& GetUnexecutedTest() const; + + private: + AZStd::vector m_executionFailures; + AZStd::vector m_launcherFailures; + AZStd::vector m_unexecutionTests; + }; + + //! Represents the report for a failed regular test sequence run without test impact analysis. + class RegularSequenceFailure + : public SequenceFailure + { + public: + RegularSequenceFailure( + AZStd::vector&& executionFailures, + AZStd::vector&& launcherFailures, + AZStd::vector&& testRunFailures, + AZStd::vector&& unexecutionTests); + + //! Returns the test targets that contain failing tests. + const AZStd::vector& GetTestRunFailures() const; + + private: + AZStd::vector m_testRunFailures; + }; + + //! Represents the report for a failed test sequence run with test impact analysis. + class ImpactAnalysisSequenceFailure + : public SequenceFailure + { + public: + ImpactAnalysisSequenceFailure( + AZStd::vector&& executionFailures, + AZStd::vector&& launcherFailures, + AZStd::vector&& selectedTestRunFailures, + AZStd::vector&& discardedTestRunFailures, + AZStd::vector&& unexecutionTests); + + //! Returns the test targets that were selected to run but contain failing tests. + const AZStd::vector GetSelectedTestRunFailures() const; + + //! Returns the test targets that were not selected but still run but contain failing tests. + const AZStd::vector GetDiscardedTestRunFailures() const; + + private: + AZStd::vector m_selectedTestRunFailures; + AZStd::vector m_discardedTestRunFailures; + }; + } // namespace Client +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientTestRun.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientTestRun.h new file mode 100644 index 0000000000..7c525e00ba --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientTestRun.h @@ -0,0 +1,46 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include + +#pragma once + +namespace TestImpact +{ + namespace Client + { + //! Result of a test run. + enum class TestRunResult + { + NotRun, //!< The test run was not executed due to the test sequence terminating prematurely. + FailedToExecute, //!< The test run failed to execute either due to the target binary missing or incorrect arguments. + Timeout, //!< The test run timed out whilst in flight before being able to complete its run. + TestFailures, //!< The test run completed its run but there were failing tests. + AllTestsPass //!< The test run completed its run and all tests passed. + }; + + class TestRun + { + public: + TestRun(const AZStd::string& name, TestRunResult result, AZStd::chrono::milliseconds duration); + const AZStd::string& GetTargetName() const; + TestRunResult GetResult() const; + AZStd::chrono::milliseconds GetDuration() const; + + private: + AZStd::string m_targetName; + TestRunResult m_result; + AZStd::chrono::milliseconds m_duration; + }; + } // namespace Client +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientTestSelection.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientTestSelection.h new file mode 100644 index 0000000000..a42a67f0d7 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientTestSelection.h @@ -0,0 +1,47 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include + +#pragma once + +namespace TestImpact +{ + namespace Client + { + //! The set of test targets selected to run regardless of whether or not the test targets are to be exclude either for being on the master exclude + //! list and/or being part of a test suite excluded from this run. + //! @note Only the included test targets will be run. The excluded test targetss, although selected, will not be run. + class TestRunSelection + { + public: + TestRunSelection(AZStd::vector&& includedTests, AZStd::vector&& excludedTests); + + //! Returns the test runs that were selected to be run and will actually be run. + const AZStd::vector& GetIncludededTestRuns() const; + + //! Returns the test runs that were selected to be run but will not actually be run. + const AZStd::vector& GetExcludedTestRuns() const; + + //! Returns the number of selected test runs that will be run. + size_t GetNumIncludedTestRuns() const; + + //! Returns the number of selected test runs that will not be run. + size_t GetNumNumExcludedTestRuns() const; + + private: + AZStd::vector m_includedTestRuns; + AZStd::vector m_excludedTestRuns; + }; + } // namespace Client +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactConfiguration.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactConfiguration.h new file mode 100644 index 0000000000..1a4ae73ce0 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactConfiguration.h @@ -0,0 +1,137 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include +#include + +#include +#include + +namespace TestImpact +{ + //! Meta-data about the configuration. + struct ConfigMeta + { + AZStd::string m_platform; //!< The platform for which the configuration pertains to. + }; + + //! Repository configuration. + struct RepoConfig + { + RepoPath m_root; //!< The absolute path to the repository root. + }; + + //! Test impact analysis framework workspace configuration. + struct WorkspaceConfig + { + //! Temporary workspace configuration. + struct Temp + { + //! Paths relative to root. + struct RelativePaths + { + RepoPath m_artifactDirectory; //!< Path to read and write runtime artifacts to and from. + }; + + RepoPath m_root; //!< Path to the temporary workspace (cleaned prior to use). + RelativePaths m_relativePaths; + }; + + struct Persistent + { + //! Paths relative to root. + struct RelativePaths + { + RepoPath m_sparTIAFile; //!< Path to the test impact analysis data. + RepoPath m_enumerationCacheDirectory; //!< Path to the test enumerations cache. + }; + + RepoPath m_root; //!< Path to the persistent workspace tracked by the repository. + RelativePaths m_relativePaths; + }; + + Temp m_temp; + Persistent m_persistent; + }; + + //! Build target descriptor configuration. + struct BuildTargetDescriptorConfig + { + RepoPath m_mappingDirectory; //!< Path to the source to target mapping files. + AZStd::vector m_staticInclusionFilters; //!< File extensions to include for static files. + AZStd::string m_inputOutputPairer; //!< Regex for matching autogen input files with autogen outputs files. + AZStd::vector m_inputInclusionFilters; //!< File extensions fo include for autogen input files. + }; + + //! Dependency graph configuration. + struct DependencyGraphDataConfig + { + RepoPath m_graphDirectory; //!< Path to the dependency graph files. + AZStd::string m_targetDependencyFileMatcher; //!< Regex for matching dependency graph files to build targets. + AZStd::string m_targetVertexMatcher; //!< Regex form matching dependency graph vertices to build targets. + }; + + //! Test target meta configuration. + struct TestTargetMetaConfig + { + RepoPath m_metaFile; //!< Path to the master test target meta file. + }; + + //! Test engine configuration. + struct TestEngineConfig + { + //!< Test runner configuration. + struct TestRunner + { + RepoPath m_binary; //!< Path to the test runner binary. + }; + + //!< Test instrumentation configuration. + struct Instrumentation + { + RepoPath m_binary; //!< Path to the test instrumentation binary. + }; + + TestRunner m_testRunner; + Instrumentation m_instrumentation; + }; + + //!< Build target configuration. + struct TargetConfig + { + //!< Test target sharding configuration. + struct ShardedTarget + { + AZStd::string m_name; //!< Name of test target this sharding configuration applies to. + ShardConfiguration m_configuration; //!< The shard configuration to use. + }; + + RepoPath m_outputDirectory; //!< Path to the test target binary directory. + AZStd::vector m_excludedTestTargets; //!< Test targets to always exclude from test run sequences. + AZStd::vector m_shardedTestTargets; //!< Test target shard configurations (opt-in). + }; + + //! Runtime configuration. + struct RuntimeConfig + { + ConfigMeta m_meta; + RepoConfig m_repo; + WorkspaceConfig m_workspace; + BuildTargetDescriptorConfig m_buildTargetDescriptor; + DependencyGraphDataConfig m_dependencyGraphData; + TestTargetMetaConfig m_testTargetMeta; + TestEngineConfig m_testEngine; + TargetConfig m_target; + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactConfigurationException.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactConfigurationException.h new file mode 100644 index 0000000000..15d7913997 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactConfigurationException.h @@ -0,0 +1,26 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +namespace TestImpact +{ + //! Exception for configuration operations. + class ConfigurationException + : public Exception + { + public: + using Exception::Exception; + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRepoPath.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRepoPath.h new file mode 100644 index 0000000000..e7e0ba685f --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRepoPath.h @@ -0,0 +1,42 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include +#include + +namespace TestImpact +{ + //! Wrapper class to ensure that all paths have the same path separator regardless of how they are sourced. This is critical + //! to the test impact analysis data as otherwise querying/retrieving test impact analysis data for the same source albeit + //! with different path separators will be considered different files entirely. + class RepoPath + : public AZ::IO::Path + { + public: + constexpr RepoPath() = default; + constexpr RepoPath(const RepoPath&) = default; + constexpr RepoPath(RepoPath&&) = default; + constexpr RepoPath(const string_type&) noexcept; + constexpr RepoPath(const value_type*) noexcept; + constexpr RepoPath(const AZ::IO::PathView&); + constexpr RepoPath(const AZ::IO::Path&); + + RepoPath& operator=(const RepoPath&) noexcept = default; + RepoPath& operator=(const string_type&) noexcept; + RepoPath& operator=(const value_type*) noexcept; + RepoPath& operator=(const AZ::IO::Path& str) noexcept; + + using AZ::IO::Path::operator AZ::IO::PathView; + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h new file mode 100644 index 0000000000..a42da8dc38 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h @@ -0,0 +1,183 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace TestImpact +{ + class DynamicDependencyMap; + class TestEngine; + class TestTarget; + + //! Callback for a test sequence that isn't using test impact analysis to determine selected tests. + //! @param tests The tests that will be run for this sequence. + using TestSequenceStartCallback = AZStd::function; + + //! Callback for a test sequence using test impact analysis. + //! @param selectedTests The tests that have been selected for this run by test impact analysis. + //! @param discardedTests The tests that have been rejected for this run by test impact analysis. + //! @param draftedTests The tests that have been drafted in for this run due to requirements outside of test impact analysis + //! (e.g. test targets that have been added to the repository since the last test impact analysis sequence or test that failed + //! to execute previously). + //! These tests will be run with coverage instrumentation. + //! @note discardedTests and draftedTests may contain overlapping tests. + using ImpactAnalysisTestSequenceStartCallback = AZStd::function&& discardedTests, + AZStd::vector&& draftedTests)>; + + //! Callback for a test sequence using test impact analysis. + //! @param selectedTests The tests that have been selected for this run by test impact analysis. + //! @param discardedTests The tests that have been rejected for this run by test impact analysis. + //! These tests will not be run without coverage instrumentation unless there is an entry in the draftedTests list. + //! @param draftedTests The tests that have been drafted in for this run due to requirements outside of test impact analysis + //! (e.g. test targets that have been added to the repository since the last test impact analysis sequence or test that failed + //! to execute previously). + //! @note discardedTests and draftedTests may contain overlapping tests. + using SafeImpactAnalysisTestSequenceStartCallback = AZStd::function&& draftedTests)>; + + //! Callback for end of a test sequence. + //! @param failureReport The test runs that failed for any reason during this sequence. + //! @param duration The total duration of this test sequence. + using TestSequenceCompleteCallback = AZStd::function; + + //! Callback for end of a test impact analysis test sequence. + //! @param failureReport The test runs that failed for any reason during this sequence. + //! @param duration The total duration of this test sequence. + using ImpactAnalysisTestSequenceCompleteCallback = AZStd::function; + + //! Callback for test runs that have completed for any reason + //! test The test that has completed. + using TestCompleteCallback = AZStd::function; + + //! The API exposed to the client responsible for all test runs and persistent data management. + class Runtime + { + public: + //! Constructs a runtime with the specified configuration and policies. + //! @param config The configuration used for this runtime instance. + //! @param executionFailurePolicy Determines how to handle test targets that fail to execute. + //! @param executionFailureDraftingPolicy Determines how test targets that previously failed to execute are drafted into subsequent test sequences. + //! @param testFailurePolicy Determines how to handle test targets that report test failures. + //! @param integrationFailurePolicy Determines how to handle instances where the build system model and/or test impact analysis data is compromised. + //! @param testShardingPolicy Determines how to handle test targets that have opted in to test sharding. + Runtime( + RuntimeConfig&& config, + Policy::ExecutionFailure executionFailurePolicy, + Policy::ExecutionFailureDrafting executionFailureDraftingPolicy, + Policy::TestFailure testFailurePolicy, + Policy::IntegrityFailure integrationFailurePolicy, + Policy::TestSharding testShardingPolicy, + TargetOutputCapture targetOutputCapture, + AZStd::optional maxConcurrency = AZStd::nullopt); + + ~Runtime(); + + //! Runs a test sequence where all tests with a matching suite in the suite filter and also not on the excluded list are selected. + //! @param suitesFilter The test suites that will be included in the test selection. + //! @param testTargetTimeout The maximum duration individual test targets may be in flight for (infinite if empty). + //! @param testTargetTimeout The maximum duration the entire test sequence may run for (infinite if empty). + //! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests. + //! @param testSequenceCompleteCallback The client function to be called after the test sequence has completed. + //! @param testRunCompleteCallback The client function to be called after an individual test run has compelted. + TestSequenceResult RegularTestSequence( + const AZStd::unordered_set suitesFilter, + AZStd::optional testTargetTimeout, + AZStd::optional globalTimeout, + AZStd::optional testSequenceStartCallback, + AZStd::optional testSequenceCompleteCallback, + AZStd::optional testRunCompleteCallback); + + //! Runs a test sequence where tests are selected according to test impact analysis so long as they are not on the excluded list. + //! @param changeList The change list used to determine the tests to select. + //! @param testPrioritizationPolicy Determines how selected tests will be prioritized. + //! @param testTargetTimeout The maximum duration individual test targets may be in flight for (infinite if empty). + //! @param testTargetTimeout The maximum duration the entire test sequence may run for (infinite if empty). + //! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests. + //! @param testSequenceCompleteCallback The client function to be called after the test sequence has completed. + //! @param testRunCompleteCallback The client function to be called after an individual test run has compelted. + TestSequenceResult ImpactAnalysisTestSequence( + const ChangeList& changeList, + Policy::TestPrioritization testPrioritizationPolicy, + AZStd::optional testTargetTimeout, + AZStd::optional globalTimeout, + AZStd::optional testSequenceStartCallback, + AZStd::optional testSequenceCompleteCallback, + AZStd::optional testRunCompleteCallback); + + //! Runs a test sequence as per the ImpactAnalysisTestSequence where the tests not selected are also run (albeit without instrumentation). + //! @param changeList The change list used to determine the tests to select. + //! @param suitesFilter The test suites that will be included in the test selection. + //! @param testPrioritizationPolicy Determines how selected tests will be prioritized. + //! @param testTargetTimeout The maximum duration individual test targets may be in flight for (infinite if empty). + //! @param testTargetTimeout The maximum duration the entire test sequence may run for (infinite if empty). + //! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests. + //! @param testSequenceCompleteCallback The client function to be called after the test sequence has completed. + //! @param testRunCompleteCallback The client function to be called after an individual test run has compelted. + TestSequenceResult SafeImpactAnalysisTestSequence( + const ChangeList& changeList, + const AZStd::unordered_set suitesFilter, + Policy::TestPrioritization testPrioritizationPolicy, + AZStd::optional testTargetTimeout, + AZStd::optional globalTimeout, + AZStd::optional testSequenceStartCallback, + AZStd::optional testSequenceCompleteCallback, + AZStd::optional testRunCompleteCallback); + + //! Runs all tests not on the excluded list and uses their coverage data to seed the test impact analysis data (ant existing data will be overwritten). + //! @param testTargetTimeout The maximum duration the entire test sequence may run for (infinite if empty). + //! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests. + //! @param testSequenceCompleteCallback The client function to be called after the test sequence has completed. + //! @param testRunCompleteCallback The client function to be called after an individual test run has compelted. + TestSequenceResult SeededTestSequence( + AZStd::optional globalTimeout, + AZStd::optional testSequenceStartCallback, + AZStd::optional testSequenceCompleteCallback, + AZStd::optional testRunCompleteCallback); + + //! Returns true if the runtime has test impact analysis data (either preexisting or generated). + bool HasImpactAnalysisData() const; + + private: + RuntimeConfig m_config; + Policy::ExecutionFailure m_executionFailurePolicy; + Policy::ExecutionFailureDrafting m_executionFailureDraftingPolicy; + Policy::TestFailure m_testFailurePolicy; + Policy::IntegrityFailure m_integrationFailurePolicy; + Policy::TestSharding m_testShardingPolicy; + TargetOutputCapture m_targetOutputCapture; + size_t m_maxConcurrency; + AZStd::unique_ptr m_dynamicDependencyMap; + AZStd::unique_ptr m_testEngine; + AZStd::unordered_set m_testTargetExcludeList; + AZStd::unordered_set m_testTargetShardList; + bool m_hasImpactAnalysisData = false; + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntimeException.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntimeException.h new file mode 100644 index 0000000000..f178df685d --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntimeException.h @@ -0,0 +1,26 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +namespace TestImpact +{ + //! Exception for runtime related exceptions. + class RuntimeException + : public Exception + { + public: + using Exception::Exception; + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactTestSequence.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactTestSequence.h new file mode 100644 index 0000000000..b9030d67e0 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactTestSequence.h @@ -0,0 +1,91 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +namespace TestImpact +{ + namespace Policy + { + //! Policy for handling of test targets that fail to execute (e.g. due to the binary not being found). + //! @note Test targets that fail to execute will be tagged such that their execution can be attempted at a later date. This is + //! important as otherwise it would be erroneously assumed that they cover no sources due to having no entries in the dynamic + //! dependency map. + enum class ExecutionFailure + { + Abort, //!< Abort the test sequence and report a failure. + Continue, //!< Continue the test sequence but treat the execution failures as test failures after the run. + Ignore //!< Continue the test sequence and ignore the execution failures. + }; + + //! Policy for reattempting the execution of test targets that failed to execute in previous runs. + enum class ExecutionFailureDrafting + { + Never, //!< Do not attempt to execute historic execution failures. + Always //!< Reattempt the exectution of historic execution failures. + }; + + //! Policy for prioritizing selected tests. + enum class TestPrioritization + { + None, //!< Do not attempt any test prioritization. + DependencyLocality //!< Prioritize test targets according to the locality of the production targets they cover in the build dependency graph. + }; + + //! Policy for handling test targets that report failing tests. + enum class TestFailure + { + Abort, //!< Abort the test sequence and report the test failure. + Continue //!< Continue the test sequence and report the test failures after the run. + }; + + //! Policy for handling integrity failures of the dynamic dependency map and the source to target mappings. + enum class IntegrityFailure + { + Abort, //!< Abort the test sequence and report the test failure. + Continue //!< Continue the test sequence and report the test failures after the run. + }; + + //! Policy for sharding test targets that have been marked for test sharding. + enum class TestSharding + { + Never, //!< Do not shard any test targets. + Always //!< Shard all test targets that have been marked for test sharding. + }; + } + + //! Standard output capture of test target runs. + enum class TargetOutputCapture + { + None, //!< Do not capture any output. + StdOut, //!< Send captured output to standard output + File, //!< Write captured output to file. + StdOutAndFile //!< Send captured output to standard output and write to file. + }; + + enum class ShardConfiguration + { + Never, //!< Never shard this test target. + FixtureContiguous, //!< Each shard contains contiguous fixtures of tests (safest but least optimal). + TestContiguous, //!< Each shard contains contiguous tests agnostic of fixtures. + FixtureInterleaved, //!< Fixtures of tests are interleaved across shards. + TestInterleaved //!< Tests are interlaced across shards agnostic of fixtures (fastest but prone to inter-test dependency problems). + }; + + //! Result of a test sequence that was run. + enum class TestSequenceResult + { + Success, //! All tests ran with no failures. + Failure, //! One or more tests failed and/or timed out and/or failed to launch (if execution failure policy is not Ignore) and/or an integrity failure was encountered. + Timeout //! The global timeout for the sequence was exceeded. + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactUtils.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactUtils.h new file mode 100644 index 0000000000..bd62e70a0d --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactUtils.h @@ -0,0 +1,60 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include + +#include +#include +#include + +#pragma once + +namespace TestImpact +{ + //! Attempts to read the contents of the specified file into a string. + //! @tparam ExceptionType The exception type to throw upon failure. + //! @param path The path to the file to read the contents of. + //! @returns The contents of the file. + template + AZStd::string ReadFileContents(const RepoPath& path) + { + const auto fileSize = AZ::IO::SystemFile::Length(path.c_str()); + AZ_TestImpact_Eval(fileSize > 0, ExceptionType, AZStd::string::format("File %s does not exist", path.c_str())); + + AZStd::vector buffer(fileSize + 1); + buffer[fileSize] = '\0'; + AZ_TestImpact_Eval(AZ::IO::SystemFile::Read(path.c_str(), buffer.data()), ExceptionType, AZStd::string::format("Could not read contents of file %s", path.c_str())); + + return AZStd::string(buffer.begin(), buffer.end()); + } + + //! Attempts to write the contents of the specified string to a file. + //! @tparam ExceptionType The exception type to throw upon failure. + //! @param contents The contents to write to the file. + //! @param path The path to the file to write the contents to. + template + void WriteFileContents(const AZStd::string& contents, const RepoPath& path) + { + AZ::IO::SystemFile file; + const AZStd::vector bytes(contents.begin(), contents.end()); + AZ_TestImpact_Eval( + file.Open(path.c_str(), + AZ::IO::SystemFile::SF_OPEN_CREATE | AZ::IO::SystemFile::SF_OPEN_CREATE_PATH | AZ::IO::SystemFile::SF_OPEN_WRITE_ONLY), + ExceptionType, + AZStd::string::format("Couldn't open file %s for writing", path.c_str())); + + AZ_TestImpact_Eval(file.Write(bytes.data(), bytes.size()), ExceptionType, AZStd::string::format("Couldn't write contents for file %s", path.c_str())); + + return; + } +} // namespace TestImpact From b33569638e9a377a13dbcd8058f11afda5dd462f Mon Sep 17 00:00:00 2001 From: jonawals Date: Thu, 20 May 2021 10:06:14 +0100 Subject: [PATCH 020/102] Address PR comments --- .../TestImpactClientTestSelection.h | 4 ++-- .../TestImpactFramework/TestImpactConfiguration.h | 11 +++++------ .../TestImpactFramework/TestImpactTestSequence.h | 1 + .../Include/TestImpactFramework/TestImpactUtils.h | 5 ++++- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientTestSelection.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientTestSelection.h index a42a67f0d7..f251e7117d 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientTestSelection.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientTestSelection.h @@ -19,9 +19,9 @@ namespace TestImpact { namespace Client { - //! The set of test targets selected to run regardless of whether or not the test targets are to be exclude either for being on the master exclude + //! The set of test targets selected to run regardless of whether or not the test targets are to be excluded either for being on the primary exclude //! list and/or being part of a test suite excluded from this run. - //! @note Only the included test targets will be run. The excluded test targetss, although selected, will not be run. + //! @note Only the included test targets will be run. The excluded test targets, although selected, will not be run. class TestRunSelection { public: diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactConfiguration.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactConfiguration.h index 1a4ae73ce0..1327ddbdff 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactConfiguration.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactConfiguration.h @@ -85,19 +85,19 @@ namespace TestImpact //! Test target meta configuration. struct TestTargetMetaConfig { - RepoPath m_metaFile; //!< Path to the master test target meta file. + RepoPath m_metaFile; //!< Path to the test target meta file. }; //! Test engine configuration. struct TestEngineConfig { - //!< Test runner configuration. + //! Test runner configuration. struct TestRunner { RepoPath m_binary; //!< Path to the test runner binary. }; - //!< Test instrumentation configuration. + //! Test instrumentation configuration. struct Instrumentation { RepoPath m_binary; //!< Path to the test instrumentation binary. @@ -107,10 +107,10 @@ namespace TestImpact Instrumentation m_instrumentation; }; - //!< Build target configuration. + //! Build target configuration. struct TargetConfig { - //!< Test target sharding configuration. + //! Test target sharding configuration. struct ShardedTarget { AZStd::string m_name; //!< Name of test target this sharding configuration applies to. @@ -122,7 +122,6 @@ namespace TestImpact AZStd::vector m_shardedTestTargets; //!< Test target shard configurations (opt-in). }; - //! Runtime configuration. struct RuntimeConfig { ConfigMeta m_meta; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactTestSequence.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactTestSequence.h index b9030d67e0..8b1b54e7f4 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactTestSequence.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactTestSequence.h @@ -72,6 +72,7 @@ namespace TestImpact StdOutAndFile //!< Send captured output to standard output and write to file. }; + //! Configuration for test targets that opt in to test sharding. enum class ShardConfiguration { Never, //!< Never shard this test target. diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactUtils.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactUtils.h index bd62e70a0d..8580582a12 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactUtils.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactUtils.h @@ -33,7 +33,10 @@ namespace TestImpact AZStd::vector buffer(fileSize + 1); buffer[fileSize] = '\0'; - AZ_TestImpact_Eval(AZ::IO::SystemFile::Read(path.c_str(), buffer.data()), ExceptionType, AZStd::string::format("Could not read contents of file %s", path.c_str())); + AZ_TestImpact_Eval( + AZ::IO::SystemFile::Read(path.c_str(), buffer.data()), + ExceptionType, + AZStd::string::format("Could not read contents of file %s", path.c_str())); return AZStd::string(buffer.begin(), buffer.end()); } From feb2909508a19dbe61f3917d185f34591bd4a476 Mon Sep 17 00:00:00 2001 From: jonawals Date: Thu, 20 May 2021 16:38:56 +0100 Subject: [PATCH 021/102] Address PR comments --- .../TestImpactClientFailureReport.h | 8 ++++---- .../TestImpactFramework/TestImpactConfiguration.h | 5 +++-- .../Include/TestImpactFramework/TestImpactRepoPath.h | 2 +- .../Include/TestImpactFramework/TestImpactRuntime.h | 12 ++++++------ .../TestImpactFramework/TestImpactTestSequence.h | 6 +++--- .../Include/TestImpactFramework/TestImpactUtils.h | 5 ++--- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientFailureReport.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientFailureReport.h index e5cadcbbd2..d2ab863d86 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientFailureReport.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientFailureReport.h @@ -116,7 +116,7 @@ namespace TestImpact SequenceFailure( AZStd::vector&& executionFailures, AZStd::vector&& launcherFailures, - AZStd::vector&& unexecutionTests); + AZStd::vector&& unexecutedTests); //! Returns the test targets in this sequence that failed to execute. const AZStd::vector& GetExecutionFailures() const; @@ -130,7 +130,7 @@ namespace TestImpact private: AZStd::vector m_executionFailures; AZStd::vector m_launcherFailures; - AZStd::vector m_unexecutionTests; + AZStd::vector m_unexecutedTestsTests; }; //! Represents the report for a failed regular test sequence run without test impact analysis. @@ -142,7 +142,7 @@ namespace TestImpact AZStd::vector&& executionFailures, AZStd::vector&& launcherFailures, AZStd::vector&& testRunFailures, - AZStd::vector&& unexecutionTests); + AZStd::vector&& unexecutedTests); //! Returns the test targets that contain failing tests. const AZStd::vector& GetTestRunFailures() const; @@ -161,7 +161,7 @@ namespace TestImpact AZStd::vector&& launcherFailures, AZStd::vector&& selectedTestRunFailures, AZStd::vector&& discardedTestRunFailures, - AZStd::vector&& unexecutionTests); + AZStd::vector&& unexecutedTests); //! Returns the test targets that were selected to run but contain failing tests. const AZStd::vector GetSelectedTestRunFailures() const; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactConfiguration.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactConfiguration.h index 1327ddbdff..3406789c9e 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactConfiguration.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactConfiguration.h @@ -48,7 +48,8 @@ namespace TestImpact RelativePaths m_relativePaths; }; - struct Persistent + //! Active persistent data workspace configuration. + struct Active { //! Paths relative to root. struct RelativePaths @@ -62,7 +63,7 @@ namespace TestImpact }; Temp m_temp; - Persistent m_persistent; + Active m_active; }; //! Build target descriptor configuration. diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRepoPath.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRepoPath.h index e7e0ba685f..ccba3f677d 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRepoPath.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRepoPath.h @@ -26,7 +26,7 @@ namespace TestImpact public: constexpr RepoPath() = default; constexpr RepoPath(const RepoPath&) = default; - constexpr RepoPath(RepoPath&&) = default; + constexpr RepoPath(RepoPath&&) noexcept = default; constexpr RepoPath(const string_type&) noexcept; constexpr RepoPath(const value_type*) noexcept; constexpr RepoPath(const AZ::IO::PathView&); diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h index a42da8dc38..8221eba834 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h @@ -73,7 +73,7 @@ namespace TestImpact //! @param duration The total duration of this test sequence. using ImpactAnalysisTestSequenceCompleteCallback = AZStd::function; - //! Callback for test runs that have completed for any reason + //! Callback for test runs that have completed for any reason. //! test The test that has completed. using TestCompleteCallback = AZStd::function; @@ -106,7 +106,7 @@ namespace TestImpact //! @param testTargetTimeout The maximum duration the entire test sequence may run for (infinite if empty). //! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests. //! @param testSequenceCompleteCallback The client function to be called after the test sequence has completed. - //! @param testRunCompleteCallback The client function to be called after an individual test run has compelted. + //! @param testRunCompleteCallback The client function to be called after an individual test run has completed. TestSequenceResult RegularTestSequence( const AZStd::unordered_set suitesFilter, AZStd::optional testTargetTimeout, @@ -122,7 +122,7 @@ namespace TestImpact //! @param testTargetTimeout The maximum duration the entire test sequence may run for (infinite if empty). //! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests. //! @param testSequenceCompleteCallback The client function to be called after the test sequence has completed. - //! @param testRunCompleteCallback The client function to be called after an individual test run has compelted. + //! @param testRunCompleteCallback The client function to be called after an individual test run has completed. TestSequenceResult ImpactAnalysisTestSequence( const ChangeList& changeList, Policy::TestPrioritization testPrioritizationPolicy, @@ -140,7 +140,7 @@ namespace TestImpact //! @param testTargetTimeout The maximum duration the entire test sequence may run for (infinite if empty). //! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests. //! @param testSequenceCompleteCallback The client function to be called after the test sequence has completed. - //! @param testRunCompleteCallback The client function to be called after an individual test run has compelted. + //! @param testRunCompleteCallback The client function to be called after an individual test run has completed. TestSequenceResult SafeImpactAnalysisTestSequence( const ChangeList& changeList, const AZStd::unordered_set suitesFilter, @@ -155,7 +155,7 @@ namespace TestImpact //! @param testTargetTimeout The maximum duration the entire test sequence may run for (infinite if empty). //! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests. //! @param testSequenceCompleteCallback The client function to be called after the test sequence has completed. - //! @param testRunCompleteCallback The client function to be called after an individual test run has compelted. + //! @param testRunCompleteCallback The client function to be called after an individual test run has completed. TestSequenceResult SeededTestSequence( AZStd::optional globalTimeout, AZStd::optional testSequenceStartCallback, @@ -173,7 +173,7 @@ namespace TestImpact Policy::IntegrityFailure m_integrationFailurePolicy; Policy::TestSharding m_testShardingPolicy; TargetOutputCapture m_targetOutputCapture; - size_t m_maxConcurrency; + size_t m_maxConcurrency = 0; AZStd::unique_ptr m_dynamicDependencyMap; AZStd::unique_ptr m_testEngine; AZStd::unordered_set m_testTargetExcludeList; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactTestSequence.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactTestSequence.h index 8b1b54e7f4..751db43987 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactTestSequence.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactTestSequence.h @@ -85,8 +85,8 @@ namespace TestImpact //! Result of a test sequence that was run. enum class TestSequenceResult { - Success, //! All tests ran with no failures. - Failure, //! One or more tests failed and/or timed out and/or failed to launch (if execution failure policy is not Ignore) and/or an integrity failure was encountered. - Timeout //! The global timeout for the sequence was exceeded. + Success, //!< All tests ran with no failures. + Failure, //!< One or more tests failed and/or timed out and/or failed to launch and/or an integrity failure was encountered. + Timeout //!< The global timeout for the sequence was exceeded. }; } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactUtils.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactUtils.h index 8580582a12..d734d7a70e 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactUtils.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactUtils.h @@ -56,8 +56,7 @@ namespace TestImpact ExceptionType, AZStd::string::format("Couldn't open file %s for writing", path.c_str())); - AZ_TestImpact_Eval(file.Write(bytes.data(), bytes.size()), ExceptionType, AZStd::string::format("Couldn't write contents for file %s", path.c_str())); - - return; + AZ_TestImpact_Eval( + file.Write(bytes.data(), bytes.size()), ExceptionType, AZStd::string::format("Couldn't write contents for file %s", path.c_str())); } } // namespace TestImpact From d732b7d0ced6a8b01defbd19c331ad286194ce76 Mon Sep 17 00:00:00 2001 From: jonawals Date: Fri, 21 May 2021 09:00:57 +0100 Subject: [PATCH 022/102] General refactor of runtime classes. --- ...TestImpactBuildTargetDescriptorFactory.cpp | 31 +++--- .../TestImpactBuildTargetDescriptorFactory.h | 8 +- .../TestImpactTestTargetMetaMapFactory.cpp | 10 +- .../Static/TestImpactBuildTargetDescriptor.h | 11 ++- .../Static/TestImpactTestTargetMeta.h | 3 + .../TestImpactDynamicDependencyMap.cpp | 32 ++++--- .../TestImpactDynamicDependencyMap.h | 9 +- .../TestImpactSourceCoveringTestsList.cpp | 24 ++++- .../TestImpactSourceCoveringTestsList.h | 13 ++- .../Dependency/TestImpactSourceDependency.cpp | 4 +- .../Dependency/TestImpactSourceDependency.h | 6 +- .../Process/TestImpactWin32_Process.cpp | 2 +- .../Process/JobRunner/TestImpactProcessJob.h | 96 +++---------------- .../JobRunner/TestImpactProcessJobInfo.h | 24 +++-- .../JobRunner/TestImpactProcessJobMeta.cpp | 61 ++++++++++++ .../JobRunner/TestImpactProcessJobMeta.h | 68 +++++++++++++ .../JobRunner/TestImpactProcessJobRunner.h | 13 ++- .../Scheduler/TestImpactProcessScheduler.cpp | 14 +-- .../Scheduler/TestImpactProcessScheduler.h | 18 ++-- .../Source/Process/TestImpactProcessInfo.cpp | 6 +- .../Source/Process/TestImpactProcessInfo.h | 11 ++- .../Source/Target/TestImpactBuildTarget.cpp | 2 +- .../Source/Target/TestImpactBuildTarget.h | 2 +- .../Source/Target/TestImpactBuildTargetList.h | 11 ++- .../Source/Target/TestImpactTestTarget.cpp | 10 ++ .../Code/Source/Target/TestImpactTestTarget.h | 6 ++ 26 files changed, 320 insertions(+), 175 deletions(-) create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJobMeta.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJobMeta.h diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactBuildTargetDescriptorFactory.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactBuildTargetDescriptorFactory.cpp index 5e1f2bc890..1bb88912d1 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactBuildTargetDescriptorFactory.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactBuildTargetDescriptorFactory.cpp @@ -10,18 +10,19 @@ * */ +#include + #include #include -#include #include #include namespace TestImpact { AutogenSources PairAutogenSources( - const AZStd::vector& inputSources, - const AZStd::vector& outputSources, + const AZStd::vector& inputSources, + const AZStd::vector& outputSources, const AZStd::string& autogenMatcher) { AutogenSources autogenSources; @@ -65,8 +66,8 @@ namespace TestImpact BuildTargetDescriptor BuildTargetDescriptorFactory( const AZStd::string& buildTargetData, - const AZStd::vector& staticSourceExtensionExcludes, - const AZStd::vector& autogenInputExtensionExcludes, + const AZStd::vector& staticSourceExtensionIncludes, + const AZStd::vector& autogenInputExtensionIncludes, const AZStd::string& autogenMatcher) { // Keys for pertinent JSON node and attribute names @@ -118,14 +119,14 @@ namespace TestImpact const auto& staticSources = sources[Keys[StaticKey]].GetArray(); if (!staticSources.Empty()) { - buildTargetDescriptor.m_sources.m_staticSources = AZStd::vector(); + buildTargetDescriptor.m_sources.m_staticSources = AZStd::vector(); for (const auto& source : staticSources) { - const AZ::IO::Path sourcePath = AZ::IO::Path(source.GetString()); + const RepoPath sourcePath = RepoPath(source.GetString()); if (AZStd::find( - staticSourceExtensionExcludes.begin(), staticSourceExtensionExcludes.end(), sourcePath.Extension().Native()) == - staticSourceExtensionExcludes.end()) + staticSourceExtensionIncludes.begin(), staticSourceExtensionIncludes.end(), sourcePath.Extension().Native()) != + staticSourceExtensionIncludes.end()) { buildTargetDescriptor.m_sources.m_staticSources.emplace_back(AZStd::move(sourcePath)); } @@ -139,17 +140,17 @@ namespace TestImpact AZ_TestImpact_Eval( !inputSources.Empty() && !outputSources.Empty(), ArtifactException, "Autogen malformed, input or output sources are empty"); - AZStd::vector inputPaths; - AZStd::vector outputPaths; + AZStd::vector inputPaths; + AZStd::vector outputPaths; inputPaths.reserve(inputSources.Size()); outputPaths.reserve(outputSources.Size()); for (const auto& source : inputSources) { - const AZ::IO::Path sourcePath = AZ::IO::Path(source.GetString()); + const RepoPath sourcePath = RepoPath(source.GetString()); if (AZStd::find( - autogenInputExtensionExcludes.begin(), autogenInputExtensionExcludes.end(), sourcePath.Extension().Native()) == - autogenInputExtensionExcludes.end()) + autogenInputExtensionIncludes.begin(), autogenInputExtensionIncludes.end(), sourcePath.Extension().Native()) != + autogenInputExtensionIncludes.end()) { inputPaths.emplace_back(AZStd::move(sourcePath)); } @@ -157,7 +158,7 @@ namespace TestImpact for (const auto& source : outputSources) { - outputPaths.emplace_back(AZStd::move(AZ::IO::Path(source.GetString()))); + outputPaths.emplace_back(AZStd::move(RepoPath(source.GetString()))); } buildTargetDescriptor.m_sources.m_autogenSources = PairAutogenSources(inputPaths, outputPaths, autogenMatcher); diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactBuildTargetDescriptorFactory.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactBuildTargetDescriptorFactory.h index b23e00478a..cf4e431a14 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactBuildTargetDescriptorFactory.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactBuildTargetDescriptorFactory.h @@ -21,13 +21,13 @@ namespace TestImpact { //! Constructs a build target artifact from the specified build target data. //! @param buildTargetData The raw build target data in JSON format. - //! @param staticSourceExcludes The list of file extensions to exclude for static sources. - //! @param autogenInputExtentsionExcludes The list of file extensions to exclude for autogen input sources. + //! @param staticSourceIncludes The list of file extensions to include for static sources. + //! @param autogenInputExtentsionIncludes The list of file extensions to include for autogen input sources. //! @param autogenMatcher The regex pattern used to match autogen input filenames with output filenames. //! @return The constructed build target artifact. BuildTargetDescriptor BuildTargetDescriptorFactory( const AZStd::string& buildTargetData, - const AZStd::vector& staticSourceExtentsionExcludes, - const AZStd::vector& autogenInputExtentsionExcludes, + const AZStd::vector& staticSourceExtentsionIncludes, + const AZStd::vector& autogenInputExtentsionIncludes, const AZStd::string& autogenMatcher); } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestTargetMetaMapFactory.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestTargetMetaMapFactory.cpp index db4d5f8563..8bbb5200bb 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestTargetMetaMapFactory.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestTargetMetaMapFactory.cpp @@ -31,7 +31,9 @@ namespace TestImpact "launch_method", "test_runner", "stand_alone", - "name" + "name", + "command", + "timeout" }; enum @@ -43,7 +45,9 @@ namespace TestImpact LaunchMethodKey, TestRunnerKey, StandAloneKey, - NameKey + NameKey, + CommandKey, + TimeoutKey }; AZ_TestImpact_Eval(!masterTestListData.empty(), ArtifactException, "test meta-data cannot be empty"); @@ -61,6 +65,8 @@ namespace TestImpact { TestTargetMeta testMeta; testMeta.m_suite = test[Keys[SuiteKey]].GetString(); + testMeta.m_customArgs = test[Keys[CommandKey]].GetString(); + testMeta.m_timeout = AZStd::chrono::seconds{ test[Keys[TimeoutKey]].GetUint() }; if (const auto buildTypeString = test[Keys[LaunchMethodKey]].GetString(); strcmp(buildTypeString, Keys[TestRunnerKey]) == 0) { diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactBuildTargetDescriptor.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactBuildTargetDescriptor.h index e247b663f2..49820a4bdb 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactBuildTargetDescriptor.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactBuildTargetDescriptor.h @@ -12,7 +12,8 @@ #pragma once -#include +#include + #include #include #include @@ -22,8 +23,8 @@ namespace TestImpact //! Pairing between a given autogen input source and the generated output source(s). struct AutogenPairs { - AZStd::string m_input; - AZStd::vector m_outputs; + RepoPath m_input; + AZStd::vector m_outputs; }; using AutogenSources = AZStd::vector; @@ -31,7 +32,7 @@ namespace TestImpact //! Representation of a given built target's source list. struct TargetSources { - AZStd::vector m_staticSources; //!< Source files used to build this target (if any). + AZStd::vector m_staticSources; //!< Source files used to build this target (if any). AutogenSources m_autogenSources; //!< Autogen source files (if any). }; @@ -40,7 +41,7 @@ namespace TestImpact { AZStd::string m_name; //!< Build target name. AZStd::string m_outputName; //!< Output name (sans extension) of build target binary. - AZ::IO::Path m_path; //!< Path to build target location in source tree (relative to repository root). + RepoPath m_path; //!< Path to build target location in source tree (relative to repository root). }; //! Artifact produced by the build system for each build target. Contains source and output information about said targets. diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactTestTargetMeta.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactTestTargetMeta.h index 57ae2fe894..bef9d9a44f 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactTestTargetMeta.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Static/TestImpactTestTargetMeta.h @@ -14,6 +14,7 @@ #include #include +#include namespace TestImpact { @@ -28,6 +29,8 @@ namespace TestImpact struct TestTargetMeta { AZStd::string m_suite; + AZStd::string m_customArgs; + AZStd::chrono::milliseconds m_timeout = AZStd::chrono::milliseconds{ 0 }; LaunchMethod m_launchMethod = LaunchMethod::TestRunner; }; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp index f99300ad4e..93fb5ee366 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp @@ -25,7 +25,7 @@ namespace TestImpact { for (const auto& source : target->GetSources().m_staticSources) { - if (auto mapping = m_sourceDependencyMap.find(source); + if (auto mapping = m_sourceDependencyMap.find(source.String()); mapping != m_sourceDependencyMap.end()) { // This is an existing entry in the dependency map so update the parent build targets with this target @@ -43,7 +43,7 @@ namespace TestImpact { for (const auto& output : autogen.m_outputs) { - m_autogenInputToOutputMap[autogen.m_input].push_back(output); + m_autogenInputToOutputMap[autogen.m_input.String()].push_back(output.String()); } } }; @@ -138,11 +138,11 @@ namespace TestImpact { // Autogen input files are not compiled sources and thus supplying coverage data for them makes no sense AZ_TestImpact_Eval( - m_autogenInputToOutputMap.find(sourceCoverage.GetPath()) == m_autogenInputToOutputMap.end(), + m_autogenInputToOutputMap.find(sourceCoverage.GetPath().c_str()) == m_autogenInputToOutputMap.end(), DependencyException, AZStd::string::format("Couldn't replace source coverage for %s, source file is an autogen input file", sourceCoverage.GetPath().c_str()).c_str()); - auto [it, inserted] = m_sourceDependencyMap.insert(sourceCoverage.GetPath()); + auto [it, inserted] = m_sourceDependencyMap.insert(sourceCoverage.GetPath().String()); auto& [key, sourceDependency] = *it; // Clear any existing coverage for the delta @@ -178,22 +178,26 @@ namespace TestImpact } } - void DynamicDependencyMap::ClearSourceCoverage(const AZStd::vector& paths) + void DynamicDependencyMap::ClearSourceCoverage(const AZStd::vector& paths) { for (const auto& path : paths) { - if (const auto outputSources = m_autogenInputToOutputMap.find(path); + if (const auto outputSources = m_autogenInputToOutputMap.find(path.String()); outputSources != m_autogenInputToOutputMap.end()) { // Clearing the coverage data of an autogen input source instead clears the coverage data of its output sources for (const auto& outputSource : outputSources->second) { - ReplaceSourceCoverage(SourceCoveringTestsList({ SourceCoveringTests(outputSource) })); + AZStd::vector coverage; + coverage.emplace_back(RepoPath(outputSource)); + ReplaceSourceCoverage(SourceCoveringTestsList(AZStd::move(coverage))); } } else { - ReplaceSourceCoverage(SourceCoveringTestsList({ SourceCoveringTests(path, { }) })); + AZStd::vector coverage; + coverage.emplace_back(RepoPath(path)); + ReplaceSourceCoverage(SourceCoveringTestsList(AZStd::move(coverage))); } } } @@ -221,7 +225,7 @@ namespace TestImpact return coveringTestTargets; } - AZStd::optional DynamicDependencyMap::GetSourceDependency(const AZStd::string& path) const + AZStd::optional DynamicDependencyMap::GetSourceDependency(const RepoPath& path) const { AZStd::unordered_set parentTargets; AZStd::unordered_set coveringTestTargets; @@ -243,7 +247,7 @@ namespace TestImpact } }; - if (const auto outputSources = m_autogenInputToOutputMap.find(path); outputSources != m_autogenInputToOutputMap.end()) + if (const auto outputSources = m_autogenInputToOutputMap.find(path.String()); outputSources != m_autogenInputToOutputMap.end()) { // Consolidate the parentage and coverage of each of the autogen input file's generated output files for (const auto& outputSource : outputSources->second) @@ -253,7 +257,7 @@ namespace TestImpact } else { - getSourceDependency(path); + getSourceDependency(path.String()); } if (!parentTargets.empty() || !coveringTestTargets.empty()) @@ -264,7 +268,7 @@ namespace TestImpact return AZStd::nullopt; } - SourceDependency DynamicDependencyMap::GetSourceDependencyOrThrow(const AZStd::string& path) const + SourceDependency DynamicDependencyMap::GetSourceDependencyOrThrow(const RepoPath& path) const { auto sourceDependency = GetSourceDependency(path); AZ_TestImpact_Eval(sourceDependency.has_value(), DependencyException, AZStd::string::format("Couldn't find source %s", path.c_str()).c_str()); @@ -282,7 +286,7 @@ namespace TestImpact souceCoveringTests.push_back(testTarget->GetName()); } - coverage.push_back(SourceCoveringTests(path, AZStd::move(souceCoveringTests))); + coverage.push_back(SourceCoveringTests(RepoPath(path), AZStd::move(souceCoveringTests))); } return SourceCoveringTestsList(AZStd::move(coverage)); @@ -310,7 +314,7 @@ namespace TestImpact // Keep track of the coverage to delete as a post step rather than deleting it in situ so that erroneous change lists // do not corrupt the dynamic dependency map - AZStd::vector coverageToDelete; + AZStd::vector coverageToDelete; // Create operations for (const auto& createdFile : changeList.m_createdFiles) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.h index 527a6555c6..6faef76452 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.h @@ -12,7 +12,8 @@ #pragma once -#include +#include + #include #include #include @@ -74,10 +75,10 @@ namespace TestImpact //! Gets the source dependency for the specified source file. //! @note Autogen input source dependencies are the consolidated source dependencies of all of their generated output sources. //! @returns If found, the source dependency information for the specified source file, otherwise empty. - AZStd::optional GetSourceDependency(const AZStd::string& path) const; + AZStd::optional GetSourceDependency(const RepoPath& path) const; //! Gets the source dependency for the specified source file or throw DependencyException. - SourceDependency GetSourceDependencyOrThrow(const AZStd::string& path) const; + SourceDependency GetSourceDependencyOrThrow(const RepoPath& path) const; //! Replaces the source coverage of the specified sources with the specified source coverage. //! @note The covering targets for the parent test target(s) will not be pruned if those covering targets are removed. @@ -99,7 +100,7 @@ namespace TestImpact private: //! Clears the source coverage of the specified sources. //! @note The covering targets for the parent test target(s) will not be pruned if those covering targets are removed. - void ClearSourceCoverage(const AZStd::vector& paths); + void ClearSourceCoverage(const AZStd::vector& paths); //! The sorted list of unique production targets in the repository. ProductionTargetList m_productionTargets; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsList.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsList.cpp index 792eb1f0b2..101bdf862c 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsList.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsList.cpp @@ -16,18 +16,36 @@ namespace TestImpact { - SourceCoveringTests::SourceCoveringTests(const AZStd::string& path) + AZStd::vector ExtractTargetsFromSet(AZStd::unordered_set&& coveringTestTargets) + { + AZStd::vector testTargets; + testTargets.reserve(coveringTestTargets.size()); + for (auto it = coveringTestTargets.begin(); it != coveringTestTargets.end(); ) + { + testTargets.push_back(std::move(coveringTestTargets.extract(it++).value())); + } + + return testTargets; + } + + SourceCoveringTests::SourceCoveringTests(const RepoPath& path) : m_path(path) { } - SourceCoveringTests::SourceCoveringTests(const AZStd::string& path, AZStd::vector&& coveringTestTargets) + SourceCoveringTests::SourceCoveringTests(const RepoPath& path, AZStd::vector&& coveringTestTargets) : m_path(path) , m_coveringTestTargets(AZStd::move(coveringTestTargets)) { } - const AZStd::string& SourceCoveringTests::GetPath() const + SourceCoveringTests::SourceCoveringTests(const RepoPath& path, AZStd::unordered_set&& coveringTestTargets) + : m_path(path) + , m_coveringTestTargets(ExtractTargetsFromSet(AZStd::move(coveringTestTargets))) + { + } + + const RepoPath& SourceCoveringTests::GetPath() const { return m_path; } diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsList.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsList.h index 52f3a7c3f6..f501ccefd8 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsList.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsList.h @@ -12,7 +12,10 @@ #pragma once +#include + #include +#include #include namespace TestImpact @@ -21,11 +24,13 @@ namespace TestImpact class SourceCoveringTests { public: - explicit SourceCoveringTests(const AZStd::string& path); - SourceCoveringTests(const AZStd::string& path, AZStd::vector&& coveringTestTargets); + //SourceCoveringTests(const SourceCoveringTests&); + explicit SourceCoveringTests(const RepoPath& path); + SourceCoveringTests(const RepoPath& path, AZStd::vector&& coveringTestTargets); + SourceCoveringTests(const RepoPath& path, AZStd::unordered_set&& coveringTestTargets); //! Returns the path of this source file. - const AZStd::string& GetPath() const; + const RepoPath& GetPath() const; //! Returns the number of unresolved test targets covering this source file. size_t GetNumCoveringTestTargets() const; @@ -33,7 +38,7 @@ namespace TestImpact //! Returns the unresolved test targets covering this source file. const AZStd::vector& GetCoveringTestTargets() const; private: - AZStd::string m_path; //!< The path of this source file. + RepoPath m_path; //!< The path of this source file. AZStd::vector m_coveringTestTargets; //!< The unresolved test targets that cover this source file. }; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceDependency.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceDependency.cpp index 39193d4dd9..7d4e76acd1 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceDependency.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceDependency.cpp @@ -52,14 +52,14 @@ namespace TestImpact } SourceDependency::SourceDependency( - const AZStd::string& path, + const RepoPath& path, DependencyData&& dependencyData) : m_path(path) , m_dependencyData(AZStd::move(dependencyData)) { } - const AZStd::string& SourceDependency::GetPath() const + const RepoPath& SourceDependency::GetPath() const { return m_path; } diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceDependency.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceDependency.h index a3b6783d13..d80c4a0540 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceDependency.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceDependency.h @@ -71,11 +71,11 @@ namespace TestImpact { public: SourceDependency( - const AZStd::string& path, + const RepoPath& path, DependencyData&& dependencyData); //! Returns the path of this source file. - const AZStd::string& GetPath() const; + const RepoPath& GetPath() const; //! Returns the number of parent build targets this source belongs to. size_t GetNumParentTargets() const; @@ -89,7 +89,7 @@ namespace TestImpact //! Returns the test targets covering this source file. const AZStd::unordered_set& GetCoveringTestTargets() const; private: - AZStd::string m_path; //!< The path of this source file. + RepoPath m_path; //!< The path of this source file. DependencyData m_dependencyData; //!< The dependency data for this source file. }; } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/Process/TestImpactWin32_Process.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/Process/TestImpactWin32_Process.cpp index 0a8a8643f3..06dd4e846f 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/Process/TestImpactWin32_Process.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/Process/TestImpactWin32_Process.cpp @@ -69,7 +69,7 @@ namespace TestImpact NULL, NULL, &si, &pi)) { - throw ProcessException("Couldn't create process"); + throw ProcessException(AZStd::string::format("Couldn't create process with args: %s", args.c_str())); } ReleaseChildPipes(); diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJob.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJob.h index d9dd4870dd..4710cf2141 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJob.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJob.h @@ -13,77 +13,42 @@ #pragma once #include -#include - -#include -#include +#include namespace TestImpact { - //! Result of a job that was run. - enum class JobResult - { - NotExecuted, //!< The job was not executed (e.g. the job runner terminated before the job could be executed). - FailedToExecute, //!< The job failed to execute (e.g. due to the arguments used to execute the job being invalid). - Terminated, //!< The job was terminated by the job runner (e.g. job or runner timeout exceeded while job was in-flight). - ExecutedWithFailure, //!< The job was executed but exited in an erroneous state (the underlying process returned non-zero). - ExecutedWithSuccess //!< The job was executed and exited in a successful state (the underlying processes returned zero). - }; - - //! The meta-data for a given job. - struct JobMeta - { - JobResult m_result = JobResult::NotExecuted; - AZStd::optional - m_startTime; //!< The time, relative to the job runner start, that this job started. - AZStd::optional m_duration; //!< The duration that this job took to complete. - AZStd::optional m_returnCode; //!< The return code of the underlying processes of this job. - }; - //! Representation of a unit of work to be performed by a process. //! @tparam JobInfoT The JobInfo structure containing the information required to run this job. //! @tparam JobPayloadT The resulting output of the processed artifact produced by this job. template class Job + : public JobMetaContainer { public: using Info = JobInfoT; using Payload = JobPayloadT; //! Constructor with r-values for the specific use case of the job runner. - Job(Info jobInfo, JobMeta&& jobMeta, AZStd::optional&& payload); + Job(const Info& jobInfo, JobMeta&& jobMeta, AZStd::optional&& payload); //! Returns the job info associated with this job. const Info& GetJobInfo() const; - //! Returns the result of this job. - JobResult GetResult() const; - - //! Returns the start time, relative to the job runner start, that this job started. - AZStd::chrono::high_resolution_clock::time_point GetStartTime() const; - - //! Returns the end time, relative to the job runner start, that this job ended. - AZStd::chrono::high_resolution_clock::time_point GetEndTime() const; - - //! Returns the duration that this job took to complete. - AZStd::chrono::milliseconds GetDuration() const; - - //! Returns the return code of the underlying processes of this job. - AZStd::optional GetReturnCode() const; - //! Returns the payload produced by this job. const AZStd::optional& GetPayload() const; + //! Facilitates the client consuming the payload. + AZStd::optional&& ReleasePayload(); + private: Info m_jobInfo; - JobMeta m_meta; AZStd::optional m_payload; }; template - Job::Job(Info jobInfo, JobMeta&& jobMeta, AZStd::optional&& payload) - : m_jobInfo(jobInfo) - , m_meta(AZStd::move(jobMeta)) + Job::Job(const Info& jobInfo, JobMeta&& jobMeta, AZStd::optional&& payload) + : JobMetaContainer(AZStd::move(jobMeta)) + , m_jobInfo(jobInfo) , m_payload(AZStd::move(payload)) { } @@ -94,46 +59,15 @@ namespace TestImpact return m_jobInfo; } - template - JobResult Job::GetResult() const - { - return m_meta.m_result; - } - - template - AZStd::optional Job::GetReturnCode() const - { - return m_meta.m_returnCode; - } - - template - AZStd::chrono::high_resolution_clock::time_point Job::GetStartTime() const - { - return m_meta.m_startTime.value_or(AZStd::chrono::high_resolution_clock::time_point()); - } - - template - AZStd::chrono::high_resolution_clock::time_point Job::GetEndTime() const - { - if (m_meta.m_startTime.has_value() && m_meta.m_duration.has_value()) - { - return m_meta.m_startTime.value() + m_meta.m_duration.value(); - } - else - { - return AZStd::chrono::high_resolution_clock::time_point(); - } - } - - template - AZStd::chrono::milliseconds Job::GetDuration() const - { - return m_meta.m_duration.value_or(AZStd::chrono::milliseconds{0}); - } - template const AZStd::optional& Job::GetPayload() const { return m_payload; } + + template + AZStd::optional&& Job::ReleasePayload() + { + return AZStd::move(m_payload); + } } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJobInfo.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJobInfo.h index fd9d76652a..fbdad82c12 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJobInfo.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJobInfo.h @@ -24,6 +24,7 @@ namespace TestImpact { public: using IdType = size_t; + using CommandType = AZStd::string; //! Client-provided identifier to distinguish between different jobs. //! @note Ids of different job types are not interchangeable. @@ -32,30 +33,37 @@ namespace TestImpact IdType m_value; }; + //! Command used my ProcessScheduler to execute this job. + //! @note Commands of different job types are not interchangeable. + struct Command + { + CommandType m_args; + }; + //! Constructs the job information with any additional information required by the job. //! @param jobId The client-provided unique identifier for the job. - //! @param args The arguments used to launch the process running the job. + //! @param command The command used to launch the process running the job. //! @param additionalInfo The arguments to be provided to the additional information data structure. template - JobInfo(Id jobId, const AZStd::string& args, AdditionalInfoArgs&&... additionalInfo); + JobInfo(Id jobId, const Command& command, AdditionalInfoArgs&&... additionalInfo); //! Returns the id of this job. Id GetId() const; //! Returns the command arguments used to execute this job. - const AZStd::string& GetArgs() const; + const Command& GetCommand() const; private: Id m_id; - AZStd::string m_args; + Command m_command; }; template template - JobInfo::JobInfo(Id jobId, const AZStd::string& args, AdditionalInfoArgs&&... additionalInfo) + JobInfo::JobInfo(Id jobId, const Command& command, AdditionalInfoArgs&&... additionalInfo) : AdditionalInfo{std::forward(additionalInfo)...} , m_id(jobId) - , m_args(args) + , m_command(command) { } @@ -66,8 +74,8 @@ namespace TestImpact } template - const AZStd::string& JobInfo::GetArgs() const + const typename JobInfo::Command& JobInfo::GetCommand() const { - return m_args; + return m_command; } } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJobMeta.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJobMeta.cpp new file mode 100644 index 0000000000..ba6ad4c585 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJobMeta.cpp @@ -0,0 +1,61 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +namespace TestImpact +{ + JobMetaContainer::JobMetaContainer(const JobMeta& jobMeta) + : m_meta(jobMeta) + { + } + + JobMetaContainer::JobMetaContainer(JobMeta&& jobMeta) + : m_meta(AZStd::move(jobMeta)) + { + } + + JobResult JobMetaContainer::GetJobResult() const + { + return m_meta.m_result; + } + + AZStd::optional JobMetaContainer::GetReturnCode() const + { + return m_meta.m_returnCode; + } + + AZStd::chrono::high_resolution_clock::time_point JobMetaContainer::GetStartTime() const + { + return m_meta.m_startTime.value_or(AZStd::chrono::high_resolution_clock::time_point()); + } + + AZStd::chrono::high_resolution_clock::time_point JobMetaContainer::GetEndTime() const + { + if (m_meta.m_startTime.has_value() && m_meta.m_duration.has_value()) + { + return m_meta.m_startTime.value() + m_meta.m_duration.value(); + } + else + { + return AZStd::chrono::high_resolution_clock::time_point(); + } + } + + AZStd::chrono::milliseconds JobMetaContainer::GetDuration() const + { + return m_meta.m_duration.value_or(AZStd::chrono::milliseconds{ 0 }); + } + +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJobMeta.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJobMeta.h new file mode 100644 index 0000000000..6e7ce97468 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJobMeta.h @@ -0,0 +1,68 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +#include +#include + +namespace TestImpact +{ + //! Result of a job that was run. + enum class JobResult + { + NotExecuted, //!< The job was not executed (e.g. the job runner terminated before the job could be executed). + FailedToExecute, //!< The job failed to execute (e.g. due to the arguments used to execute the job being invalid). + Timeout, //!< The job was terminated by the job runner (e.g. job timeout exceeded while job was in-flight). + Terminated, //!< The job was terminated by the job runner (e.g. global timeout exceeded while job was in-flight). + ExecutedWithFailure, //!< The job was executed but exited in an erroneous state (the underlying process returned non-zero). + ExecutedWithSuccess //!< The job was executed and exited in a successful state (the underlying processes returned zero). + }; + + //! The meta-data for a given job. + struct JobMeta + { + JobResult m_result = JobResult::NotExecuted; + AZStd::optional + m_startTime; //!< The time, relative to the job runner start, that this job started. + AZStd::optional m_duration; //!< The duration that this job took to complete. + AZStd::optional m_returnCode; //!< The return code of the underlying processes of this job. + }; + + class JobMetaContainer + { + public: + JobMetaContainer(const JobMeta& jobMeta); + JobMetaContainer(JobMeta&& jobMeta); + + //! Returns the result of this job. + JobResult GetJobResult() const; + + //! Returns the start time, relative to the job runner start, that this job started. + AZStd::chrono::high_resolution_clock::time_point GetStartTime() const; + + //! Returns the end time, relative to the job runner start, that this job ended. + AZStd::chrono::high_resolution_clock::time_point GetEndTime() const; + + //! Returns the duration that this job took to complete. + AZStd::chrono::milliseconds GetDuration() const; + + //! Returns the return code of the underlying processes of this job. + AZStd::optional GetReturnCode() const; + + private: + JobMeta m_meta; + }; +} // namespace TestImpact + diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJobRunner.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJobRunner.h index 8b9e4fa69e..0c30accc84 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJobRunner.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJobRunner.h @@ -13,7 +13,6 @@ #pragma once #include -#include #include #include @@ -27,7 +26,7 @@ namespace TestImpact //! @param meta The meta-data about the job run. //! @param std The standard output and standard error of the process running the job. template - using JobCallback = AZStd::function; + using JobCallback = AZStd::function; //! The payloads produced by the job-specific payload producer in the form of a map associating each job id with the job's payload. template @@ -116,7 +115,7 @@ namespace TestImpact const auto* jobInfo = &jobInfos[jobIndex]; const auto jobId = jobInfo->GetId().m_value; metas.emplace(jobId, AZStd::pair{JobMeta{}, jobInfo}); - processes.emplace_back(jobId, m_stdOutRouting, m_stdErrRouting, jobInfo->GetArgs()); + processes.emplace_back(jobId, m_stdOutRouting, m_stdErrRouting, jobInfo->GetCommand().m_args); } // Wrapper around low-level process launch callback to gather job meta-data and present a simplified callback interface to the client @@ -134,7 +133,7 @@ namespace TestImpact else { meta.m_startTime = createTime; - return CallbackResult::Continue; + return ProcessCallbackResult::Continue; } }; @@ -153,10 +152,14 @@ namespace TestImpact { meta.m_result = JobResult::ExecutedWithSuccess; } - else if (exitCondition == ExitCondition::Terminated || exitCondition == ExitCondition::Timeout) + else if (exitCondition == ExitCondition::Terminated) { meta.m_result = JobResult::Terminated; } + else if (exitCondition == ExitCondition::Timeout) + { + meta.m_result = JobResult::Timeout; + } else { meta.m_result = JobResult::ExecutedWithFailure; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/Scheduler/TestImpactProcessScheduler.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/Scheduler/TestImpactProcessScheduler.cpp index 15230b1a46..04a68bf612 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/Scheduler/TestImpactProcessScheduler.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/Scheduler/TestImpactProcessScheduler.cpp @@ -58,7 +58,7 @@ namespace TestImpact for (auto& process : m_processPool) { - if (PopAndLaunch(process) == CallbackResult::Abort) + if (PopAndLaunch(process) == ProcessCallbackResult::Abort) { TerminateAllProcesses(ExitCondition::Terminated); return; @@ -110,7 +110,7 @@ namespace TestImpact const auto exitTime = AZStd::chrono::high_resolution_clock::now(); // Inform the client that the processes has exited - if (CallbackResult::Abort == m_processExitCallback( + if (ProcessCallbackResult::Abort == m_processExitCallback( processId, ExitCondition::Gracefull, returnCode, @@ -124,7 +124,7 @@ namespace TestImpact else if (!m_processQueue.empty()) { // This slot in the pool is free so launch one of the processes waiting in the queue - if (PopAndLaunch(processInFlight) == CallbackResult::Abort) + if (PopAndLaunch(processInFlight) == ProcessCallbackResult::Abort) { // Client chose to abort the scheduler TerminateAllProcesses(ExitCondition::Terminated); @@ -150,7 +150,7 @@ namespace TestImpact const ReturnCode returnCode = processInFlight.m_process->GetReturnCode().value(); processInFlight.m_process.reset(); - if (CallbackResult::Abort == m_processExitCallback( + if (ProcessCallbackResult::Abort == m_processExitCallback( processId, ExitCondition::Timeout, returnCode, @@ -172,7 +172,7 @@ namespace TestImpact // Queue is empty, no more processes to launch if (!m_processQueue.empty()) { - if (PopAndLaunch(processInFlight) == CallbackResult::Abort) + if (PopAndLaunch(processInFlight) == ProcessCallbackResult::Abort) { // Client chose to abort the scheduler TerminateAllProcesses(ExitCondition::Terminated); @@ -194,7 +194,7 @@ namespace TestImpact } } - CallbackResult ProcessScheduler::PopAndLaunch(ProcessInFlight& processInFlight) + ProcessCallbackResult ProcessScheduler::PopAndLaunch(ProcessInFlight& processInFlight) { auto processInfo = m_processQueue.front(); m_processQueue.pop(); @@ -251,7 +251,7 @@ namespace TestImpact if (isCallingBackToClient) { const auto exitTime = AZStd::chrono::high_resolution_clock::now(); - if (CallbackResult::Abort == m_processExitCallback( + if (ProcessCallbackResult::Abort == m_processExitCallback( processInFlight.m_process->GetProcessInfo().GetId(), exitStatus, returnCode, diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/Scheduler/TestImpactProcessScheduler.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/Scheduler/TestImpactProcessScheduler.h index 5531c78397..27171d25ee 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/Scheduler/TestImpactProcessScheduler.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/Scheduler/TestImpactProcessScheduler.h @@ -12,11 +12,10 @@ #pragma once +#include + #include -#include - -#include #include #include #include @@ -43,12 +42,19 @@ namespace TestImpact Timeout = ProcessTimeoutErrorCode //!< The process was terminated by the scheduler due to exceeding runtime limit. }; + //! Client result for process scheduler callbacks. + enum class ProcessCallbackResult : bool + { + Continue, //!< Continune scheduling. + Abort //!< Abort scheduling immediately. + }; + //! Callback for process launch attempt. //! @param processId The id of the process that attempted to launch. //! @param launchResult The result of the process launch attempt. //! @param createTime The timestamp of the process launch attempt. using ProcessLaunchCallback = - AZStd::function; @@ -60,7 +66,7 @@ namespace TestImpact //! @param std The standard output and standard error of the process. //! @param createTime The timestamp of the process exit. using ProcessExitCallback = - AZStd::function +#include + #include #include @@ -64,9 +65,9 @@ namespace TestImpact ProcessId processId, StdOutputRouting stdOut, StdErrorRouting stdErr, - const AZ::IO::Path& processPath, + const RepoPath& processPath, const AZStd::string& startupArgs = ""); - ProcessInfo(ProcessId processId, const AZ::IO::Path& processPath, const AZStd::string& startupArgs = ""); + ProcessInfo(ProcessId processId, const RepoPath& processPath, const AZStd::string& startupArgs = ""); //! Returns the identifier of this process. ProcessId GetId() const; @@ -78,7 +79,7 @@ namespace TestImpact bool ParentHasStdError() const; // Returns the path to the process binary. - const AZ::IO::Path& GetProcessPath() const; + const RepoPath& GetProcessPath() const; //! Returns the command line arguments used to launch the process. const AZStd::string& GetStartupArgs() const; @@ -87,7 +88,7 @@ namespace TestImpact const ProcessId m_id; const bool m_parentHasStdOutput; const bool m_parentHasStdErr; - const AZ::IO::Path m_processPath; + const RepoPath m_processPath; const AZStd::string m_startupArgs; }; } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactBuildTarget.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactBuildTarget.cpp index bce6fa25d6..54161899ae 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactBuildTarget.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactBuildTarget.cpp @@ -31,7 +31,7 @@ namespace TestImpact return m_buildMetaData.m_outputName; } - const AZ::IO::Path& BuildTarget::GetPath() const + const RepoPath& BuildTarget::GetPath() const { return m_buildMetaData.m_path; } diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactBuildTarget.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactBuildTarget.h index 0cd6369472..d5d32b8ff5 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactBuildTarget.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactBuildTarget.h @@ -49,7 +49,7 @@ namespace TestImpact const AZStd::string& GetOutputName() const; //! Returns the path in the source tree to the build target location. - const AZ::IO::Path& GetPath() const; + const RepoPath& GetPath() const; //! Returns the build target's sources. const TargetSources& GetSources() const; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactBuildTargetList.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactBuildTargetList.h index dfd102b55f..9731b68d61 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactBuildTargetList.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactBuildTargetList.h @@ -42,6 +42,9 @@ namespace TestImpact //! Returns the target with the specified name or throws if target not found. const Target* GetTargetOrThrow(const AZStd::string& name) const; + //! Returns true if the specified target is in the list, otherwise false. + bool HasTarget(const AZStd::string& name) const; + // Returns the number of targets in the list. size_t GetNumTargets() const; @@ -71,7 +74,7 @@ namespace TestImpact m_targets.reserve(descriptors.size()); for (auto&& descriptor : descriptors) { - m_targets.emplace_back(Target(AZStd::move(descriptor))); + m_targets.emplace_back(AZStd::move(Target(AZStd::move(descriptor)))); } } @@ -122,4 +125,10 @@ namespace TestImpact AZ_TestImpact_Eval(target, TargetException, AZStd::string::format("Couldn't find target %s", name.c_str()).c_str()); return target; } + + template + bool BuildTargetList::HasTarget(const AZStd::string& name) const + { + return GetTarget(name) != nullptr; + } } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactTestTarget.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactTestTarget.cpp index 0189bc9e78..1890863981 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactTestTarget.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactTestTarget.cpp @@ -25,6 +25,16 @@ namespace TestImpact return m_testMetaData.m_suite; } + const AZStd::string& TestTarget::GetCustomArgs() const + { + return m_testMetaData.m_customArgs; + } + + AZStd::chrono::milliseconds TestTarget::GetTimeout() const + { + return m_testMetaData.m_timeout; + } + LaunchMethod TestTarget::GetLaunchMethod() const { return m_testMetaData.m_launchMethod; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactTestTarget.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactTestTarget.h index b6f44e7cc1..e8449dde3d 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactTestTarget.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactTestTarget.h @@ -29,6 +29,12 @@ namespace TestImpact //! Returns the test target suite. const AZStd::string& GetSuite() const; + //! Returns the launcher custom arguments. + const AZStd::string& GetCustomArgs() const; + + //! Returns the test run timeout. + AZStd::chrono::milliseconds GetTimeout() const; + //! Returns the test target launch method. LaunchMethod GetLaunchMethod() const; From c477c699ef601e1a279ae8a8e21c50485775c83b Mon Sep 17 00:00:00 2001 From: jonawals Date: Fri, 21 May 2021 09:11:53 +0100 Subject: [PATCH 023/102] Additional refactoring --- .../TestImpactClientFailureReport.h | 2 +- .../Dependency/TestImpactDynamicDependencyMap.cpp | 10 +++------- .../Process/JobRunner/TestImpactProcessJobMeta.h | 3 +-- .../Code/Source/Target/TestImpactBuildTargetList.h | 2 +- 4 files changed, 6 insertions(+), 11 deletions(-) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientFailureReport.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientFailureReport.h index d2ab863d86..02b1169e93 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientFailureReport.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientFailureReport.h @@ -130,7 +130,7 @@ namespace TestImpact private: AZStd::vector m_executionFailures; AZStd::vector m_launcherFailures; - AZStd::vector m_unexecutedTestsTests; + AZStd::vector m_unexecutedTests; }; //! Represents the report for a failed regular test sequence run without test impact analysis. diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp index 93fb5ee366..31b3485c70 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp @@ -138,7 +138,7 @@ namespace TestImpact { // Autogen input files are not compiled sources and thus supplying coverage data for them makes no sense AZ_TestImpact_Eval( - m_autogenInputToOutputMap.find(sourceCoverage.GetPath().c_str()) == m_autogenInputToOutputMap.end(), + m_autogenInputToOutputMap.find(sourceCoverage.GetPath().String()) == m_autogenInputToOutputMap.end(), DependencyException, AZStd::string::format("Couldn't replace source coverage for %s, source file is an autogen input file", sourceCoverage.GetPath().c_str()).c_str()); @@ -188,16 +188,12 @@ namespace TestImpact // Clearing the coverage data of an autogen input source instead clears the coverage data of its output sources for (const auto& outputSource : outputSources->second) { - AZStd::vector coverage; - coverage.emplace_back(RepoPath(outputSource)); - ReplaceSourceCoverage(SourceCoveringTestsList(AZStd::move(coverage))); + ReplaceSourceCoverage(SourceCoveringTestsList(AZStd::vector{ SourceCoveringTests(RepoPath(outputSource)) })); } } else { - AZStd::vector coverage; - coverage.emplace_back(RepoPath(path)); - ReplaceSourceCoverage(SourceCoveringTestsList(AZStd::move(coverage))); + ReplaceSourceCoverage(SourceCoveringTestsList(AZStd::vector{ SourceCoveringTests(RepoPath(path)) })); } } } diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJobMeta.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJobMeta.h index 6e7ce97468..874934b54d 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJobMeta.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJobMeta.h @@ -34,8 +34,7 @@ namespace TestImpact struct JobMeta { JobResult m_result = JobResult::NotExecuted; - AZStd::optional - m_startTime; //!< The time, relative to the job runner start, that this job started. + AZStd::optional m_startTime; //!< The time, relative to the job runner start, that this job started. AZStd::optional m_duration; //!< The duration that this job took to complete. AZStd::optional m_returnCode; //!< The return code of the underlying processes of this job. }; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactBuildTargetList.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactBuildTargetList.h index 9731b68d61..79f93f7779 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactBuildTargetList.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Target/TestImpactBuildTargetList.h @@ -74,7 +74,7 @@ namespace TestImpact m_targets.reserve(descriptors.size()); for (auto&& descriptor : descriptors) { - m_targets.emplace_back(AZStd::move(Target(AZStd::move(descriptor)))); + m_targets.emplace_back(Target(AZStd::move(descriptor))); } } From 5113d3f756396a02ce6bb1e560300298731fe222 Mon Sep 17 00:00:00 2001 From: jonawals Date: Fri, 21 May 2021 09:36:01 +0100 Subject: [PATCH 024/102] REname JobMetaContainer to JobMetaWrapper --- .../Process/JobRunner/TestImpactProcessJob.h | 4 ++-- .../Process/JobRunner/TestImpactProcessJobMeta.cpp | 14 +++++++------- .../Process/JobRunner/TestImpactProcessJobMeta.h | 7 ++++--- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJob.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJob.h index 4710cf2141..f1629adda4 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJob.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJob.h @@ -22,7 +22,7 @@ namespace TestImpact //! @tparam JobPayloadT The resulting output of the processed artifact produced by this job. template class Job - : public JobMetaContainer + : public JobMetaWrapper { public: using Info = JobInfoT; @@ -47,7 +47,7 @@ namespace TestImpact template Job::Job(const Info& jobInfo, JobMeta&& jobMeta, AZStd::optional&& payload) - : JobMetaContainer(AZStd::move(jobMeta)) + : JobMetaWrapper(AZStd::move(jobMeta)) , m_jobInfo(jobInfo) , m_payload(AZStd::move(payload)) { diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJobMeta.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJobMeta.cpp index ba6ad4c585..022115c042 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJobMeta.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJobMeta.cpp @@ -16,32 +16,32 @@ namespace TestImpact { - JobMetaContainer::JobMetaContainer(const JobMeta& jobMeta) + JobMetaWrapper::JobMetaWrapper(const JobMeta& jobMeta) : m_meta(jobMeta) { } - JobMetaContainer::JobMetaContainer(JobMeta&& jobMeta) + JobMetaWrapper::JobMetaWrapper(JobMeta&& jobMeta) : m_meta(AZStd::move(jobMeta)) { } - JobResult JobMetaContainer::GetJobResult() const + JobResult JobMetaWrapper::GetJobResult() const { return m_meta.m_result; } - AZStd::optional JobMetaContainer::GetReturnCode() const + AZStd::optional JobMetaWrapper::GetReturnCode() const { return m_meta.m_returnCode; } - AZStd::chrono::high_resolution_clock::time_point JobMetaContainer::GetStartTime() const + AZStd::chrono::high_resolution_clock::time_point JobMetaWrapper::GetStartTime() const { return m_meta.m_startTime.value_or(AZStd::chrono::high_resolution_clock::time_point()); } - AZStd::chrono::high_resolution_clock::time_point JobMetaContainer::GetEndTime() const + AZStd::chrono::high_resolution_clock::time_point JobMetaWrapper::GetEndTime() const { if (m_meta.m_startTime.has_value() && m_meta.m_duration.has_value()) { @@ -53,7 +53,7 @@ namespace TestImpact } } - AZStd::chrono::milliseconds JobMetaContainer::GetDuration() const + AZStd::chrono::milliseconds JobMetaWrapper::GetDuration() const { return m_meta.m_duration.value_or(AZStd::chrono::milliseconds{ 0 }); } diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJobMeta.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJobMeta.h index 874934b54d..c8f459ba6f 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJobMeta.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJobMeta.h @@ -39,11 +39,12 @@ namespace TestImpact AZStd::optional m_returnCode; //!< The return code of the underlying processes of this job. }; - class JobMetaContainer + //! Wrapper for job meta structure to inheritance/aggregation without being coupled to the JobInfo or Job classes. + class JobMetaWrapper { public: - JobMetaContainer(const JobMeta& jobMeta); - JobMetaContainer(JobMeta&& jobMeta); + JobMetaWrapper(const JobMeta& jobMeta); + JobMetaWrapper(JobMeta&& jobMeta); //! Returns the result of this job. JobResult GetJobResult() const; From 638f40b7dd23bbb07c268291a3b0c6233fe55ec2 Mon Sep 17 00:00:00 2001 From: jonawals Date: Fri, 21 May 2021 12:19:19 +0100 Subject: [PATCH 025/102] Add proper exhange to payload release --- .../Code/Source/Process/JobRunner/TestImpactProcessJob.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJob.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJob.h index f1629adda4..47afe1ca3e 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJob.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJob.h @@ -38,7 +38,8 @@ namespace TestImpact const AZStd::optional& GetPayload() const; //! Facilitates the client consuming the payload. - AZStd::optional&& ReleasePayload(); + //! @note It is valid for a job life cycle to continue after having released its payload. + AZStd::optional ReleasePayload(); private: Info m_jobInfo; @@ -66,8 +67,8 @@ namespace TestImpact } template - AZStd::optional&& Job::ReleasePayload() + AZStd::optional Job::ReleasePayload() { - return AZStd::move(m_payload); + return AZStd::exchange(m_payload, AZStd::nullopt); } } // namespace TestImpact From e68ac30bcb24dbf83d1ed25ec8b01fb1ba3af0f1 Mon Sep 17 00:00:00 2001 From: jonawals Date: Fri, 21 May 2021 13:11:24 +0100 Subject: [PATCH 026/102] Move test runners --- .../Source/Test/Job/TestImpactTestJobCommon.h | 38 ---- .../Test/Job/TestImpactTestJobException.h | 26 --- .../Enumeration/TestImpactTestEnumeration.h | 2 +- .../TestImpactTestEnumerationException.h | 0 .../TestImpactTestEnumerationSerializer.cpp | 4 +- .../TestImpactTestEnumerationSerializer.h | 2 +- .../Enumeration/TestImpactTestEnumerator.cpp | 22 +-- .../Enumeration/TestImpactTestEnumerator.h | 16 +- .../TestImpactTestJobInfoGenerator.cpp | 167 ++++++++++++++++++ .../TestImpactTestJobInfoGenerator.h | 67 +++++++ .../JobRunner}/TestImpactTestJobRunner.h | 16 +- .../Run/TestImpactInstrumentedTestRunner.cpp | 42 +++-- .../Run/TestImpactInstrumentedTestRunner.h | 20 ++- .../Run/TestImpactTestCoverage.cpp | 63 ++++++- .../Run/TestImpactTestCoverage.h | 11 +- .../Run/TestImpactTestRun.cpp | 68 ++++++- .../Run/TestImpactTestRun.h | 15 +- .../Run/TestImpactTestRunException.h | 0 .../Run/TestImpactTestRunJobData.cpp | 6 +- .../Run/TestImpactTestRunJobData.h | 8 +- .../Run/TestImpactTestRunSerializer.cpp | 4 +- .../Run/TestImpactTestRunSerializer.h | 2 +- .../Run/TestImpactTestRunner.cpp | 21 ++- .../Run/TestImpactTestRunner.h | 6 +- .../TestImpactTestSuiteContainer.h | 80 ++++++++- 25 files changed, 562 insertions(+), 144 deletions(-) delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Job/TestImpactTestJobCommon.h delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Job/TestImpactTestJobException.h rename Code/Tools/TestImpactFramework/Runtime/Code/Source/{Test => TestEngine}/Enumeration/TestImpactTestEnumeration.h (93%) rename Code/Tools/TestImpactFramework/Runtime/Code/Source/{Test => TestEngine}/Enumeration/TestImpactTestEnumerationException.h (100%) rename Code/Tools/TestImpactFramework/Runtime/Code/Source/{Test => TestEngine}/Enumeration/TestImpactTestEnumerationSerializer.cpp (96%) rename Code/Tools/TestImpactFramework/Runtime/Code/Source/{Test => TestEngine}/Enumeration/TestImpactTestEnumerationSerializer.h (94%) rename Code/Tools/TestImpactFramework/Runtime/Code/Source/{Test => TestEngine}/Enumeration/TestImpactTestEnumerator.cpp (87%) rename Code/Tools/TestImpactFramework/Runtime/Code/Source/{Test => TestEngine}/Enumeration/TestImpactTestEnumerator.h (90%) create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobInfoGenerator.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobInfoGenerator.h rename Code/Tools/TestImpactFramework/Runtime/Code/Source/{Test/Job => TestEngine/JobRunner}/TestImpactTestJobRunner.h (92%) rename Code/Tools/TestImpactFramework/Runtime/Code/Source/{Test => TestEngine}/Run/TestImpactInstrumentedTestRunner.cpp (69%) rename Code/Tools/TestImpactFramework/Runtime/Code/Source/{Test => TestEngine}/Run/TestImpactInstrumentedTestRunner.h (82%) rename Code/Tools/TestImpactFramework/Runtime/Code/Source/{Test => TestEngine}/Run/TestImpactTestCoverage.cpp (57%) rename Code/Tools/TestImpactFramework/Runtime/Code/Source/{Test => TestEngine}/Run/TestImpactTestCoverage.h (84%) rename Code/Tools/TestImpactFramework/Runtime/Code/Source/{Test => TestEngine}/Run/TestImpactTestRun.cpp (53%) rename Code/Tools/TestImpactFramework/Runtime/Code/Source/{Test => TestEngine}/Run/TestImpactTestRun.h (78%) rename Code/Tools/TestImpactFramework/Runtime/Code/Source/{Test => TestEngine}/Run/TestImpactTestRunException.h (100%) rename Code/Tools/TestImpactFramework/Runtime/Code/Source/{Test => TestEngine}/Run/TestImpactTestRunJobData.cpp (78%) rename Code/Tools/TestImpactFramework/Runtime/Code/Source/{Test => TestEngine}/Run/TestImpactTestRunJobData.h (77%) rename Code/Tools/TestImpactFramework/Runtime/Code/Source/{Test => TestEngine}/Run/TestImpactTestRunSerializer.cpp (98%) rename Code/Tools/TestImpactFramework/Runtime/Code/Source/{Test => TestEngine}/Run/TestImpactTestRunSerializer.h (95%) rename Code/Tools/TestImpactFramework/Runtime/Code/Source/{Test => TestEngine}/Run/TestImpactTestRunner.cpp (73%) rename Code/Tools/TestImpactFramework/Runtime/Code/Source/{Test => TestEngine}/Run/TestImpactTestRunner.h (93%) rename Code/Tools/TestImpactFramework/Runtime/Code/Source/{Test => TestEngine}/TestImpactTestSuiteContainer.h (57%) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Job/TestImpactTestJobCommon.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Job/TestImpactTestJobCommon.h deleted file mode 100644 index 03bbd91454..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Job/TestImpactTestJobCommon.h +++ /dev/null @@ -1,38 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include - -#include -#include -#include -#include - -#pragma once - -namespace TestImpact -{ - template - AZStd::string ReadFileContents(const AZ::IO::Path& file) - { - static_assert(AZStd::is_base_of::value, "Exception must be a TestImpact exception or derived type"); - - const auto fileSize = AZ::IO::SystemFile::Length(file.c_str()); - AZ_TestImpact_Eval(fileSize > 0, ExceptionType, AZStd::string::format("File %s does not exist", file.c_str())); - - AZStd::vector buffer(fileSize + 1); - buffer[fileSize] = '\0'; - AZ_TestImpact_Eval(AZ::IO::SystemFile::Read(file.c_str(), buffer.data()), ExceptionType, "Could not read file contents"); - - return AZStd::string(buffer.begin(), buffer.end()); - } -} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Job/TestImpactTestJobException.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Job/TestImpactTestJobException.h deleted file mode 100644 index 263d70322e..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Job/TestImpactTestJobException.h +++ /dev/null @@ -1,26 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#pragma once - -#include - -namespace TestImpact -{ - //! Exception for test job related operations. - class TestJobException - : public Exception - { - public: - using Exception::Exception; - }; -} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumeration.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumeration.h similarity index 93% rename from Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumeration.h rename to Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumeration.h index c12abec006..709c5d27d0 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumeration.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumeration.h @@ -13,7 +13,7 @@ #pragma once #include -#include +#include namespace TestImpact { diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerationException.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerationException.h similarity index 100% rename from Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerationException.h rename to Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerationException.h diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerationSerializer.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerationSerializer.cpp similarity index 96% rename from Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerationSerializer.cpp rename to Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerationSerializer.cpp index efed04970d..05a13a0703 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerationSerializer.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerationSerializer.cpp @@ -10,8 +10,8 @@ * */ -#include -#include +#include +#include #include #include diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerationSerializer.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerationSerializer.h similarity index 94% rename from Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerationSerializer.h rename to Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerationSerializer.h index 422308d862..06bd224b50 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerationSerializer.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerationSerializer.h @@ -12,7 +12,7 @@ #pragma once -#include +#include #include diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerator.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.cpp similarity index 87% rename from Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerator.cpp rename to Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.cpp index 1964f8876e..882c65c516 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerator.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.cpp @@ -10,11 +10,13 @@ * */ +#include + #include -#include -#include -#include -#include +#include +#include +#include +#include #include @@ -23,7 +25,7 @@ namespace TestImpact namespace { void WriteCacheFile( - const TestEnumeration& enumeration, const AZ::IO::Path& path, Bitwise::CacheExceptionPolicy cacheExceptionPolicy) + const TestEnumeration& enumeration, const RepoPath& path, Bitwise::CacheExceptionPolicy cacheExceptionPolicy) { const AZStd::string cacheJSON = SerializeTestEnumeration(enumeration); const AZStd::vector cacheBytes(cacheJSON.begin(), cacheJSON.end()); @@ -48,7 +50,7 @@ namespace TestImpact } } - AZStd::optional ReadCacheFile(const AZ::IO::Path& path, Bitwise::CacheExceptionPolicy cacheExceptionPolicy) + AZStd::optional ReadCacheFile(const RepoPath& path, Bitwise::CacheExceptionPolicy cacheExceptionPolicy) { AZ::IO::SystemFile cacheFile; AZStd::string cacheJSON; @@ -83,18 +85,18 @@ namespace TestImpact } } // namespace - TestEnumeration ParseTestEnumerationFile(const AZ::IO::Path& enumerationFile) + TestEnumeration ParseTestEnumerationFile(const RepoPath& enumerationFile) { return TestEnumeration(GTest::TestEnumerationSuitesFactory(ReadFileContents(enumerationFile))); } - TestEnumerationJobData::TestEnumerationJobData(const AZ::IO::Path& enumerationArtifact, AZStd::optional&& cache) + TestEnumerationJobData::TestEnumerationJobData(const RepoPath& enumerationArtifact, AZStd::optional&& cache) : m_enumerationArtifact(enumerationArtifact) , m_cache(AZStd::move(cache)) { } - const AZ::IO::Path& TestEnumerationJobData::GetEnumerationArtifactPath() const + const RepoPath& TestEnumerationJobData::GetEnumerationArtifactPath() const { return m_enumerationArtifact; } @@ -163,7 +165,7 @@ namespace TestImpact const auto& [meta, jobInfo] = jobData; if (meta.m_result == JobResult::ExecutedWithSuccess) { - const auto& enumeration = enumerations[jobId] = ParseTestEnumerationFile(jobInfo->GetEnumerationArtifactPath()); + const auto& enumeration = (enumerations[jobId] = ParseTestEnumerationFile(jobInfo->GetEnumerationArtifactPath())); // Write out the enumeration to a cache file if we have a cache write policy for this job if (jobInfo->GetCache().has_value() && jobInfo->GetCache()->m_policy == JobData::CachePolicy::Write) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerator.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.h similarity index 90% rename from Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerator.h rename to Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.h index a005a32b0d..63d0b82d3e 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerator.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.h @@ -12,11 +12,11 @@ #pragma once -#include -#include -#include +#include -#include +#include +#include +#include namespace TestImpact { @@ -35,19 +35,19 @@ namespace TestImpact struct Cache { CachePolicy m_policy; - AZ::IO::Path m_file; + RepoPath m_file; }; - TestEnumerationJobData(const AZ::IO::Path& enumerationArtifact, AZStd::optional&& cache); + TestEnumerationJobData(const RepoPath& enumerationArtifact, AZStd::optional&& cache); //! Returns the path to the enumeration artifact produced by the test target. - const AZ::IO::Path& GetEnumerationArtifactPath() const; + const RepoPath& GetEnumerationArtifactPath() const; //! Returns the cache details for this job. const AZStd::optional& GetCache() const; private: - AZ::IO::Path m_enumerationArtifact; //!< Path to enumeration artifact to be processed. + RepoPath m_enumerationArtifact; //!< Path to enumeration artifact to be processed. AZStd::optional m_cache = AZStd::nullopt; //!< No caching takes place if cache is empty. }; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobInfoGenerator.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobInfoGenerator.cpp new file mode 100644 index 0000000000..aeb3fcfeb3 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobInfoGenerator.cpp @@ -0,0 +1,167 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include + +namespace TestImpact +{ + static constexpr char* const standAloneExtension = ".exe"; // these are os specific so move these to the platform dir + static constexpr char* const testRunnerExtension = ".dll"; // these are os specific so move these to the platform dir + + // SPLIT INTO COMMAND GENERATOR (INSTRUMENT, RUNNER, ENUM, RESULTS) + + TestJobInfoGenerator::TestJobInfoGenerator( + const RepoPath& sourceDir, + const RepoPath& targetBinaryDir, + const RepoPath& cacheDir, + const RepoPath& artifactDir, + const RepoPath& testRunnerBinary, + const RepoPath& instrumentBinary) + : m_sourceDir(sourceDir) + , m_targetBinaryDir(targetBinaryDir) + , m_cacheDir(cacheDir) + , m_artifactDir(artifactDir) + , m_testRunnerBinary(testRunnerBinary) + , m_instrumentBinary(instrumentBinary) + { + } + + AZStd::string TestJobInfoGenerator::GenerateLaunchArgument(const TestTarget* testTarget) const + { + if (testTarget->GetLaunchMethod() == LaunchMethod::StandAlone) + { + return AZStd::string::format( + "%s%s %s", + (m_targetBinaryDir / testTarget->GetOutputName()).c_str(), + standAloneExtension, + testTarget->GetCustomArgs().c_str()).c_str(); + } + else + { + return AZStd::string::format( + "\"%s\" \"%s%s\" %s", + m_testRunnerBinary.c_str(), + (m_targetBinaryDir / testTarget->GetOutputName()).c_str(), + testRunnerExtension, + testTarget->GetCustomArgs().c_str()).c_str(); + } + } + + RepoPath TestJobInfoGenerator::GenerateTargetEnumerationCacheFilePath(const TestTarget* testTarget) const + { + return AZStd::string::format("%s.cache", (m_artifactDir / testTarget->GetName()).c_str()); + } + + RepoPath TestJobInfoGenerator::GenerateTargetEnumerationArtifactFilePath(const TestTarget* testTarget) const + { + return AZStd::string::format("%s.Enumeration.xml", (m_artifactDir / testTarget->GetName()).c_str()); + } + + RepoPath TestJobInfoGenerator::GenerateTargetRunArtifactFilePath(const TestTarget* testTarget) const + { + return AZStd::string::format("%s.Run.xml", (m_artifactDir / testTarget->GetName()).c_str()); + } + + RepoPath TestJobInfoGenerator::GenerateTargetCoverageArtifactFilePath(const TestTarget* testTarget) const + { + return AZStd::string::format("%s.Coverage.xml", (m_artifactDir / testTarget->GetName()).c_str()); + } + + const RepoPath& TestJobInfoGenerator::GetCacheDir() const + { + return m_cacheDir; + } + + const RepoPath& TestJobInfoGenerator::GetArtifactDir() const + { + return m_artifactDir; + } + + TestEnumerator::JobInfo TestJobInfoGenerator::GenerateTestEnumerationJobInfo( + const TestTarget* testTarget, + TestEnumerator::JobInfo::Id jobId, + TestEnumerator::JobInfo::CachePolicy cachePolicy) const + { + using Command = TestEnumerator::Command; + using JobInfo = TestEnumerator::JobInfo; + using JobData = TestEnumerator::JobData; + using Cache = TestEnumerator::JobData::Cache; + + const auto enumerationArtifact = GenerateTargetEnumerationArtifactFilePath(testTarget); + const Command args = + { + AZStd::string::format( + "%s --gtest_list_tests --gtest_output=xml:\"%s\"", + GenerateLaunchArgument(testTarget).c_str(), + enumerationArtifact.c_str()) + }; + + return JobInfo(jobId, args, JobData(enumerationArtifact, Cache{ cachePolicy, GenerateTargetEnumerationCacheFilePath(testTarget) })); + } + + TestRunner::JobInfo TestJobInfoGenerator::GenerateRegularTestRunJobInfo( + const TestTarget* testTarget, + TestRunner::JobInfo::Id jobId) const + { + using Command = TestRunner::Command; + using JobInfo = TestRunner::JobInfo; + using JobData = TestRunner::JobData; + + const auto runArtifact = GenerateTargetRunArtifactFilePath(testTarget); + const Command args = + { + AZStd::string::format( + "%s --gtest_output=xml:\"%s\"", + GenerateLaunchArgument(testTarget).c_str(), + runArtifact.c_str()) + }; + + return JobInfo(jobId, args, JobData(runArtifact)); + } + + InstrumentedTestRunner::JobInfo TestJobInfoGenerator::GenerateInstrumentedTestRunJobInfo( + const TestTarget* testTarget, + InstrumentedTestRunner::JobInfo::Id jobId, + CoverageLevel coverageLevel) const + { + using Command = InstrumentedTestRunner::Command; + using JobInfo = InstrumentedTestRunner::JobInfo; + using JobData = InstrumentedTestRunner::JobData; + + const auto coverageArtifact = GenerateTargetCoverageArtifactFilePath(testTarget); + const auto runArtifact = GenerateTargetRunArtifactFilePath(testTarget); + const Command args = + { + AZStd::string::format( + "\"%s\" " // 1. Instrumented test runner + "--coverage_level %s " // 2. Coverage level + "--export_type cobertura:\"%s\" " // 3. Test coverage artifact path + "--modules \"%s\" " // 4. Modules path + "--excluded_modules \"%s\" " // 5. Exclude modules + "--sources \"%s\" -- " // 6. Sources path + "%s " // 7. Launch command + "--gtest_output=xml:\"%s\"", // 8. Result artifact + + m_instrumentBinary.c_str(), // 1. Instrumented test runner + (coverageLevel == CoverageLevel::Line ? "line" : "source"), // 2. Coverage level + coverageArtifact.c_str(), // 3. Test coverage artifact path + m_targetBinaryDir.c_str(), // 4. Modules path + m_testRunnerBinary.c_str(), // 5. Exclude modules + m_sourceDir.c_str(), // 6. Sources path + GenerateLaunchArgument(testTarget).c_str(), // 7. Launch command + runArtifact.c_str()) // 8. Result artifact + }; + + return JobInfo(jobId, args, JobData(runArtifact, coverageArtifact)); + } +} diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobInfoGenerator.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobInfoGenerator.h new file mode 100644 index 0000000000..5d55904edd --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobInfoGenerator.h @@ -0,0 +1,67 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +#include +#include +#include +#include + +namespace TestImpact +{ + class TestTarget; + + class TestJobInfoGenerator + { + public: + TestJobInfoGenerator( + const RepoPath& sourceDir, + const RepoPath& targetBinaryDir, + const RepoPath& cacheDir, + const RepoPath& artifactDir, + const RepoPath& testRunnerBinary, + const RepoPath& instrumentBinary); + + const RepoPath& GetCacheDir() const; + const RepoPath& GetArtifactDir() const; + + TestEnumerator::JobInfo GenerateTestEnumerationJobInfo( + const TestTarget* testTarget, + TestEnumerator::JobInfo::Id jobId, + TestEnumerator::JobInfo::CachePolicy cachePolicy) const; + + TestRunner::JobInfo GenerateRegularTestRunJobInfo( + const TestTarget* testTarget, + TestRunner::JobInfo::Id jobId) const; + + InstrumentedTestRunner::JobInfo GenerateInstrumentedTestRunJobInfo( + const TestTarget* testTarget, + InstrumentedTestRunner::JobInfo::Id jobId, + CoverageLevel coverageLevel) const; + private: + AZStd::string GenerateLaunchArgument(const TestTarget* testTarget) const; + RepoPath GenerateTargetEnumerationCacheFilePath(const TestTarget* testTarget) const; + RepoPath GenerateTargetEnumerationArtifactFilePath(const TestTarget* testTarget) const; + RepoPath GenerateTargetRunArtifactFilePath(const TestTarget* testTarget) const; + RepoPath GenerateTargetCoverageArtifactFilePath(const TestTarget* testTarget) const; + + RepoPath m_sourceDir; + RepoPath m_targetBinaryDir; + RepoPath m_cacheDir; + RepoPath m_artifactDir; + RepoPath m_testRunnerBinary; + RepoPath m_instrumentBinary; + }; +} diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Job/TestImpactTestJobRunner.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobRunner.h similarity index 92% rename from Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Job/TestImpactTestJobRunner.h rename to Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobRunner.h index a4a0b19845..c60a93e21d 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Job/TestImpactTestJobRunner.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobRunner.h @@ -12,11 +12,10 @@ #pragma once -#include +#include #include #include -#include #include #include @@ -45,6 +44,7 @@ namespace TestImpact public: using JobData = AdditionalInfo; using JobInfo = JobInfo; + using Command = typename JobInfo::Command; using JobPayload = Payload; using Job = Job; using ClientJobCallback = AZStd::function; @@ -111,20 +111,22 @@ namespace TestImpact // Callback to handle job exception policies and client/derived callbacks const auto jobCallback = [this, &jobExceptionPolicy](const JobInfo& jobInfo, const JobMeta& meta, StdContent&& std) { + auto callbackResult = ProcessCallbackResult::Continue; if (meta.m_result == JobResult::FailedToExecute && IsFlagSet(jobExceptionPolicy, JobExceptionPolicy::OnFailedToExecute)) { - throw TestJobException("Job failed to execute"); + callbackResult = ProcessCallbackResult::Abort; } else if (meta.m_result == JobResult::ExecutedWithFailure && IsFlagSet(jobExceptionPolicy, JobExceptionPolicy::OnExecutedWithFailure)) { - throw TestJobException("Job executed with failure"); + callbackResult = ProcessCallbackResult::Abort; } if (m_derivedJobCallback.has_value()) { - if (const auto result = (*m_derivedJobCallback)(jobInfo, meta, AZStd::move(std)); result != CallbackResult::Continue) + if (const auto result = (*m_derivedJobCallback)(jobInfo, meta, AZStd::move(std)); + result == ProcessCallbackResult::Abort) { - return result; + callbackResult = ProcessCallbackResult::Abort; } } @@ -133,7 +135,7 @@ namespace TestImpact (*m_clientJobCallback)(jobInfo, meta); } - return CallbackResult::Continue; + return callbackResult; }; return m_jobRunner.Execute(jobInfos, jobCallback, payloadMapProducer); diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactInstrumentedTestRunner.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.cpp similarity index 69% rename from Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactInstrumentedTestRunner.cpp rename to Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.cpp index 3b7a088fb4..0d27d56af1 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactInstrumentedTestRunner.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.cpp @@ -10,31 +10,32 @@ * */ +#include + #include #include -#include -#include -#include -#include +#include +#include +#include #include namespace TestImpact { - InstrumentedTestRunJobData::InstrumentedTestRunJobData(const AZ::IO::Path& resultsArtifact, const AZ::IO::Path& coverageArtifact) + InstrumentedTestRunJobData::InstrumentedTestRunJobData(const RepoPath& resultsArtifact, const RepoPath& coverageArtifact) : TestRunJobData(resultsArtifact) , m_coverageArtifact(coverageArtifact) { } - const AZ::IO::Path& InstrumentedTestRunJobData::GetCoverageArtifactPath() const + const RepoPath& InstrumentedTestRunJobData::GetCoverageArtifactPath() const { return m_coverageArtifact; } InstrumentedTestRunner::JobPayload ParseTestRunAndCoverageFiles( - const AZ::IO::Path& runFile, - const AZ::IO::Path& coverageFile, + const RepoPath& runFile, + const RepoPath& coverageFile, AZStd::chrono::milliseconds duration, InstrumentedTestRunner::CoverageExceptionPolicy coverageExceptionPolicy) { @@ -44,7 +45,7 @@ namespace TestImpact { AZ_TestImpact_Eval( !IsFlagSet(coverageExceptionPolicy, Bitwise::CoverageExceptionPolicy::OnEmptyCoverage), TestRunException, - "No coverage data generated"); + AZStd::string::format("No coverage data generated for '%s'", coverageFile.c_str())); } TestCoverage coverage(AZStd::move(moduleCoverages)); @@ -73,11 +74,24 @@ namespace TestImpact const auto& [meta, jobInfo] = jobData; if (meta.m_result == JobResult::ExecutedWithSuccess || meta.m_result == JobResult::ExecutedWithFailure) { - runs[jobId] = ParseTestRunAndCoverageFiles( - jobInfo->GetRunArtifactPath(), - jobInfo->GetCoverageArtifactPath(), - meta.m_duration.value(), - coverageExceptionPolicy); + try + { + runs[jobId] = ParseTestRunAndCoverageFiles( + jobInfo->GetRunArtifactPath(), + jobInfo->GetCoverageArtifactPath(), + meta.m_duration.value(), + coverageExceptionPolicy); + } + catch (const Exception& e) + { + AZ_Warning("RunInstrumentedTests", false, e.what()); + runs[jobId] = AZStd::nullopt; + + if (coverageExceptionPolicy == CoverageExceptionPolicy::OnEmptyCoverage) + { + break; + } + } } } diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactInstrumentedTestRunner.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.h similarity index 82% rename from Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactInstrumentedTestRunner.h rename to Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.h index e6f059b5d3..3b47f53128 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactInstrumentedTestRunner.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.h @@ -12,24 +12,25 @@ #pragma once -#include -#include -#include -#include +#include +#include +#include +#include namespace TestImpact { //! Per-job data for instrumented test runs. - class InstrumentedTestRunJobData : public TestRunJobData + class InstrumentedTestRunJobData + : public TestRunJobData { public: - InstrumentedTestRunJobData(const AZ::IO::Path& resultsArtifact, const AZ::IO::Path& coverageArtifact); + InstrumentedTestRunJobData(const RepoPath& resultsArtifact, const RepoPath& coverageArtifact); //! Returns the path to the coverage artifact produced by the test target. - const AZ::IO::Path& GetCoverageArtifactPath() const; + const RepoPath& GetCoverageArtifactPath() const; private: - AZ::IO::Path m_coverageArtifact; //!< Path to coverage data. + RepoPath m_coverageArtifact; //!< Path to coverage data. }; namespace Bitwise @@ -43,7 +44,8 @@ namespace TestImpact } // namespace Bitwise //! Runs a batch of test targets to determine the test coverage and passes/failures. - class InstrumentedTestRunner : public TestJobRunner> + class InstrumentedTestRunner + : public TestJobRunner> { using JobRunner = TestJobRunner>; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestCoverage.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestCoverage.cpp similarity index 57% rename from Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestCoverage.cpp rename to Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestCoverage.cpp index 0d50dbbf9a..6135383863 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestCoverage.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestCoverage.cpp @@ -10,16 +10,75 @@ * */ -#include +#include #include #include namespace TestImpact { - TestCoverage::TestCoverage(AZStd::vector&& moduleCoverages) + TestCoverage::TestCoverage(const TestCoverage& other) + : m_modules(other.m_modules) + , m_sourcesCovered(other.m_sourcesCovered) + , m_coverageLevel(other.m_coverageLevel) + { + } + + TestCoverage::TestCoverage(TestCoverage&& other) noexcept + : m_modules(AZStd::move(other.m_modules)) + , m_sourcesCovered(AZStd::move(other.m_sourcesCovered)) + { + AZStd::swap(m_coverageLevel, other.m_coverageLevel); + other.~TestCoverage(); + } + + TestCoverage::TestCoverage(const AZStd::vector& moduleCoverages) + : m_modules(moduleCoverages) + { + CalculateTestMetrics(); + } + + TestCoverage::TestCoverage(AZStd::vector&& moduleCoverages) noexcept : m_modules(AZStd::move(moduleCoverages)) { + CalculateTestMetrics(); + } + + TestCoverage::~TestCoverage() + { + m_modules.clear(); + m_coverageLevel.reset(); + m_sourcesCovered.clear(); + } + + TestCoverage& TestCoverage::operator=(const TestCoverage& other) + { + if (this != &other) + { + this->~TestCoverage(); + new(this)TestCoverage(other); + } + + return *this; + } + + TestCoverage& TestCoverage::operator=(TestCoverage&& other) noexcept + { + if (this != &other) + { + this->~TestCoverage(); + new(this)TestCoverage(AZStd::move(other)); + other.~TestCoverage(); + } + + return *this; + } + + void TestCoverage::CalculateTestMetrics() + { + m_coverageLevel.reset(); + m_sourcesCovered.clear(); + for (const auto& moduleCovered : m_modules) { for (const auto& sourceCovered : moduleCovered.m_sources) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestCoverage.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestCoverage.h similarity index 84% rename from Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestCoverage.h rename to Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestCoverage.h index 7253a29df0..c672814f7c 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestCoverage.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestCoverage.h @@ -29,7 +29,14 @@ namespace TestImpact class TestCoverage { public: - TestCoverage(AZStd::vector&& moduleCoverages); + TestCoverage(const TestCoverage&); + TestCoverage(TestCoverage&&) noexcept; + TestCoverage(AZStd::vector&& moduleCoverages) noexcept; + TestCoverage(const AZStd::vector& moduleCoverages); + ~TestCoverage(); + + TestCoverage& operator=(const TestCoverage&); + TestCoverage& operator=(TestCoverage&&) noexcept; //! Returns the number of unique sources covered. size_t GetNumSourcesCovered() const; @@ -47,6 +54,8 @@ namespace TestImpact AZStd::optional GetCoverageLevel() const; private: + void CalculateTestMetrics(); + AZStd::vector m_modules; AZStd::vector m_sourcesCovered; AZStd::optional m_coverageLevel; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRun.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRun.cpp similarity index 53% rename from Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRun.cpp rename to Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRun.cpp index 1ba2a3e849..9ee1569f0e 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRun.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRun.cpp @@ -14,10 +14,76 @@ namespace TestImpact { - TestRun::TestRun(AZStd::vector&& testSuites, AZStd::chrono::milliseconds duration) + TestRun::TestRun(const TestRun& other) + : TestSuiteContainer(other) + { + CalculateTestMetrics(); + } + + TestRun::TestRun(TestRun&& other) noexcept + : TestSuiteContainer(AZStd::move(other)) + , m_numRuns(other.m_numRuns) + , m_numNotRuns(other.m_numNotRuns) + , m_numPasses(other.m_numPasses) + , m_numFailures(other.m_numFailures) + , m_duration(other.m_duration) + { + other.~TestRun(); + } + + TestRun::TestRun(AZStd::vector&& testSuites, AZStd::chrono::milliseconds duration) noexcept : TestSuiteContainer(AZStd::move(testSuites)) , m_duration(duration) { + CalculateTestMetrics(); + } + + TestRun::TestRun(const AZStd::vector& testSuites, AZStd::chrono::milliseconds duration) + : TestSuiteContainer(testSuites) + , m_duration(duration) + { + CalculateTestMetrics(); + } + + TestRun::~TestRun() + { + m_numRuns = 0; + m_numNotRuns = 0; + m_numPasses = 0; + m_numFailures = 0; + m_duration = AZStd::chrono::milliseconds{ 0 }; + } + + TestRun& TestRun::operator=(TestRun&& other) noexcept + { + if (this != &other) + { + this->~TestRun(); + new(this)TestRun(AZStd::move(other)); + other.~TestRun(); + } + + return *this; + } + + TestRun& TestRun::operator=(const TestRun& other) + { + if (this != &other) + { + this->~TestRun(); + new(this)TestRun(other); + } + + return *this; + } + + void TestRun::CalculateTestMetrics() + { + m_numRuns = 0; + m_numNotRuns = 0; + m_numPasses = 0; + m_numFailures = 0; + for (const auto& suite : m_testSuites) { for (const auto& test : suite.m_tests) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRun.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRun.h similarity index 78% rename from Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRun.h rename to Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRun.h index 75d2140a68..f2eeac14fa 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRun.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRun.h @@ -13,7 +13,7 @@ #pragma once #include -#include +#include namespace TestImpact { @@ -24,7 +24,14 @@ namespace TestImpact using TestSuiteContainer = TestSuiteContainer; public: - TestRun(AZStd::vector&& testSuites, AZStd::chrono::milliseconds duration); + TestRun(const TestRun&); + TestRun(TestRun&&) noexcept; + TestRun(const AZStd::vector& testSuites, AZStd::chrono::milliseconds duration); + TestRun(AZStd::vector&& testSuites, AZStd::chrono::milliseconds duration) noexcept; + ~TestRun(); + + TestRun& operator=(const TestRun&); + TestRun& operator=(TestRun&&) noexcept; //! Returns the total number of tests that were run. size_t GetNumRuns() const; @@ -42,10 +49,12 @@ namespace TestImpact AZStd::chrono::milliseconds GetDuration() const; private: + void CalculateTestMetrics(); + size_t m_numRuns = 0; size_t m_numNotRuns = 0; size_t m_numPasses = 0; size_t m_numFailures = 0; - AZStd::chrono::milliseconds m_duration = AZStd::chrono::milliseconds{0}; + AZStd::chrono::milliseconds m_duration = AZStd::chrono::milliseconds{0}; // this might be removed... }; } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunException.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunException.h similarity index 100% rename from Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunException.h rename to Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunException.h diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunJobData.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunJobData.cpp similarity index 78% rename from Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunJobData.cpp rename to Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunJobData.cpp index fa40b30263..a7ed446a25 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunJobData.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunJobData.cpp @@ -10,16 +10,16 @@ * */ -#include +#include namespace TestImpact { - TestRunJobData::TestRunJobData(const AZ::IO::Path& resultsArtifact) + TestRunJobData::TestRunJobData(const RepoPath& resultsArtifact) : m_runArtifact(resultsArtifact) { } - const AZ::IO::Path& TestRunJobData::GetRunArtifactPath() const + const RepoPath& TestRunJobData::GetRunArtifactPath() const { return m_runArtifact; } diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunJobData.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunJobData.h similarity index 77% rename from Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunJobData.h rename to Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunJobData.h index 802aa0d30f..a2033b5945 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunJobData.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunJobData.h @@ -12,7 +12,7 @@ #pragma once -#include +#include namespace TestImpact { @@ -20,12 +20,12 @@ namespace TestImpact class TestRunJobData { public: - TestRunJobData(const AZ::IO::Path& resultsArtifact); + TestRunJobData(const RepoPath& resultsArtifact); //! Returns the path to the test run artifact produced by the test target. - const AZ::IO::Path& GetRunArtifactPath() const; + const RepoPath& GetRunArtifactPath() const; private: - AZ::IO::Path m_runArtifact; //!< Path to results data. + RepoPath m_runArtifact; //!< Path to results data. }; } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunSerializer.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunSerializer.cpp similarity index 98% rename from Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunSerializer.cpp rename to Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunSerializer.cpp index 7f29dd77a9..7fe8789d22 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunSerializer.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunSerializer.cpp @@ -10,8 +10,8 @@ * */ -#include -#include +#include +#include #include #include diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunSerializer.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunSerializer.h similarity index 95% rename from Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunSerializer.h rename to Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunSerializer.h index 5978b27105..a73e675559 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunSerializer.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunSerializer.h @@ -12,7 +12,7 @@ #pragma once -#include +#include #include diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunner.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunner.cpp similarity index 73% rename from Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunner.cpp rename to Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunner.cpp index dde7dd7d47..ebb846f406 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunner.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunner.cpp @@ -10,17 +10,18 @@ * */ +#include + #include -#include -#include -#include -#include +#include +#include +#include #include namespace TestImpact { - TestRun ParseTestRunFile(const AZ::IO::Path& runFile, AZStd::chrono::milliseconds duration) + TestRun ParseTestRunFile(const RepoPath& runFile, AZStd::chrono::milliseconds duration) { return TestRun(GTest::TestRunSuitesFactory(ReadFileContents(runFile)), duration); } @@ -46,7 +47,15 @@ namespace TestImpact const auto& [meta, jobInfo] = jobData; if (meta.m_result == JobResult::ExecutedWithSuccess || meta.m_result == JobResult::ExecutedWithFailure) { - runs[jobId] = ParseTestRunFile(jobInfo->GetRunArtifactPath(), meta.m_duration.value()); + try + { + runs[jobId] = ParseTestRunFile(jobInfo->GetRunArtifactPath(), meta.m_duration.value()); + } + catch (const Exception& e) + { + AZ_Warning("RunTests", false, e.what()); + runs[jobId] = AZStd::nullopt; + } } } diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunner.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunner.h similarity index 93% rename from Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunner.h rename to Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunner.h index 0bf66601fe..379c6b424e 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunner.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunner.h @@ -12,9 +12,9 @@ #pragma once -#include -#include -#include +#include +#include +#include namespace TestImpact { diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/TestImpactTestSuiteContainer.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestSuiteContainer.h similarity index 57% rename from Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/TestImpactTestSuiteContainer.h rename to Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestSuiteContainer.h index c41d76bd04..4f3fdebb08 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/TestImpactTestSuiteContainer.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestSuiteContainer.h @@ -22,7 +22,14 @@ namespace TestImpact class TestSuiteContainer { public: - TestSuiteContainer(AZStd::vector&& testSuites); + TestSuiteContainer(const TestSuiteContainer&); + TestSuiteContainer(TestSuiteContainer&&) noexcept; + TestSuiteContainer(const AZStd::vector& testSuites); + TestSuiteContainer(AZStd::vector&& testSuites) noexcept; + virtual ~TestSuiteContainer(); + + TestSuiteContainer& operator=(const TestSuiteContainer&); + TestSuiteContainer& operator=(TestSuiteContainer&&) noexcept; //! Returns the test suites in this container. const AZStd::vector& GetTestSuites() const; @@ -39,6 +46,9 @@ namespace TestImpact //! Returns the total number of disabled tests across all test suites. size_t GetNumDisabledTests() const; + private: + void CalculateTestMetrics(); + protected: AZStd::vector m_testSuites; size_t m_numDisabledTests = 0; @@ -46,9 +56,75 @@ namespace TestImpact }; template - TestSuiteContainer::TestSuiteContainer(AZStd::vector&& testSuites) + TestSuiteContainer::TestSuiteContainer(TestSuiteContainer&& other) noexcept + : m_testSuites(AZStd::move(other.m_testSuites)) + , m_numDisabledTests(other.m_numDisabledTests) + , m_numEnabledTests(other.m_numEnabledTests) + { + other.~TestSuiteContainer(); + } + + template + TestSuiteContainer::TestSuiteContainer(const TestSuiteContainer& other) + : m_testSuites(other.m_testSuites.begin(), other.m_testSuites.end()) + , m_numDisabledTests(other.m_numDisabledTests) + , m_numEnabledTests(other.m_numEnabledTests) + { + } + + template + TestSuiteContainer::~TestSuiteContainer() + { + m_testSuites.clear(); + m_numDisabledTests = 0; + m_numEnabledTests = 0; + } + + template + TestSuiteContainer::TestSuiteContainer(AZStd::vector&& testSuites) noexcept : m_testSuites(std::move(testSuites)) { + CalculateTestMetrics(); + } + + template + TestSuiteContainer::TestSuiteContainer(const AZStd::vector& testSuites) + : m_testSuites(testSuites) + { + CalculateTestMetrics(); + } + + template + TestSuiteContainer& TestSuiteContainer::operator=(TestSuiteContainer&& other) noexcept + { + if (this != &other) + { + this->~TestSuiteContainer(); + new(this)TestSuiteContainer(AZStd::move(other)); + other.~TestSuiteContainer(); + } + + return *this; + } + + template + TestSuiteContainer& TestSuiteContainer::operator=(const TestSuiteContainer& other) + { + if (this != &other) + { + this->~TestSuiteContainer(); + new(this)TestSuiteContainer(other); + } + + return *this; + } + + template + void TestSuiteContainer::CalculateTestMetrics() + { + m_numDisabledTests = 0; + m_numEnabledTests = 0; + for (const auto& suite : m_testSuites) { if (suite.m_enabled) From f08de28cd819e38b995c4337984ab7a1cc804993 Mon Sep 17 00:00:00 2001 From: jonawals Date: Fri, 21 May 2021 13:13:54 +0100 Subject: [PATCH 027/102] Remove JobInfoGenerator from this PR --- .../TestImpactTestJobInfoGenerator.cpp | 167 ------------------ .../TestImpactTestJobInfoGenerator.h | 67 ------- 2 files changed, 234 deletions(-) delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobInfoGenerator.cpp delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobInfoGenerator.h diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobInfoGenerator.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobInfoGenerator.cpp deleted file mode 100644 index aeb3fcfeb3..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobInfoGenerator.cpp +++ /dev/null @@ -1,167 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include -#include - -namespace TestImpact -{ - static constexpr char* const standAloneExtension = ".exe"; // these are os specific so move these to the platform dir - static constexpr char* const testRunnerExtension = ".dll"; // these are os specific so move these to the platform dir - - // SPLIT INTO COMMAND GENERATOR (INSTRUMENT, RUNNER, ENUM, RESULTS) - - TestJobInfoGenerator::TestJobInfoGenerator( - const RepoPath& sourceDir, - const RepoPath& targetBinaryDir, - const RepoPath& cacheDir, - const RepoPath& artifactDir, - const RepoPath& testRunnerBinary, - const RepoPath& instrumentBinary) - : m_sourceDir(sourceDir) - , m_targetBinaryDir(targetBinaryDir) - , m_cacheDir(cacheDir) - , m_artifactDir(artifactDir) - , m_testRunnerBinary(testRunnerBinary) - , m_instrumentBinary(instrumentBinary) - { - } - - AZStd::string TestJobInfoGenerator::GenerateLaunchArgument(const TestTarget* testTarget) const - { - if (testTarget->GetLaunchMethod() == LaunchMethod::StandAlone) - { - return AZStd::string::format( - "%s%s %s", - (m_targetBinaryDir / testTarget->GetOutputName()).c_str(), - standAloneExtension, - testTarget->GetCustomArgs().c_str()).c_str(); - } - else - { - return AZStd::string::format( - "\"%s\" \"%s%s\" %s", - m_testRunnerBinary.c_str(), - (m_targetBinaryDir / testTarget->GetOutputName()).c_str(), - testRunnerExtension, - testTarget->GetCustomArgs().c_str()).c_str(); - } - } - - RepoPath TestJobInfoGenerator::GenerateTargetEnumerationCacheFilePath(const TestTarget* testTarget) const - { - return AZStd::string::format("%s.cache", (m_artifactDir / testTarget->GetName()).c_str()); - } - - RepoPath TestJobInfoGenerator::GenerateTargetEnumerationArtifactFilePath(const TestTarget* testTarget) const - { - return AZStd::string::format("%s.Enumeration.xml", (m_artifactDir / testTarget->GetName()).c_str()); - } - - RepoPath TestJobInfoGenerator::GenerateTargetRunArtifactFilePath(const TestTarget* testTarget) const - { - return AZStd::string::format("%s.Run.xml", (m_artifactDir / testTarget->GetName()).c_str()); - } - - RepoPath TestJobInfoGenerator::GenerateTargetCoverageArtifactFilePath(const TestTarget* testTarget) const - { - return AZStd::string::format("%s.Coverage.xml", (m_artifactDir / testTarget->GetName()).c_str()); - } - - const RepoPath& TestJobInfoGenerator::GetCacheDir() const - { - return m_cacheDir; - } - - const RepoPath& TestJobInfoGenerator::GetArtifactDir() const - { - return m_artifactDir; - } - - TestEnumerator::JobInfo TestJobInfoGenerator::GenerateTestEnumerationJobInfo( - const TestTarget* testTarget, - TestEnumerator::JobInfo::Id jobId, - TestEnumerator::JobInfo::CachePolicy cachePolicy) const - { - using Command = TestEnumerator::Command; - using JobInfo = TestEnumerator::JobInfo; - using JobData = TestEnumerator::JobData; - using Cache = TestEnumerator::JobData::Cache; - - const auto enumerationArtifact = GenerateTargetEnumerationArtifactFilePath(testTarget); - const Command args = - { - AZStd::string::format( - "%s --gtest_list_tests --gtest_output=xml:\"%s\"", - GenerateLaunchArgument(testTarget).c_str(), - enumerationArtifact.c_str()) - }; - - return JobInfo(jobId, args, JobData(enumerationArtifact, Cache{ cachePolicy, GenerateTargetEnumerationCacheFilePath(testTarget) })); - } - - TestRunner::JobInfo TestJobInfoGenerator::GenerateRegularTestRunJobInfo( - const TestTarget* testTarget, - TestRunner::JobInfo::Id jobId) const - { - using Command = TestRunner::Command; - using JobInfo = TestRunner::JobInfo; - using JobData = TestRunner::JobData; - - const auto runArtifact = GenerateTargetRunArtifactFilePath(testTarget); - const Command args = - { - AZStd::string::format( - "%s --gtest_output=xml:\"%s\"", - GenerateLaunchArgument(testTarget).c_str(), - runArtifact.c_str()) - }; - - return JobInfo(jobId, args, JobData(runArtifact)); - } - - InstrumentedTestRunner::JobInfo TestJobInfoGenerator::GenerateInstrumentedTestRunJobInfo( - const TestTarget* testTarget, - InstrumentedTestRunner::JobInfo::Id jobId, - CoverageLevel coverageLevel) const - { - using Command = InstrumentedTestRunner::Command; - using JobInfo = InstrumentedTestRunner::JobInfo; - using JobData = InstrumentedTestRunner::JobData; - - const auto coverageArtifact = GenerateTargetCoverageArtifactFilePath(testTarget); - const auto runArtifact = GenerateTargetRunArtifactFilePath(testTarget); - const Command args = - { - AZStd::string::format( - "\"%s\" " // 1. Instrumented test runner - "--coverage_level %s " // 2. Coverage level - "--export_type cobertura:\"%s\" " // 3. Test coverage artifact path - "--modules \"%s\" " // 4. Modules path - "--excluded_modules \"%s\" " // 5. Exclude modules - "--sources \"%s\" -- " // 6. Sources path - "%s " // 7. Launch command - "--gtest_output=xml:\"%s\"", // 8. Result artifact - - m_instrumentBinary.c_str(), // 1. Instrumented test runner - (coverageLevel == CoverageLevel::Line ? "line" : "source"), // 2. Coverage level - coverageArtifact.c_str(), // 3. Test coverage artifact path - m_targetBinaryDir.c_str(), // 4. Modules path - m_testRunnerBinary.c_str(), // 5. Exclude modules - m_sourceDir.c_str(), // 6. Sources path - GenerateLaunchArgument(testTarget).c_str(), // 7. Launch command - runArtifact.c_str()) // 8. Result artifact - }; - - return JobInfo(jobId, args, JobData(runArtifact, coverageArtifact)); - } -} diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobInfoGenerator.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobInfoGenerator.h deleted file mode 100644 index 5d55904edd..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobInfoGenerator.h +++ /dev/null @@ -1,67 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#pragma once - -#include - -#include -#include -#include -#include - -namespace TestImpact -{ - class TestTarget; - - class TestJobInfoGenerator - { - public: - TestJobInfoGenerator( - const RepoPath& sourceDir, - const RepoPath& targetBinaryDir, - const RepoPath& cacheDir, - const RepoPath& artifactDir, - const RepoPath& testRunnerBinary, - const RepoPath& instrumentBinary); - - const RepoPath& GetCacheDir() const; - const RepoPath& GetArtifactDir() const; - - TestEnumerator::JobInfo GenerateTestEnumerationJobInfo( - const TestTarget* testTarget, - TestEnumerator::JobInfo::Id jobId, - TestEnumerator::JobInfo::CachePolicy cachePolicy) const; - - TestRunner::JobInfo GenerateRegularTestRunJobInfo( - const TestTarget* testTarget, - TestRunner::JobInfo::Id jobId) const; - - InstrumentedTestRunner::JobInfo GenerateInstrumentedTestRunJobInfo( - const TestTarget* testTarget, - InstrumentedTestRunner::JobInfo::Id jobId, - CoverageLevel coverageLevel) const; - private: - AZStd::string GenerateLaunchArgument(const TestTarget* testTarget) const; - RepoPath GenerateTargetEnumerationCacheFilePath(const TestTarget* testTarget) const; - RepoPath GenerateTargetEnumerationArtifactFilePath(const TestTarget* testTarget) const; - RepoPath GenerateTargetRunArtifactFilePath(const TestTarget* testTarget) const; - RepoPath GenerateTargetCoverageArtifactFilePath(const TestTarget* testTarget) const; - - RepoPath m_sourceDir; - RepoPath m_targetBinaryDir; - RepoPath m_cacheDir; - RepoPath m_artifactDir; - RepoPath m_testRunnerBinary; - RepoPath m_instrumentBinary; - }; -} From d917388bf9fc31df0c175f120daca53e826944e3 Mon Sep 17 00:00:00 2001 From: jonawals Date: Fri, 21 May 2021 13:16:38 +0100 Subject: [PATCH 028/102] Remove dead comment --- .../Runtime/Code/Source/TestEngine/Run/TestImpactTestRun.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRun.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRun.h index f2eeac14fa..a0acb37061 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRun.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRun.h @@ -55,6 +55,6 @@ namespace TestImpact size_t m_numNotRuns = 0; size_t m_numPasses = 0; size_t m_numFailures = 0; - AZStd::chrono::milliseconds m_duration = AZStd::chrono::milliseconds{0}; // this might be removed... + AZStd::chrono::milliseconds m_duration = AZStd::chrono::milliseconds{0}; }; } // namespace TestImpact From 7c28480307746b6571acae5f059589f2fcbbe605 Mon Sep 17 00:00:00 2001 From: jonawals Date: Fri, 21 May 2021 14:09:12 +0100 Subject: [PATCH 029/102] Address PR comments --- .../TestEngine/Run/TestImpactTestCoverage.cpp | 19 ++++++----------- .../TestEngine/Run/TestImpactTestCoverage.h | 1 - .../TestEngine/Run/TestImpactTestRun.cpp | 10 --------- .../Source/TestEngine/Run/TestImpactTestRun.h | 1 - .../TestEngine/TestImpactTestSuiteContainer.h | 21 ++++++------------- 5 files changed, 12 insertions(+), 40 deletions(-) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestCoverage.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestCoverage.cpp index 6135383863..453b1fa962 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestCoverage.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestCoverage.cpp @@ -29,7 +29,6 @@ namespace TestImpact , m_sourcesCovered(AZStd::move(other.m_sourcesCovered)) { AZStd::swap(m_coverageLevel, other.m_coverageLevel); - other.~TestCoverage(); } TestCoverage::TestCoverage(const AZStd::vector& moduleCoverages) @@ -44,19 +43,13 @@ namespace TestImpact CalculateTestMetrics(); } - TestCoverage::~TestCoverage() - { - m_modules.clear(); - m_coverageLevel.reset(); - m_sourcesCovered.clear(); - } - TestCoverage& TestCoverage::operator=(const TestCoverage& other) { if (this != &other) { - this->~TestCoverage(); - new(this)TestCoverage(other); + m_modules = other.m_modules; + m_sourcesCovered = other.m_sourcesCovered; + m_coverageLevel = other.m_coverageLevel; } return *this; @@ -66,9 +59,9 @@ namespace TestImpact { if (this != &other) { - this->~TestCoverage(); - new(this)TestCoverage(AZStd::move(other)); - other.~TestCoverage(); + m_modules = AZStd::move(other.m_modules); + m_sourcesCovered = other.m_sourcesCovered; + m_coverageLevel = other.m_coverageLevel; } return *this; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestCoverage.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestCoverage.h index c672814f7c..e7599fb145 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestCoverage.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestCoverage.h @@ -33,7 +33,6 @@ namespace TestImpact TestCoverage(TestCoverage&&) noexcept; TestCoverage(AZStd::vector&& moduleCoverages) noexcept; TestCoverage(const AZStd::vector& moduleCoverages); - ~TestCoverage(); TestCoverage& operator=(const TestCoverage&); TestCoverage& operator=(TestCoverage&&) noexcept; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRun.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRun.cpp index 9ee1569f0e..3796bbfe7e 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRun.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRun.cpp @@ -28,7 +28,6 @@ namespace TestImpact , m_numFailures(other.m_numFailures) , m_duration(other.m_duration) { - other.~TestRun(); } TestRun::TestRun(AZStd::vector&& testSuites, AZStd::chrono::milliseconds duration) noexcept @@ -45,15 +44,6 @@ namespace TestImpact CalculateTestMetrics(); } - TestRun::~TestRun() - { - m_numRuns = 0; - m_numNotRuns = 0; - m_numPasses = 0; - m_numFailures = 0; - m_duration = AZStd::chrono::milliseconds{ 0 }; - } - TestRun& TestRun::operator=(TestRun&& other) noexcept { if (this != &other) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRun.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRun.h index a0acb37061..f3a1ba93de 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRun.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRun.h @@ -28,7 +28,6 @@ namespace TestImpact TestRun(TestRun&&) noexcept; TestRun(const AZStd::vector& testSuites, AZStd::chrono::milliseconds duration); TestRun(AZStd::vector&& testSuites, AZStd::chrono::milliseconds duration) noexcept; - ~TestRun(); TestRun& operator=(const TestRun&); TestRun& operator=(TestRun&&) noexcept; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestSuiteContainer.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestSuiteContainer.h index 4f3fdebb08..5991640c46 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestSuiteContainer.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestSuiteContainer.h @@ -26,7 +26,6 @@ namespace TestImpact TestSuiteContainer(TestSuiteContainer&&) noexcept; TestSuiteContainer(const AZStd::vector& testSuites); TestSuiteContainer(AZStd::vector&& testSuites) noexcept; - virtual ~TestSuiteContainer(); TestSuiteContainer& operator=(const TestSuiteContainer&); TestSuiteContainer& operator=(TestSuiteContainer&&) noexcept; @@ -61,7 +60,6 @@ namespace TestImpact , m_numDisabledTests(other.m_numDisabledTests) , m_numEnabledTests(other.m_numEnabledTests) { - other.~TestSuiteContainer(); } template @@ -72,14 +70,6 @@ namespace TestImpact { } - template - TestSuiteContainer::~TestSuiteContainer() - { - m_testSuites.clear(); - m_numDisabledTests = 0; - m_numEnabledTests = 0; - } - template TestSuiteContainer::TestSuiteContainer(AZStd::vector&& testSuites) noexcept : m_testSuites(std::move(testSuites)) @@ -99,9 +89,9 @@ namespace TestImpact { if (this != &other) { - this->~TestSuiteContainer(); - new(this)TestSuiteContainer(AZStd::move(other)); - other.~TestSuiteContainer(); + m_testSuites = AZStd::move(other.m_testSuites); + m_numDisabledTests = other.m_numDisabledTests; + m_numEnabledTests = other.m_numEnabledTests; } return *this; @@ -112,8 +102,9 @@ namespace TestImpact { if (this != &other) { - this->~TestSuiteContainer(); - new(this)TestSuiteContainer(other); + m_testSuites = other.m_testSuites; + m_numDisabledTests = other.m_numDisabledTests; + m_numEnabledTests = other.m_numEnabledTests; } return *this; From 084e459a632b4a1417e719c4e4ff8b3c9cdb61f6 Mon Sep 17 00:00:00 2001 From: jonawals Date: Fri, 21 May 2021 17:41:19 +0100 Subject: [PATCH 030/102] Add missing change from PR feedback --- .../TestEngine/Run/TestImpactTestRun.cpp | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRun.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRun.cpp index 3796bbfe7e..6a361ca9d0 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRun.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRun.cpp @@ -16,6 +16,11 @@ namespace TestImpact { TestRun::TestRun(const TestRun& other) : TestSuiteContainer(other) + , m_numRuns(other.m_numRuns) + , m_numNotRuns(other.m_numNotRuns) + , m_numPasses(other.m_numPasses) + , m_numFailures(other.m_numFailures) + , m_duration(other.m_duration) { CalculateTestMetrics(); } @@ -48,9 +53,12 @@ namespace TestImpact { if (this != &other) { - this->~TestRun(); - new(this)TestRun(AZStd::move(other)); - other.~TestRun(); + TestSuiteContainer::operator=(AZStd::move(other)); + m_numRuns = other.m_numRuns; + m_numNotRuns = other.m_numNotRuns; + m_numPasses = other.m_numPasses; + m_numFailures = other.m_numFailures; + m_duration = other.m_duration; } return *this; @@ -60,8 +68,12 @@ namespace TestImpact { if (this != &other) { - this->~TestRun(); - new(this)TestRun(other); + TestSuiteContainer::operator=(other); + m_numRuns = other.m_numRuns; + m_numNotRuns = other.m_numNotRuns; + m_numPasses = other.m_numPasses; + m_numFailures = other.m_numFailures; + m_duration = other.m_duration; } return *this; From 95a30b0feea76583bd32dece9eeebd41b9ddaef9 Mon Sep 17 00:00:00 2001 From: jonawals Date: Fri, 21 May 2021 19:39:50 +0100 Subject: [PATCH 031/102] Add missing file from refactor --- .../Code/Source/Artifact/Dynamic/TestImpactCoverage.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Dynamic/TestImpactCoverage.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Dynamic/TestImpactCoverage.h index 54b78d682e..4e1c435841 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Dynamic/TestImpactCoverage.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Dynamic/TestImpactCoverage.h @@ -12,6 +12,8 @@ #pragma once +#include + #include #include @@ -27,14 +29,14 @@ namespace TestImpact //! Coverage information about a particular source file. struct SourceCoverage { - AZStd::string m_path; //!< Source file path. + RepoPath m_path; //!< Source file path. AZStd::vector m_coverage; //!< Source file line coverage (empty if source level coverage only). }; //! Coverage information about a particular module (executable, shared library). struct ModuleCoverage { - AZStd::string m_path; //!< Module path. + RepoPath m_path; //!< Module path. AZStd::vector m_sources; //!< Sources of this module that are covered. }; } // namespace TestImpact From 597f0905c5250f8417dfe19aa5c56c9b526ba06d Mon Sep 17 00:00:00 2001 From: jonawals Date: Fri, 21 May 2021 20:13:10 +0100 Subject: [PATCH 032/102] Refactor ProcessScheduler with Execute method --- .../JobRunner/TestImpactProcessJobRunner.h | 78 +++++------- .../Scheduler/TestImpactProcessScheduler.cpp | 118 +++++++++++++----- .../Scheduler/TestImpactProcessScheduler.h | 56 ++++----- .../Enumeration/TestImpactTestEnumerator.cpp | 32 +++-- .../Enumeration/TestImpactTestEnumerator.h | 23 ++-- .../JobRunner/TestImpactTestJobRunner.h | 70 +++++------ .../Run/TestImpactInstrumentedTestRunner.cpp | 26 ++-- .../Run/TestImpactInstrumentedTestRunner.h | 25 ++-- .../TestEngine/Run/TestImpactTestCoverage.cpp | 2 +- .../TestEngine/Run/TestImpactTestCoverage.h | 4 +- .../TestEngine/Run/TestImpactTestRunner.cpp | 26 ++-- .../TestEngine/Run/TestImpactTestRunner.h | 21 ++-- 12 files changed, 272 insertions(+), 209 deletions(-) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJobRunner.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJobRunner.h index 0c30accc84..384111b823 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJobRunner.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJobRunner.h @@ -50,32 +50,29 @@ namespace TestImpact { public: //! Constructs the job runner with the specified parameters to constrain job runs. + //! @param maxConcurrentProcesses he maximum number of concurrent jobs in-flight. + JobRunner(size_t maxConcurrentProcesses); + + //! Executes the specified jobs and returns the products of their labor. + //! @param jobs The arguments (and other pertinent information) required for each job to be run. //! @param stdOutRouting The standard output routing to be specified for all jobs. //! @param stdErrRouting The standard error routing to be specified for all jobs. - //! @param maxConcurrentProcesses he maximum number of concurrent jobs in-flight. - //! @param processTimeout The maximum duration a job may be in-flight before being forcefully terminated (nullopt if no timeout). - //! @param scheduleTimeout The maximum duration the scheduler may run before forcefully terminating all in-flight jobs (nullopt if - //! no timeout). - JobRunner( + //! @param jobTimeout The maximum duration a job may be in-flight before being forcefully terminated (nullopt if no timeout). + //! @param runnerTimeout The maximum duration the scheduler may run before forcefully terminating all in-flight jobs (nullopt if no timeout). + //! @param payloadMapProducer The client callback to be called when all jobs have finished to transform the work produced by each job into the desired output. + //! @param jobCallback The client callback to be called when each job changes state. + //! @return The result of the run sequence and the jobs with their associated payloads. + AZStd::pair> Execute( + const AZStd::vector& jobs, + PayloadMapProducer payloadMapProducer, StdOutputRouting stdOutRouting, StdErrorRouting stdErrRouting, - size_t maxConcurrentProcesses, - AZStd::optional processTimeout, - AZStd::optional scheduleTimeout); - - //! Executes the specified jobs and returns the products of their labor. - //! @note: the job and payload callbacks are specified here rather than in the constructor to allow clients to use capturing lambdas - //! should they desire to. - //! @param jobs The arguments (and other pertinent information) required for each job to be run. - //! @param jobCallback The client callback to be called when each job changes state. - //! @param payloadMapProducer The client callback to be called when all jobs have finished to transform the work produced by each - //! job into the desired output. - AZStd::vector Execute( - const AZStd::vector& jobs, JobCallback jobCallback, - PayloadMapProducer payloadMapProducer); + AZStd::optional jobTimeout, + AZStd::optional runnerTimeout, + JobCallback jobCallback); private: - size_t m_maxConcurrentProcesses = 0; //!< Maximum number of concurrent jobs being executed at a given time. + ProcessScheduler m_processScheduler; StdOutputRouting m_stdOutRouting; //!< Standard output routing from each job process to job runner. StdErrorRouting m_stdErrRouting; //!< Standard error routing from each job process to job runner AZStd::optional m_jobTimeout; //!< Maximum time a job can run for before being forcefully terminated. @@ -83,25 +80,20 @@ namespace TestImpact }; template - JobRunner::JobRunner( - StdOutputRouting stdOutRouting, - StdErrorRouting stdErrRouting, - size_t maxConcurrentProcesses, - AZStd::optional jobTimeout, - AZStd::optional runnerTimeout) - : m_maxConcurrentProcesses(maxConcurrentProcesses) - , m_stdOutRouting(stdOutRouting) - , m_stdErrRouting(stdErrRouting) - , m_jobTimeout(jobTimeout) - , m_runnerTimeout(runnerTimeout) + JobRunner::JobRunner(size_t maxConcurrentProcesses) + : m_processScheduler(maxConcurrentProcesses) { } template - AZStd::vector JobRunner::Execute( + AZStd::pair> JobRunner::Execute( const AZStd::vector& jobInfos, - JobCallback jobCallback, - PayloadMapProducer payloadMapProducer) + PayloadMapProducer payloadMapProducer, + StdOutputRouting stdOutRouting, + StdErrorRouting stdErrRouting, + AZStd::optional jobTimeout, + AZStd::optional runnerTimeout, + JobCallback jobCallback) { AZStd::vector processes; AZStd::unordered_map> metas; @@ -115,11 +107,11 @@ namespace TestImpact const auto* jobInfo = &jobInfos[jobIndex]; const auto jobId = jobInfo->GetId().m_value; metas.emplace(jobId, AZStd::pair{JobMeta{}, jobInfo}); - processes.emplace_back(jobId, m_stdOutRouting, m_stdErrRouting, jobInfo->GetCommand().m_args); + processes.emplace_back(jobId, stdOutRouting, stdErrRouting, jobInfo->GetCommand().m_args); } // Wrapper around low-level process launch callback to gather job meta-data and present a simplified callback interface to the client - const auto processLaunchCallback = [&jobCallback, &jobInfos, &metas]( + const ProcessLaunchCallback processLaunchCallback = [&jobCallback, &jobInfos, &metas]( TestImpact::ProcessId pid, TestImpact::LaunchResult launchResult, AZStd::chrono::high_resolution_clock::time_point createTime) @@ -138,7 +130,7 @@ namespace TestImpact }; // Wrapper around low-level process exit callback to gather job meta-data and present a simplified callback interface to the client - const auto processExitCallback = [&jobCallback, &jobInfos, &metas]( + const ProcessExitCallback processExitCallback = [&jobCallback, &jobInfos, &metas]( TestImpact::ProcessId pid, TestImpact::ExitCondition exitCondition, TestImpact::ReturnCode returnCode, @@ -169,14 +161,12 @@ namespace TestImpact }; // Schedule all jobs for execution - ProcessScheduler scheduler( + const auto result = m_processScheduler.Execute( processes, + jobTimeout, + runnerTimeout, processLaunchCallback, - processExitCallback, - m_maxConcurrentProcesses, - m_jobTimeout, - m_runnerTimeout - ); + processExitCallback); // Hand off the jobs to the client for payload generation auto payloadMap = payloadMapProducer(metas); @@ -188,6 +178,6 @@ namespace TestImpact jobs.emplace_back(JobT(jobInfo, AZStd::move(metas.at(jobId).first), AZStd::move(payloadMap[jobId]))); } - return jobs; + return { result, jobs }; } } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/Scheduler/TestImpactProcessScheduler.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/Scheduler/TestImpactProcessScheduler.cpp index 04a68bf612..8d6c5c6739 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/Scheduler/TestImpactProcessScheduler.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/Scheduler/TestImpactProcessScheduler.cpp @@ -18,7 +18,7 @@ namespace TestImpact { - struct ProcessScheduler::ProcessInFlight + struct ProcessInFlight { AZStd::unique_ptr m_process; AZStd::optional m_startTime; @@ -26,29 +26,64 @@ namespace TestImpact AZStd::string m_stdError; }; - ProcessScheduler::ProcessScheduler( - const AZStd::vector& processes, - const ProcessLaunchCallback& processLaunchCallback, - const ProcessExitCallback& processExitCallback, + class ProcessScheduler::ExecutionState + { + public: + ExecutionState( + size_t maxConcurrentProcesses, + AZStd::optional processTimeout, + AZStd::optional scheduleTimeout, + ProcessLaunchCallback& processLaunchCallback, + ProcessExitCallback& processExitCallback); + ~ExecutionState(); + + ProcessSchedulerResult MonitorProcesses(const AZStd::vector& processes); + void TerminateAllProcesses(ExitCondition exitStatus); + private: + ProcessCallbackResult PopAndLaunch(ProcessInFlight& processInFlight); + StdContent ConsumeProcessStdContent(ProcessInFlight& processInFlight); + void AccumulateProcessStdContent(ProcessInFlight& processInFlight); + + size_t m_maxConcurrentProcesses = 0; + ProcessLaunchCallback m_processLaunchCallback; + ProcessExitCallback m_processExitCallback; + AZStd::optional m_processTimeout; + AZStd::optional m_scheduleTimeout; + AZStd::chrono::high_resolution_clock::time_point m_startTime; + AZStd::vector m_processPool; + AZStd::queue m_processQueue; + }; + + ProcessScheduler::ExecutionState::ExecutionState( size_t maxConcurrentProcesses, AZStd::optional processTimeout, - AZStd::optional scheduleTimeout) - : m_processCreateCallback(processLaunchCallback) + AZStd::optional scheduleTimeout, + ProcessLaunchCallback& processLaunchCallback, + ProcessExitCallback& processExitCallback) + : m_maxConcurrentProcesses(maxConcurrentProcesses) + , m_processLaunchCallback(processLaunchCallback) , m_processExitCallback(processExitCallback) , m_processTimeout(processTimeout) , m_scheduleTimeout(scheduleTimeout) - , m_startTime(AZStd::chrono::high_resolution_clock::now()) { - AZ_TestImpact_Eval(maxConcurrentProcesses != 0, ProcessException, "Max Number of concurrent processes in flight cannot be 0"); - AZ_TestImpact_Eval(!processes.empty(), ProcessException, "Number of processes to launch cannot be 0"); AZ_TestImpact_Eval( !m_processTimeout.has_value() || m_processTimeout->count() > 0, ProcessException, "Process timeout must be empty or non-zero value"); AZ_TestImpact_Eval( !m_scheduleTimeout.has_value() || m_scheduleTimeout->count() > 0, ProcessException, "Scheduler timeout must be empty or non-zero value"); + } - const size_t numConcurrentProcesses = AZStd::min(processes.size(), maxConcurrentProcesses); + ProcessScheduler::ExecutionState::~ExecutionState() + { + TerminateAllProcesses(ExitCondition::Terminated); + } + + ProcessSchedulerResult ProcessScheduler::ExecutionState::MonitorProcesses(const AZStd::vector& processes) + { + AZ_TestImpact_Eval(!processes.empty(), ProcessException, "Number of processes to launch cannot be 0"); + m_startTime = AZStd::chrono::high_resolution_clock::now(); + const size_t numConcurrentProcesses = AZStd::min(processes.size(), m_maxConcurrentProcesses); m_processPool.resize(numConcurrentProcesses); for (const auto& process : processes) @@ -61,20 +96,10 @@ namespace TestImpact if (PopAndLaunch(process) == ProcessCallbackResult::Abort) { TerminateAllProcesses(ExitCondition::Terminated); - return; + return ProcessSchedulerResult::Graceful; } } - MonitorProcesses(); - } - - ProcessScheduler::~ProcessScheduler() - { - TerminateAllProcesses(ExitCondition::Terminated); - } - - void ProcessScheduler::MonitorProcesses() - { while (true) { // Check to see whether or not the scheduling has exceeded its specified runtime @@ -86,7 +111,7 @@ namespace TestImpact { // Runtime exceeded, terminate all proccesses and schedule no further TerminateAllProcesses(ExitCondition::Timeout); - return; + return ProcessSchedulerResult::Timeout; } } @@ -98,7 +123,7 @@ namespace TestImpact { if (processInFlight.m_process) { - // Process is alive (note: not necessarilly currently running) + // Process is alive (note: not necessarily currently running) AccumulateProcessStdContent(processInFlight); const ProcessId processId = processInFlight.m_process->GetProcessInfo().GetId(); @@ -119,7 +144,7 @@ namespace TestImpact { // Client chose to abort the scheduler TerminateAllProcesses(ExitCondition::Terminated); - return; + return ProcessSchedulerResult::Graceful; } else if (!m_processQueue.empty()) { @@ -128,7 +153,7 @@ namespace TestImpact { // Client chose to abort the scheduler TerminateAllProcesses(ExitCondition::Terminated); - return; + return ProcessSchedulerResult::Graceful; } else { @@ -157,9 +182,9 @@ namespace TestImpact ConsumeProcessStdContent(processInFlight), exitTime)) { - // Flight time exceeded, terminate this process + // Client chose to abort the scheduler TerminateAllProcesses(ExitCondition::Terminated); - return; + return ProcessSchedulerResult::Graceful; } } @@ -176,7 +201,7 @@ namespace TestImpact { // Client chose to abort the scheduler TerminateAllProcesses(ExitCondition::Terminated); - return; + return ProcessSchedulerResult::Graceful; } else { @@ -192,9 +217,11 @@ namespace TestImpact break; } } + + return ProcessSchedulerResult::Graceful; } - ProcessCallbackResult ProcessScheduler::PopAndLaunch(ProcessInFlight& processInFlight) + ProcessCallbackResult ProcessScheduler::ExecutionState::PopAndLaunch(ProcessInFlight& processInFlight) { auto processInfo = m_processQueue.front(); m_processQueue.pop(); @@ -212,17 +239,17 @@ namespace TestImpact createResult = LaunchResult::Failure; } - return m_processCreateCallback(processInfo.GetId(), createResult, createTime); + return m_processLaunchCallback(processInfo.GetId(), createResult, createTime); } - void ProcessScheduler::AccumulateProcessStdContent(ProcessInFlight& processInFlight) + void ProcessScheduler::ExecutionState::AccumulateProcessStdContent(ProcessInFlight& processInFlight) { // Accumulate the stdout/stderr so we don't deadlock with the process waiting for the pipe to empty before finishing processInFlight.m_stdOutput += processInFlight.m_process->ConsumeStdOut().value_or(""); processInFlight.m_stdError += processInFlight.m_process->ConsumeStdErr().value_or(""); } - StdContent ProcessScheduler::ConsumeProcessStdContent(ProcessInFlight& processInFlight) + StdContent ProcessScheduler::ExecutionState::ConsumeProcessStdContent(ProcessInFlight& processInFlight) { return { @@ -235,7 +262,7 @@ namespace TestImpact }; } - void ProcessScheduler::TerminateAllProcesses(ExitCondition exitStatus) + void ProcessScheduler::ExecutionState::TerminateAllProcesses(ExitCondition exitStatus) { bool isCallingBackToClient = true; const ReturnCode returnCode = static_cast(exitStatus); @@ -267,4 +294,27 @@ namespace TestImpact } } } + + ProcessScheduler::ProcessScheduler(size_t maxConcurrentProcesses) + : m_maxConcurrentProcesses(maxConcurrentProcesses) + { + AZ_TestImpact_Eval(maxConcurrentProcesses != 0, ProcessException, "Max Number of concurrent processes in flight cannot be 0"); + } + + ProcessScheduler::~ProcessScheduler() = default; + + ProcessSchedulerResult ProcessScheduler::Execute( + const AZStd::vector& processes, + AZStd::optional processTimeout, + AZStd::optional scheduleTimeout, + ProcessLaunchCallback processLaunchCallback, + ProcessExitCallback processExitCallback) + { + AZ_TestImpact_Eval(!m_executionState, ProcessException, "Couldn't execute schedule, schedule already in progress"); + m_executionState = AZStd::make_unique( + m_maxConcurrentProcesses, processTimeout, scheduleTimeout, processLaunchCallback, processExitCallback); + const auto result = m_executionState->MonitorProcesses(processes); + m_executionState.reset(); + return result; + } } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/Scheduler/TestImpactProcessScheduler.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/Scheduler/TestImpactProcessScheduler.h index 27171d25ee..7f562fb99e 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/Scheduler/TestImpactProcessScheduler.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/Scheduler/TestImpactProcessScheduler.h @@ -22,6 +22,7 @@ #include #include #include +#include namespace TestImpact { @@ -49,6 +50,13 @@ namespace TestImpact Abort //!< Abort scheduling immediately. }; + //! Result of the process scheduling sequence. + enum class ProcessSchedulerResult : bool + { + Graceful, //!< The scheduler completed its run without incident or was terminated gracefully in response to a client callback result. + Timeout //!< The scheduler aborted its run prematurely due to its runtime exceeding the scheduler timeout value. + }; + //! Callback for process launch attempt. //! @param processId The id of the process that attempted to launch. //! @param launchResult The result of the process launch attempt. @@ -79,38 +87,28 @@ namespace TestImpact { public: //! Constructs the scheduler with the specified batch of processes. - //! @param processes The batch of processes to schedule. - //! @param processLaunchCallback The process launch callback function. - //! @param processExitCallback The process exit callback function. //! @param maxConcurrentProcesses The maximum number of concurrent processes in-flight. - //! @param processTimeout The maximum duration a process may be in-flight for before being forcefully terminated. - //! @param scheduleTimeout The maximum duration the scheduler may run before forcefully terminating all in-flight processes. - //! processes and abandoning any queued processes. - ProcessScheduler( - const AZStd::vector& processes, - const ProcessLaunchCallback& processLaunchCallback, - const ProcessExitCallback& processExitCallback, - size_t maxConcurrentProcesses, - AZStd::optional processTimeout, - AZStd::optional scheduleTimeout); - + ProcessScheduler(size_t maxConcurrentProcesses); ~ProcessScheduler(); + //! Executes the specified processes and calls the client callbacks (if any) as each process progresses in its life cycle. + //! @note Multiple subsequent calls to Execute are permitted. + //! @param processes The batch of processes to schedule. + //! @param processTimeout The maximum duration a process may be in-flight for before being forcefully terminated. + //! @param scheduleTimeout The maximum duration the scheduler may run before forcefully terminating all in-flight processes. + //! @param processLaunchCallback The process launch callback function. + //! @param processExitCallback The process exit callback function. + //! @returns The state that triggered the end of the schedule sequence. + ProcessSchedulerResult Execute( + const AZStd::vector& processes, + AZStd::optional processTimeout, + AZStd::optional scheduleTimeout, + ProcessLaunchCallback processLaunchCallback, + ProcessExitCallback processExitCallback); + private: - struct ProcessInFlight; - - void MonitorProcesses(); - ProcessCallbackResult PopAndLaunch(ProcessInFlight& processInFlight); - void TerminateAllProcesses(ExitCondition exitStatus); - StdContent ConsumeProcessStdContent(ProcessInFlight& processInFlight); - void AccumulateProcessStdContent(ProcessInFlight& processInFlight); - - const ProcessLaunchCallback m_processCreateCallback; - const ProcessExitCallback m_processExitCallback; - const AZStd::optional m_processTimeout; - const AZStd::optional m_scheduleTimeout; - const AZStd::chrono::high_resolution_clock::time_point m_startTime; - AZStd::vector m_processPool; - AZStd::queue m_processQueue; + class ExecutionState; + AZStd::unique_ptr m_executionState; + size_t m_maxConcurrentProcesses = 0; }; } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.cpp index 882c65c516..09cea87d41 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.cpp @@ -106,19 +106,18 @@ namespace TestImpact return m_cache; } - TestEnumerator::TestEnumerator( - AZStd::optional clientCallback, - size_t maxConcurrentEnumerations, - AZStd::optional enumerationTimeout, - AZStd::optional enumeratorTimeout) - : JobRunner(clientCallback, AZStd::nullopt, StdOutputRouting::None, StdErrorRouting::None, maxConcurrentEnumerations, enumerationTimeout, enumeratorTimeout) + TestEnumerator::TestEnumerator(size_t maxConcurrentEnumerations) + : JobRunner(maxConcurrentEnumerations) { } - - AZStd::vector TestEnumerator::Enumerate( + + AZStd::pair> TestEnumerator::Enumerate( const AZStd::vector& jobInfos, CacheExceptionPolicy cacheExceptionPolicy, - JobExceptionPolicy jobExceptionPolicy) + JobExceptionPolicy jobExceptionPolicy, + AZStd::optional enumerationTimeout, + AZStd::optional enumeratorTimeout, + AZStd::optional clientCallback) { AZStd::vector cachedJobs; AZStd::vector jobQueue; @@ -179,14 +178,23 @@ namespace TestImpact }; // Generate the enumeration results for the jobs that weren't cached - auto jobs = ExecuteJobs(jobQueue, payloadGenerator, jobExceptionPolicy); - + auto [result, jobs] = ExecuteJobs( + jobQueue, + jobExceptionPolicy, + payloadGenerator, + StdOutputRouting::None, + StdErrorRouting::None, + enumerationTimeout, + enumeratorTimeout, + clientCallback, + AZStd::nullopt); + // We need to add the cached jobs to the completed job list even though they technically weren't executed for (auto&& job : cachedJobs) { jobs.emplace_back(AZStd::move(job)); } - return jobs; + return { result, jobs }; } } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.h index 63d0b82d3e..287dd9e2d9 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.h @@ -73,22 +73,23 @@ namespace TestImpact using CacheExceptionPolicy = Bitwise::CacheExceptionPolicy; //! Constructs a test enumerator with the specified parameters common to all enumeration job runs of this enumerator. - //! @param clientCallback The optional client callback to be called whenever an enumeration job changes state. //! @param maxConcurrentEnumerations The maximum number of enumerations to be in flight at any given time. - //! @param enumerationTimeout The maximum duration an enumeration may be in-flight for before being forcefully terminated. - //! @param enumeratorTimeout The maximum duration the enumerator may run before forcefully terminating all in-flight enumerations. - TestEnumerator( - AZStd::optional clientCallback, - size_t maxConcurrentEnumerations, - AZStd::optional enumerationTimeout, - AZStd::optional enumeratorTimeout); + TestEnumerator(size_t maxConcurrentEnumerations); //! Executes the specified test enumeration jobs according to the specified cache and job exception policies. //! @param jobInfos The enumeration jobs to execute. //! @param cacheExceptionPolicy The cache exception policy to be used for this run. //! @param jobExceptionPolicy The enumeration job exception policy to be used for this run. - //! @return the test enumeration jobs with their associated test enumeration payloads. - AZStd::vector Enumerate( - const AZStd::vector& jobInfos, CacheExceptionPolicy cacheExceptionPolicy, JobExceptionPolicy jobExceptionPolicy); + //! @param enumerationTimeout The maximum duration an enumeration may be in-flight for before being forcefully terminated. + //! @param enumeratorTimeout The maximum duration the enumerator may run before forcefully terminating all in-flight enumerations. + //! @param clientCallback The optional client callback to be called whenever an enumeration job changes state. + //! @return The result of the run sequence and the enumeration jobs with their associated test enumeration payloads. + AZStd::pair> Enumerate( + const AZStd::vector& jobInfos, + CacheExceptionPolicy cacheExceptionPolicy, + JobExceptionPolicy jobExceptionPolicy, + AZStd::optional enumerationTimeout, + AZStd::optional enumeratorTimeout, + AZStd::optional clientCallback); }; } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobRunner.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobRunner.h index c60a93e21d..91f4d4e39c 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobRunner.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobRunner.h @@ -53,32 +53,31 @@ namespace TestImpact using JobDataMap = JobDataMap; //! Constructs the job runner with the specified parameters common to all job runs of this runner. - //! @param clientCallback The optional callback function provided by the client to be called upon job state change. - //! @param clientCallback The optional callback function provided by the derived job runner to be called upon job state change. - //! @param stdOutRouting The standard output routing from the underlying job processes to the derived runner. - //! @param stdErrorRouting The standard error routing from the underlying job processes to the derived runner. //! @param maxConcurrentJobs The maximum number of jobs to be in flight at any given time. - //! @param jobTimeout The maximum duration a job may be in-flight for before being forcefully terminated (nullopt if no timeout). - //! @param runnerTimeout The maximum duration the runner may run before forcefully terminating all in-flight jobs (nullopt if no - //! timeout). - TestJobRunner( - AZStd::optional clientCallback, - AZStd::optional derivedJobCallback, - StdOutputRouting stdOutRouting, - StdErrorRouting stdErrRouting, - size_t maxConcurrentJobs, - AZStd::optional jobTimeout, - AZStd::optional runnerTimeout); + TestJobRunner(size_t maxConcurrentJobs); protected: //! Runs the specified jobs and returns the completed payloads produced by each job. //! @param jobInfos The batch of jobs to execute. - //! @param payloadMapProducer The client callback for producing the payload map based on the completed job data. //! @param jobExceptionPolicy The job execution policy for this job run. - AZStd::vector ExecuteJobs( + //! @param payloadMapProducer The client callback for producing the payload map based on the completed job data. + //! @param stdOutRouting The standard output routing from the underlying job processes to the derived runner. + //! @param stdErrorRouting The standard error routing from the underlying job processes to the derived runner. + //! @param jobTimeout The maximum duration a job may be in-flight for before being forcefully terminated (nullopt if no timeout). + //! @param runnerTimeout The maximum duration the runner may run before forcefully terminating all in-flight jobs (nullopt if no timeout). + //! @param clientCallback The optional callback function provided by the client to be called upon job state change. + //! @param clientCallback The optional callback function provided by the derived job runner to be called upon job state change. + //! @returns The result of the run sequence and the jobs that the sequence produced. + AZStd::pair> ExecuteJobs( const AZStd::vector& jobInfos, + JobExceptionPolicy jobExceptionPolicy, PayloadMapProducer payloadMapProducer, - JobExceptionPolicy jobExceptionPolicy); + StdOutputRouting stdOutRouting, + StdErrorRouting stdErrRouting, + AZStd::optional jobTimeout, + AZStd::optional runnerTimeout, + AZStd::optional clientCallback, + AZStd::optional derivedJobCallback); const AZStd::optional m_clientJobCallback; @@ -88,28 +87,25 @@ namespace TestImpact }; template - TestJobRunner::TestJobRunner( - AZStd::optional clientCallback, - AZStd::optional derivedJobCallback, - StdOutputRouting stdOutRouting, - StdErrorRouting stdErrRouting, - size_t maxConcurrentJobs, - AZStd::optional jobTimeout, - AZStd::optional runnerTimeout) - : m_jobRunner(stdOutRouting, stdErrRouting, maxConcurrentJobs, jobTimeout, runnerTimeout) - , m_clientJobCallback(clientCallback) - , m_derivedJobCallback(derivedJobCallback) + TestJobRunner::TestJobRunner(size_t maxConcurrentJobs) + : m_jobRunner(maxConcurrentJobs) { } template - AZStd::vector::Job> TestJobRunner::ExecuteJobs( + AZStd::pair::Job>> TestJobRunner::ExecuteJobs( const AZStd::vector& jobInfos, + JobExceptionPolicy jobExceptionPolicy, PayloadMapProducer payloadMapProducer, - JobExceptionPolicy jobExceptionPolicy) + StdOutputRouting stdOutRouting, + StdErrorRouting stdErrRouting, + AZStd::optional jobTimeout, + AZStd::optional runnerTimeout, + AZStd::optional clientCallback, + AZStd::optional derivedJobCallback) { // Callback to handle job exception policies and client/derived callbacks - const auto jobCallback = [this, &jobExceptionPolicy](const JobInfo& jobInfo, const JobMeta& meta, StdContent&& std) + const auto jobCallback = [&clientCallback, &derivedJobCallback, &jobExceptionPolicy](const JobInfo& jobInfo, const JobMeta& meta, StdContent&& std) { auto callbackResult = ProcessCallbackResult::Continue; if (meta.m_result == JobResult::FailedToExecute && IsFlagSet(jobExceptionPolicy, JobExceptionPolicy::OnFailedToExecute)) @@ -121,23 +117,23 @@ namespace TestImpact callbackResult = ProcessCallbackResult::Abort; } - if (m_derivedJobCallback.has_value()) + if (derivedJobCallback.has_value()) { - if (const auto result = (*m_derivedJobCallback)(jobInfo, meta, AZStd::move(std)); + if (const auto result = (*derivedJobCallback)(jobInfo, meta, AZStd::move(std)); result == ProcessCallbackResult::Abort) { callbackResult = ProcessCallbackResult::Abort; } } - if (m_clientJobCallback.has_value()) + if (clientCallback.has_value()) { - (*m_clientJobCallback)(jobInfo, meta); + (*clientCallback)(jobInfo, meta); } return callbackResult; }; - return m_jobRunner.Execute(jobInfos, jobCallback, payloadMapProducer); + return m_jobRunner.Execute(jobInfos, payloadMapProducer, stdOutRouting, stdErrRouting, jobTimeout, runnerTimeout, jobCallback); } } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.cpp index 0d27d56af1..c0373e89f5 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.cpp @@ -52,19 +52,18 @@ namespace TestImpact return {AZStd::move(run), AZStd::move(coverage)}; } - InstrumentedTestRunner::InstrumentedTestRunner( - AZStd::optional clientCallback, - size_t maxConcurrentRuns, - AZStd::optional runTimeout, - AZStd::optional runnerTimeout) - : JobRunner(clientCallback, AZStd::nullopt, StdOutputRouting::None, StdErrorRouting::None, maxConcurrentRuns, runTimeout, runnerTimeout) + InstrumentedTestRunner::InstrumentedTestRunner(size_t maxConcurrentRuns) + : JobRunner(maxConcurrentRuns) { } - AZStd::vector InstrumentedTestRunner::RunInstrumentedTests( + AZStd::pair> InstrumentedTestRunner::RunInstrumentedTests( const AZStd::vector& jobInfos, CoverageExceptionPolicy coverageExceptionPolicy, - JobExceptionPolicy jobExceptionPolicy) + JobExceptionPolicy jobExceptionPolicy, + AZStd::optional runTimeout, + AZStd::optional runnerTimeout, + AZStd::optional clientCallback) { const auto payloadGenerator = [this, coverageExceptionPolicy](const JobDataMap& jobDataMap) { @@ -98,6 +97,15 @@ namespace TestImpact return runs; }; - return ExecuteJobs(jobInfos, payloadGenerator, jobExceptionPolicy); + return ExecuteJobs( + jobInfos, + jobExceptionPolicy, + payloadGenerator, + StdOutputRouting::None, + StdErrorRouting::None, + runTimeout, + runnerTimeout, + clientCallback, + AZStd::nullopt); } } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.h index 3b47f53128..5545843e35 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.h @@ -53,21 +53,24 @@ namespace TestImpact using CoverageExceptionPolicy = Bitwise::CoverageExceptionPolicy; //! Constructs an instrumented test runner with the specified parameters common to all job runs of this runner. - //! @param clientCallback The optional client callback to be called whenever a run job changes state. - //! @param maxConcurrentRuns The maximum number of runs to be in flight at any given time. - //! @param runTimeout The maximum duration a run may be in-flight for before being forcefully terminated. - //! @param runnerTimeout The maximum duration the runner may run before forcefully terminating all in-flight runs. - InstrumentedTestRunner( - AZStd::optional clientCallback, size_t maxConcurrentRuns, - AZStd::optional runTimeout, AZStd::optional runnerTimeout); - + //! @param maxConcurrentRuns The maximum number of runs to be in flight at any given time. + InstrumentedTestRunner(size_t maxConcurrentRuns); + //! Executes the specified instrumented test run jobs according to the specified job exception policies. //! @param jobInfos The test run jobs to execute. //! @param CoverageExceptionPolicy The coverage exception policy to be used for this run. //! @param jobExceptionPolicy The test run job exception policy to be used for this run (use //! TestJobExceptionPolicy::OnFailedToExecute to throw on test failures). - //! @return the instrumented test run jobs with their associated test run and test coverage payloads. - AZStd::vector RunInstrumentedTests( - const AZStd::vector& jobInfos, CoverageExceptionPolicy coverageExceptionPolicy, JobExceptionPolicy jobExceptionPolicy); + //! @param runTimeout The maximum duration a run may be in-flight for before being forcefully terminated. + //! @param runnerTimeout The maximum duration the runner may run before forcefully terminating all in-flight runs. + //! @param clientCallback The optional client callback to be called whenever a run job changes state. + //! @return The result of the run sequence and the instrumented run jobs with their associated test run and coverage payloads. + AZStd::pair> RunInstrumentedTests( + const AZStd::vector& jobInfos, + CoverageExceptionPolicy coverageExceptionPolicy, + JobExceptionPolicy jobExceptionPolicy, + AZStd::optional runTimeout, + AZStd::optional runnerTimeout, + AZStd::optional clientCallback); }; } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestCoverage.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestCoverage.cpp index 453b1fa962..4e82857cb2 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestCoverage.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestCoverage.cpp @@ -103,7 +103,7 @@ namespace TestImpact return m_modules.size(); } - const AZStd::vector& TestCoverage::GetSourcesCovered() const + const AZStd::vector& TestCoverage::GetSourcesCovered() const { return m_sourcesCovered; } diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestCoverage.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestCoverage.h index e7599fb145..8de5fbb6c7 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestCoverage.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestCoverage.h @@ -44,7 +44,7 @@ namespace TestImpact size_t GetNumModulesCovered() const; //! Returns the sorted set of unique sources covered (empty if no coverage). - const AZStd::vector& GetSourcesCovered() const; + const AZStd::vector& GetSourcesCovered() const; //! Returns the modules covered (empty if no coverage). const AZStd::vector& GetModuleCoverages() const; @@ -56,7 +56,7 @@ namespace TestImpact void CalculateTestMetrics(); AZStd::vector m_modules; - AZStd::vector m_sourcesCovered; + AZStd::vector m_sourcesCovered; AZStd::optional m_coverageLevel; }; } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunner.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunner.cpp index ebb846f406..27780a5f95 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunner.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunner.cpp @@ -26,18 +26,17 @@ namespace TestImpact return TestRun(GTest::TestRunSuitesFactory(ReadFileContents(runFile)), duration); } - TestRunner::TestRunner( - AZStd::optional clientCallback, - size_t maxConcurrentRuns, - AZStd::optional runTimeout, - AZStd::optional runnerTimeout) - : JobRunner(clientCallback, AZStd::nullopt, StdOutputRouting::None, StdErrorRouting::None, maxConcurrentRuns, runTimeout, runnerTimeout) + TestRunner::TestRunner(size_t maxConcurrentRuns) + : JobRunner(maxConcurrentRuns) { } - AZStd::vector TestRunner::RunTests( + AZStd::pair> TestRunner::RunTests( const AZStd::vector& jobInfos, - JobExceptionPolicy jobExceptionPolicy) + JobExceptionPolicy jobExceptionPolicy, + AZStd::optional runTimeout, + AZStd::optional runnerTimeout, + AZStd::optional clientCallback) { const auto payloadGenerator = [this](const JobDataMap& jobDataMap) { @@ -62,6 +61,15 @@ namespace TestImpact return runs; }; - return ExecuteJobs(jobInfos, payloadGenerator, jobExceptionPolicy); + return ExecuteJobs( + jobInfos, + jobExceptionPolicy, + payloadGenerator, + StdOutputRouting::None, + StdErrorRouting::None, + runTimeout, + runnerTimeout, + clientCallback, + AZStd::nullopt); } } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunner.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunner.h index 379c6b424e..597ce2d2f8 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunner.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunner.h @@ -26,21 +26,22 @@ namespace TestImpact public: //! Constructs a test runner with the specified parameters common to all job runs of this runner. - //! @param clientCallback The optional client callback to be called whenever a run job changes state. //! @param maxConcurrentRuns The maximum number of runs to be in flight at any given time. - //! @param runTimeout The maximum duration a run may be in-flight for before being forcefully terminated. - //! @param runnerTimeout The maximum duration the runner may run before forcefully terminating all in-flight runs. - TestRunner( - AZStd::optional clientCallback, - size_t maxConcurrentRuns, - AZStd::optional runTimeout, - AZStd::optional runnerTimeout); + TestRunner(size_t maxConcurrentRuns); //! Executes the specified test run jobs according to the specified job exception policies. //! @param jobInfos The test run jobs to execute. //! @param jobExceptionPolicy The test run job exception policy to be used for this run (use //! TestJobExceptionPolicy::OnFailedToExecute to throw on test failures). - //! @return the test run jobs with their associated test run payloads. - AZStd::vector RunTests(const AZStd::vector& jobInfos, JobExceptionPolicy jobExceptionPolicy); + //! @param runTimeout The maximum duration a run may be in-flight for before being forcefully terminated. + //! @param runnerTimeout The maximum duration the runner may run before forcefully terminating all in-flight runs. + //! @param clientCallback The optional client callback to be called whenever a run job changes state. + //! @return The result of the run sequence and the run jobs with their associated test run payloads. + AZStd::pair> RunTests( + const AZStd::vector& jobInfos, + JobExceptionPolicy jobExceptionPolicy, + AZStd::optional runTimeout, + AZStd::optional runnerTimeout, + AZStd::optional clientCallback); }; } // namespace TestImpact From 787b44db49396487ef295b83d4664d50bcd11f96 Mon Sep 17 00:00:00 2001 From: jonawals Date: Mon, 24 May 2021 14:15:32 +0100 Subject: [PATCH 033/102] Address PR comments --- .../Process/JobRunner/TestImpactProcessJobRunner.h | 2 +- .../Process/Scheduler/TestImpactProcessScheduler.cpp | 11 ++++++----- .../Process/Scheduler/TestImpactProcessScheduler.h | 5 +++-- .../TestEngine/Enumeration/TestImpactTestEnumerator.h | 2 +- .../TestEngine/JobRunner/TestImpactTestJobRunner.h | 2 +- .../TestEngine/Run/TestImpactInstrumentedTestRunner.h | 2 +- .../Code/Source/TestEngine/Run/TestImpactTestRunner.h | 2 +- 7 files changed, 14 insertions(+), 12 deletions(-) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJobRunner.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJobRunner.h index 384111b823..7d726cecc1 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJobRunner.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/JobRunner/TestImpactProcessJobRunner.h @@ -51,7 +51,7 @@ namespace TestImpact public: //! Constructs the job runner with the specified parameters to constrain job runs. //! @param maxConcurrentProcesses he maximum number of concurrent jobs in-flight. - JobRunner(size_t maxConcurrentProcesses); + explicit JobRunner(size_t maxConcurrentProcesses); //! Executes the specified jobs and returns the products of their labor. //! @param jobs The arguments (and other pertinent information) required for each job to be run. diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/Scheduler/TestImpactProcessScheduler.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/Scheduler/TestImpactProcessScheduler.cpp index 8d6c5c6739..065caf54bd 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/Scheduler/TestImpactProcessScheduler.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/Scheduler/TestImpactProcessScheduler.cpp @@ -95,8 +95,9 @@ namespace TestImpact { if (PopAndLaunch(process) == ProcessCallbackResult::Abort) { + // Client chose to abort the scheduler TerminateAllProcesses(ExitCondition::Terminated); - return ProcessSchedulerResult::Graceful; + return ProcessSchedulerResult::UserAborted; } } @@ -144,7 +145,7 @@ namespace TestImpact { // Client chose to abort the scheduler TerminateAllProcesses(ExitCondition::Terminated); - return ProcessSchedulerResult::Graceful; + return ProcessSchedulerResult::UserAborted; } else if (!m_processQueue.empty()) { @@ -153,7 +154,7 @@ namespace TestImpact { // Client chose to abort the scheduler TerminateAllProcesses(ExitCondition::Terminated); - return ProcessSchedulerResult::Graceful; + return ProcessSchedulerResult::UserAborted; } else { @@ -184,7 +185,7 @@ namespace TestImpact { // Client chose to abort the scheduler TerminateAllProcesses(ExitCondition::Terminated); - return ProcessSchedulerResult::Graceful; + return ProcessSchedulerResult::UserAborted; } } @@ -201,7 +202,7 @@ namespace TestImpact { // Client chose to abort the scheduler TerminateAllProcesses(ExitCondition::Terminated); - return ProcessSchedulerResult::Graceful; + return ProcessSchedulerResult::UserAborted; } else { diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/Scheduler/TestImpactProcessScheduler.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/Scheduler/TestImpactProcessScheduler.h index 7f562fb99e..769bf59ff8 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/Scheduler/TestImpactProcessScheduler.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Process/Scheduler/TestImpactProcessScheduler.h @@ -51,9 +51,10 @@ namespace TestImpact }; //! Result of the process scheduling sequence. - enum class ProcessSchedulerResult : bool + enum class ProcessSchedulerResult : AZ::u8 { Graceful, //!< The scheduler completed its run without incident or was terminated gracefully in response to a client callback result. + UserAborted, //!< The scheduler aborted prematurely due to the user returning an abort value from thier callback handler. Timeout //!< The scheduler aborted its run prematurely due to its runtime exceeding the scheduler timeout value. }; @@ -88,7 +89,7 @@ namespace TestImpact public: //! Constructs the scheduler with the specified batch of processes. //! @param maxConcurrentProcesses The maximum number of concurrent processes in-flight. - ProcessScheduler(size_t maxConcurrentProcesses); + explicit ProcessScheduler(size_t maxConcurrentProcesses); ~ProcessScheduler(); //! Executes the specified processes and calls the client callbacks (if any) as each process progresses in its life cycle. diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.h index 287dd9e2d9..fedaf300f7 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.h @@ -74,7 +74,7 @@ namespace TestImpact //! Constructs a test enumerator with the specified parameters common to all enumeration job runs of this enumerator. //! @param maxConcurrentEnumerations The maximum number of enumerations to be in flight at any given time. - TestEnumerator(size_t maxConcurrentEnumerations); + explicit TestEnumerator(size_t maxConcurrentEnumerations); //! Executes the specified test enumeration jobs according to the specified cache and job exception policies. //! @param jobInfos The enumeration jobs to execute. diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobRunner.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobRunner.h index 91f4d4e39c..3078114ef4 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobRunner.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobRunner.h @@ -54,7 +54,7 @@ namespace TestImpact //! Constructs the job runner with the specified parameters common to all job runs of this runner. //! @param maxConcurrentJobs The maximum number of jobs to be in flight at any given time. - TestJobRunner(size_t maxConcurrentJobs); + explicit TestJobRunner(size_t maxConcurrentJobs); protected: //! Runs the specified jobs and returns the completed payloads produced by each job. diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.h index 5545843e35..42b718b241 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.h @@ -54,7 +54,7 @@ namespace TestImpact //! Constructs an instrumented test runner with the specified parameters common to all job runs of this runner. //! @param maxConcurrentRuns The maximum number of runs to be in flight at any given time. - InstrumentedTestRunner(size_t maxConcurrentRuns); + explicit InstrumentedTestRunner(size_t maxConcurrentRuns); //! Executes the specified instrumented test run jobs according to the specified job exception policies. //! @param jobInfos The test run jobs to execute. diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunner.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunner.h index 597ce2d2f8..1fdbc16d58 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunner.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunner.h @@ -27,7 +27,7 @@ namespace TestImpact public: //! Constructs a test runner with the specified parameters common to all job runs of this runner. //! @param maxConcurrentRuns The maximum number of runs to be in flight at any given time. - TestRunner(size_t maxConcurrentRuns); + explicit TestRunner(size_t maxConcurrentRuns); //! Executes the specified test run jobs according to the specified job exception policies. //! @param jobInfos The test run jobs to execute. From bdbbedda71c9e60c5c3cc200b9c130fd249811fc Mon Sep 17 00:00:00 2001 From: jonawals Date: Mon, 24 May 2021 15:00:21 +0100 Subject: [PATCH 034/102] Add test engine --- .../TestImpactTestEnumerationSerializer.cpp | 4 +- .../Enumeration/TestImpactTestEnumerator.cpp | 71 ++-- .../Enumeration/TestImpactTestEnumerator.h | 4 +- .../Run/TestImpactInstrumentedTestRunner.cpp | 29 +- .../Run/TestImpactInstrumentedTestRunner.h | 14 - .../Run/TestImpactTestRunSerializer.cpp | 4 +- .../TestEngine/Run/TestImpactTestRunner.cpp | 11 +- .../TestEngine/TestImpactTestEngine.cpp | 382 ++++++++++++++++++ .../Source/TestEngine/TestImpactTestEngine.h | 126 ++++++ .../TestImpactTestEngineEnumeration.cpp | 27 ++ .../TestImpactTestEngineEnumeration.h | 32 ++ ...tion.h => TestImpactTestEngineException.h} | 4 +- .../TestImpactTestEngineInstrumentedRun.cpp | 52 +++ .../TestImpactTestEngineInstrumentedRun.h | 32 ++ .../TestEngine/TestImpactTestEngineJob.cpp | 40 ++ .../TestEngine/TestImpactTestEngineJob.h | 41 ++ ...n.h => TestImpactTestEngineRegularRun.cpp} | 19 +- .../TestImpactTestEngineRegularRun.h | 34 ++ 18 files changed, 820 insertions(+), 106 deletions(-) create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineEnumeration.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineEnumeration.h rename Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/{Run/TestImpactTestRunException.h => TestImpactTestEngineException.h} (86%) create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineInstrumentedRun.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineInstrumentedRun.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJob.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJob.h rename Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/{Enumeration/TestImpactTestEnumerationException.h => TestImpactTestEngineRegularRun.cpp} (61%) create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineRegularRun.h diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerationSerializer.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerationSerializer.cpp index 05a13a0703..d1e7925cc0 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerationSerializer.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerationSerializer.cpp @@ -10,7 +10,7 @@ * */ -#include +#include #include #include @@ -82,7 +82,7 @@ namespace TestImpact if (doc.Parse<0>(testEnumString.c_str()).HasParseError()) { - throw TestEnumerationException("Could not parse enumeration data"); + throw TestEngineException("Could not parse enumeration data"); } for (const auto& suite : doc[TestEnumFields::Keys[TestEnumFields::SuitesKey]].GetArray()) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.cpp index 09cea87d41..f7b6d4a264 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.cpp @@ -13,7 +13,7 @@ #include #include -#include +#include #include #include #include @@ -36,7 +36,7 @@ namespace TestImpact AZ::IO::SystemFile::SF_OPEN_CREATE | AZ::IO::SystemFile::SF_OPEN_CREATE_PATH | AZ::IO::SystemFile::SF_OPEN_WRITE_ONLY)) { AZ_TestImpact_Eval( - !IsFlagSet(cacheExceptionPolicy, Bitwise::CacheExceptionPolicy::OnCacheWriteFailure), TestEnumerationException, + !IsFlagSet(cacheExceptionPolicy, Bitwise::CacheExceptionPolicy::OnCacheWriteFailure), TestEngineException, "Couldn't open cache file for writing"); return; } @@ -44,50 +44,16 @@ namespace TestImpact if (cacheFile.Write(cacheBytes.data(), cacheBytes.size()) == 0) { AZ_TestImpact_Eval( - !IsFlagSet(cacheExceptionPolicy, Bitwise::CacheExceptionPolicy::OnCacheWriteFailure), TestEnumerationException, + !IsFlagSet(cacheExceptionPolicy, Bitwise::CacheExceptionPolicy::OnCacheWriteFailure), TestEngineException, "Couldn't write cache file data"); return; } } - - AZStd::optional ReadCacheFile(const RepoPath& path, Bitwise::CacheExceptionPolicy cacheExceptionPolicy) - { - AZ::IO::SystemFile cacheFile; - AZStd::string cacheJSON; - if (!cacheFile.Open(path.c_str(), AZ::IO::SystemFile::SF_OPEN_READ_ONLY)) - { - AZ_TestImpact_Eval( - !IsFlagSet(cacheExceptionPolicy, Bitwise::CacheExceptionPolicy::OnCacheNotExist), TestEnumerationException, - "Couldn't locate cache file"); - return AZStd::nullopt; - } - - const AZ::IO::SystemFile::SizeType length = cacheFile.Length(); - if (length == 0) - { - AZ_TestImpact_Eval( - !IsFlagSet(cacheExceptionPolicy, Bitwise::CacheExceptionPolicy::OnCacheReadFailure), TestEnumerationException, - "Cache file is empty"); - return AZStd::nullopt; - } - - cacheFile.Seek(0, AZ::IO::SystemFile::SF_SEEK_BEGIN); - cacheJSON.resize(length); - if (cacheFile.Read(length, cacheJSON.data()) != length) - { - AZ_TestImpact_Eval( - !IsFlagSet(cacheExceptionPolicy, Bitwise::CacheExceptionPolicy::OnCacheReadFailure), TestEnumerationException, - "Couldn't read cache file"); - return AZStd::nullopt; - } - - return DeserializeTestEnumeration(cacheJSON); - } } // namespace TestEnumeration ParseTestEnumerationFile(const RepoPath& enumerationFile) { - return TestEnumeration(GTest::TestEnumerationSuitesFactory(ReadFileContents(enumerationFile))); + return TestEnumeration(GTest::TestEnumerationSuitesFactory(ReadFileContents(enumerationFile))); } TestEnumerationJobData::TestEnumerationJobData(const RepoPath& enumerationArtifact, AZStd::optional&& cache) @@ -128,7 +94,16 @@ namespace TestImpact if (jobInfo.GetCache().has_value() && jobInfo.GetCache()->m_policy == JobData::CachePolicy::Read) { JobMeta meta; - auto enumeration = ReadCacheFile(jobInfo.GetCache()->m_file, cacheExceptionPolicy); + AZStd::optional enumeration; + + try + { + enumeration = TestEnumeration(DeserializeTestEnumeration(ReadFileContents(jobInfo.GetCache()->m_file))); + } + catch (const TestEngineException& e) + { + AZ_Printf("Enumerate", "Enumeration cache error: %s", e.what()); + } // Even though cached jobs don't get executed we still give the client the opportunity to handle the job state // change in order to make the caching process transparent to the client @@ -164,12 +139,20 @@ namespace TestImpact const auto& [meta, jobInfo] = jobData; if (meta.m_result == JobResult::ExecutedWithSuccess) { - const auto& enumeration = (enumerations[jobId] = ParseTestEnumerationFile(jobInfo->GetEnumerationArtifactPath())); - - // Write out the enumeration to a cache file if we have a cache write policy for this job - if (jobInfo->GetCache().has_value() && jobInfo->GetCache()->m_policy == JobData::CachePolicy::Write) + try { - WriteCacheFile(enumeration.value(), jobInfo->GetCache()->m_file, cacheExceptionPolicy); + const auto& enumeration = (enumerations[jobId] = ParseTestEnumerationFile(jobInfo->GetEnumerationArtifactPath())); + + // Write out the enumeration to a cache file if we have a cache write policy for this job + if (jobInfo->GetCache().has_value() && jobInfo->GetCache()->m_policy == JobData::CachePolicy::Write) + { + WriteCacheFile(enumeration.value(), jobInfo->GetCache()->m_file, cacheExceptionPolicy); + } + } + catch (const Exception& e) + { + AZ_Warning("Enumerate", false, e.what()); + enumerations[jobId] = AZStd::nullopt; } } } diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.h index fedaf300f7..c802b92529 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.h @@ -57,9 +57,7 @@ namespace TestImpact enum class CacheExceptionPolicy { Never = 0, //! Never throw. - OnCacheNotExist = 1, //! Throw when a cache read policy was in place but a cache file for this job doesn't exist. - OnCacheReadFailure = 1 << 1, //! Throw when a cache read policy is in place but the cache file could not be read. - OnCacheWriteFailure = 1 << 2 //! Throw when a cache write policy is in place but the cache file could not be written. + OnCacheWriteFailure = 1 //! Throw when a cache write policy is in place but the cache file could not be written. }; } // namespace Bitwise diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.cpp index c0373e89f5..3b109c3a77 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.cpp @@ -14,8 +14,8 @@ #include #include +#include #include -#include #include #include @@ -36,18 +36,10 @@ namespace TestImpact InstrumentedTestRunner::JobPayload ParseTestRunAndCoverageFiles( const RepoPath& runFile, const RepoPath& coverageFile, - AZStd::chrono::milliseconds duration, - InstrumentedTestRunner::CoverageExceptionPolicy coverageExceptionPolicy) + AZStd::chrono::milliseconds duration) { - TestRun run(GTest::TestRunSuitesFactory(ReadFileContents(runFile)), duration); - AZStd::vector moduleCoverages = Cobertura::ModuleCoveragesFactory(ReadFileContents(coverageFile)); - if (moduleCoverages.empty()) - { - AZ_TestImpact_Eval( - !IsFlagSet(coverageExceptionPolicy, Bitwise::CoverageExceptionPolicy::OnEmptyCoverage), TestRunException, - AZStd::string::format("No coverage data generated for '%s'", coverageFile.c_str())); - } - + TestRun run(GTest::TestRunSuitesFactory(ReadFileContents(runFile)), duration); + AZStd::vector moduleCoverages = Cobertura::ModuleCoveragesFactory(ReadFileContents(coverageFile)); TestCoverage coverage(AZStd::move(moduleCoverages)); return {AZStd::move(run), AZStd::move(coverage)}; } @@ -59,13 +51,12 @@ namespace TestImpact AZStd::pair> InstrumentedTestRunner::RunInstrumentedTests( const AZStd::vector& jobInfos, - CoverageExceptionPolicy coverageExceptionPolicy, JobExceptionPolicy jobExceptionPolicy, AZStd::optional runTimeout, AZStd::optional runnerTimeout, AZStd::optional clientCallback) { - const auto payloadGenerator = [this, coverageExceptionPolicy](const JobDataMap& jobDataMap) + const auto payloadGenerator = [this](const JobDataMap& jobDataMap) { PayloadMap runs; for (const auto& [jobId, jobData] : jobDataMap) @@ -78,18 +69,12 @@ namespace TestImpact runs[jobId] = ParseTestRunAndCoverageFiles( jobInfo->GetRunArtifactPath(), jobInfo->GetCoverageArtifactPath(), - meta.m_duration.value(), - coverageExceptionPolicy); + meta.m_duration.value()); } catch (const Exception& e) { - AZ_Warning("RunInstrumentedTests", false, e.what()); + AZ_Printf("RunInstrumentedTests", e.what()); runs[jobId] = AZStd::nullopt; - - if (coverageExceptionPolicy == CoverageExceptionPolicy::OnEmptyCoverage) - { - break; - } } } } diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.h index 42b718b241..b2d7f84b6d 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.h @@ -33,16 +33,6 @@ namespace TestImpact RepoPath m_coverageArtifact; //!< Path to coverage data. }; - namespace Bitwise - { - //! Exception policy for test coverage artifacts. - enum class CoverageExceptionPolicy - { - Never = 0, //! Never throw. - OnEmptyCoverage = 1 //! Throw when no coverage data was produced. - }; - } // namespace Bitwise - //! Runs a batch of test targets to determine the test coverage and passes/failures. class InstrumentedTestRunner : public TestJobRunner> @@ -50,15 +40,12 @@ namespace TestImpact using JobRunner = TestJobRunner>; public: - using CoverageExceptionPolicy = Bitwise::CoverageExceptionPolicy; - //! Constructs an instrumented test runner with the specified parameters common to all job runs of this runner. //! @param maxConcurrentRuns The maximum number of runs to be in flight at any given time. explicit InstrumentedTestRunner(size_t maxConcurrentRuns); //! Executes the specified instrumented test run jobs according to the specified job exception policies. //! @param jobInfos The test run jobs to execute. - //! @param CoverageExceptionPolicy The coverage exception policy to be used for this run. //! @param jobExceptionPolicy The test run job exception policy to be used for this run (use //! TestJobExceptionPolicy::OnFailedToExecute to throw on test failures). //! @param runTimeout The maximum duration a run may be in-flight for before being forcefully terminated. @@ -67,7 +54,6 @@ namespace TestImpact //! @return The result of the run sequence and the instrumented run jobs with their associated test run and coverage payloads. AZStd::pair> RunInstrumentedTests( const AZStd::vector& jobInfos, - CoverageExceptionPolicy coverageExceptionPolicy, JobExceptionPolicy jobExceptionPolicy, AZStd::optional runTimeout, AZStd::optional runnerTimeout, diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunSerializer.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunSerializer.cpp index 7fe8789d22..15d3cc04b1 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunSerializer.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunSerializer.cpp @@ -10,7 +10,7 @@ * */ -#include +#include #include #include @@ -142,7 +142,7 @@ namespace TestImpact if (doc.Parse<0>(testEnumString.c_str()).HasParseError()) { - throw TestRunException("Could not parse enumeration data"); + throw TestEngineException("Could not parse enumeration data"); } // Run duration diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunner.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunner.cpp index 27780a5f95..401084c934 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunner.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunner.cpp @@ -13,7 +13,7 @@ #include #include -#include +#include #include #include @@ -21,11 +21,6 @@ namespace TestImpact { - TestRun ParseTestRunFile(const RepoPath& runFile, AZStd::chrono::milliseconds duration) - { - return TestRun(GTest::TestRunSuitesFactory(ReadFileContents(runFile)), duration); - } - TestRunner::TestRunner(size_t maxConcurrentRuns) : JobRunner(maxConcurrentRuns) { @@ -48,11 +43,11 @@ namespace TestImpact { try { - runs[jobId] = ParseTestRunFile(jobInfo->GetRunArtifactPath(), meta.m_duration.value()); + runs[jobId] = TestRun(GTest::TestRunSuitesFactory(ReadFileContents(jobInfo->GetRunArtifactPath())), meta.m_duration.value()); } catch (const Exception& e) { - AZ_Warning("RunTests", false, e.what()); + AZ_Printf("RunTests", e.what()); runs[jobId] = AZStd::nullopt; } } diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.cpp new file mode 100644 index 0000000000..5046b3f8ee --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.cpp @@ -0,0 +1,382 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace TestImpact +{ + // Known error codes for test instrumentation, test runner and unit test library + // This could be refactored into a generic solution agnostic of the tool and library specific details + namespace ErrorCodes + { + namespace OpenCppCoverage + { + static constexpr ReturnCode InvalidArgs = -1618178468; + } + + namespace GTest + { + static constexpr ReturnCode Unsuccessful = 1; + } + + namespace AZTestRunner + { + static constexpr ReturnCode InvalidArgs = 101; + static constexpr ReturnCode FailedToFindTargetBinary = 102; + static constexpr ReturnCode SymbolNotFound = 103; + static constexpr ReturnCode ModuleSkipped = 104; + } + } + + namespace + { + // Calculate the sequence result by analysing the state of the test targets that were run. + template + TestSequenceResult CalculateSequenceResult( + ProcessSchedulerResult result, + const AZStd::vector& engineJobs, + Policy::ExecutionFailure executionFailurePolicy) + { + if (result == ProcessSchedulerResult::Timeout) + { + // Test job runner timing out overrules all other possible sequence results + return TestSequenceResult::Timeout; + } + + bool hasExecutionFailures = false; + bool hasTestFailures = false; + for (const auto& engineJob : engineJobs) + { + switch (engineJob.GetTestResult()) + { + case Client::TestRunResult::FailedToExecute: + { + hasExecutionFailures = true; + break; + } + case Client::TestRunResult::Timeout: + case Client::TestRunResult::TestFailures: + { + hasTestFailures = true; + break; + } + default: + { + continue; + } + } + } + + // Execution failure can be considered test passes if a permissive execution failure policy is used, otherwise they are failures + if ((hasExecutionFailures && executionFailurePolicy != Policy::ExecutionFailure::Ignore) || hasTestFailures) + { + return TestSequenceResult::Failure; + } + else + { + return TestSequenceResult::Success; + } + } + + // Deduces the run result for a given test target based on how the process exited and known return values + Client::TestRunResult GetClientTestRunResultForMeta(const JobMeta& meta) + { + // Attempt to determine why a given test target executed successfully but return with an error code + if (meta.m_returnCode.has_value()) + { + switch (meta.m_returnCode.value()) + { + // We will consider test targets that technically execute but their launcher or unit test library return a know error + // code that pertains to incorrect argument usage as test targets that failed to execute + case ErrorCodes::OpenCppCoverage::InvalidArgs: + case ErrorCodes::AZTestRunner::InvalidArgs: + case ErrorCodes::AZTestRunner::FailedToFindTargetBinary: + case ErrorCodes::AZTestRunner::ModuleSkipped: + case ErrorCodes::AZTestRunner::SymbolNotFound: + return Client::TestRunResult::FailedToExecute; + // The trivial case: the test target has failing tests + case ErrorCodes::GTest::Unsuccessful: + return Client::TestRunResult::TestFailures; + default: + break; + } + } + + switch (meta.m_result) + { + // If the test target executed successfully but returned in an unknown abnormal state it's probably because a test caused + // an unhandled exception, segfault or any other of the weird and wonderful ways a badly behaving test can terminate + case JobResult::ExecutedWithFailure: + return Client::TestRunResult::TestFailures; + // The trivial case: all of the tests in the test target passed + case JobResult::ExecutedWithSuccess: + return Client::TestRunResult::AllTestsPass; + // NotExecuted happens when a test is queued for launch but the test runner terminates the sequence (either due to client abort + // or due to the sequence timer expiring) whereas Terminated happens when the aforementioned scenarios happen when the test target + // is in flight + case JobResult::NotExecuted: + case JobResult::Terminated: + return Client::TestRunResult::NotRun; + // The individual timer for the test target expired + case JobResult::Timeout: + return Client::TestRunResult::Timeout; + default: + throw(TestEngineException(AZStd::string::format("Unexpected job result: %u", static_cast(meta.m_result)))); + } + } + + // Map for storing the test engine job data of completed test target runs + template + using TestEngineJobMap = AZStd::unordered_map; + + // Helper trait for identifying the test engine job specialization for a given test job runner + template + struct TestJobRunnerTrait + {}; + + // Helper function for getting the type directly of the test job runner trait + template + using TestEngineJobType = typename TestJobRunnerTrait::TestEngineJobType; + + // Type trait for the test enumerator + template<> + struct TestJobRunnerTrait + { + using TestEngineJobType = TestEngineEnumeration; + }; + + // Type trait for the test runner + template<> + struct TestJobRunnerTrait + { + using TestEngineJobType = TestEngineRegularRun; + }; + + // Type trait for the instrumented test runner + template<> + struct TestJobRunnerTrait + { + using TestEngineJobType = TestEngineInstrumentedRun; + }; + + // Functor for handling test job runner callbacks + template + class TestJobRunnerCallbackHandler + { + using IdType = typename TestJobRunner::JobInfo::IdType; + using JobInfo = typename TestJobRunner::JobInfo; + public: + TestJobRunnerCallbackHandler( + const AZStd::vector& testTargets, + TestEngineJobMap* engineJobs, + AZStd::optional* callback) + : m_testTargets(testTargets) + , m_engineJobs(engineJobs) + , m_callback(callback) + { + } + + void operator()(const typename JobInfo& jobInfo, const TestImpact::JobMeta& meta) + { + const auto id = jobInfo.GetId().m_value; + const auto& args = jobInfo.GetCommand().m_args; + const auto* target = m_testTargets[id]; + const auto result = GetClientTestRunResultForMeta(meta); + + // Place the test engine job associated with this test run into the map along with its client test run result so + // that it can be retrieved when the sequence has ended (and any associated artifacts processed) + const auto& [it, success] = m_engineJobs->emplace(id, TestEngineJob(target, args, meta, result)); + + if (m_callback->has_value()) + { + (*m_callback).value()(it->second); + } + } + + private: + const AZStd::vector& m_testTargets; + TestEngineJobMap* m_engineJobs; + AZStd::optional* m_callback; + }; + + // Helper function to compile the run type specific test engine jobs from their associated jobs and payloads + template + AZStd::vector> CompileTestEngineRuns( + const AZStd::vector& testTargets, + AZStd::vector& runnerjobs, + TestEngineJobMap&& engineJobs) + { + AZStd::vector> engineRuns; + engineRuns.reserve(testTargets.size()); + + for (auto& job : runnerjobs) + { + const auto id = job.GetJobInfo().GetId().m_value; + if (auto it = engineJobs.find(id); + it != engineJobs.end()) + { + // An entry in the test engine job map means that this job was acted upon (an attempt to execute, successful or otherwise) + auto& engineJob = it->second; + TestEngineJobType run(AZStd::move(engineJob), job.ReleasePayload()); + engineRuns.push_back(AZStd::move(run)); + } + else + { + // No entry in the test engine job map means that this job never had the opportunity to be acted upon (the sequence + // was terminated whilst this job was still queued up for execution) + const auto& args = job.GetJobInfo().GetCommand().m_args; + const auto* target = testTargets[id]; + TestEngineJobType run(TestEngineJob(target, args, {}, Client::TestRunResult::NotRun), {}); + engineRuns.push_back(AZStd::move(run)); + } + } + + return engineRuns; + } + + Bitwise::TestJobExceptionPolicy GetTestJobExceptionPolicy( + Policy::ExecutionFailure executionFailurePolicy, Policy::TestFailure testFailurePolicy) + { + auto jobExecutionPolicy = Bitwise::TestJobExceptionPolicy::Never; + if (executionFailurePolicy == Policy::ExecutionFailure::Abort) + { + jobExecutionPolicy |= Bitwise::TestJobExceptionPolicy::OnFailedToExecute; + } + + if (testFailurePolicy == Policy::TestFailure::Abort) + { + jobExecutionPolicy |= Bitwise::TestJobExceptionPolicy::OnExecutedWithFailure; + } + + return jobExecutionPolicy; + } + } + + TestEngine::TestEngine( + const RepoPath& sourceDir, + const RepoPath& targetBinaryDir, + const RepoPath& cacheDir, + const RepoPath& artifactDir, + const RepoPath& testRunnerBinary, + const RepoPath& instrumentBinary, + size_t maxConcurrentRuns) + : m_maxConcurrentRuns(maxConcurrentRuns) + , m_testJobInfoGenerator(AZStd::make_unique( + sourceDir, targetBinaryDir, cacheDir, artifactDir, testRunnerBinary, instrumentBinary)) + , m_testEnumerator(AZStd::make_unique(maxConcurrentRuns)) + , m_instrumentedTestRunner(AZStd::make_unique(maxConcurrentRuns)) + , m_testRunner(AZStd::make_unique(maxConcurrentRuns)) + { + } + + TestEngine::~TestEngine() = default; + + AZStd::pair> TestEngine::UpdateEnumerationCache( + const AZStd::vector& testTargets, + Policy::ExecutionFailure executionFailurePolicy, + AZStd::optional testTargetTimeout, + AZStd::optional globalTimeout, + AZStd::optional callback) + { + TestEngineJobMap engineJobs; + const auto jobInfos = m_testJobInfoGenerator->GenerateTestEnumerationJobInfos(testTargets, TestEnumerator::JobInfo::CachePolicy::Write); + + const auto jobExecutionPolicy = executionFailurePolicy == Policy::ExecutionFailure::Abort + ? (TestEnumerator::JobExceptionPolicy::OnExecutedWithFailure | TestEnumerator::JobExceptionPolicy::OnFailedToExecute) + : TestEnumerator::JobExceptionPolicy::Never; + + auto [result, runnerJobs] = m_testEnumerator->Enumerate( + jobInfos, + TestEnumerator::CacheExceptionPolicy::OnCacheWriteFailure, + jobExecutionPolicy, + testTargetTimeout, + globalTimeout, + TestJobRunnerCallbackHandler(testTargets, &engineJobs, &callback)); + + auto engineRuns = CompileTestEngineRuns(testTargets, runnerJobs, AZStd::move(engineJobs)); + return { CalculateSequenceResult(result, engineRuns, executionFailurePolicy), AZStd::move(engineRuns) }; + } + + AZStd::pair> TestEngine::RegularRun( + const AZStd::vector& testTargets, + [[maybe_unused]]Policy::TestSharding testShardingPolicy, + Policy::ExecutionFailure executionFailurePolicy, + Policy::TestFailure testFailurePolicy, + [[maybe_unused]]Policy::TargetOutputCapture targetOutputCapture, + AZStd::optional testTargetTimeout, + AZStd::optional globalTimeout, + AZStd::optional callback) + { + TestEngineJobMap engineJobs; + const auto jobInfos = m_testJobInfoGenerator->GenerateRegularTestRunJobInfos(testTargets); + + TestJobRunnerCallbackHandler jobCallback(testTargets, &engineJobs, &callback); + auto [result, runnerJobs] = m_testRunner->RunTests( + jobInfos, + GetTestJobExceptionPolicy(executionFailurePolicy, testFailurePolicy), + testTargetTimeout, + globalTimeout, + jobCallback); + + auto engineRuns = CompileTestEngineRuns(testTargets, runnerJobs, AZStd::move(engineJobs)); + return { CalculateSequenceResult(result, engineRuns, executionFailurePolicy), AZStd::move(engineRuns) }; + } + + AZStd::pair> TestEngine::InstrumentedRun( + const AZStd::vector& testTargets, + [[maybe_unused]] Policy::TestSharding testShardingPolicy, + Policy::ExecutionFailure executionFailurePolicy, + Policy::IntegrityFailure integrityFailurePolicy, + Policy::TestFailure testFailurePolicy, + [[maybe_unused]]Policy::TargetOutputCapture targetOutputCapture, + AZStd::optional testTargetTimeout, + AZStd::optional globalTimeout, + AZStd::optional callback) + { + TestEngineJobMap engineJobs; + const auto jobInfos = m_testJobInfoGenerator->GenerateInstrumentedTestRunJobInfos(testTargets, CoverageLevel::Source); + + auto [result, runnerJobs] = m_instrumentedTestRunner->RunInstrumentedTests( + jobInfos, + GetTestJobExceptionPolicy(executionFailurePolicy, testFailurePolicy), + testTargetTimeout, + globalTimeout, + TestJobRunnerCallbackHandler(testTargets, &engineJobs, &callback)); + + auto engineRuns = CompileTestEngineRuns(testTargets, runnerJobs, AZStd::move(engineJobs)); + + // Now that we know the true result of successful jobs that return non-zero we can deduce if we have any integrity failures + // where a test target ran and completed its tests without incident yet failed to produce coverage data + if (integrityFailurePolicy == Policy::IntegrityFailure::Abort) + { + for (const auto& engineRun : engineRuns) + { + if (const auto testResult = engineRun.GetTestResult(); + testResult == Client::TestRunResult::AllTestsPass || testResult == Client::TestRunResult::TestFailures) + { + AZ_TestImpact_Eval(engineRun.GetTestCoverge().has_value(), TestEngineException, AZStd::string::format( + "Test target %s completed its test run but failed to produce coverage data", engineRun.GetTestTarget()->GetName().c_str())); + } + } + } + + return { CalculateSequenceResult(result, engineRuns, executionFailurePolicy), AZStd::move(engineRuns) }; + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.h new file mode 100644 index 0000000000..f6fb6cec11 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.h @@ -0,0 +1,126 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include +#include +#include + +#include +#include +#include + +#include +#include + +namespace TestImpact +{ + class TestTarget; + class TestJobInfoGenerator; + class TestEnumerator; + class InstrumentedTestRunner; + class TestRunner; + + //! Callback for when a given test engine job completes. + using TestEngineJobCompleteCallback = AZStd::function; + + //! Provides the front end for performing test enumerations and test runs. + class TestEngine + { + public: + //! Configures the test engine with the necessary path information for launching test targets and managing the artifacts they produce. + //! @param sourceDir Root path where source files are found (including subfolders). + //! @param targetBinaryDir Path to where the test target binaries are found. + //! @param cacheDir Path to the persistent folder where test target enumerations are cached. + //! @param artifactDir Path to the transient directory where test artifacts are produced. + //! @param testRunnerBinary Path to the binary responsible for launching test targets that have the TestRunner launch method. + //! @param instrumentBinary Path to the binary responsible for launching test targets with test coverage instrumentation. + //! @param maxConcurrentRuns The maximum number of concurrent test targets that can be in flight at any given moment. + TestEngine( + const RepoPath& sourceDir, + const RepoPath& targetBinaryDir, + const RepoPath& cacheDir, + const RepoPath& artifactDir, + const RepoPath& testRunnerBinary, + const RepoPath& instrumentBinary, + size_t maxConcurrentRuns); + + ~TestEngine(); + + //! Updates the cached enumerations for the specified test targets. + //! @note Whilst test runs will make use of this cache for test target sharding it is the responsibility of the client to + //! ensure any stale caches are up to date by calling this function. No attempt to maintain internal consistency will be made + //! by the test engine itself. + //! @param testTargets The test targets to enumerate. + //! @param executionFailurePolicy The policy for how enumeration execution failures should be handled. + //! @param testTargetTimeout The maximum duration a test target may be in-flight for before being forcefully terminated (infinite if empty). + //! @param globalTimeout The maximum duration the enumeration sequence may run before being forcefully terminated (infinite if empty). + //! @param callback The client callback function to handle completed test target enumerations. + //! @ returns The sequence result and the enumerations for the target that were enumerated. + AZStd::pair < TestSequenceResult, AZStd::vector> UpdateEnumerationCache( + const AZStd::vector& testTargets, + Policy::ExecutionFailure executionFailurePolicy, + AZStd::optional testTargetTimeout, + AZStd::optional globalTimeout, + AZStd::optional callback); + + //! Performs a test run without any instrumentation and, for each test target, returns the test run results and metrics about the run. + //! @param testTargets The test targets to run. + //! @param testShardingPolicy Test sharding policy to use for test targets in this run. + //! @param executionFailurePolicy Policy for how test execution failures should be handled. + //! @param testFailurePolicy Policy for how test targets with failing tests should be handled. + //! @param targetOutputCapture Policy for how test target standard output should be captured and handled. + //! @param testTargetTimeout The maximum duration a test target may be in-flight for before being forcefully terminated (infinite if empty). + //! @param globalTimeout The maximum duration the enumeration sequence may run before being forcefully terminated (infinite if empty). + //! @param callback The client callback function to handle completed test target runs. + //! @ returns The sequence result and the test run results for the test targets that were run. + [[nodiscard]]AZStd::pair> RegularRun( + const AZStd::vector& testTargets, + Policy::TestSharding testShardingPolicy, + Policy::ExecutionFailure executionFailurePolicy, + Policy::TestFailure testFailurePolicy, + Policy::TargetOutputCapture targetOutputCapture, + AZStd::optional testTargetTimeout, + AZStd::optional globalTimeout, + AZStd::optional callback); + + //! Performs a test run with instrumentation and, for each test target, returns the test run results, coverage data and metrics about the run. + //! @param testTargets The test targets to run. + //! @param testShardingPolicy Test sharding policy to use for test targets in this run. + //! @param executionFailurePolicy Policy for how test execution failures should be handled. + //! @param integrityFailurePolicy Policy for how integrty failures of the test impact data and source tree model should be handled. + //! @param testFailurePolicy Policy for how test targets with failing tests should be handled. + //! @param targetOutputCapture Policy for how test target standard output should be captured and handled. + //! @param testTargetTimeout The maximum duration a test target may be in-flight for before being forcefully terminated (infinite if empty). + //! @param globalTimeout The maximum duration the enumeration sequence may run before being forcefully terminated (infinite if empty). + //! @param callback The client callback function to handle completed test target runs. + //! @ returns The sequence result and the test run results and test coverages for the test targets that were run. + [[nodiscard]]AZStd::pair> InstrumentedRun( + const AZStd::vector& testTargets, + Policy::TestSharding testShardingPolicy, + Policy::ExecutionFailure executionFailurePolicy, + Policy::IntegrityFailure integrityFailurePolicy, + Policy::TestFailure testFailurePolicy, + Policy::TargetOutputCapture targetOutputCapture, + AZStd::optional testTargetTimeout, + AZStd::optional globalTimeout, + AZStd::optional callback); + + private: + size_t m_maxConcurrentRuns = 0; + AZStd::unique_ptr m_testJobInfoGenerator; + AZStd::unique_ptr m_testEnumerator; + AZStd::unique_ptr m_instrumentedTestRunner; + AZStd::unique_ptr m_testRunner; + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineEnumeration.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineEnumeration.cpp new file mode 100644 index 0000000000..3c114d1b33 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineEnumeration.cpp @@ -0,0 +1,27 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include + +namespace TestImpact +{ + TestEngineEnumeration::TestEngineEnumeration(TestEngineJob&& job, AZStd::optional&& enumeration) + : TestEngineJob(AZStd::move(job)) + , m_enumeration(AZStd::move(enumeration)) + { + } + + const AZStd::optional& TestEngineEnumeration::GetTestEnumeration() const + { + return m_enumeration; + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineEnumeration.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineEnumeration.h new file mode 100644 index 0000000000..28473a08bc --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineEnumeration.h @@ -0,0 +1,32 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include +#include + +namespace TestImpact +{ + //! Represents the generated test enumeration data for a test engine enumeration. + class TestEngineEnumeration + : public TestEngineJob + { + public: + TestEngineEnumeration(TestEngineJob&& job, AZStd::optional&& enumeration); + + //! Returns the test enumeration payload for this job (if any). + const AZStd::optional& GetTestEnumeration() const; + private: + AZStd::optional m_enumeration; + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunException.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineException.h similarity index 86% rename from Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunException.h rename to Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineException.h index ec13a17f57..cf10b68b24 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunException.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineException.h @@ -16,8 +16,8 @@ namespace TestImpact { - //! Exception for test runs and test run related operations. - class TestRunException : public Exception + //! Exception for test engine runs and related operations. + class TestEngineException : public Exception { public: using Exception::Exception; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineInstrumentedRun.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineInstrumentedRun.cpp new file mode 100644 index 0000000000..b19ec723d1 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineInstrumentedRun.cpp @@ -0,0 +1,52 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +namespace TestImpact +{ + namespace + { + AZStd::optional ReleaseTestRun(AZStd::optional>&& testRunAndCoverage) + { + if (testRunAndCoverage.has_value()) + { + return AZStd::move(testRunAndCoverage.value().first); + } + + return AZStd::nullopt; + } + + AZStd::optional ReleaseTestCoverage(AZStd::optional>&& testRunAndCoverage) + { + if (testRunAndCoverage.has_value()) + { + return AZStd::move(testRunAndCoverage.value().second); + } + + return AZStd::nullopt; + } + } + + TestEngineInstrumentedRun::TestEngineInstrumentedRun(TestEngineJob&& testJob, AZStd::optional>&& testRunAndCoverage) + : TestEngineRegularRun(AZStd::move(testJob), ReleaseTestRun(AZStd::move(testRunAndCoverage))) + , m_testCoverage(ReleaseTestCoverage(AZStd::move(testRunAndCoverage))) + { + } + + const AZStd::optional& TestEngineInstrumentedRun::GetTestCoverge() const + { + return m_testCoverage; + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineInstrumentedRun.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineInstrumentedRun.h new file mode 100644 index 0000000000..b1edd943a4 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineInstrumentedRun.h @@ -0,0 +1,32 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include +#include + +namespace TestImpact +{ + //! Represents the generated test run and coverage data for an instrumented regular test engine run. + class TestEngineInstrumentedRun + : public TestEngineRegularRun + { + public: + TestEngineInstrumentedRun(TestEngineJob&& testJob, AZStd::optional>&& testRunAndCoverage); + + //! Returns the test coverage payload for this job (if any). + const AZStd::optional& GetTestCoverge() const; + private: + AZStd::optional m_testCoverage; + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJob.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJob.cpp new file mode 100644 index 0000000000..26aa71adcd --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJob.cpp @@ -0,0 +1,40 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include + +namespace TestImpact +{ + TestEngineJob::TestEngineJob(const TestTarget* testTarget, const AZStd::string& commandString, const JobMeta& jobMeta, Client::TestRunResult testResult) + : JobMetaWrapper(jobMeta) + , m_testTarget(testTarget) + , m_commandString(commandString) + , m_testResult(testResult) + { + } + + const TestTarget* TestEngineJob::GetTestTarget() const + { + return m_testTarget; + } + + const AZStd::string& TestEngineJob::GetCommandString() const + { + return m_commandString; + } + + Client::TestRunResult TestEngineJob::GetTestResult() const + { + return m_testResult; + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJob.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJob.h new file mode 100644 index 0000000000..00d0c7d099 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJob.h @@ -0,0 +1,41 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +namespace TestImpact +{ + class TestTarget; + + //! Represents the meta-data describing a test engine run. + class TestEngineJob + : public JobMetaWrapper + { + public: + TestEngineJob(const TestTarget* testTarget, const AZStd::string& commandString, const JobMeta& jobMeta, Client::TestRunResult testResult); + + //! Returns the test target that was run for this job. + const TestTarget* GetTestTarget() const; + + //! Returns the result of the job that was run. + Client::TestRunResult GetTestResult() const; + + //! Returns the command string that was used to execute this job. + const AZStd::string& GetCommandString() const; + private: + const TestTarget* m_testTarget; + AZStd::string m_commandString; + Client::TestRunResult m_testResult; + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerationException.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineRegularRun.cpp similarity index 61% rename from Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerationException.h rename to Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineRegularRun.cpp index 0c9d27df7a..0090689e8b 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerationException.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineRegularRun.cpp @@ -10,17 +10,18 @@ * */ -#pragma once - -#include +#include namespace TestImpact { - //! Exception for test enumerations and test enumeration related operations. - class TestEnumerationException - : public Exception + TestEngineRegularRun::TestEngineRegularRun(TestEngineJob&& testJob, AZStd::optional&& testRun) + : TestEngineJob(AZStd::move(testJob)) + , m_testRun(AZStd::move(testRun)) { - public: - using Exception::Exception; - }; + } + + const AZStd::optional& TestEngineRegularRun::GetTestRun() const + { + return m_testRun; + } } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineRegularRun.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineRegularRun.h new file mode 100644 index 0000000000..3c65830094 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineRegularRun.h @@ -0,0 +1,34 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +#include +#include + +namespace TestImpact +{ + //! Represents the generated test run data for a regular test engine run. + class TestEngineRegularRun + : public TestEngineJob + { + public: + TestEngineRegularRun(TestEngineJob&& testJob, AZStd::optional&& testRun); + + //! Returns the test run payload for this job (if any). + const AZStd::optional& GetTestRun() const; + private: + AZStd::optional m_testRun; + }; +} // namespace TestImpact From 0ef519d11d5b7d7fd6d3924a88f469afd0088dea Mon Sep 17 00:00:00 2001 From: jonawals Date: Tue, 25 May 2021 12:55:11 +0100 Subject: [PATCH 035/102] Address PR comments --- .../Runtime/Code/Source/TestEngine/TestImpactTestEngine.h | 6 +++--- .../TestEngine/TestImpactTestEngineInstrumentedRun.cpp | 8 ++++---- .../TestEngine/TestImpactTestEngineInstrumentedRun.h | 1 + .../Code/Source/TestEngine/TestImpactTestEngineJob.h | 1 + 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.h index f6fb6cec11..c67a10aa4c 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.h @@ -67,7 +67,7 @@ namespace TestImpact //! @param globalTimeout The maximum duration the enumeration sequence may run before being forcefully terminated (infinite if empty). //! @param callback The client callback function to handle completed test target enumerations. //! @ returns The sequence result and the enumerations for the target that were enumerated. - AZStd::pair < TestSequenceResult, AZStd::vector> UpdateEnumerationCache( + AZStd::pair> UpdateEnumerationCache( const AZStd::vector& testTargets, Policy::ExecutionFailure executionFailurePolicy, AZStd::optional testTargetTimeout, @@ -84,7 +84,7 @@ namespace TestImpact //! @param globalTimeout The maximum duration the enumeration sequence may run before being forcefully terminated (infinite if empty). //! @param callback The client callback function to handle completed test target runs. //! @ returns The sequence result and the test run results for the test targets that were run. - [[nodiscard]]AZStd::pair> RegularRun( + [[nodiscard]] AZStd::pair> RegularRun( const AZStd::vector& testTargets, Policy::TestSharding testShardingPolicy, Policy::ExecutionFailure executionFailurePolicy, @@ -105,7 +105,7 @@ namespace TestImpact //! @param globalTimeout The maximum duration the enumeration sequence may run before being forcefully terminated (infinite if empty). //! @param callback The client callback function to handle completed test target runs. //! @ returns The sequence result and the test run results and test coverages for the test targets that were run. - [[nodiscard]]AZStd::pair> InstrumentedRun( + [[nodiscard]] AZStd::pair> InstrumentedRun( const AZStd::vector& testTargets, Policy::TestSharding testShardingPolicy, Policy::ExecutionFailure executionFailurePolicy, diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineInstrumentedRun.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineInstrumentedRun.cpp index b19ec723d1..20c80c5f5f 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineInstrumentedRun.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineInstrumentedRun.cpp @@ -18,7 +18,7 @@ namespace TestImpact { namespace { - AZStd::optional ReleaseTestRun(AZStd::optional>&& testRunAndCoverage) + AZStd::optional ReleaseTestRun(AZStd::optional>& testRunAndCoverage) { if (testRunAndCoverage.has_value()) { @@ -28,7 +28,7 @@ namespace TestImpact return AZStd::nullopt; } - AZStd::optional ReleaseTestCoverage(AZStd::optional>&& testRunAndCoverage) + AZStd::optional ReleaseTestCoverage(AZStd::optional>& testRunAndCoverage) { if (testRunAndCoverage.has_value()) { @@ -40,8 +40,8 @@ namespace TestImpact } TestEngineInstrumentedRun::TestEngineInstrumentedRun(TestEngineJob&& testJob, AZStd::optional>&& testRunAndCoverage) - : TestEngineRegularRun(AZStd::move(testJob), ReleaseTestRun(AZStd::move(testRunAndCoverage))) - , m_testCoverage(ReleaseTestCoverage(AZStd::move(testRunAndCoverage))) + : TestEngineRegularRun(AZStd::move(testJob), ReleaseTestRun(testRunAndCoverage)) + , m_testCoverage(ReleaseTestCoverage(testRunAndCoverage)) { } diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineInstrumentedRun.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineInstrumentedRun.h index b1edd943a4..efecebcbd4 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineInstrumentedRun.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineInstrumentedRun.h @@ -26,6 +26,7 @@ namespace TestImpact //! Returns the test coverage payload for this job (if any). const AZStd::optional& GetTestCoverge() const; + private: AZStd::optional m_testCoverage; }; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJob.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJob.h index 00d0c7d099..753a4cd494 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJob.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJob.h @@ -33,6 +33,7 @@ namespace TestImpact //! Returns the command string that was used to execute this job. const AZStd::string& GetCommandString() const; + private: const TestTarget* m_testTarget; AZStd::string m_commandString; From 828aeb2147e73b25fbde5ce890fd89ee9c03137d Mon Sep 17 00:00:00 2001 From: jonawals Date: Tue, 25 May 2021 13:20:43 +0100 Subject: [PATCH 036/102] Add JobInfoGenerator --- .../TestImpactWin32_TestTargetExtension.cpp | 46 +++++ .../Windows/platform_windows_files.cmake | 1 + .../TestImpactTestJobInfoGenerator.cpp | 191 ++++++++++++++++++ .../TestImpactTestJobInfoGenerator.h | 108 ++++++++++ .../JobRunner/TestImpactTestTargetExtension.h | 23 +++ 5 files changed, 369 insertions(+) create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/TestEngine/JobRunner/TestImpactWin32_TestTargetExtension.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobInfoGenerator.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobInfoGenerator.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestTargetExtension.h diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/TestEngine/JobRunner/TestImpactWin32_TestTargetExtension.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/TestEngine/JobRunner/TestImpactWin32_TestTargetExtension.cpp new file mode 100644 index 0000000000..bd380ac2a7 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/TestEngine/JobRunner/TestImpactWin32_TestTargetExtension.cpp @@ -0,0 +1,46 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include +#include +#include + +namespace TestImpact +{ + AZStd::string GetTestTargetExtension(const TestTarget* testTarget) + { + static constexpr char* const standAloneExtension = ".exe"; + static constexpr char* const testRunnerExtension = ".dll"; + + switch (const auto launchMethod = testTarget->GetLaunchMethod(); launchMethod) + { + case LaunchMethod::StandAlone: + { + return standAloneExtension; + } + case LaunchMethod::TestRunner: + { + return testRunnerExtension; + } + default: + { + throw TestEngineException( + AZStd::string::format( + "Unexpected launch method for target %s: %u", + testTarget->GetName().c_str(), + aznumeric_cast(launchMethod))); + } + } + } +} diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/platform_windows_files.cmake b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/platform_windows_files.cmake index 32f996cdf8..5f5e8fc3d1 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/platform_windows_files.cmake +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/platform_windows_files.cmake @@ -16,4 +16,5 @@ set(FILES Process/TestImpactWin32_Handle.h Process/TestImpactWin32_Pipe.cpp Process/TestImpactWin32_Pipe.h + TestEngine/JobRunner/TestImpactWin32_TestTargetExtension.cpp ) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobInfoGenerator.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobInfoGenerator.cpp new file mode 100644 index 0000000000..0cf717e88e --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobInfoGenerator.cpp @@ -0,0 +1,191 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include +#include + +namespace TestImpact +{ + TestJobInfoGenerator::TestJobInfoGenerator( + const RepoPath& sourceDir, + const RepoPath& targetBinaryDir, + const RepoPath& cacheDir, + const RepoPath& artifactDir, + const RepoPath& testRunnerBinary, + const RepoPath& instrumentBinary) + : m_sourceDir(sourceDir) + , m_targetBinaryDir(targetBinaryDir) + , m_cacheDir(cacheDir) + , m_artifactDir(artifactDir) + , m_testRunnerBinary(testRunnerBinary) + , m_instrumentBinary(instrumentBinary) + { + } + + AZStd::string TestJobInfoGenerator::GenerateLaunchArgument(const TestTarget* testTarget) const + { + if (testTarget->GetLaunchMethod() == LaunchMethod::StandAlone) + { + return AZStd::string::format( + "%s%s %s", + (m_targetBinaryDir / testTarget->GetOutputName()).c_str(), + GetTestTargetExtension(testTarget).c_str(), + testTarget->GetCustomArgs().c_str()).c_str(); + } + else + { + return AZStd::string::format( + "\"%s\" \"%s%s\" %s", + m_testRunnerBinary.c_str(), + (m_targetBinaryDir / testTarget->GetOutputName()).c_str(), + GetTestTargetExtension(testTarget).c_str(), + testTarget->GetCustomArgs().c_str()).c_str(); + } + } + + RepoPath TestJobInfoGenerator::GenerateTargetEnumerationCacheFilePath(const TestTarget* testTarget) const + { + return AZStd::string::format("%s.cache", (m_artifactDir / testTarget->GetName()).c_str()); + } + + RepoPath TestJobInfoGenerator::GenerateTargetEnumerationArtifactFilePath(const TestTarget* testTarget) const + { + return AZStd::string::format("%s.Enumeration.xml", (m_artifactDir / testTarget->GetName()).c_str()); + } + + RepoPath TestJobInfoGenerator::GenerateTargetRunArtifactFilePath(const TestTarget* testTarget) const + { + return AZStd::string::format("%s.Run.xml", (m_artifactDir / testTarget->GetName()).c_str()); + } + + RepoPath TestJobInfoGenerator::GenerateTargetCoverageArtifactFilePath(const TestTarget* testTarget) const + { + return AZStd::string::format("%s.Coverage.xml", (m_artifactDir / testTarget->GetName()).c_str()); + } + + TestEnumerator::JobInfo TestJobInfoGenerator::GenerateTestEnumerationJobInfo( + const TestTarget* testTarget, + TestEnumerator::JobInfo::Id jobId, + TestEnumerator::JobInfo::CachePolicy cachePolicy) const + { + using Command = TestEnumerator::Command; + using JobInfo = TestEnumerator::JobInfo; + using JobData = TestEnumerator::JobData; + using Cache = TestEnumerator::JobData::Cache; + + const auto enumerationArtifact = GenerateTargetEnumerationArtifactFilePath(testTarget); + const Command args = + { + AZStd::string::format( + "%s --gtest_list_tests --gtest_output=xml:\"%s\"", + GenerateLaunchArgument(testTarget).c_str(), + enumerationArtifact.c_str()) + }; + + return JobInfo(jobId, args, JobData(enumerationArtifact, Cache{ cachePolicy, GenerateTargetEnumerationCacheFilePath(testTarget) })); + } + + TestRunner::JobInfo TestJobInfoGenerator::GenerateRegularTestRunJobInfo( + const TestTarget* testTarget, + TestRunner::JobInfo::Id jobId) const + { + using Command = TestRunner::Command; + using JobInfo = TestRunner::JobInfo; + using JobData = TestRunner::JobData; + + const auto runArtifact = GenerateTargetRunArtifactFilePath(testTarget); + const Command args = + { + AZStd::string::format( + "%s --gtest_output=xml:\"%s\"", + GenerateLaunchArgument(testTarget).c_str(), + runArtifact.c_str()) + }; + + return JobInfo(jobId, args, JobData(runArtifact)); + } + + InstrumentedTestRunner::JobInfo TestJobInfoGenerator::GenerateInstrumentedTestRunJobInfo( + const TestTarget* testTarget, + InstrumentedTestRunner::JobInfo::Id jobId, + CoverageLevel coverageLevel) const + { + using Command = InstrumentedTestRunner::Command; + using JobInfo = InstrumentedTestRunner::JobInfo; + using JobData = InstrumentedTestRunner::JobData; + + const auto coverageArtifact = GenerateTargetCoverageArtifactFilePath(testTarget); + const auto runArtifact = GenerateTargetRunArtifactFilePath(testTarget); + const Command args = + { + AZStd::string::format( + "\"%s\" " // 1. Instrumented test runner + "--coverage_level %s " // 2. Coverage level + "--export_type cobertura:\"%s\" " // 3. Test coverage artifact path + "--modules \"%s\" " // 4. Modules path + "--excluded_modules \"%s\" " // 5. Exclude modules + "--sources \"%s\" -- " // 6. Sources path + "%s " // 7. Launch command + "--gtest_output=xml:\"%s\"", // 8. Result artifact + + m_instrumentBinary.c_str(), // 1. Instrumented test runner + (coverageLevel == CoverageLevel::Line ? "line" : "source"), // 2. Coverage level + coverageArtifact.c_str(), // 3. Test coverage artifact path + m_targetBinaryDir.c_str(), // 4. Modules path + m_testRunnerBinary.c_str(), // 5. Exclude modules + m_sourceDir.c_str(), // 6. Sources path + GenerateLaunchArgument(testTarget).c_str(), // 7. Launch command + runArtifact.c_str()) // 8. Result artifact + }; + + return JobInfo(jobId, args, JobData(runArtifact, coverageArtifact)); + } + + AZStd::vector TestJobInfoGenerator::GenerateTestEnumerationJobInfos( + const AZStd::vector& testTargets, + TestEnumerator::JobInfo::CachePolicy cachePolicy) const + { + AZStd::vector jobInfos; + for (size_t jobId = 0; jobId < testTargets.size(); jobId++) + { + jobInfos.push_back(GenerateTestEnumerationJobInfo(testTargets[jobId], { jobId }, cachePolicy)); + } + + return jobInfos; + } + + AZStd::vector TestJobInfoGenerator::GenerateRegularTestRunJobInfos( + const AZStd::vector& testTargets) const + { + AZStd::vector jobInfos; + for (size_t jobId = 0; jobId < testTargets.size(); jobId++) + { + jobInfos.push_back(GenerateRegularTestRunJobInfo(testTargets[jobId], { jobId })); + } + + return jobInfos; + } + + AZStd::vector TestJobInfoGenerator::GenerateInstrumentedTestRunJobInfos( + const AZStd::vector& testTargets, + CoverageLevel coverageLevel) const + { + AZStd::vector jobInfos; + for (size_t jobId = 0; jobId < testTargets.size(); jobId++) + { + jobInfos.push_back(GenerateInstrumentedTestRunJobInfo(testTargets[jobId], { jobId }, coverageLevel)); + } + + return jobInfos; + } +} diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobInfoGenerator.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobInfoGenerator.h new file mode 100644 index 0000000000..0cae419e3a --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobInfoGenerator.h @@ -0,0 +1,108 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +#include +#include +#include +#include + +#include + +namespace TestImpact +{ + class TestTarget; + + // Generates job information for the different test job runner types. + class TestJobInfoGenerator + { + public: + //! Configures the test job info generator with the necessary path information for launching test targets. + //! @param sourceDir Root path where source files are found (including subfolders). + //! @param targetBinaryDir Path to where the test target binaries are found. + //! @param cacheDir Path to the persistent folder where test target enumerations are cached. + //! @param artifactDir Path to the transient directory where test artifacts are produced. + //! @param testRunnerBinary Path to the binary responsible for launching test targets that have the TestRunner launch method. + //! @param instrumentBinary Path to the binary responsible for launching test targets with test coverage instrumentation. + TestJobInfoGenerator( + const RepoPath& sourceDir, + const RepoPath& targetBinaryDir, + const RepoPath& cacheDir, + const RepoPath& artifactDir, + const RepoPath& testRunnerBinary, + const RepoPath& instrumentBinary); + + //! Generates a the information for a test enumeration job. + //! @param testTarget The test target to generate the job information for. + //! @param jobId The id to assign for this job. + //! @param cachePolicy The cache policy to use for this job. + TestEnumerator::JobInfo GenerateTestEnumerationJobInfo( + const TestTarget* testTarget, + TestEnumerator::JobInfo::Id jobId, + TestEnumerator::JobInfo::CachePolicy cachePolicy) const; + + //! Generates a the information for a test run job. + //! @param testTarget The test target to generate the job information for. + //! @param jobId The id to assign for this job. + TestRunner::JobInfo GenerateRegularTestRunJobInfo( + const TestTarget* testTarget, + TestRunner::JobInfo::Id jobId) const; + + //! Generates a the information for an instrumented test run job. + //! @param testTarget The test target to generate the job information for. + //! @param jobId The id to assign for this job. + //! @param coverageLevel The coverage level to use for this job. + InstrumentedTestRunner::JobInfo GenerateInstrumentedTestRunJobInfo( + const TestTarget* testTarget, + InstrumentedTestRunner::JobInfo::Id jobId, + CoverageLevel coverageLevel) const; + + //! Generates the information for the batch of test enumeration jobs. + AZStd::vector GenerateTestEnumerationJobInfos( + const AZStd::vector& testTargets, + TestEnumerator::JobInfo::CachePolicy cachePolicy) const; + + //! Generates the information for the batch of test run jobs. + AZStd::vector GenerateRegularTestRunJobInfos( + const AZStd::vector& testTargets) const; + + //! Generates the information for the batch of instrumented test run jobs. + AZStd::vector GenerateInstrumentedTestRunJobInfos( + const AZStd::vector& testTargets, + CoverageLevel coverageLevel) const; + private: + //! Generates the command string to launch the specified test target. + AZStd::string GenerateLaunchArgument(const TestTarget* testTarget) const; + + //! Generates the path to the enumeration cache file for the specified test target. + RepoPath GenerateTargetEnumerationCacheFilePath(const TestTarget* testTarget) const; + + //! Generates the path to the enumeration artifact file for the specified test target. + RepoPath GenerateTargetEnumerationArtifactFilePath(const TestTarget* testTarget) const; + + //! Generates the path to the test run artifact file for the specified test target. + RepoPath GenerateTargetRunArtifactFilePath(const TestTarget* testTarget) const; + + //! Generates the path to the test coverage artifact file for the specified test target. + RepoPath GenerateTargetCoverageArtifactFilePath(const TestTarget* testTarget) const; + + RepoPath m_sourceDir; + RepoPath m_targetBinaryDir; + RepoPath m_cacheDir; + RepoPath m_artifactDir; + RepoPath m_testRunnerBinary; + RepoPath m_instrumentBinary; + }; +} diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestTargetExtension.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestTargetExtension.h new file mode 100644 index 0000000000..9251164052 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestTargetExtension.h @@ -0,0 +1,23 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +namespace TestImpact +{ + class TestTarget; + + //! Returns the binary file extension for the specified test target. + AZStd::string GetTestTargetExtension(const TestTarget* testTarget); +} From c93a2f06ff7f5944f390be98206367d23b649055 Mon Sep 17 00:00:00 2001 From: jonawals Date: Tue, 25 May 2021 15:37:31 +0100 Subject: [PATCH 037/102] Address PR comments --- .../JobRunner/TestImpactWin32_TestTargetExtension.cpp | 2 -- .../JobRunner/TestImpactTestJobInfoGenerator.cpp | 3 +++ .../TestEngine/JobRunner/TestImpactTestJobInfoGenerator.h | 8 ++++---- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/TestEngine/JobRunner/TestImpactWin32_TestTargetExtension.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/TestEngine/JobRunner/TestImpactWin32_TestTargetExtension.cpp index bd380ac2a7..12c78f91cf 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/TestEngine/JobRunner/TestImpactWin32_TestTargetExtension.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/TestEngine/JobRunner/TestImpactWin32_TestTargetExtension.cpp @@ -10,8 +10,6 @@ * */ -#pragma once - #include #include #include diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobInfoGenerator.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobInfoGenerator.cpp index 0cf717e88e..7e8e65e940 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobInfoGenerator.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobInfoGenerator.cpp @@ -156,6 +156,7 @@ namespace TestImpact TestEnumerator::JobInfo::CachePolicy cachePolicy) const { AZStd::vector jobInfos; + jobInfos.reserve(testTargets.size()); for (size_t jobId = 0; jobId < testTargets.size(); jobId++) { jobInfos.push_back(GenerateTestEnumerationJobInfo(testTargets[jobId], { jobId }, cachePolicy)); @@ -168,6 +169,7 @@ namespace TestImpact const AZStd::vector& testTargets) const { AZStd::vector jobInfos; + jobInfos.reserve(testTargets.size()); for (size_t jobId = 0; jobId < testTargets.size(); jobId++) { jobInfos.push_back(GenerateRegularTestRunJobInfo(testTargets[jobId], { jobId })); @@ -181,6 +183,7 @@ namespace TestImpact CoverageLevel coverageLevel) const { AZStd::vector jobInfos; + jobInfos.reserve(testTargets.size()); for (size_t jobId = 0; jobId < testTargets.size(); jobId++) { jobInfos.push_back(GenerateInstrumentedTestRunJobInfo(testTargets[jobId], { jobId }, coverageLevel)); diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobInfoGenerator.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobInfoGenerator.h index 0cae419e3a..1bd06f6372 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobInfoGenerator.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobInfoGenerator.h @@ -25,7 +25,7 @@ namespace TestImpact { class TestTarget; - // Generates job information for the different test job runner types. + //! Generates job information for the different test job runner types. class TestJobInfoGenerator { public: @@ -44,7 +44,7 @@ namespace TestImpact const RepoPath& testRunnerBinary, const RepoPath& instrumentBinary); - //! Generates a the information for a test enumeration job. + //! Generates the information for a test enumeration job. //! @param testTarget The test target to generate the job information for. //! @param jobId The id to assign for this job. //! @param cachePolicy The cache policy to use for this job. @@ -53,14 +53,14 @@ namespace TestImpact TestEnumerator::JobInfo::Id jobId, TestEnumerator::JobInfo::CachePolicy cachePolicy) const; - //! Generates a the information for a test run job. + //! Generates the information for a test run job. //! @param testTarget The test target to generate the job information for. //! @param jobId The id to assign for this job. TestRunner::JobInfo GenerateRegularTestRunJobInfo( const TestTarget* testTarget, TestRunner::JobInfo::Id jobId) const; - //! Generates a the information for an instrumented test run job. + //! Generates the information for an instrumented test run job. //! @param testTarget The test target to generate the job information for. //! @param jobId The id to assign for this job. //! @param coverageLevel The coverage level to use for this job. From f9425992f86fe20fa822c761233cd71e34020c52 Mon Sep 17 00:00:00 2001 From: jonawals Date: Thu, 27 May 2021 11:37:34 +0100 Subject: [PATCH 038/102] Runtime refactor --- .../TestImpactClientFailureReport.h | 72 +++----------- .../TestImpactClientTestSelection.h | 6 +- ...estImpactUtils.h => TestImpactFileUtils.h} | 22 +++++ .../TestImpactFramework/TestImpactRepoPath.h | 68 ++++++++++++-- .../TestImpactFramework/TestImpactRuntime.h | 77 +++++++++++---- .../TestImpactTestSequence.h | 18 ++-- .../TestImpactDynamicDependencyMap.cpp | 66 +++++++++++-- .../TestImpactDynamicDependencyMap.h | 14 ++- .../TestImpactSourceCoveringTestsList.cpp | 2 +- ...estImpactSourceCoveringTestsSerializer.cpp | 93 +++++++++++++++++++ .../TestImpactSourceCoveringTestsSerializer.h | 26 ++++++ .../TestImpactTestSelectorAndPrioritizer.cpp | 4 +- .../TestImpactTestSelectorAndPrioritizer.h | 13 +-- .../TestImpactWin32_TestEngineJobFailure.cpp | 35 +++++++ .../TestImpactTestEngineJobFailure.cpp | 82 ++++++++++++++++ .../TestImpactTestEngineJobFailure.h | 29 ++++++ 16 files changed, 513 insertions(+), 114 deletions(-) rename Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/{TestImpactUtils.h => TestImpactFileUtils.h} (77%) create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsSerializer.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsSerializer.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/TestEngine/TestImpactWin32_TestEngineJobFailure.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJobFailure.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJobFailure.h diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientFailureReport.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientFailureReport.h index 02b1169e93..19e98a7d1f 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientFailureReport.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientFailureReport.h @@ -44,19 +44,6 @@ namespace TestImpact AZStd::string m_commandString; }; - //! Represents a test target that terminated abnormally. - class LauncherFailure - : public ExecutionFailure - { - public: - LauncherFailure(const AZStd::string& targetName, const AZStd::string& command, int returnCode); - - //! The return code of the test target that terminated abnormally. - int GetReturnCode() const; - private: - int m_returnCode; - }; - //! Represents an individual test of a test target that failed. class TestFailure { @@ -107,6 +94,7 @@ namespace TestImpact private: AZStd::vector m_testCaseFailures; + size_t m_numTestFailures = 0; }; //! Base class for reporting failing test sequences. @@ -115,63 +103,27 @@ namespace TestImpact public: SequenceFailure( AZStd::vector&& executionFailures, - AZStd::vector&& launcherFailures, + AZStd::vector&& testRunFailures, + AZStd::vector&& timedOutTests, AZStd::vector&& unexecutedTests); //! Returns the test targets in this sequence that failed to execute. const AZStd::vector& GetExecutionFailures() const; - //! Returns the test targets in this sequence that terminated abnormally. - const AZStd::vector& GetLauncherFailures() const; - - //! Returns the test targets in this sequence that were not executed due to the sequence terminating prematurely. - const AZStd::vector& GetUnexecutedTest() const; - - private: - AZStd::vector m_executionFailures; - AZStd::vector m_launcherFailures; - AZStd::vector m_unexecutedTests; - }; - - //! Represents the report for a failed regular test sequence run without test impact analysis. - class RegularSequenceFailure - : public SequenceFailure - { - public: - RegularSequenceFailure( - AZStd::vector&& executionFailures, - AZStd::vector&& launcherFailures, - AZStd::vector&& testRunFailures, - AZStd::vector&& unexecutedTests); - //! Returns the test targets that contain failing tests. const AZStd::vector& GetTestRunFailures() const; + //! Returns the test targets in this sequence that were terminated for exceeding their allotted flight time. + const AZStd::vector& GetTimedOutTests() const; + + //! Returns the test targets in this sequence that were not executed due to the sequence terminating prematurely. + const AZStd::vector& GetUnexecutedTests() const; + private: + AZStd::vector m_executionFailures; AZStd::vector m_testRunFailures; - }; - - //! Represents the report for a failed test sequence run with test impact analysis. - class ImpactAnalysisSequenceFailure - : public SequenceFailure - { - public: - ImpactAnalysisSequenceFailure( - AZStd::vector&& executionFailures, - AZStd::vector&& launcherFailures, - AZStd::vector&& selectedTestRunFailures, - AZStd::vector&& discardedTestRunFailures, - AZStd::vector&& unexecutedTests); - - //! Returns the test targets that were selected to run but contain failing tests. - const AZStd::vector GetSelectedTestRunFailures() const; - - //! Returns the test targets that were not selected but still run but contain failing tests. - const AZStd::vector GetDiscardedTestRunFailures() const; - - private: - AZStd::vector m_selectedTestRunFailures; - AZStd::vector m_discardedTestRunFailures; + AZStd::vector m_timedOutTests; + AZStd::vector m_unexecutedTests; }; } // namespace Client } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientTestSelection.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientTestSelection.h index f251e7117d..76e6461206 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientTestSelection.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientTestSelection.h @@ -25,6 +25,7 @@ namespace TestImpact class TestRunSelection { public: + TestRunSelection(const AZStd::vector& includedTests, const AZStd::vector& excludedTests); TestRunSelection(AZStd::vector&& includedTests, AZStd::vector&& excludedTests); //! Returns the test runs that were selected to be run and will actually be run. @@ -37,7 +38,10 @@ namespace TestImpact size_t GetNumIncludedTestRuns() const; //! Returns the number of selected test runs that will not be run. - size_t GetNumNumExcludedTestRuns() const; + size_t GetNumExcludedTestRuns() const; + + //! Returns the total number of tests runs selected regardless of whether or not they will actually be run. + size_t GetTotalNumTests() const; private: AZStd::vector m_includedTestRuns; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactUtils.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactFileUtils.h similarity index 77% rename from Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactUtils.h rename to Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactFileUtils.h index d734d7a70e..aa64840e78 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactUtils.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactFileUtils.h @@ -59,4 +59,26 @@ namespace TestImpact AZ_TestImpact_Eval( file.Write(bytes.data(), bytes.size()), ExceptionType, AZStd::string::format("Couldn't write contents for file %s", path.c_str())); } + + //! Delete the files that match the pattern from the specified directory. + //! @param path The path to the directory to pattern match the files for deletion. + //! @param pattern The pattern to match files for deletion. + inline void DeleteFiles(const RepoPath& path, const AZStd::string& pattern) + { + AZ::IO::SystemFile::FindFiles(AZStd::string::format("%s/%s", path.c_str(), pattern.c_str()).c_str(), + [&path](const char* file, bool isFile) + { + if (isFile) + { + AZ::IO::SystemFile::Delete(AZStd::string::format("%s/%s", path.c_str(), file).c_str()); + } + + return true; + }); + } + + inline void DeleteFile(const RepoPath& path) + { + DeleteFiles(path.ParentPath(), path.Filename().Native()); + } } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRepoPath.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRepoPath.h index ccba3f677d..80f82e4785 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRepoPath.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRepoPath.h @@ -21,22 +21,78 @@ namespace TestImpact //! to the test impact analysis data as otherwise querying/retrieving test impact analysis data for the same source albeit //! with different path separators will be considered different files entirely. class RepoPath - : public AZ::IO::Path { public: + using string_type = AZ::IO::Path::string_type; + using string_view_type = AZ::IO::Path::string_view_type; + using value_type = AZ::IO::Path::value_type; + constexpr RepoPath() = default; constexpr RepoPath(const RepoPath&) = default; constexpr RepoPath(RepoPath&&) noexcept = default; - constexpr RepoPath(const string_type&) noexcept; - constexpr RepoPath(const value_type*) noexcept; - constexpr RepoPath(const AZ::IO::PathView&); - constexpr RepoPath(const AZ::IO::Path&); + constexpr RepoPath::RepoPath(const string_type & path) noexcept; + constexpr RepoPath::RepoPath(const string_view_type& path) noexcept; + constexpr RepoPath::RepoPath(const value_type* path) noexcept; + constexpr RepoPath::RepoPath(const AZ::IO::PathView& path); + constexpr RepoPath::RepoPath(const AZ::IO::Path& path); RepoPath& operator=(const RepoPath&) noexcept = default; RepoPath& operator=(const string_type&) noexcept; RepoPath& operator=(const value_type*) noexcept; RepoPath& operator=(const AZ::IO::Path& str) noexcept; - using AZ::IO::Path::operator AZ::IO::PathView; + const char* c_str() const { return m_path.c_str(); } + AZStd::string String() const { return m_path.String(); } + constexpr AZ::IO::PathView Stem() const { return m_path.Stem(); } + constexpr AZ::IO::PathView Extension() const { return m_path.Extension(); } + constexpr bool empty() const { return m_path.empty(); } + constexpr AZ::IO::PathView ParentPath() const { return m_path.ParentPath(); } + constexpr AZ::IO::PathView Filename() const { return m_path.Filename(); } + AZ::IO::Path LexicallyRelative(const RepoPath& base) const { return m_path.LexicallyRelative(base.m_path); } + [[nodiscard]] bool IsRelativeTo(const RepoPath& base) const { return m_path.IsRelativeTo(base.m_path); } + constexpr AZ::IO::PathView RootName() const { return m_path.RootName(); } + constexpr AZ::IO::PathView RelativePath() const { return m_path.RelativePath(); } + + friend RepoPath operator/(const RepoPath& lhs, const AZ::IO::PathView& rhs); + friend RepoPath operator/(const RepoPath& lhs, AZStd::string_view rhs); + friend RepoPath operator/(const RepoPath& lhs, const typename value_type* rhs); + friend RepoPath operator/(const RepoPath& lhs, const RepoPath& rhs); + + RepoPath& operator/=(const AZ::IO::PathView& rhs); + RepoPath& operator/=(AZStd::string_view rhs); + RepoPath& operator/=(const typename value_type* rhs); + RepoPath& operator/=(const RepoPath& rhs); + + friend bool operator==(const RepoPath& lhs, const RepoPath& rhs) noexcept; + friend bool operator!=(const RepoPath& lhs, const RepoPath& rhs) noexcept; + friend bool operator<(const RepoPath& lhs, const RepoPath& rhs) noexcept; + + private: + AZ::IO::Path m_path; }; + + constexpr RepoPath::RepoPath(const string_type& path) noexcept + : m_path(AZ::IO::Path(path).MakePreferred()) + { + } + + constexpr RepoPath::RepoPath(const string_view_type& path) noexcept + : m_path(AZ::IO::Path(path).MakePreferred()) + { + } + + constexpr RepoPath::RepoPath(const value_type* path) noexcept + : m_path(AZ::IO::Path(path).MakePreferred()) + { + } + + constexpr RepoPath::RepoPath(const AZ::IO::PathView& path) + : m_path(AZ::IO::Path(path).MakePreferred()) + { + } + + constexpr RepoPath::RepoPath(const AZ::IO::Path& path) + : m_path(AZ::IO::Path(path).MakePreferred()) + { + } } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h index 8221eba834..31b2d1edce 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h @@ -29,9 +29,12 @@ namespace TestImpact { + class ChangeDependencyList; class DynamicDependencyMap; + class TestSelectorAndPrioritizer; class TestEngine; class TestTarget; + class SourceCoveringTestsList; //! Callback for a test sequence that isn't using test impact analysis to determine selected tests. //! @param tests The tests that will be run for this sequence. @@ -66,16 +69,22 @@ namespace TestImpact //! Callback for end of a test sequence. //! @param failureReport The test runs that failed for any reason during this sequence. //! @param duration The total duration of this test sequence. - using TestSequenceCompleteCallback = AZStd::function; + using TestSequenceCompleteCallback = AZStd::function; //! Callback for end of a test impact analysis test sequence. - //! @param failureReport The test runs that failed for any reason during this sequence. + //! @param selectedFailureReport The selected test runs that failed for any reason during this sequence. + //! @param discardedFailureReport The discarded test runs that failed for any reason during this sequence. //! @param duration The total duration of this test sequence. - using ImpactAnalysisTestSequenceCompleteCallback = AZStd::function; + using SafeTestSequenceCompleteCallback = AZStd::function; //! Callback for test runs that have completed for any reason. - //! test The test that has completed. - using TestCompleteCallback = AZStd::function; + //! selectedTests The test that has completed. + using TestRunCompleteCallback = AZStd::function; //! The API exposed to the client responsible for all test runs and persistent data management. class Runtime @@ -95,7 +104,7 @@ namespace TestImpact Policy::TestFailure testFailurePolicy, Policy::IntegrityFailure integrationFailurePolicy, Policy::TestSharding testShardingPolicy, - TargetOutputCapture targetOutputCapture, + Policy::TargetOutputCapture targetOutputCapture, AZStd::optional maxConcurrency = AZStd::nullopt); ~Runtime(); @@ -103,78 +112,110 @@ namespace TestImpact //! Runs a test sequence where all tests with a matching suite in the suite filter and also not on the excluded list are selected. //! @param suitesFilter The test suites that will be included in the test selection. //! @param testTargetTimeout The maximum duration individual test targets may be in flight for (infinite if empty). - //! @param testTargetTimeout The maximum duration the entire test sequence may run for (infinite if empty). + //! @param globalTimeout The maximum duration the entire test sequence may run for (infinite if empty). //! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests. //! @param testSequenceCompleteCallback The client function to be called after the test sequence has completed. //! @param testRunCompleteCallback The client function to be called after an individual test run has completed. + //! @returns TestSequenceResult RegularTestSequence( const AZStd::unordered_set suitesFilter, AZStd::optional testTargetTimeout, AZStd::optional globalTimeout, AZStd::optional testSequenceStartCallback, AZStd::optional testSequenceCompleteCallback, - AZStd::optional testRunCompleteCallback); + AZStd::optional testRunCompleteCallback); //! Runs a test sequence where tests are selected according to test impact analysis so long as they are not on the excluded list. //! @param changeList The change list used to determine the tests to select. //! @param testPrioritizationPolicy Determines how selected tests will be prioritized. //! @param testTargetTimeout The maximum duration individual test targets may be in flight for (infinite if empty). - //! @param testTargetTimeout The maximum duration the entire test sequence may run for (infinite if empty). + //! @param globalTimeout The maximum duration the entire test sequence may run for (infinite if empty). //! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests. //! @param testSequenceCompleteCallback The client function to be called after the test sequence has completed. //! @param testRunCompleteCallback The client function to be called after an individual test run has completed. + //! @returns TestSequenceResult ImpactAnalysisTestSequence( const ChangeList& changeList, Policy::TestPrioritization testPrioritizationPolicy, AZStd::optional testTargetTimeout, AZStd::optional globalTimeout, AZStd::optional testSequenceStartCallback, - AZStd::optional testSequenceCompleteCallback, - AZStd::optional testRunCompleteCallback); + AZStd::optional testSequenceCompleteCallback, + AZStd::optional testRunCompleteCallback); //! Runs a test sequence as per the ImpactAnalysisTestSequence where the tests not selected are also run (albeit without instrumentation). //! @param changeList The change list used to determine the tests to select. //! @param suitesFilter The test suites that will be included in the test selection. //! @param testPrioritizationPolicy Determines how selected tests will be prioritized. //! @param testTargetTimeout The maximum duration individual test targets may be in flight for (infinite if empty). - //! @param testTargetTimeout The maximum duration the entire test sequence may run for (infinite if empty). + //! @param globalTimeout The maximum duration the entire test sequence may run for (infinite if empty). //! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests. //! @param testSequenceCompleteCallback The client function to be called after the test sequence has completed. //! @param testRunCompleteCallback The client function to be called after an individual test run has completed. - TestSequenceResult SafeImpactAnalysisTestSequence( + //! @returns + AZStd::pair SafeImpactAnalysisTestSequence( const ChangeList& changeList, const AZStd::unordered_set suitesFilter, Policy::TestPrioritization testPrioritizationPolicy, AZStd::optional testTargetTimeout, AZStd::optional globalTimeout, AZStd::optional testSequenceStartCallback, - AZStd::optional testSequenceCompleteCallback, - AZStd::optional testRunCompleteCallback); + AZStd::optional testSequenceCompleteCallback, + AZStd::optional testRunCompleteCallback); //! Runs all tests not on the excluded list and uses their coverage data to seed the test impact analysis data (ant existing data will be overwritten). - //! @param testTargetTimeout The maximum duration the entire test sequence may run for (infinite if empty). + //! @param testTargetTimeout The maximum duration individual test targets may be in flight for (infinite if empty). + //! @param globalTimeout The maximum duration the entire test sequence may run for (infinite if empty). //! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests. //! @param testSequenceCompleteCallback The client function to be called after the test sequence has completed. //! @param testRunCompleteCallback The client function to be called after an individual test run has completed. + //! TestSequenceResult SeededTestSequence( + AZStd::optional testTargetTimeout, AZStd::optional globalTimeout, AZStd::optional testSequenceStartCallback, AZStd::optional testSequenceCompleteCallback, - AZStd::optional testRunCompleteCallback); + AZStd::optional testRunCompleteCallback); //! Returns true if the runtime has test impact analysis data (either preexisting or generated). bool HasImpactAnalysisData() const; private: + //! Updates the test enumeration cache for test targets that had sources modified by a given change list. + //! @param changeDependencyList The resolved change dependency list generated for the change list. + void EnumerateMutatedTestTargets(const ChangeDependencyList& changeDependencyList); + + //! Selects the test targets covering a given change list and updates the enumeration cache of the test targets with sources + //! modified in that change list. + //! @param changeList The change list for which the covering tests and enumeration cache updates will be generated for. + //! @param testPrioritizationPolicy The test prioritization strategy to use for the selected test targets. + //! @returns The pair of selected test targets and discarded test targets. + AZStd::pair, AZStd::vector> SelectCoveringTestTargetsAndUpdateEnumerationCache( + const ChangeList& changeList, + Policy::TestPrioritization testPrioritizationPolicy); + + //! Selects the test targets from the specified list of test targets that are not on the test target exclusion list. + //! @param testTargets The list of test targets to select from. + //! @returns The subset of test targets in the specified list that are not on the target exclude list. + AZStd::pair, AZStd::vector> SelectTestTargetsByExcludeList( + AZStd::vector testTargets) const; + + //! Prepares the dynamic dependency map for a seed update by clearing all existing data and deleting the file that will be serialized. + void ClearDynamicDependencyMapAndRemoveExistingFile(); + + //! Updates the dynamic dependency map and serializes the entire map to disk. + void UpdateAndSerializeDynamicDependencyMap(const SourceCoveringTestsList& sourceCoverageTestsList); + RuntimeConfig m_config; Policy::ExecutionFailure m_executionFailurePolicy; Policy::ExecutionFailureDrafting m_executionFailureDraftingPolicy; Policy::TestFailure m_testFailurePolicy; Policy::IntegrityFailure m_integrationFailurePolicy; Policy::TestSharding m_testShardingPolicy; - TargetOutputCapture m_targetOutputCapture; + Policy::TargetOutputCapture m_targetOutputCapture; size_t m_maxConcurrency = 0; AZStd::unique_ptr m_dynamicDependencyMap; + AZStd::unique_ptr m_testSelectorAndPrioritizer; AZStd::unique_ptr m_testEngine; AZStd::unordered_set m_testTargetExcludeList; AZStd::unordered_set m_testTargetShardList; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactTestSequence.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactTestSequence.h index 751db43987..594c997c45 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactTestSequence.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactTestSequence.h @@ -61,16 +61,16 @@ namespace TestImpact Never, //!< Do not shard any test targets. Always //!< Shard all test targets that have been marked for test sharding. }; - } - //! Standard output capture of test target runs. - enum class TargetOutputCapture - { - None, //!< Do not capture any output. - StdOut, //!< Send captured output to standard output - File, //!< Write captured output to file. - StdOutAndFile //!< Send captured output to standard output and write to file. - }; + //! Standard output capture of test target runs. + enum class TargetOutputCapture + { + None, //!< Do not capture any output. + StdOut, //!< Send captured output to standard output + File, //!< Write captured output to file. + StdOutAndFile //!< Send captured output to standard output and write to file. + }; + } //! Configuration for test targets that opt in to test sharding. enum class ShardConfiguration diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp index 31b3485c70..919c10d049 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp @@ -34,7 +34,7 @@ namespace TestImpact else { // This is a new entry on the dependency map so create an entry with this parent target and no covering targets - m_sourceDependencyMap.emplace(source, DependencyData{ {target}, {} }); + m_sourceDependencyMap.emplace(source.String(), DependencyData{ {target}, {} }); } } @@ -56,6 +56,7 @@ namespace TestImpact for (const auto& target : m_testTargets.GetTargets()) { mapBuildTargetSources(&target); + m_testTargetSourceCoverageCount[&target] = 0; } } @@ -142,10 +143,21 @@ namespace TestImpact DependencyException, AZStd::string::format("Couldn't replace source coverage for %s, source file is an autogen input file", sourceCoverage.GetPath().c_str()).c_str()); - auto [it, inserted] = m_sourceDependencyMap.insert(sourceCoverage.GetPath().String()); - auto& [key, sourceDependency] = *it; + auto [sourceDependencyIt, inserted] = m_sourceDependencyMap.insert(sourceCoverage.GetPath().String()); + auto& [key, sourceDependency] = *sourceDependencyIt; - // Clear any existing coverage for the delta + // Knock down the source coverage count for the test targets and clear any existing coverage for the delta + for (const auto& testTarget : sourceDependency.m_coveringTestTargets) + { + if (auto coveringTestTargetIt = m_testTargetSourceCoverageCount.find(testTarget); + coveringTestTargetIt != m_testTargetSourceCoverageCount.end()) + { + if (coveringTestTargetIt->second > 0) + { + coveringTestTargetIt->second--; + } + } + } sourceDependency.m_coveringTestTargets.clear(); // Update the dependency with any new coverage data @@ -157,10 +169,13 @@ namespace TestImpact // Source to covering test target mapping sourceDependency.m_coveringTestTargets.insert(testTarget); + // Test target covering sources count + m_testTargetSourceCoverageCount[testTarget]++; + // Build target to covering test target mapping for (const auto& parentTarget : sourceDependency.m_parentTargets) - { - m_buildTargetCoverage[parentTarget.GetBuildTarget()].insert(testTarget); + { + m_buildTargetCoverage[parentTarget.GetBuildTarget()].insert(testTarget); } } else @@ -173,7 +188,7 @@ namespace TestImpact // If the new coverage data results in a parentless and coverageless entry, consider it a dead entry and remove accordingly if (sourceDependency.m_coveringTestTargets.empty() && sourceDependency.m_parentTargets.empty()) { - m_sourceDependencyMap.erase(it); + m_sourceDependencyMap.erase(sourceDependencyIt); } } } @@ -198,6 +213,14 @@ namespace TestImpact } } + void DynamicDependencyMap::ClearAllSourceCoverage() + { + for (const auto& [path, coverage] : m_sourceDependencyMap) + { + ReplaceSourceCoverage(SourceCoveringTestsList(AZStd::vector{ SourceCoveringTests(RepoPath(path)) })); + } + } + const ProductionTargetList& DynamicDependencyMap::GetProductionTargetList() const { return m_productionTargets; @@ -405,4 +428,33 @@ namespace TestImpact return ChangeDependencyList(AZStd::move(createDependencies), AZStd::move(updateDependencies), AZStd::move(deleteDependencies)); } + + AZStd::vector DynamicDependencyMap::GetCoveringTests() const + { + AZStd::vector covering; + for (const auto& [testTarget, coveringSources] : m_testTargetSourceCoverageCount) + { + if (coveringSources > 0) + { + covering.push_back(testTarget); + } + } + + return covering; + } + + AZStd::vector DynamicDependencyMap::GetNotCoveringTests() const + { + AZStd::vector notCovering; + for(const auto& [testTarget, coveringSources] : m_testTargetSourceCoverageCount) + { + if(coveringSources == 0) + { + notCovering.push_back(testTarget); + } + } + + return notCovering; + } + } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.h index 6faef76452..f209e22823 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.h @@ -85,18 +85,27 @@ namespace TestImpact //! @param sourceCoverageDelta The source coverage delta to replace in the dependency map. void ReplaceSourceCoverage(const SourceCoveringTestsList& sourceCoverageDelta); + //! Clears all of the existing source coverage in the dependency map. + void ClearAllSourceCoverage(); + //! Exports the coverage of all sources in the dependency map. SourceCoveringTestsList ExportSourceCoverage() const; //! Gets the list of orphaned source files in the dependency map that have coverage data but belong to no parent build targets. AZStd::vector GetOrphanSourceFiles() const; - //! Applies the specified change list to the dynamic dependency map and resolves the change list to a change dependency list + //! Applies the specified change list to the dependency map and resolves the change list to a change dependency list //! containing the updated source dependencies for each source file in the change list. //! @param changeList The change list to apply and resolve. //! @returns The change list as resolved to the appropriate source dependencies. [[nodiscard]] ChangeDependencyList ApplyAndResoveChangeList(const ChangeList& changeList); + //! Returns the test targets that cover one or more sources in the repository. + AZStd::vector GetCoveringTests() const; + + //! Returns the test targets that do not cover any sources in the repository. + AZStd::vector GetNotCoveringTests() const; + private: //! Clears the source coverage of the specified sources. //! @note The covering targets for the parent test target(s) will not be pruned if those covering targets are removed. @@ -116,5 +125,8 @@ namespace TestImpact //! Mapping of autogen input sources to their generated output sources. AZStd::unordered_map> m_autogenInputToOutputMap; + + //! Number of sources that each test target in the repository covers. + AZStd::unordered_map m_testTargetSourceCoverageCount; }; } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsList.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsList.cpp index 101bdf862c..ce8bf64678 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsList.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsList.cpp @@ -65,7 +65,7 @@ namespace TestImpact { AZStd::sort(m_coverage.begin(), m_coverage.end(), [](const SourceCoveringTests& lhs, const SourceCoveringTests& rhs) { - return lhs.GetPath() < rhs.GetPath(); + return lhs.GetPath().String() < rhs.GetPath().String(); }); } diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsSerializer.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsSerializer.cpp new file mode 100644 index 0000000000..7b89f3bda9 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsSerializer.cpp @@ -0,0 +1,93 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include + +#include +#include +#include +#include + +namespace TestImpact +{ + // Tag used to indicate that a given line is the name o a covering test target + constexpr char TargetTag = '-'; + + AZStd::string SerializeSourceCoveringTestsList(const SourceCoveringTestsList& sourceCoveringTestsList) + { + AZStd::string output; + output.reserve(1U << 24); // Reserve approx. 16Mib as the outputs can be quite large + + for (const auto& source : sourceCoveringTestsList.GetCoverage()) + { + // Source file + output += source.GetPath().String(); + output += "\n"; + + // Covering test targets + for (const auto& testTarget : source.GetCoveringTestTargets()) + { + output += AZStd::string::format("%c%s\n", TargetTag, testTarget.c_str()); + } + } + + // Add the newline so the deserializer can properly terminate on the last read line + output += "\n"; + + return output; + } + + SourceCoveringTestsList DeserializeSourceCoveringTestsList(const AZStd::string& sourceCoveringTestsListString) + { + AZStd::vector sourceCoveringTests; + AZStd::string source; + AZStd::vector coveringTests; + sourceCoveringTests.reserve(1U << 16); // Reserve for approx. 65k source files + const AZStd::string delim = "\n"; + auto start = 0U; + auto end = sourceCoveringTestsListString.find(delim); + + while (end != AZStd::string::npos) + { + const auto line = sourceCoveringTestsListString.substr(start, end - start); + if (line.starts_with(TargetTag)) + { + // This is a test target covering the most recent source discovered + coveringTests.push_back(line.substr(1, line.length() - 1)); + } + else + { + // This is a new source file so assign the accumulated test targets to the current source file before proceeding + if (!coveringTests.empty()) + { + sourceCoveringTests.push_back(SourceCoveringTests(source, AZStd::move(coveringTests))); + coveringTests.clear(); + } + + source = line; + } + + start = end + delim.length(); + end = sourceCoveringTestsListString.find(delim, start); + } + + // Ensure we properly assign the accumulated test targets to the most recent source discovered + if (!coveringTests.empty()) + { + sourceCoveringTests.push_back(SourceCoveringTests(source, AZStd::move(coveringTests))); + coveringTests.clear(); + } + + return SourceCoveringTestsList(AZStd::move(sourceCoveringTests)); + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsSerializer.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsSerializer.h new file mode 100644 index 0000000000..901e43233c --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsSerializer.h @@ -0,0 +1,26 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +#include + +namespace TestImpact +{ + //! Serializes the specified source covering tests list to plain text format. + AZStd::string SerializeSourceCoveringTestsList(const SourceCoveringTestsList& sourceCoveringTestsList); + + //! Deserializes a source covering tests list from the specified source covering tests data in plain text format. + SourceCoveringTestsList DeserializeSourceCoveringTestsList(const AZStd::string& sourceCoveringTestsListString); +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactTestSelectorAndPrioritizer.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactTestSelectorAndPrioritizer.cpp index 549ef0c108..35e6679fdc 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactTestSelectorAndPrioritizer.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactTestSelectorAndPrioritizer.cpp @@ -25,7 +25,7 @@ namespace TestImpact } AZStd::vector TestSelectorAndPrioritizer::SelectTestTargets( - const ChangeDependencyList& changeDependencyList, TestSelectionStrategy testSelectionStrategy) + const ChangeDependencyList& changeDependencyList, Policy::TestPrioritization testSelectionStrategy) { const auto selectedTestTargetAndDependerMap = SelectTestTargets(changeDependencyList); const auto prioritizedSelectedTests = PrioritizeSelectedTestTargets(selectedTestTargetAndDependerMap, testSelectionStrategy); @@ -210,7 +210,7 @@ namespace TestImpact AZStd::vector TestSelectorAndPrioritizer::PrioritizeSelectedTestTargets( const SelectedTestTargetAndDependerMap& selectedTestTargetAndDependerMap, - [[maybe_unused]]TestSelectionStrategy testSelectionStrategy) + [[maybe_unused]] Policy::TestPrioritization testSelectionStrategy) { AZStd::vector selectedTestTargets; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactTestSelectorAndPrioritizer.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactTestSelectorAndPrioritizer.h index c742176073..6ee3a616c4 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactTestSelectorAndPrioritizer.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactTestSelectorAndPrioritizer.h @@ -12,6 +12,8 @@ #pragma once +#include + #include #include @@ -25,13 +27,6 @@ namespace TestImpact class BuildTarget; class TestTarget; - //! Strategy for selecting tests given a set of source changes. - enum class TestSelectionStrategy : bool - { - SelectOnly, //!< Select tests only, do not attempt prioritization of those selected tests. - SelectAndPriotitize //!< Select tests and prioritize according to dependency graph locality of coverer and coveree. - }; - //! Map of build targets and their dependency graph data. //! For test targets, the dependency graph data is that of the build targets which the test target depends on. //! For production targets, the dependency graph is that of the build targets that depend on it (dependers). @@ -52,7 +47,7 @@ namespace TestImpact //! Select the covering test targets for the given set of source changes and optionally prioritizes said test selection. //! @param changeDependencyList The resolved list of source dependencies for the CRUD source changes. //! @param testSelectionStrategy The test selection and prioritization strategy to apply to the given CRUD source changes. - AZStd::vector SelectTestTargets(const ChangeDependencyList& changeDependencyList, TestSelectionStrategy testSelectionStrategy); + AZStd::vector SelectTestTargets(const ChangeDependencyList& changeDependencyList, Policy::TestPrioritization testSelectionStrategy); private: //! Map of selected test targets and the production targets they cover for the given set of source changes. @@ -69,7 +64,7 @@ namespace TestImpact //! @param testSelectionStrategy The test selection strategy to prioritize the selected tests. //! @returns The selected tests either in either arbitrary order or in prioritized with highest priority first. AZStd::vector PrioritizeSelectedTestTargets( - const SelectedTestTargetAndDependerMap& selectedTestTargetAndDependerMap, TestSelectionStrategy testSelectionStrategy); + const SelectedTestTargetAndDependerMap& selectedTestTargetAndDependerMap, Policy::TestPrioritization testSelectionStrategy); const DynamicDependencyMap* m_dynamicDependencyMap; DependencyGraphDataMap m_dependencyGraphDataMap; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/TestEngine/TestImpactWin32_TestEngineJobFailure.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/TestEngine/TestImpactWin32_TestEngineJobFailure.cpp new file mode 100644 index 0000000000..578130882d --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/TestEngine/TestImpactWin32_TestEngineJobFailure.cpp @@ -0,0 +1,35 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include + +namespace TestImpact +{ + // Known error codes for test instrumentation + namespace ErrorCodes + { + namespace OpenCppCoverage + { + static constexpr ReturnCode InvalidArgs = 0x9F8C8E5C; + } + } + + AZStd::optional CheckForKnownTestInstrumentErrorCode(ReturnCode returnCode) + { + if (returnCode == ErrorCodes::OpenCppCoverage::InvalidArgs) + { + return Client::TestRunResult::FailedToExecute; + } + + return AZStd::nullopt; + } +} diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJobFailure.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJobFailure.cpp new file mode 100644 index 0000000000..debf1406f2 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJobFailure.cpp @@ -0,0 +1,82 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include + +namespace TestImpact +{ + // Known error codes for test runner and test library + namespace ErrorCodes + { + namespace GTest + { + static constexpr ReturnCode Unsuccessful = 1; + } + + namespace AZTestRunner + { + static constexpr ReturnCode InvalidArgs = 101; + static constexpr ReturnCode FailedToFindTargetBinary = 102; + static constexpr ReturnCode SymbolNotFound = 103; + static constexpr ReturnCode ModuleSkipped = 104; + } + } + + AZStd::optional CheckForKnownTestRunnerErrorCode(int returnCode) + { + switch (returnCode) + { + // We will consider test targets that technically execute but their launcher or unit test library return a know error + // code that pertains to incorrect argument usage as test targets that failed to execute + case ErrorCodes::AZTestRunner::InvalidArgs: + case ErrorCodes::AZTestRunner::FailedToFindTargetBinary: + case ErrorCodes::AZTestRunner::ModuleSkipped: + case ErrorCodes::AZTestRunner::SymbolNotFound: + return Client::TestRunResult::FailedToExecute; + default: + return AZStd::nullopt; + } + } + + AZStd::optional CheckForKnownTestLibraryErrorCode(int returnCode) + { + if (returnCode == ErrorCodes::GTest::Unsuccessful) + { + return Client::TestRunResult::TestFailures; + } + + return AZStd::nullopt; + } + + AZStd::optional CheckForAnyKnownErrorCode(ReturnCode returnCode) + { + if (const auto result = CheckForKnownTestInstrumentErrorCode(returnCode); + result != AZStd::nullopt) + { + return result.value(); + } + + if (const auto result = CheckForKnownTestRunnerErrorCode(returnCode); + result != AZStd::nullopt) + { + return result.value(); + } + + if (const auto result = CheckForKnownTestLibraryErrorCode(returnCode); + result != AZStd::nullopt) + { + return result.value(); + } + + return AZStd::nullopt; + } +} diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJobFailure.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJobFailure.h new file mode 100644 index 0000000000..aef4e962c5 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJobFailure.h @@ -0,0 +1,29 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +#include + +#include + +namespace TestImpact +{ + AZStd::optional CheckForKnownTestInstrumentErrorCode(ReturnCode returnCode); + AZStd::optional CheckForKnownTestRunnerErrorCode(ReturnCode returnCode); + AZStd::optional CheckForKnownTestLibraryErrorCode(ReturnCode returnCode); + + AZStd::optional CheckForAnyKnownErrorCode(ReturnCode returnCode); + +} From de38e5ad35895caff54edf0152cf9cc0d438bc42 Mon Sep 17 00:00:00 2001 From: jonawals Date: Thu, 27 May 2021 11:46:18 +0100 Subject: [PATCH 039/102] Add missing API comments --- .../Code/Source/TestEngine/TestImpactTestEngineJobFailure.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJobFailure.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJobFailure.h index aef4e962c5..8e88571957 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJobFailure.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJobFailure.h @@ -20,10 +20,16 @@ namespace TestImpact { + //! Checks for known test instrumentation error return codes and returns the corresponding client test run result or empty. AZStd::optional CheckForKnownTestInstrumentErrorCode(ReturnCode returnCode); + + //! Checks for known test runner error return codes and returns the corresponding client test run result or empty. AZStd::optional CheckForKnownTestRunnerErrorCode(ReturnCode returnCode); + + //! Checks for known test library error return codes and returns the corresponding client test run result or empty. AZStd::optional CheckForKnownTestLibraryErrorCode(ReturnCode returnCode); + //! Checks for all known error return codes and returns the corresponding client test run result or empty. AZStd::optional CheckForAnyKnownErrorCode(ReturnCode returnCode); } From a91e9caaf9861d7b99eee717f9c09ba8ebb7775e Mon Sep 17 00:00:00 2001 From: jonawals Date: Thu, 27 May 2021 12:09:28 +0100 Subject: [PATCH 040/102] Add missing API comment to DeleteFile --- .../Code/Include/TestImpactFramework/TestImpactFileUtils.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactFileUtils.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactFileUtils.h index aa64840e78..ffecd27c23 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactFileUtils.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactFileUtils.h @@ -77,8 +77,9 @@ namespace TestImpact }); } - inline void DeleteFile(const RepoPath& path) + //! Deletes the specified file. + inline void DeleteFile(const RepoPath& file) { - DeleteFiles(path.ParentPath(), path.Filename().Native()); + DeleteFiles(file.ParentPath(), file.Filename().Native()); } } // namespace TestImpact From c3ea2d18baf38dc8a3fb64b22f9c8d06502254fb Mon Sep 17 00:00:00 2001 From: jonawals Date: Thu, 27 May 2021 12:15:19 +0100 Subject: [PATCH 041/102] Remove extra whitespace --- .../Code/Source/TestEngine/TestImpactTestEngineJobFailure.h | 1 - 1 file changed, 1 deletion(-) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJobFailure.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJobFailure.h index 8e88571957..799e8fc718 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJobFailure.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJobFailure.h @@ -31,5 +31,4 @@ namespace TestImpact //! Checks for all known error return codes and returns the corresponding client test run result or empty. AZStd::optional CheckForAnyKnownErrorCode(ReturnCode returnCode); - } From 8965061b9b6f890f3bfc60a9bb9886f801e1a4ac Mon Sep 17 00:00:00 2001 From: jonawals Date: Thu, 27 May 2021 13:21:11 +0100 Subject: [PATCH 042/102] Address PR comments --- .../TestImpactFramework/TestImpactClientFailureReport.h | 2 +- .../TestImpactFramework/TestImpactClientTestSelection.h | 2 +- .../Code/Include/TestImpactFramework/TestImpactRepoPath.h | 4 ++-- .../Code/Include/TestImpactFramework/TestImpactRuntime.h | 2 +- .../Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp | 4 ++-- .../Dependency/TestImpactSourceCoveringTestsSerializer.cpp | 2 +- .../TestEngine/TestImpactWin32_TestEngineJobFailure.cpp | 2 +- .../Code/Source/TestEngine/TestImpactTestEngineJobFailure.cpp | 2 +- .../Code/Source/TestEngine/TestImpactTestEngineJobFailure.h | 2 +- 9 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientFailureReport.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientFailureReport.h index 19e98a7d1f..0a482e8c51 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientFailureReport.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientFailureReport.h @@ -113,7 +113,7 @@ namespace TestImpact //! Returns the test targets that contain failing tests. const AZStd::vector& GetTestRunFailures() const; - //! Returns the test targets in this sequence that were terminated for exceeding their allotted flight time. + //! Returns the test targets in this sequence that were terminated for exceeding their allotted runtime. const AZStd::vector& GetTimedOutTests() const; //! Returns the test targets in this sequence that were not executed due to the sequence terminating prematurely. diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientTestSelection.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientTestSelection.h index 76e6461206..7720bbc5ad 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientTestSelection.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientTestSelection.h @@ -40,7 +40,7 @@ namespace TestImpact //! Returns the number of selected test runs that will not be run. size_t GetNumExcludedTestRuns() const; - //! Returns the total number of tests runs selected regardless of whether or not they will actually be run. + //! Returns the total number of test runs selected regardless of whether or not they will actually be run. size_t GetTotalNumTests() const; private: diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRepoPath.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRepoPath.h index 80f82e4785..9ac457e428 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRepoPath.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRepoPath.h @@ -30,7 +30,7 @@ namespace TestImpact constexpr RepoPath() = default; constexpr RepoPath(const RepoPath&) = default; constexpr RepoPath(RepoPath&&) noexcept = default; - constexpr RepoPath::RepoPath(const string_type & path) noexcept; + constexpr RepoPath::RepoPath(const string_type& path) noexcept; constexpr RepoPath::RepoPath(const string_view_type& path) noexcept; constexpr RepoPath::RepoPath(const value_type* path) noexcept; constexpr RepoPath::RepoPath(const AZ::IO::PathView& path); @@ -53,11 +53,11 @@ namespace TestImpact constexpr AZ::IO::PathView RootName() const { return m_path.RootName(); } constexpr AZ::IO::PathView RelativePath() const { return m_path.RelativePath(); } + // Wrappers around the AZ::IO::Path concatenation operator friend RepoPath operator/(const RepoPath& lhs, const AZ::IO::PathView& rhs); friend RepoPath operator/(const RepoPath& lhs, AZStd::string_view rhs); friend RepoPath operator/(const RepoPath& lhs, const typename value_type* rhs); friend RepoPath operator/(const RepoPath& lhs, const RepoPath& rhs); - RepoPath& operator/=(const AZ::IO::PathView& rhs); RepoPath& operator/=(AZStd::string_view rhs); RepoPath& operator/=(const typename value_type* rhs); diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h index 31b2d1edce..266bd5ef7a 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h @@ -83,7 +83,7 @@ namespace TestImpact AZStd::chrono::milliseconds duration)>; //! Callback for test runs that have completed for any reason. - //! selectedTests The test that has completed. + //! @param selectedTests The test that has completed. using TestRunCompleteCallback = AZStd::function; //! The API exposed to the client responsible for all test runs and persistent data management. diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp index 919c10d049..826fa0a87d 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp @@ -174,8 +174,8 @@ namespace TestImpact // Build target to covering test target mapping for (const auto& parentTarget : sourceDependency.m_parentTargets) - { - m_buildTargetCoverage[parentTarget.GetBuildTarget()].insert(testTarget); + { + m_buildTargetCoverage[parentTarget.GetBuildTarget()].insert(testTarget); } } else diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsSerializer.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsSerializer.cpp index 7b89f3bda9..5143f9b73c 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsSerializer.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsSerializer.cpp @@ -20,7 +20,7 @@ namespace TestImpact { - // Tag used to indicate that a given line is the name o a covering test target + // Tag used to indicate whether a given line is the name or a covering test target constexpr char TargetTag = '-'; AZStd::string SerializeSourceCoveringTestsList(const SourceCoveringTestsList& sourceCoveringTestsList) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/TestEngine/TestImpactWin32_TestEngineJobFailure.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/TestEngine/TestImpactWin32_TestEngineJobFailure.cpp index 578130882d..6561293d5a 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/TestEngine/TestImpactWin32_TestEngineJobFailure.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/TestEngine/TestImpactWin32_TestEngineJobFailure.cpp @@ -32,4 +32,4 @@ namespace TestImpact return AZStd::nullopt; } -} +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJobFailure.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJobFailure.cpp index debf1406f2..931e3a272b 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJobFailure.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJobFailure.cpp @@ -79,4 +79,4 @@ namespace TestImpact return AZStd::nullopt; } -} +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJobFailure.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJobFailure.h index 799e8fc718..969aad9c84 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJobFailure.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJobFailure.h @@ -31,4 +31,4 @@ namespace TestImpact //! Checks for all known error return codes and returns the corresponding client test run result or empty. AZStd::optional CheckForAnyKnownErrorCode(ReturnCode returnCode); -} +} // namespace TestImpact From 52e61fa3cac8b7bae1ae78019dc3306895669cb7 Mon Sep 17 00:00:00 2001 From: jonawals Date: Thu, 27 May 2021 13:24:48 +0100 Subject: [PATCH 043/102] Remove JobExceptionPolicy and move knoen error handling --- .../TestImpactWin32_TestTargetExtension.cpp | 2 +- .../Enumeration/TestImpactTestEnumerator.cpp | 105 ++++++++---------- .../Enumeration/TestImpactTestEnumerator.h | 14 --- .../TestImpactTestJobInfoGenerator.cpp | 12 +- .../JobRunner/TestImpactTestJobRunner.h | 41 ++----- .../Run/TestImpactInstrumentedTestRunner.cpp | 4 +- .../Run/TestImpactInstrumentedTestRunner.h | 1 - .../TestEngine/Run/TestImpactTestRunner.cpp | 4 +- .../TestEngine/Run/TestImpactTestRunner.h | 1 - .../TestEngine/TestImpactTestEngine.cpp | 91 +++++---------- .../Source/TestEngine/TestImpactTestEngine.h | 1 + 11 files changed, 88 insertions(+), 188 deletions(-) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/TestEngine/JobRunner/TestImpactWin32_TestTargetExtension.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/TestEngine/JobRunner/TestImpactWin32_TestTargetExtension.cpp index 12c78f91cf..14a1a28ba7 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/TestEngine/JobRunner/TestImpactWin32_TestTargetExtension.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/TestEngine/JobRunner/TestImpactWin32_TestTargetExtension.cpp @@ -41,4 +41,4 @@ namespace TestImpact } } } -} +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.cpp index f7b6d4a264..afc40036e3 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.cpp @@ -10,7 +10,7 @@ * */ -#include +#include #include #include @@ -22,35 +22,6 @@ namespace TestImpact { - namespace - { - void WriteCacheFile( - const TestEnumeration& enumeration, const RepoPath& path, Bitwise::CacheExceptionPolicy cacheExceptionPolicy) - { - const AZStd::string cacheJSON = SerializeTestEnumeration(enumeration); - const AZStd::vector cacheBytes(cacheJSON.begin(), cacheJSON.end()); - AZ::IO::SystemFile cacheFile; - - if (!cacheFile.Open( - path.c_str(), - AZ::IO::SystemFile::SF_OPEN_CREATE | AZ::IO::SystemFile::SF_OPEN_CREATE_PATH | AZ::IO::SystemFile::SF_OPEN_WRITE_ONLY)) - { - AZ_TestImpact_Eval( - !IsFlagSet(cacheExceptionPolicy, Bitwise::CacheExceptionPolicy::OnCacheWriteFailure), TestEngineException, - "Couldn't open cache file for writing"); - return; - } - - if (cacheFile.Write(cacheBytes.data(), cacheBytes.size()) == 0) - { - AZ_TestImpact_Eval( - !IsFlagSet(cacheExceptionPolicy, Bitwise::CacheExceptionPolicy::OnCacheWriteFailure), TestEngineException, - "Couldn't write cache file data"); - return; - } - } - } // namespace - TestEnumeration ParseTestEnumerationFile(const RepoPath& enumerationFile) { return TestEnumeration(GTest::TestEnumerationSuitesFactory(ReadFileContents(enumerationFile))); @@ -79,8 +50,6 @@ namespace TestImpact AZStd::pair> TestEnumerator::Enumerate( const AZStd::vector& jobInfos, - CacheExceptionPolicy cacheExceptionPolicy, - JobExceptionPolicy jobExceptionPolicy, AZStd::optional enumerationTimeout, AZStd::optional enumeratorTimeout, AZStd::optional clientCallback) @@ -88,50 +57,67 @@ namespace TestImpact AZStd::vector cachedJobs; AZStd::vector jobQueue; - for (const auto& jobInfo : jobInfos) + for (auto jobInfo = jobInfos.begin(); jobInfo != jobInfos.end(); ++jobInfo) { // If this job has a cache read policy attempt to read the cache - if (jobInfo.GetCache().has_value() && jobInfo.GetCache()->m_policy == JobData::CachePolicy::Read) + if (jobInfo->GetCache().has_value()) { - JobMeta meta; - AZStd::optional enumeration; + if (jobInfo->GetCache()->m_policy == JobData::CachePolicy::Read) + { + JobMeta meta; + AZStd::optional enumeration; - try - { - enumeration = TestEnumeration(DeserializeTestEnumeration(ReadFileContents(jobInfo.GetCache()->m_file))); - } - catch (const TestEngineException& e) - { - AZ_Printf("Enumerate", "Enumeration cache error: %s", e.what()); - } - - // Even though cached jobs don't get executed we still give the client the opportunity to handle the job state - // change in order to make the caching process transparent to the client - if (enumeration.has_value()) - { - if (m_clientJobCallback.has_value()) + try { - (*m_clientJobCallback)(jobInfo, meta); + enumeration = TestEnumeration(DeserializeTestEnumeration(ReadFileContents(jobInfo->GetCache()->m_file))); + } + catch (const TestEngineException& e) + { + AZ_Printf("Enumerate", "Enumeration cache error: %s", e.what()); + DeleteFile(jobInfo->GetCache()->m_file); } - // Cache read successfully, this job will not be placed in the job queue - cachedJobs.emplace_back(Job(jobInfo, AZStd::move(meta), AZStd::move(enumeration))); + // Even though cached jobs don't get executed we still give the client the opportunity to handle the job state + // change in order to make the caching process transparent to the client + if (enumeration.has_value()) + { + // Cache read successfully, this job will not be placed in the job queue + cachedJobs.emplace_back(Job(*jobInfo, AZStd::move(meta), AZStd::move(enumeration))); + + if (m_clientJobCallback.has_value() && (*m_clientJobCallback)(*jobInfo, meta) == ProcessCallbackResult::Abort) + { + // Client chose to abort so we will copy over the existing cache enumerations and fill the rest with blanks + AZStd::vector jobs(cachedJobs); + for (auto emptyJobInfo = ++jobInfo; emptyJobInfo != jobInfos.end(); ++emptyJobInfo) + { + jobs.emplace_back(Job(*emptyJobInfo, {}, AZStd::nullopt)); + } + + return { ProcessSchedulerResult::UserAborted, jobs }; + } + } + else + { + // The cache read failed and exception policy for cache read failures is not to throw so instead place this + // job in the job queue + jobQueue.emplace_back(*jobInfo); + } } else { - // The cache read failed and exception policy for cache read failures is not to throw so instead place this job in the - // job queue - jobQueue.emplace_back(jobInfo); + // This job has no cache read policy so delete the cache and place in job queue + DeleteFile(jobInfo->GetCache()->m_file); + jobQueue.emplace_back(*jobInfo); } } else { // This job has no cache read policy so place in job queue - jobQueue.emplace_back(jobInfo); + jobQueue.emplace_back(*jobInfo); } } - const auto payloadGenerator = [this, cacheExceptionPolicy](const JobDataMap& jobDataMap) + const auto payloadGenerator = [this](const JobDataMap& jobDataMap) { PayloadMap enumerations; for (const auto& [jobId, jobData] : jobDataMap) @@ -146,7 +132,7 @@ namespace TestImpact // Write out the enumeration to a cache file if we have a cache write policy for this job if (jobInfo->GetCache().has_value() && jobInfo->GetCache()->m_policy == JobData::CachePolicy::Write) { - WriteCacheFile(enumeration.value(), jobInfo->GetCache()->m_file, cacheExceptionPolicy); + WriteFileContents(SerializeTestEnumeration(enumeration.value()), jobInfo->GetCache()->m_file); } } catch (const Exception& e) @@ -163,7 +149,6 @@ namespace TestImpact // Generate the enumeration results for the jobs that weren't cached auto [result, jobs] = ExecuteJobs( jobQueue, - jobExceptionPolicy, payloadGenerator, StdOutputRouting::None, StdErrorRouting::None, diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.h index c802b92529..196700998f 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.h @@ -51,16 +51,6 @@ namespace TestImpact AZStd::optional m_cache = AZStd::nullopt; //!< No caching takes place if cache is empty. }; - namespace Bitwise - { - //! Exception policy for test enumeration cache reads/writes. - enum class CacheExceptionPolicy - { - Never = 0, //! Never throw. - OnCacheWriteFailure = 1 //! Throw when a cache write policy is in place but the cache file could not be written. - }; - } // namespace Bitwise - //! Enumerate a batch of test targets to determine the test suites and fixtures they contain, caching the results where applicable. class TestEnumerator : public TestJobRunner @@ -68,8 +58,6 @@ namespace TestImpact using JobRunner = TestJobRunner; public: - using CacheExceptionPolicy = Bitwise::CacheExceptionPolicy; - //! Constructs a test enumerator with the specified parameters common to all enumeration job runs of this enumerator. //! @param maxConcurrentEnumerations The maximum number of enumerations to be in flight at any given time. explicit TestEnumerator(size_t maxConcurrentEnumerations); @@ -84,8 +72,6 @@ namespace TestImpact //! @return The result of the run sequence and the enumeration jobs with their associated test enumeration payloads. AZStd::pair> Enumerate( const AZStd::vector& jobInfos, - CacheExceptionPolicy cacheExceptionPolicy, - JobExceptionPolicy jobExceptionPolicy, AZStd::optional enumerationTimeout, AZStd::optional enumeratorTimeout, AZStd::optional clientCallback); diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobInfoGenerator.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobInfoGenerator.cpp index 7e8e65e940..356688a128 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobInfoGenerator.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobInfoGenerator.cpp @@ -38,7 +38,7 @@ namespace TestImpact { return AZStd::string::format( "%s%s %s", - (m_targetBinaryDir / testTarget->GetOutputName()).c_str(), + (m_targetBinaryDir / RepoPath(testTarget->GetOutputName())).c_str(), GetTestTargetExtension(testTarget).c_str(), testTarget->GetCustomArgs().c_str()).c_str(); } @@ -47,7 +47,7 @@ namespace TestImpact return AZStd::string::format( "\"%s\" \"%s%s\" %s", m_testRunnerBinary.c_str(), - (m_targetBinaryDir / testTarget->GetOutputName()).c_str(), + (m_targetBinaryDir / RepoPath(testTarget->GetOutputName())).c_str(), GetTestTargetExtension(testTarget).c_str(), testTarget->GetCustomArgs().c_str()).c_str(); } @@ -55,22 +55,22 @@ namespace TestImpact RepoPath TestJobInfoGenerator::GenerateTargetEnumerationCacheFilePath(const TestTarget* testTarget) const { - return AZStd::string::format("%s.cache", (m_artifactDir / testTarget->GetName()).c_str()); + return AZStd::string::format("%s.cache", (m_cacheDir / RepoPath(testTarget->GetName())).c_str()); } RepoPath TestJobInfoGenerator::GenerateTargetEnumerationArtifactFilePath(const TestTarget* testTarget) const { - return AZStd::string::format("%s.Enumeration.xml", (m_artifactDir / testTarget->GetName()).c_str()); + return AZStd::string::format("%s.Enumeration.xml", (m_artifactDir / RepoPath(testTarget->GetName())).c_str()); } RepoPath TestJobInfoGenerator::GenerateTargetRunArtifactFilePath(const TestTarget* testTarget) const { - return AZStd::string::format("%s.Run.xml", (m_artifactDir / testTarget->GetName()).c_str()); + return AZStd::string::format("%s.Run.xml", (m_artifactDir / RepoPath(testTarget->GetName())).c_str()); } RepoPath TestJobInfoGenerator::GenerateTargetCoverageArtifactFilePath(const TestTarget* testTarget) const { - return AZStd::string::format("%s.Coverage.xml", (m_artifactDir / testTarget->GetName()).c_str()); + return AZStd::string::format("%s.Coverage.xml", (m_artifactDir / RepoPath(testTarget->GetName())).c_str()); } TestEnumerator::JobInfo TestJobInfoGenerator::GenerateTestEnumerationJobInfo( diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobRunner.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobRunner.h index 3078114ef4..0c1c8189e3 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobRunner.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobRunner.h @@ -12,8 +12,6 @@ #pragma once -#include - #include #include @@ -23,17 +21,6 @@ namespace TestImpact { - namespace Bitwise - { - //! Exception policy for test jobs (and derived jobs). - enum class TestJobExceptionPolicy - { - Never = 0, //!< Never throw. - OnFailedToExecute = 1, //!< Throw when a job fails to execute. - OnExecutedWithFailure = 1 << 1 //!< Throw when a job returns with an error code. - }; - } // namespace Bitwise - //! Base class for test related job runners. //! @tparam AdditionalInfo The data structure containing the information additional to the command arguments necessary to execute and //! complete a job. @@ -47,9 +34,8 @@ namespace TestImpact using Command = typename JobInfo::Command; using JobPayload = Payload; using Job = Job; - using ClientJobCallback = AZStd::function; + using ClientJobCallback = AZStd::function; using DerivedJobCallback = JobCallback; - using JobExceptionPolicy = Bitwise::TestJobExceptionPolicy; using JobDataMap = JobDataMap; //! Constructs the job runner with the specified parameters common to all job runs of this runner. @@ -70,7 +56,6 @@ namespace TestImpact //! @returns The result of the run sequence and the jobs that the sequence produced. AZStd::pair> ExecuteJobs( const AZStd::vector& jobInfos, - JobExceptionPolicy jobExceptionPolicy, PayloadMapProducer payloadMapProducer, StdOutputRouting stdOutRouting, StdErrorRouting stdErrRouting, @@ -95,7 +80,6 @@ namespace TestImpact template AZStd::pair::Job>> TestJobRunner::ExecuteJobs( const AZStd::vector& jobInfos, - JobExceptionPolicy jobExceptionPolicy, PayloadMapProducer payloadMapProducer, StdOutputRouting stdOutRouting, StdErrorRouting stdErrRouting, @@ -105,30 +89,21 @@ namespace TestImpact AZStd::optional derivedJobCallback) { // Callback to handle job exception policies and client/derived callbacks - const auto jobCallback = [&clientCallback, &derivedJobCallback, &jobExceptionPolicy](const JobInfo& jobInfo, const JobMeta& meta, StdContent&& std) + const auto jobCallback = [&clientCallback, &derivedJobCallback](const JobInfo& jobInfo, const JobMeta& meta, StdContent&& std) { auto callbackResult = ProcessCallbackResult::Continue; - if (meta.m_result == JobResult::FailedToExecute && IsFlagSet(jobExceptionPolicy, JobExceptionPolicy::OnFailedToExecute)) - { - callbackResult = ProcessCallbackResult::Abort; - } - else if (meta.m_result == JobResult::ExecutedWithFailure && IsFlagSet(jobExceptionPolicy, JobExceptionPolicy::OnExecutedWithFailure)) - { - callbackResult = ProcessCallbackResult::Abort; - } - if (derivedJobCallback.has_value()) { - if (const auto result = (*derivedJobCallback)(jobInfo, meta, AZStd::move(std)); - result == ProcessCallbackResult::Abort) - { - callbackResult = ProcessCallbackResult::Abort; - } + callbackResult = (*derivedJobCallback)(jobInfo, meta, AZStd::move(std)); } if (clientCallback.has_value()) { - (*clientCallback)(jobInfo, meta); + if (const auto result = (*clientCallback)(jobInfo, meta); + result == ProcessCallbackResult::Abort) + { + callbackResult = ProcessCallbackResult::Abort; + } } return callbackResult; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.cpp index 3b109c3a77..aa8ff04657 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.cpp @@ -10,7 +10,7 @@ * */ -#include +#include #include #include @@ -51,7 +51,6 @@ namespace TestImpact AZStd::pair> InstrumentedTestRunner::RunInstrumentedTests( const AZStd::vector& jobInfos, - JobExceptionPolicy jobExceptionPolicy, AZStd::optional runTimeout, AZStd::optional runnerTimeout, AZStd::optional clientCallback) @@ -84,7 +83,6 @@ namespace TestImpact return ExecuteJobs( jobInfos, - jobExceptionPolicy, payloadGenerator, StdOutputRouting::None, StdErrorRouting::None, diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.h index b2d7f84b6d..2f3e6eb98e 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.h @@ -54,7 +54,6 @@ namespace TestImpact //! @return The result of the run sequence and the instrumented run jobs with their associated test run and coverage payloads. AZStd::pair> RunInstrumentedTests( const AZStd::vector& jobInfos, - JobExceptionPolicy jobExceptionPolicy, AZStd::optional runTimeout, AZStd::optional runnerTimeout, AZStd::optional clientCallback); diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunner.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunner.cpp index 401084c934..7ac13f85b2 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunner.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunner.cpp @@ -10,7 +10,7 @@ * */ -#include +#include #include #include @@ -28,7 +28,6 @@ namespace TestImpact AZStd::pair> TestRunner::RunTests( const AZStd::vector& jobInfos, - JobExceptionPolicy jobExceptionPolicy, AZStd::optional runTimeout, AZStd::optional runnerTimeout, AZStd::optional clientCallback) @@ -58,7 +57,6 @@ namespace TestImpact return ExecuteJobs( jobInfos, - jobExceptionPolicy, payloadGenerator, StdOutputRouting::None, StdErrorRouting::None, diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunner.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunner.h index 1fdbc16d58..eaa7b6de1d 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunner.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunner.h @@ -39,7 +39,6 @@ namespace TestImpact //! @return The result of the run sequence and the run jobs with their associated test run payloads. AZStd::pair> RunTests( const AZStd::vector& jobInfos, - JobExceptionPolicy jobExceptionPolicy, AZStd::optional runTimeout, AZStd::optional runnerTimeout, AZStd::optional clientCallback); diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.cpp index 5046b3f8ee..31c0d7c5de 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.cpp @@ -17,34 +17,14 @@ #include #include #include +#include #include +#include + namespace TestImpact { - // Known error codes for test instrumentation, test runner and unit test library - // This could be refactored into a generic solution agnostic of the tool and library specific details - namespace ErrorCodes - { - namespace OpenCppCoverage - { - static constexpr ReturnCode InvalidArgs = -1618178468; - } - - namespace GTest - { - static constexpr ReturnCode Unsuccessful = 1; - } - - namespace AZTestRunner - { - static constexpr ReturnCode InvalidArgs = 101; - static constexpr ReturnCode FailedToFindTargetBinary = 102; - static constexpr ReturnCode SymbolNotFound = 103; - static constexpr ReturnCode ModuleSkipped = 104; - } - } - namespace { // Calculate the sequence result by analysing the state of the test targets that were run. @@ -101,21 +81,10 @@ namespace TestImpact // Attempt to determine why a given test target executed successfully but return with an error code if (meta.m_returnCode.has_value()) { - switch (meta.m_returnCode.value()) + if (const auto result = CheckForAnyKnownErrorCode(meta.m_returnCode.value()); + result != AZStd::nullopt) { - // We will consider test targets that technically execute but their launcher or unit test library return a know error - // code that pertains to incorrect argument usage as test targets that failed to execute - case ErrorCodes::OpenCppCoverage::InvalidArgs: - case ErrorCodes::AZTestRunner::InvalidArgs: - case ErrorCodes::AZTestRunner::FailedToFindTargetBinary: - case ErrorCodes::AZTestRunner::ModuleSkipped: - case ErrorCodes::AZTestRunner::SymbolNotFound: - return Client::TestRunResult::FailedToExecute; - // The trivial case: the test target has failing tests - case ErrorCodes::GTest::Unsuccessful: - return Client::TestRunResult::TestFailures; - default: - break; + return result.value(); } } @@ -186,14 +155,18 @@ namespace TestImpact TestJobRunnerCallbackHandler( const AZStd::vector& testTargets, TestEngineJobMap* engineJobs, + Policy::ExecutionFailure executionFailurePolicy, + Policy::TestFailure testFailurePolicy, AZStd::optional* callback) : m_testTargets(testTargets) , m_engineJobs(engineJobs) + , m_executionFailurePolicy(executionFailurePolicy) + , m_testFailurePolicy(testFailurePolicy) , m_callback(callback) { } - void operator()(const typename JobInfo& jobInfo, const TestImpact::JobMeta& meta) + [[nodiscard]] ProcessCallbackResult operator()(const typename JobInfo& jobInfo, const TestImpact::JobMeta& meta) { const auto id = jobInfo.GetId().m_value; const auto& args = jobInfo.GetCommand().m_args; @@ -208,11 +181,21 @@ namespace TestImpact { (*m_callback).value()(it->second); } + + if ((result == Client::TestRunResult::FailedToExecute && m_executionFailurePolicy == Policy::ExecutionFailure::Abort) || + (result == Client::TestRunResult::TestFailures && m_testFailurePolicy == Policy::TestFailure::Abort)) + { + return ProcessCallbackResult::Abort; + } + + return ProcessCallbackResult::Continue; } private: const AZStd::vector& m_testTargets; TestEngineJobMap* m_engineJobs; + Policy::ExecutionFailure m_executionFailurePolicy; + Policy::TestFailure m_testFailurePolicy; AZStd::optional* m_callback; }; @@ -250,23 +233,6 @@ namespace TestImpact return engineRuns; } - - Bitwise::TestJobExceptionPolicy GetTestJobExceptionPolicy( - Policy::ExecutionFailure executionFailurePolicy, Policy::TestFailure testFailurePolicy) - { - auto jobExecutionPolicy = Bitwise::TestJobExceptionPolicy::Never; - if (executionFailurePolicy == Policy::ExecutionFailure::Abort) - { - jobExecutionPolicy |= Bitwise::TestJobExceptionPolicy::OnFailedToExecute; - } - - if (testFailurePolicy == Policy::TestFailure::Abort) - { - jobExecutionPolicy |= Bitwise::TestJobExceptionPolicy::OnExecutedWithFailure; - } - - return jobExecutionPolicy; - } } TestEngine::TestEngine( @@ -291,24 +257,19 @@ namespace TestImpact AZStd::pair> TestEngine::UpdateEnumerationCache( const AZStd::vector& testTargets, Policy::ExecutionFailure executionFailurePolicy, + Policy::TestFailure testFailurePolicy, AZStd::optional testTargetTimeout, AZStd::optional globalTimeout, AZStd::optional callback) { TestEngineJobMap engineJobs; const auto jobInfos = m_testJobInfoGenerator->GenerateTestEnumerationJobInfos(testTargets, TestEnumerator::JobInfo::CachePolicy::Write); - - const auto jobExecutionPolicy = executionFailurePolicy == Policy::ExecutionFailure::Abort - ? (TestEnumerator::JobExceptionPolicy::OnExecutedWithFailure | TestEnumerator::JobExceptionPolicy::OnFailedToExecute) - : TestEnumerator::JobExceptionPolicy::Never; auto [result, runnerJobs] = m_testEnumerator->Enumerate( jobInfos, - TestEnumerator::CacheExceptionPolicy::OnCacheWriteFailure, - jobExecutionPolicy, testTargetTimeout, globalTimeout, - TestJobRunnerCallbackHandler(testTargets, &engineJobs, &callback)); + TestJobRunnerCallbackHandler(testTargets, &engineJobs, executionFailurePolicy, testFailurePolicy, &callback)); auto engineRuns = CompileTestEngineRuns(testTargets, runnerJobs, AZStd::move(engineJobs)); return { CalculateSequenceResult(result, engineRuns, executionFailurePolicy), AZStd::move(engineRuns) }; @@ -327,10 +288,9 @@ namespace TestImpact TestEngineJobMap engineJobs; const auto jobInfos = m_testJobInfoGenerator->GenerateRegularTestRunJobInfos(testTargets); - TestJobRunnerCallbackHandler jobCallback(testTargets, &engineJobs, &callback); + TestJobRunnerCallbackHandler jobCallback(testTargets, &engineJobs, executionFailurePolicy, testFailurePolicy, &callback); auto [result, runnerJobs] = m_testRunner->RunTests( jobInfos, - GetTestJobExceptionPolicy(executionFailurePolicy, testFailurePolicy), testTargetTimeout, globalTimeout, jobCallback); @@ -355,10 +315,9 @@ namespace TestImpact auto [result, runnerJobs] = m_instrumentedTestRunner->RunInstrumentedTests( jobInfos, - GetTestJobExceptionPolicy(executionFailurePolicy, testFailurePolicy), testTargetTimeout, globalTimeout, - TestJobRunnerCallbackHandler(testTargets, &engineJobs, &callback)); + TestJobRunnerCallbackHandler(testTargets, &engineJobs, executionFailurePolicy, testFailurePolicy, &callback)); auto engineRuns = CompileTestEngineRuns(testTargets, runnerJobs, AZStd::move(engineJobs)); diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.h index c67a10aa4c..83f57f00ef 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.h @@ -70,6 +70,7 @@ namespace TestImpact AZStd::pair> UpdateEnumerationCache( const AZStd::vector& testTargets, Policy::ExecutionFailure executionFailurePolicy, + Policy::TestFailure testFailurePolicy, AZStd::optional testTargetTimeout, AZStd::optional globalTimeout, AZStd::optional callback); From 41c8490ebbbbccb0fe68c1dff4442cca1fe945b9 Mon Sep 17 00:00:00 2001 From: jonawals Date: Thu, 27 May 2021 13:30:06 +0100 Subject: [PATCH 044/102] Add missing cache delete --- .../Source/TestEngine/Enumeration/TestImpactTestEnumerator.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.cpp index afc40036e3..eacf73efef 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.cpp @@ -113,6 +113,7 @@ namespace TestImpact else { // This job has no cache read policy so place in job queue + DeleteFile(jobInfo->GetCache()->m_file); jobQueue.emplace_back(*jobInfo); } } From ec758edd0decbfe2962d682acc3efe08c77b6cac Mon Sep 17 00:00:00 2001 From: jonawals Date: Thu, 27 May 2021 13:45:44 +0100 Subject: [PATCH 045/102] Remove iostream include --- .../Runtime/Code/Source/TestEngine/TestImpactTestEngine.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.cpp index 31c0d7c5de..b563c1846b 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.cpp @@ -21,8 +21,6 @@ #include -#include - namespace TestImpact { namespace From e82b54d59b79959b75e609690ac4430017113832 Mon Sep 17 00:00:00 2001 From: jonawals Date: Thu, 27 May 2021 14:04:16 +0100 Subject: [PATCH 046/102] Address PR comments --- .../Enumeration/TestImpactTestEnumerator.cpp | 39 ++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.cpp index eacf73efef..58ffb85cb3 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.cpp @@ -112,12 +112,49 @@ namespace TestImpact } else { - // This job has no cache read policy so place in job queue + // This job has no cache read policy so delete the cache and place in job queue DeleteFile(jobInfo->GetCache()->m_file); jobQueue.emplace_back(*jobInfo); } } + /* As per comment on PR51, this suggestion will be explored once the test coverage code for this subsystem is revisited + bool aborted = false; + for (const auto& jobInfo : jobInfos) + { + if (!jobInfo->GetCache().has_value() || + jobInfo->GetCache()->m_policy != JobData::CachePolicy::Read) + { + DeleteFile(jobInfo->GetCache()->m_file); + jobQueue.emplace_back(*jobInfo); + continue; + } + + ... // try catch part + if (!enumeration.has_value()) + { + jobQueue.emplace_back(*jobInfo); + continue; + } + + // Cache read successfully, this job will not be placed in the job queue + cachedJobs.emplace_back(Job(*jobInfo, AZStd::move(meta), AZStd::move(enumeration))); + + if (m_clientJobCallback.has_value() && (*m_clientJobCallback)(*jobInfo, meta) == ProcessCallbackResult::Abort) + { + aborted = true; // catch the index too + break; + } + } + + if (aborted) + { + // do the abortion part + + return { ProcessSchedulerResult::UserAborted, jobs }; + } + */ + const auto payloadGenerator = [this](const JobDataMap& jobDataMap) { PayloadMap enumerations; From 042a025f619a9cc3c3a9c886e4e22d80f04e40f8 Mon Sep 17 00:00:00 2001 From: jonawals Date: Thu, 27 May 2021 14:06:59 +0100 Subject: [PATCH 047/102] Private runtime implementation --- .../Source/TestImpactChangeListSerializer.cpp | 96 ++++ .../Source/TestImpactClientFailureReport.cpp | 124 +++++ .../Code/Source/TestImpactClientTestRun.cpp | 40 ++ .../Source/TestImpactClientTestSelection.cpp | 56 ++ .../Code/Source/TestImpactRepoPath.cpp | 104 ++++ .../Runtime/Code/Source/TestImpactRuntime.cpp | 486 ++++++++++++++++++ .../Code/Source/TestImpactRuntimeUtils.cpp | 133 +++++ .../Code/Source/TestImpactRuntimeUtils.h | 134 +++++ 8 files changed, 1173 insertions(+) create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactChangeListSerializer.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientFailureReport.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientTestRun.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientTestSelection.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRepoPath.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.h diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactChangeListSerializer.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactChangeListSerializer.cpp new file mode 100644 index 0000000000..ef157f1848 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactChangeListSerializer.cpp @@ -0,0 +1,96 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include + +#include +#include +#include +#include + +namespace TestImpact +{ + namespace ChangeListFields + { + // Keys for pertinent JSON node and attribute names + constexpr const char* Keys[] = + { + "createdFiles", + "updatedFiles", + "deletedFiles" + }; + + enum + { + CreateKey, + UpdateKey, + DeleteKey + }; + } // namespace + + AZStd::string SerializeChangeList(const ChangeList& changeList) + { + rapidjson::StringBuffer stringBuffer; + rapidjson::PrettyWriter writer(stringBuffer); + + const auto serializeFileList = [&writer](const char* key, const AZStd::vector& fileList) + { + writer.Key(key); + writer.StartArray(); + + for (const auto& file : fileList) + { + writer.String(file.c_str()); + } + + writer.EndArray(); + }; + + writer.StartObject(); + serializeFileList(ChangeListFields::Keys[ChangeListFields::CreateKey], changeList.m_createdFiles); + serializeFileList(ChangeListFields::Keys[ChangeListFields::UpdateKey], changeList.m_updatedFiles); + serializeFileList(ChangeListFields::Keys[ChangeListFields::DeleteKey], changeList.m_deletedFiles); + writer.EndObject(); + + return stringBuffer.GetString(); + } + + ChangeList DeserializeChangeList(const AZStd::string& changeListString) + { + ChangeList changeList; + rapidjson::Document doc; + + if (doc.Parse<0>(changeListString.c_str()).HasParseError()) + { + throw ChangeListException("Could not parse change list data"); + } + + const auto deserializeFileList = [&doc](const char* key) + { + AZStd::vector fileList; + + for (const auto& file : doc[key].GetArray()) + { + fileList.push_back(file.GetString()); + } + + return fileList; + }; + + changeList.m_createdFiles = deserializeFileList(ChangeListFields::Keys[ChangeListFields::CreateKey]); + changeList.m_updatedFiles = deserializeFileList(ChangeListFields::Keys[ChangeListFields::UpdateKey]); + changeList.m_deletedFiles = deserializeFileList(ChangeListFields::Keys[ChangeListFields::DeleteKey]); + + return changeList; + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientFailureReport.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientFailureReport.cpp new file mode 100644 index 0000000000..f39d8ee426 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientFailureReport.cpp @@ -0,0 +1,124 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include + +namespace TestImpact +{ + namespace Client + { + TargetFailure::TargetFailure(const AZStd::string& targetName) + : m_targetName(targetName) + { + } + + const AZStd::string& TargetFailure::GetTargetName() const + { + return m_targetName; + } + + ExecutionFailure::ExecutionFailure(const AZStd::string& targetName, const AZStd::string& command) + : TargetFailure(targetName) + , m_commandString(command) + { + } + + const AZStd::string& ExecutionFailure::GetCommandString() const + { + return m_commandString; + } + + TestFailure::TestFailure(const AZStd::string& testName, const AZStd::string& errorMessage) + : m_name(testName) + , m_errorMessage(errorMessage) + { + } + + const AZStd::string& TestFailure::GetName() const + { + return m_name; + } + + const AZStd::string& TestFailure::GetErrorMessage() const + { + return m_errorMessage; + } + + TestCaseFailure::TestCaseFailure(const AZStd::string& testCaseName, AZStd::vector&& testFailures) + : m_name(testCaseName) + , m_testFailures(AZStd::move(testFailures)) + { + } + + const AZStd::string& TestCaseFailure::GetName() const + { + return m_name; + } + + const AZStd::vector& TestCaseFailure::GetTestFailures() const + { + return m_testFailures; + } + + TestRunFailure::TestRunFailure(const AZStd::string& targetName, AZStd::vector&& testFailures) + : TargetFailure(targetName) + , m_testCaseFailures(AZStd::move(testFailures)) + { + for (const auto& testCase : m_testCaseFailures) + { + m_numTestFailures += testCase.GetTestFailures().size(); + } + } + + size_t TestRunFailure::GetNumTestFailures() const + { + return m_numTestFailures; + } + + const AZStd::vector& TestRunFailure::GetTestCaseFailures() const + { + return m_testCaseFailures; + } + + SequenceFailure::SequenceFailure( + AZStd::vector&& executionFailures, + AZStd::vector&& testRunFailures, + AZStd::vector&& timedOutTests, + AZStd::vector&& unexecutionTests) + : m_executionFailures(AZStd::move(executionFailures)) + , m_testRunFailures(testRunFailures) + , m_timedOutTests(AZStd::move(timedOutTests)) + , m_unexecutedTests(AZStd::move(unexecutionTests)) + { + } + + const AZStd::vector& SequenceFailure::GetExecutionFailures() const + { + return m_executionFailures; + } + + const AZStd::vector& SequenceFailure::GetTestRunFailures() const + { + return m_testRunFailures; + } + + const AZStd::vector& SequenceFailure::GetTimedOutTests() const + { + return m_timedOutTests; + } + + const AZStd::vector& SequenceFailure::GetUnexecutedTests() const + { + return m_unexecutedTests; + } + } +} diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientTestRun.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientTestRun.cpp new file mode 100644 index 0000000000..6661de5d69 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientTestRun.cpp @@ -0,0 +1,40 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +namespace TestImpact +{ + namespace Client + { + TestRun::TestRun(const AZStd::string& name, TestRunResult result, AZStd::chrono::milliseconds duration) + : m_targetName(name) + , m_result(result) + , m_duration(duration) + { + } + + const AZStd::string& TestRun::GetTargetName() const + { + return m_targetName; + } + + AZStd::chrono::milliseconds TestRun::GetDuration() const + { + return m_duration; + } + + TestRunResult TestRun::GetResult() const + { + return m_result; + } + } +} diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientTestSelection.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientTestSelection.cpp new file mode 100644 index 0000000000..f6c88893d7 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientTestSelection.cpp @@ -0,0 +1,56 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include + +namespace TestImpact +{ + namespace Client + { + TestRunSelection::TestRunSelection(const AZStd::vector& includedTests, const AZStd::vector& excludedTests) + : m_includedTestRuns(includedTests) + , m_excludedTestRuns(excludedTests) + { + } + + TestRunSelection::TestRunSelection(AZStd::vector&& includedTests, AZStd::vector&& excludedTests) + : m_includedTestRuns(AZStd::move(includedTests)) + , m_excludedTestRuns(AZStd::move(excludedTests)) + { + } + + const AZStd::vector& TestRunSelection::GetIncludededTestRuns() const + { + return m_includedTestRuns; + } + + const AZStd::vector& TestRunSelection::GetExcludedTestRuns() const + { + return m_excludedTestRuns; + } + + size_t TestRunSelection::GetNumIncludedTestRuns() const + { + return m_includedTestRuns.size(); + } + + size_t TestRunSelection::GetNumExcludedTestRuns() const + { + return m_excludedTestRuns.size(); + } + + size_t TestRunSelection::GetTotalNumTests() const + { + return GetNumIncludedTestRuns() + GetNumExcludedTestRuns(); + } + } +} diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRepoPath.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRepoPath.cpp new file mode 100644 index 0000000000..11b927247f --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRepoPath.cpp @@ -0,0 +1,104 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include + +namespace TestImpact +{ + + + RepoPath& RepoPath::operator=(const string_type& other) noexcept + { + m_path = AZ::IO::Path(other).MakePreferred(); + return *this; + } + + RepoPath& RepoPath::operator=(const value_type* other) noexcept + { + m_path = AZ::IO::Path(other).MakePreferred(); + return *this; + } + + RepoPath& RepoPath::operator=(const AZ::IO::Path& other) noexcept + { + m_path = AZ::IO::Path(other).MakePreferred(); + return *this; + } + + inline RepoPath operator/(const RepoPath& lhs, const AZ::IO::PathView& rhs) + { + RepoPath result(lhs); + result.m_path /= RepoPath(rhs).m_path; + return result; + } + + inline RepoPath operator/(const RepoPath& lhs, AZStd::string_view rhs) + { + RepoPath result(lhs); + result.m_path /= RepoPath(rhs).m_path; + return result; + } + + inline RepoPath operator/(const RepoPath& lhs, const RepoPath::value_type* rhs) + { + RepoPath result(lhs); + result.m_path /= RepoPath(rhs).m_path; + return result; + } + + inline RepoPath operator/(const RepoPath& lhs, const RepoPath& rhs) + { + RepoPath result(lhs); + result.m_path /= rhs.m_path; + return result; + } + + inline RepoPath& RepoPath::operator/=(const AZ::IO::PathView& rhs) + { + m_path /= RepoPath(rhs).m_path; + return *this; + } + + + inline RepoPath& RepoPath::operator/=(AZStd::string_view rhs) + { + m_path /= RepoPath(rhs).m_path; + return *this; + } + + inline RepoPath& RepoPath::operator/=(const RepoPath::value_type* rhs) + { + m_path /= RepoPath(rhs).m_path; + return *this; + } + + inline RepoPath& RepoPath::operator/=(const RepoPath& rhs) + { + m_path /= rhs.m_path; + return *this; + } + + inline bool operator==(const RepoPath& lhs, const RepoPath& rhs) noexcept + { + return lhs.m_path.Compare(rhs.m_path) == 0; + } + + inline bool operator!=(const RepoPath& lhs, const RepoPath& rhs) noexcept + { + return lhs.m_path.Compare(rhs.m_path) != 0; + } + + inline bool operator<([[maybe_unused]] const RepoPath& lhs, [[maybe_unused]] const RepoPath& rhs) noexcept + { + return lhs.m_path.String() < rhs.m_path.String(); + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp new file mode 100644 index 0000000000..1bc55ab179 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp @@ -0,0 +1,486 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include + +namespace TestImpact +{ + namespace + { + // Simple helper class for tracking basic timing information + class Timer + { + public: + Timer() + : m_startTime(AZStd::chrono::high_resolution_clock::now()) + { + } + + // Returns the time elapsed (in milliseconds) since the timer was instantiated + AZStd::chrono::milliseconds Elapsed() + { + const auto endTime = AZStd::chrono::high_resolution_clock::now(); + return AZStd::chrono::duration_cast(endTime - m_startTime); + } + + private: + AZStd::chrono::high_resolution_clock::time_point m_startTime; + }; + + // Handler for test run complete events + class TestRunCompleteCallbackHandler + { + public: + TestRunCompleteCallbackHandler(AZStd::optional testCompleteCallback) + : m_testCompleteCallback(testCompleteCallback) + { + } + + void operator()(const TestEngineJob& testJob) + { + if (m_testCompleteCallback.has_value()) + { + (*m_testCompleteCallback) + (Client::TestRun(testJob.GetTestTarget()->GetName(), testJob.GetTestResult(), testJob.GetDuration())); + } + } + + private: + AZStd::optional m_testCompleteCallback; + }; + } + + Runtime::Runtime( + RuntimeConfig&& config, + Policy::ExecutionFailure executionFailurePolicy, + Policy::ExecutionFailureDrafting executionFailureDraftingPolicy, + Policy::TestFailure testFailurePolicy, + Policy::IntegrityFailure integrationFailurePolicy, + Policy::TestSharding testShardingPolicy, + Policy::TargetOutputCapture targetOutputCapture, + AZStd::optional maxConcurrency) + : m_config(AZStd::move(config)) + , m_executionFailurePolicy(executionFailurePolicy) + , m_executionFailureDraftingPolicy(executionFailureDraftingPolicy) + , m_testFailurePolicy(testFailurePolicy) + , m_integrationFailurePolicy(integrationFailurePolicy) + , m_testShardingPolicy(testShardingPolicy) + , m_targetOutputCapture(targetOutputCapture) + , m_maxConcurrency(maxConcurrency.value_or(AZStd::thread::hardware_concurrency())) + { + // Construct the dynamic dependency map from the build target descriptors + m_dynamicDependencyMap = ConstructDynamicDependencyMap(m_config.m_buildTargetDescriptor, m_config.m_testTargetMeta); + + // Construct the test selector and prioritizer from the dependency graph data (NOTE: currently not implemented) + m_testSelectorAndPrioritizer = AZStd::make_unique(m_dynamicDependencyMap.get(), DependencyGraphDataMap{}); + + // Construct the target exclude list from the target configuration data + m_testTargetExcludeList = ConstructTestTargetExcludeList(m_dynamicDependencyMap->GetTestTargetList(), m_config.m_target.m_excludedTestTargets); + + // Construct the test engine with the workspace path and launcher binaries + m_testEngine = AZStd::make_unique( + m_config.m_repo.m_root, + m_config.m_target.m_outputDirectory, + m_config.m_workspace.m_active.m_relativePaths.m_enumerationCacheDirectory, + m_config.m_workspace.m_temp.m_relativePaths.m_artifactDirectory, + m_config.m_testEngine.m_testRunner.m_binary, + m_config.m_testEngine.m_instrumentation.m_binary, + m_maxConcurrency); + + try + { + // Populate the dynamic dependency map with the existing source coverage data (if any) + const auto tiaDataRaw = ReadFileContents(m_config.m_workspace.m_active.m_relativePaths.m_sparTIAFile); + const auto tiaData = DeserializeSourceCoveringTestsList(tiaDataRaw); + if (tiaData.GetNumSources()) + { + m_dynamicDependencyMap->ReplaceSourceCoverage(tiaData); + m_hasImpactAnalysisData = true; + + // Enumerate new test targets + m_testEngine->UpdateEnumerationCache( + m_dynamicDependencyMap->GetNotCoveringTests(), + Policy::ExecutionFailure::Ignore, + Policy::TestFailure::Continue, + AZStd::nullopt, + AZStd::nullopt, + AZStd::nullopt); + } + } + catch (const DependencyException& e) + { + if (integrationFailurePolicy == Policy::IntegrityFailure::Abort) + { + throw RuntimeException(e.what()); + } + } + catch ([[maybe_unused]]const Exception& e) + { + AZ_Printf("No test impact analysis data found at %s", m_config.m_workspace.m_active.m_relativePaths.m_sparTIAFile.c_str()); + } + } + + Runtime::~Runtime() = default; + + void Runtime::EnumerateMutatedTestTargets(const ChangeDependencyList& changeDependencyList) + { + AZStd::vector testTargets; + const auto addMutatedTestTargetsToEnumerationList = [this, &testTargets](const AZStd::vector& sourceDependency) + { + for (const auto& sourceDependency : sourceDependency) + { + for (const auto& parentTarget : sourceDependency.GetParentTargets()) + { + AZStd::visit([&testTargets]([[maybe_unused]] auto&& target) + { + if constexpr (IsTestTarget) + { + testTargets.push_back(target); + } + }, parentTarget.GetTarget()); + } + } + }; + + // Gather all of the test targets that have had any of their sources modified + addMutatedTestTargetsToEnumerationList(changeDependencyList.GetCreateSourceDependencies()); + addMutatedTestTargetsToEnumerationList(changeDependencyList.GetUpdateSourceDependencies()); + addMutatedTestTargetsToEnumerationList(changeDependencyList.GetDeleteSourceDependencies()); + + // Enumerate the mutated test targets to ensure their enumeration caches are up to date + m_testEngine->UpdateEnumerationCache( + testTargets, + Policy::ExecutionFailure::Ignore, + Policy::TestFailure::Continue, + AZStd::nullopt, + AZStd::nullopt, + AZStd::nullopt); + } + + AZStd::pair, AZStd::vector> Runtime::SelectCoveringTestTargetsAndUpdateEnumerationCache( + const ChangeList& changeList, + Policy::TestPrioritization testPrioritizationPolicy) + { + AZStd::vector discardedTestTargets; + + // Select and prioritize the test targets pertinent to this change list + const auto changeDependecyList = m_dynamicDependencyMap->ApplyAndResoveChangeList(changeList); + const auto selectedTestTargets = m_testSelectorAndPrioritizer->SelectTestTargets(changeDependecyList, testPrioritizationPolicy); + + // Populate a set with the selected test targets so that we can infer the discarded test target not selected for this change list + const AZStd::unordered_set selectedTestTargetSet(selectedTestTargets.begin(), selectedTestTargets.end()); + + if (m_testShardingPolicy == Policy::TestSharding::Always) + { + EnumerateMutatedTestTargets(changeDependecyList); + } + + // The test targets in the main list not in the selected test target set are the test targets not selected for this change list + for (const auto& testTarget : m_dynamicDependencyMap->GetTestTargetList().GetTargets()) + { + if (!selectedTestTargetSet.contains(&testTarget)) + { + discardedTestTargets.push_back(&testTarget); + } + } + + return { selectedTestTargets, discardedTestTargets }; + } + + AZStd::pair, AZStd::vector> Runtime::SelectTestTargetsByExcludeList( + AZStd::vector testTargets) const + { + AZStd::vector includedTestTargets; + AZStd::vector excludedTestTargets; + + if (m_testTargetExcludeList.empty()) + { + return { testTargets, {} }; + } + + for (const auto& testTarget : testTargets) + { + if (!m_testTargetExcludeList.contains(testTarget)) + { + includedTestTargets.push_back(testTarget); + } + else + { + excludedTestTargets.push_back(testTarget); + } + } + + return { includedTestTargets, excludedTestTargets }; + } + + void Runtime::ClearDynamicDependencyMapAndRemoveExistingFile() + { + DeleteFile(m_config.m_workspace.m_active.m_relativePaths.m_sparTIAFile); + m_dynamicDependencyMap->ClearAllSourceCoverage(); + } + + void Runtime::UpdateAndSerializeDynamicDependencyMap(const SourceCoveringTestsList& sourceCoverageTestsList) + { + if (!sourceCoverageTestsList.GetNumSources()) + { + return; + } + + m_dynamicDependencyMap->ReplaceSourceCoverage(sourceCoverageTestsList); + const auto sparTIA = m_dynamicDependencyMap->ExportSourceCoverage(); + const auto sparTIAData = SerializeSourceCoveringTestsList(sparTIA); + WriteFileContents(sparTIAData, m_config.m_workspace.m_active.m_relativePaths.m_sparTIAFile); + m_hasImpactAnalysisData = true; + } + + TestSequenceResult Runtime::RegularTestSequence( + const AZStd::unordered_set suitesFilter, + AZStd::optional testTargetTimeout, + AZStd::optional globalTimeout, + AZStd::optional testSequenceStartCallback, + AZStd::optional testSequenceEndCallback, + AZStd::optional testCompleteCallback) + { + Timer timer; + AZStd::vector includedTestTargets; + AZStd::vector excludedTestTargets; + + // Separate the test targets into those that are excluded by either the test filter or exclusion list and those that are not + for (const auto& testTarget : m_dynamicDependencyMap->GetTestTargetList().GetTargets()) + { + if (!m_testTargetExcludeList.contains(&testTarget)) + { + if (suitesFilter.empty()) + { + // Suite filter is empty, all tests that are not on the excluded list are included + includedTestTargets.push_back(&testTarget); + } + else if(suitesFilter.contains(testTarget.GetSuite())) + { + // Test target belonging to a suite in the suite filter are included, provided that are not on the exclude list + includedTestTargets.push_back(&testTarget); + } + else + { + // Test target not belonging to a suite in the suite filter are excluded + excludedTestTargets.push_back(&testTarget); + } + } + else + { + // Test targets on the exclude list are excluded + excludedTestTargets.push_back(&testTarget); + } + } + + // Sequence start callback + if (testSequenceStartCallback.has_value()) + { + (*testSequenceStartCallback)(Client::TestRunSelection(ExtractTestTargetNames(includedTestTargets), ExtractTestTargetNames(excludedTestTargets))); + } + + const auto [result, testJobs] = m_testEngine->RegularRun( + includedTestTargets, + m_testShardingPolicy, + m_executionFailurePolicy, + m_testFailurePolicy, + m_targetOutputCapture, + testTargetTimeout, + globalTimeout, + TestRunCompleteCallbackHandler(testCompleteCallback)); + + if (testSequenceEndCallback.has_value()) + { + (*testSequenceEndCallback)(CreateSequenceFailureReport(testJobs), timer.Elapsed()); + } + + return result; + } + + TestSequenceResult Runtime::ImpactAnalysisTestSequence( + const ChangeList& changeList, + Policy::TestPrioritization testPrioritizationPolicy, + AZStd::optional testTargetTimeout, + AZStd::optional globalTimeout, + AZStd::optional testSequenceStartCallback, + AZStd::optional testSequenceEndCallback, + AZStd::optional testCompleteCallback) + { + Timer timer; + AZStd::vector draftedTestTargets; + + auto [selectedTestTargets, discardedTestTargets] = SelectCoveringTestTargetsAndUpdateEnumerationCache(changeList, testPrioritizationPolicy); + auto [includedSelectedTestTargets, excludedSelectedTestTargets] = SelectTestTargetsByExcludeList(selectedTestTargets); + + if (testSequenceStartCallback.has_value()) + { + (*testSequenceStartCallback)( + Client::TestRunSelection(ExtractTestTargetNames(includedSelectedTestTargets), ExtractTestTargetNames(excludedSelectedTestTargets)), + ExtractTestTargetNames(discardedTestTargets), + ExtractTestTargetNames(draftedTestTargets)); + } + + const auto [result, testJobs] = m_testEngine->InstrumentedRun( + includedSelectedTestTargets, + m_testShardingPolicy, + m_executionFailurePolicy, + Policy::IntegrityFailure::Continue, + m_testFailurePolicy, + m_targetOutputCapture, + testTargetTimeout, + globalTimeout, + TestRunCompleteCallbackHandler(testCompleteCallback)); + + UpdateAndSerializeDynamicDependencyMap(CreateSourceCoveringTestFromTestCoverages(testJobs, m_config.m_repo.m_root)); + + if (testSequenceEndCallback.has_value()) + { + (*testSequenceEndCallback)(CreateSequenceFailureReport(testJobs), timer.Elapsed()); + } + + return result; + } + + AZStd::pair Runtime::SafeImpactAnalysisTestSequence( + const ChangeList& changeList, + const AZStd::unordered_set suitesFilter, + Policy::TestPrioritization testPrioritizationPolicy, + AZStd::optional testTargetTimeout, + AZStd::optional globalTimeout, + AZStd::optional testSequenceStartCallback, + AZStd::optional testSequenceEndCallback, + AZStd::optional testCompleteCallback) + { + Timer timer; + AZStd::vector draftedTestTargets; + + auto [selectedTestTargets, discardedTestTargets] = SelectCoveringTestTargetsAndUpdateEnumerationCache(changeList, testPrioritizationPolicy); + auto [includedSelectedTestTargets, excludedSelectedTestTargets] = SelectTestTargetsByExcludeList(selectedTestTargets); + auto [includedDiscardedTestTargets, excludedDiscardedTestTargets] = SelectTestTargetsByExcludeList(discardedTestTargets); + + if (testSequenceStartCallback.has_value()) + { + (*testSequenceStartCallback)( + Client::TestRunSelection(ExtractTestTargetNames(includedSelectedTestTargets), ExtractTestTargetNames(excludedSelectedTestTargets)), + Client::TestRunSelection(ExtractTestTargetNames(includedDiscardedTestTargets), ExtractTestTargetNames(excludedDiscardedTestTargets)), + ExtractTestTargetNames(draftedTestTargets)); + } + + // Impact analysis run of the selected test targets + const auto [selectedResult, selectedTestJobs] = m_testEngine->InstrumentedRun( + includedSelectedTestTargets, + m_testShardingPolicy, + m_executionFailurePolicy, + Policy::IntegrityFailure::Continue, + m_testFailurePolicy, + m_targetOutputCapture, + testTargetTimeout, + globalTimeout, + TestRunCompleteCallbackHandler(testCompleteCallback)); + + // Carry the remaining global sequence time over to the discarded test run + if (globalTimeout.has_value()) + { + const auto elapsed = timer.Elapsed(); + globalTimeout = elapsed < globalTimeout.value() ? globalTimeout.value() - elapsed : AZStd::chrono::milliseconds(0); + } + + // Regular run of the discarded test targets + const auto [discardedResult, discardedTestJobs] = m_testEngine->RegularRun( + includedDiscardedTestTargets, + m_testShardingPolicy, + m_executionFailurePolicy, + m_testFailurePolicy, + m_targetOutputCapture, + testTargetTimeout, + globalTimeout, + TestRunCompleteCallbackHandler(testCompleteCallback)); + + UpdateAndSerializeDynamicDependencyMap(CreateSourceCoveringTestFromTestCoverages(selectedTestJobs, m_config.m_repo.m_root)); + + if (testSequenceEndCallback.has_value()) + { + (*testSequenceEndCallback)( + CreateSequenceFailureReport(selectedTestJobs), + CreateSequenceFailureReport(discardedTestJobs), + timer.Elapsed()); + } + + return { selectedResult, discardedResult }; + } + + TestSequenceResult Runtime::SeededTestSequence( + AZStd::optional testTargetTimeout, + AZStd::optional globalTimeout, + AZStd::optional testSequenceStartCallback, + AZStd::optional testSequenceEndCallback, + AZStd::optional testCompleteCallback) + { + Timer timer; + AZStd::vector includedTestTargets; + AZStd::vector excludedTestTargets; + + for (const auto& testTarget : m_dynamicDependencyMap->GetTestTargetList().GetTargets()) + { + if (!m_testTargetExcludeList.contains(&testTarget)) + { + includedTestTargets.push_back(&testTarget); + } + else + { + excludedTestTargets.push_back(&testTarget); + } + } + + if (testSequenceStartCallback.has_value()) + { + (*testSequenceStartCallback)(Client::TestRunSelection(ExtractTestTargetNames(includedTestTargets), ExtractTestTargetNames(excludedTestTargets))); + } + + const auto [result, testJobs] = m_testEngine->InstrumentedRun( + includedTestTargets, + m_testShardingPolicy, + m_executionFailurePolicy, + Policy::IntegrityFailure::Continue, + m_testFailurePolicy, + m_targetOutputCapture, + testTargetTimeout, + globalTimeout, + TestRunCompleteCallbackHandler(testCompleteCallback)); + + ClearDynamicDependencyMapAndRemoveExistingFile(); + UpdateAndSerializeDynamicDependencyMap(CreateSourceCoveringTestFromTestCoverages(testJobs, m_config.m_repo.m_root)); + + if (testSequenceEndCallback.has_value()) + { + (*testSequenceEndCallback)(CreateSequenceFailureReport(testJobs), timer.Elapsed()); + } + + return result; + } + + bool Runtime::HasImpactAnalysisData() const + { + return m_hasImpactAnalysisData; + } +} diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.cpp new file mode 100644 index 0000000000..e0f77c6479 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.cpp @@ -0,0 +1,133 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include + +#include +#include +#include +#include + +#include + +namespace TestImpact +{ + TestTargetMetaMap ReadTestTargetMetaMapFile(const RepoPath& testTargetMetaConfigFile) + { + const auto masterTestListData = ReadFileContents(testTargetMetaConfigFile); + return TestTargetMetaMapFactory(masterTestListData); + } + + AZStd::vector ReadBuildTargetDescriptorFiles(const BuildTargetDescriptorConfig& buildTargetDescriptorConfig) + { + AZStd::vector buildTargetDescriptors; + for (const auto& buildTargetDescriptorFile : std::filesystem::directory_iterator(buildTargetDescriptorConfig.m_mappingDirectory.c_str())) + { + const auto buildTargetDescriptorContents = ReadFileContents(buildTargetDescriptorFile.path().string().c_str()); + auto buildTargetDescriptor = TestImpact::BuildTargetDescriptorFactory( + buildTargetDescriptorContents, + buildTargetDescriptorConfig.m_staticInclusionFilters, + buildTargetDescriptorConfig.m_inputInclusionFilters, + buildTargetDescriptorConfig.m_inputOutputPairer); + buildTargetDescriptors.emplace_back(AZStd::move(buildTargetDescriptor)); + } + + return buildTargetDescriptors; + } + + AZStd::unique_ptr ConstructDynamicDependencyMap( + const BuildTargetDescriptorConfig& buildTargetDescriptorConfig, + const TestTargetMetaConfig& testTargetMetaConfig) + { + auto testTargetmetaMap = ReadTestTargetMetaMapFile(testTargetMetaConfig.m_metaFile); + auto buildTargetDescriptors = ReadBuildTargetDescriptorFiles(buildTargetDescriptorConfig); + auto buildTargets = CompileTargetDescriptors(AZStd::move(buildTargetDescriptors), AZStd::move(testTargetmetaMap)); + auto&& [productionTargets, testTargets] = buildTargets; + return AZStd::make_unique(AZStd::move(productionTargets), AZStd::move(testTargets)); + } + + AZStd::unordered_set ConstructTestTargetExcludeList(const TestTargetList& testTargets, const AZStd::vector& excludedTestTargets) + { + AZStd::unordered_set testTargetExcludeList; + for (const auto& testTargetName : excludedTestTargets) + { + if (const auto* testTarget = testTargets.GetTarget(testTargetName); testTarget != nullptr) + { + testTargetExcludeList.insert(testTarget); + } + } + + return testTargetExcludeList; + } + + AZStd::vector ExtractTestTargetNames(const AZStd::vector testTargets) + { + AZStd::vector testNames; + AZStd::transform(testTargets.begin(), testTargets.end(), AZStd::back_inserter(testNames), [](const TestTarget* testTarget) + { + return testTarget->GetName(); + }); + + return testNames; + } + + SourceCoveringTestsList CreateSourceCoveringTestFromTestCoverages(const AZStd::vector& jobs, const RepoPath& root) + { + AZStd::unordered_map> coverage; + for (const auto& job : jobs) + { + if (const auto testResult = job.GetTestResult(); + testResult == Client::TestRunResult::AllTestsPass || testResult == Client::TestRunResult::TestFailures) + { + if (testResult == Client::TestRunResult::AllTestsPass) + { + // Passing tests should have coverage data, otherwise something is very wrong + AZ_TestImpact_Eval( + job.GetTestCoverge().has_value(), + RuntimeException, + AZStd::string::format( + "Test target '%s' completed its test run successfully but produced no coverage data", + job.GetTestTarget()->GetName().c_str())); + } + else if (!job.GetTestCoverge().has_value()) + { + // When a test run completes with failing tests but produces no coverage artifact that's typically a sign of the + // test aborting due to an unhandled exception, in which case ignore it and let it be picked up in the failure report + continue; + } + + for (const auto& source : job.GetTestCoverge().value().GetSourcesCovered()) + { + coverage[source.String()].insert(job.GetTestTarget()->GetName()); + } + } + } + + AZStd::vector sourceCoveringTests; + sourceCoveringTests.reserve(coverage.size()); + for (auto&& [source, testTargets] : coverage) + { + if (const auto sourcePath = RepoPath(source); + sourcePath.IsRelativeTo(root)) + { + sourceCoveringTests.push_back(SourceCoveringTests(RepoPath(sourcePath.LexicallyRelative(root)), AZStd::move(testTargets))); + } + else + { + AZ_Warning("TestImpact", false, "Ignoring source, source it outside of repo: %s", sourcePath.c_str()); + } + } + + return SourceCoveringTestsList(AZStd::move(sourceCoveringTests)); + } +} diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.h new file mode 100644 index 0000000000..3210ba808b --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.h @@ -0,0 +1,134 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace TestImpact +{ + //! Construct a dynamic dependency map from the build target descriptors and test target metas. + AZStd::unique_ptr ConstructDynamicDependencyMap( + const BuildTargetDescriptorConfig& buildTargetDescriptorConfig, + const TestTargetMetaConfig& testTargetMetaConfig); + + //! Constructs the resolved test target exclude list from the specified list of targets and unresolved test target exclude list. + AZStd::unordered_set ConstructTestTargetExcludeList( + const TestTargetList& testTargets, + const AZStd::vector& excludedTestTargets); + + //! Extracts the name information from the specified test targets. + AZStd::vector ExtractTestTargetNames(const AZStd::vector testTargets); + + //! Creates the consolidates source covering tests list from the test engine instrumented run jobs. + SourceCoveringTestsList CreateSourceCoveringTestFromTestCoverages( + const AZStd::vector& jobs, + const RepoPath& root); + + //! Generates a test run failure report from the specified test engine job information. + //! @tparam TestJob The test engine job type. + template + Client::TestRunFailure GenerateTestRunFailure(const TestJob& testJob) + { + if (testJob.GetTestRun().has_value()) + { + AZStd::vector testCaseFailures; + for (const auto& testSuite : testJob.GetTestRun()->GetTestSuites()) + { + AZStd::vector testFailures; + for (const auto& testCase : testSuite.m_tests) + { + if(testCase.m_result.value_or(TestRunResult::Passed) == TestRunResult::Failed) + { + testFailures.push_back(Client::TestFailure(testCase.m_name, "No error message retrieved")); + } + } + + if (!testFailures.empty()) + { + testCaseFailures.push_back(Client::TestCaseFailure(testSuite.m_name, AZStd::move(testFailures))); + } + } + + return Client::TestRunFailure(Client::TestRunFailure(testJob.GetTestTarget()->GetName(), AZStd::move(testCaseFailures))); + } + else + { + return Client::TestRunFailure(testJob.GetTestTarget()->GetName(), { }); + } + } + + //! Generates a sequence failure report from the specified list of test engine jobs. + //! @tparam TestJob The test engine job type. + template + Client::SequenceFailure GenerateSequenceFailureReport(const AZStd::vector& testJobs) + { + AZStd::vector executionFailures; + AZStd::vector testRunFailures; + AZStd::vector timedOutTestRuns; + AZStd::vector unexecutedTestRuns; + + for (const auto& testJob : testJobs) + { + switch (testJob.GetTestResult()) + { + case Client::TestRunResult::FailedToExecute: + { + executionFailures.push_back(Client::ExecutionFailure(testJob.GetTestTarget()->GetName(), testJob.GetCommandString())); + break; + } + case Client::TestRunResult::NotRun: + { + unexecutedTestRuns.push_back(testJob.GetTestTarget()->GetName()); + break; + } + case Client::TestRunResult::Timeout: + { + timedOutTestRuns.push_back(testJob.GetTestTarget()->GetName()); + break; + } + case Client::TestRunResult::AllTestsPass: + { + break; + } + case Client::TestRunResult::TestFailures: + { + testRunFailures.push_back(ExtractTestRunFailure(testJob)); + break; + } + default: + { + throw Exception( + AZStd::string::format("Unexpected client test run result: %u", static_cast(testJob.GetTestResult()))); + } + } + } + + return Client::SequenceFailure( + AZStd::move(executionFailures), + AZStd::move(testRunFailures), + AZStd::move(timedOutTestRuns), + AZStd::move(unexecutedTestRuns)); + } +} From 1936dbc0d1e1939e3b449aad86e7b71c86e88f25 Mon Sep 17 00:00:00 2001 From: jonawals Date: Thu, 27 May 2021 14:11:45 +0100 Subject: [PATCH 048/102] Fix long line length --- .../Runtime/Code/Source/TestImpactRuntimeUtils.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.cpp index e0f77c6479..87a402bd6a 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.cpp @@ -56,7 +56,8 @@ namespace TestImpact return AZStd::make_unique(AZStd::move(productionTargets), AZStd::move(testTargets)); } - AZStd::unordered_set ConstructTestTargetExcludeList(const TestTargetList& testTargets, const AZStd::vector& excludedTestTargets) + AZStd::unordered_set ConstructTestTargetExcludeList( + const TestTargetList& testTargets, const AZStd::vector& excludedTestTargets) { AZStd::unordered_set testTargetExcludeList; for (const auto& testTargetName : excludedTestTargets) From e9daed723a7496073fa29e5e65c37d4b74316d25 Mon Sep 17 00:00:00 2001 From: jonawals Date: Thu, 27 May 2021 14:19:08 +0100 Subject: [PATCH 049/102] Ensure enumeration caches are always updated --- .../Runtime/Code/Source/TestImpactRuntime.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp index 1bc55ab179..538d435d16 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp @@ -190,10 +190,8 @@ namespace TestImpact // Populate a set with the selected test targets so that we can infer the discarded test target not selected for this change list const AZStd::unordered_set selectedTestTargetSet(selectedTestTargets.begin(), selectedTestTargets.end()); - if (m_testShardingPolicy == Policy::TestSharding::Always) - { - EnumerateMutatedTestTargets(changeDependecyList); - } + // Update the enumeration caches of mutated targets regardless of the current sharding policy + EnumerateMutatedTestTargets(changeDependecyList); // The test targets in the main list not in the selected test target set are the test targets not selected for this change list for (const auto& testTarget : m_dynamicDependencyMap->GetTestTargetList().GetTargets()) From b13a7e1d3c228078f7f67ff1da3e1bed755d7165 Mon Sep 17 00:00:00 2001 From: jonawals Date: Thu, 27 May 2021 17:09:59 +0100 Subject: [PATCH 050/102] Remove whitespace --- .../Runtime/Code/Source/TestImpactRepoPath.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRepoPath.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRepoPath.cpp index 11b927247f..b2ffed2caf 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRepoPath.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRepoPath.cpp @@ -14,8 +14,6 @@ namespace TestImpact { - - RepoPath& RepoPath::operator=(const string_type& other) noexcept { m_path = AZ::IO::Path(other).MakePreferred(); From 990f90410deecda6c703bed209eb624319a24897 Mon Sep 17 00:00:00 2001 From: jonawals Date: Thu, 27 May 2021 18:21:41 +0100 Subject: [PATCH 051/102] Address PR comments --- .../Source/TestImpactClientFailureReport.cpp | 6 +++--- .../Code/Source/TestImpactClientTestRun.cpp | 4 ++-- .../Source/TestImpactClientTestSelection.cpp | 4 ++-- .../Runtime/Code/Source/TestImpactRuntime.cpp | 16 ++++++++-------- .../Runtime/Code/Source/TestImpactRuntimeUtils.h | 2 +- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientFailureReport.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientFailureReport.cpp index f39d8ee426..d8e4709f21 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientFailureReport.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientFailureReport.cpp @@ -119,6 +119,6 @@ namespace TestImpact const AZStd::vector& SequenceFailure::GetUnexecutedTests() const { return m_unexecutedTests; - } - } -} + } + } // namespace Client +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientTestRun.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientTestRun.cpp index 6661de5d69..dfc9f9d46c 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientTestRun.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientTestRun.cpp @@ -36,5 +36,5 @@ namespace TestImpact { return m_result; } - } -} + } // namespace Client +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientTestSelection.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientTestSelection.cpp index f6c88893d7..067ab53fd5 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientTestSelection.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientTestSelection.cpp @@ -52,5 +52,5 @@ namespace TestImpact { return GetNumIncludedTestRuns() + GetNumExcludedTestRuns(); } - } -} + } // namespace Client +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp index 538d435d16..9c9351244f 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp @@ -27,7 +27,7 @@ namespace TestImpact { namespace { - // Simple helper class for tracking basic timing information + //! Simple helper class for tracking basic timing information. class Timer { public: @@ -36,7 +36,7 @@ namespace TestImpact { } - // Returns the time elapsed (in milliseconds) since the timer was instantiated + //! Returns the time elapsed (in milliseconds) since the timer was instantiated AZStd::chrono::milliseconds Elapsed() { const auto endTime = AZStd::chrono::high_resolution_clock::now(); @@ -47,7 +47,7 @@ namespace TestImpact AZStd::chrono::high_resolution_clock::time_point m_startTime; }; - // Handler for test run complete events + //! Handler for test run complete events. class TestRunCompleteCallbackHandler { public: @@ -145,9 +145,9 @@ namespace TestImpact void Runtime::EnumerateMutatedTestTargets(const ChangeDependencyList& changeDependencyList) { AZStd::vector testTargets; - const auto addMutatedTestTargetsToEnumerationList = [this, &testTargets](const AZStd::vector& sourceDependency) + const auto addMutatedTestTargetsToEnumerationList = [this, &testTargets](const AZStd::vector& sourceDependencies) { - for (const auto& sourceDependency : sourceDependency) + for (const auto& sourceDependency : sourceDependencies) { for (const auto& parentTarget : sourceDependency.GetParentTargets()) { @@ -184,14 +184,14 @@ namespace TestImpact AZStd::vector discardedTestTargets; // Select and prioritize the test targets pertinent to this change list - const auto changeDependecyList = m_dynamicDependencyMap->ApplyAndResoveChangeList(changeList); - const auto selectedTestTargets = m_testSelectorAndPrioritizer->SelectTestTargets(changeDependecyList, testPrioritizationPolicy); + const auto changeDependencyList = m_dynamicDependencyMap->ApplyAndResoveChangeList(changeList); + const auto selectedTestTargets = m_testSelectorAndPrioritizer->SelectTestTargets(changeDependencyList, testPrioritizationPolicy); // Populate a set with the selected test targets so that we can infer the discarded test target not selected for this change list const AZStd::unordered_set selectedTestTargetSet(selectedTestTargets.begin(), selectedTestTargets.end()); // Update the enumeration caches of mutated targets regardless of the current sharding policy - EnumerateMutatedTestTargets(changeDependecyList); + EnumerateMutatedTestTargets(changeDependencyList); // The test targets in the main list not in the selected test target set are the test targets not selected for this change list for (const auto& testTarget : m_dynamicDependencyMap->GetTestTargetList().GetTargets()) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.h index 3210ba808b..e9d1c5425b 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.h @@ -59,7 +59,7 @@ namespace TestImpact AZStd::vector testFailures; for (const auto& testCase : testSuite.m_tests) { - if(testCase.m_result.value_or(TestRunResult::Passed) == TestRunResult::Failed) + if (testCase.m_result.value_or(TestRunResult::Passed) == TestRunResult::Failed) { testFailures.push_back(Client::TestFailure(testCase.m_name, "No error message retrieved")); } From 684ef8046fad906a0952d2832f6473696fb17629 Mon Sep 17 00:00:00 2001 From: jonawals Date: Fri, 28 May 2021 10:46:25 +0100 Subject: [PATCH 052/102] Add console command line options --- .../Frontend/Console/CMakeLists.txt | 12 - .../Frontend/Console/Code/CMakeLists.txt | 23 - ...pactframework_frontend_console_files.cmake | 14 - .../Source/TestImpactCommandLineOptions.cpp | 463 ++++++++++++++++++ .../Source/TestImpactCommandLineOptions.h | 111 +++++ .../TestImpactCommandLineOptionsException.h} | 17 +- .../TestImpactCommandLineOptionsUtils.cpp | 77 +++ .../TestImpactCommandLineOptionsUtils.h | 119 +++++ 8 files changed, 783 insertions(+), 53 deletions(-) delete mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/CMakeLists.txt delete mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/Code/CMakeLists.txt delete mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/Code/testimpactframework_frontend_console_files.cmake create mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp create mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.h rename Code/Tools/TestImpactFramework/Frontend/Console/{Code/Source/TestImpactConsole.cpp => Static/Code/Source/TestImpactCommandLineOptionsException.h} (65%) create mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.cpp create mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.h diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/CMakeLists.txt b/Code/Tools/TestImpactFramework/Frontend/Console/CMakeLists.txt deleted file mode 100644 index 20a680bce9..0000000000 --- a/Code/Tools/TestImpactFramework/Frontend/Console/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -# -# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or -# its licensors. -# -# For complete copyright and license terms please see the LICENSE at the root of this -# distribution (the "License"). All use of this software is governed by the License, -# or, if provided, by the license below or the license accompanying this file. Do not -# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# - -add_subdirectory(Code) diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Code/CMakeLists.txt b/Code/Tools/TestImpactFramework/Frontend/Console/Code/CMakeLists.txt deleted file mode 100644 index da2c707cb8..0000000000 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Code/CMakeLists.txt +++ /dev/null @@ -1,23 +0,0 @@ -# -# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or -# its licensors. -# -# For complete copyright and license terms please see the LICENSE at the root of this -# distribution (the "License"). All use of this software is governed by the License, -# or, if provided, by the license below or the license accompanying this file. Do not -# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# - -ly_add_target( - NAME TestImpact.Frontend.Console EXECUTABLE - NAMESPACE AZ - FILES_CMAKE - testimpactframework_frontend_console_files.cmake - INCLUDE_DIRECTORIES - PRIVATE - Source - BUILD_DEPENDENCIES - PRIVATE - AZ::TestImpact.Runtime.Static -) \ No newline at end of file diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Code/testimpactframework_frontend_console_files.cmake b/Code/Tools/TestImpactFramework/Frontend/Console/Code/testimpactframework_frontend_console_files.cmake deleted file mode 100644 index 19e99649a7..0000000000 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Code/testimpactframework_frontend_console_files.cmake +++ /dev/null @@ -1,14 +0,0 @@ -# -# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or -# its licensors. -# -# For complete copyright and license terms please see the LICENSE at the root of this -# distribution (the "License"). All use of this software is governed by the License, -# or, if provided, by the license below or the license accompanying this file. Do not -# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# - -set(FILES - Source/TestImpactConsole.cpp -) diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp new file mode 100644 index 0000000000..e4083641e5 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp @@ -0,0 +1,463 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include + +#include + +namespace TestImpact +{ + namespace + { + enum + { + // Options + Config, + ChangeList, + OutputChangeList, + Sequence, + TestPrioritizationPolicy, + ExecutionFailurePolicy, + ExecutionFailureDraftingPolicy, + TestFailurePolicy, + IntegrityFailurePolicy, + TestShardingPolicy, + TargetOutputCapture, + MaxConcurrency, + TestTargetTimeout, + GlobalTimeout, + SuitesFilter, + SafeMode, + // Values + None, + Seed, + Regular, + ImpactAnalysis, + ImpactAnalysisOrSeed, + Locality, + Abort, + Continue, + Ignore, + StdOut, + File, + AllSuites + }; + + constexpr const char* OptionKeys[] = + { + // Options + "config", + "changelist", + "ochangelist", + "sequence", + "ppolicy", + "epolicy", + "rexecfailures", + "fpolicy", + "ipolicy", + "shard", + "targetout", + "maxconcurrency", + "ttimeout", + "gtimeout", + "suites", + "safemode", + // Values + "none", + "seed", + "regular", + "tia", + "tiaorseed", + "locality", + "abort", + "continue", + "ignore", + "stdout", + "file", + "*" + }; + + RepoPath ParseConfigurationFile(const AZ::CommandLine& cmd) + { + return ParsePathOption(OptionKeys[Config], cmd).value_or(LY_TEST_IMPACT_DEFAULT_CONFIG_FILE); + } + + AZStd::optional ParseChangeListFile(const AZ::CommandLine& cmd) + { + return ParsePathOption(OptionKeys[ChangeList], cmd); + } + + bool ParseOutputChangeList(const AZ::CommandLine& cmd) + { + return ParseOnOffOption(OptionKeys[OutputChangeList], BinaryStateValue{ false, true }, cmd).value_or(false); + } + + TestSequenceType ParseTestSequenceType(const AZ::CommandLine& cmd) + { + const AZStd::vector> states = + { + {OptionKeys[None], TestSequenceType::None}, + {OptionKeys[Seed], TestSequenceType::Seed}, + {OptionKeys[Regular], TestSequenceType::Regular}, + {OptionKeys[ImpactAnalysis], TestSequenceType::ImpactAnalysis}, + {OptionKeys[ImpactAnalysisOrSeed], TestSequenceType::ImpactAnalysisOrSeed} + }; + + return ParseMultiStateOption(OptionKeys[Sequence], states, cmd).value_or(TestSequenceType::None); + } + + Policy::TestPrioritization ParseTestPrioritizationPolicy(const AZ::CommandLine& cmd) + { + const BinaryStateOption states = + { + {OptionKeys[None], Policy::TestPrioritization::None}, + {OptionKeys[Locality], Policy::TestPrioritization::DependencyLocality} + }; + + return ParseBinaryStateOption(OptionKeys[TestPrioritizationPolicy], states, cmd).value_or(Policy::TestPrioritization::None); + } + + Policy::ExecutionFailure ParseExecutionFailurePolicy(const AZ::CommandLine& cmd) + { + const AZStd::vector> states = + { + {OptionKeys[Abort], Policy::ExecutionFailure::Abort}, + {OptionKeys[Continue], Policy::ExecutionFailure::Continue}, + {OptionKeys[Ignore], Policy::ExecutionFailure::Ignore} + }; + return ParseMultiStateOption(OptionKeys[ExecutionFailurePolicy], states, cmd).value_or(Policy::ExecutionFailure::Continue); + } + + Policy::ExecutionFailureDrafting ParseExecutionFailureDraftingPolicy(const AZ::CommandLine& cmd) + { + const BinaryStateValue states = + { + Policy::ExecutionFailureDrafting::Never, + Policy::ExecutionFailureDrafting::Always + }; + + return ParseOnOffOption(OptionKeys[ExecutionFailureDraftingPolicy], states, cmd).value_or(Policy::ExecutionFailureDrafting::Always); + } + + Policy::TestFailure ParseTestFailurePolicy(const AZ::CommandLine& cmd) + { + const BinaryStateValue states = + { + Policy::TestFailure::Abort, + Policy::TestFailure::Continue + }; + + return ParseAbortContinueOption(OptionKeys[TestFailurePolicy], states, cmd).value_or(Policy::TestFailure::Abort); + } + + Policy::IntegrityFailure ParseIntegrityFailurePolicy(const AZ::CommandLine& cmd) + { + const BinaryStateValue states = + { + Policy::IntegrityFailure::Abort, + Policy::IntegrityFailure::Continue + }; + + return ParseAbortContinueOption(OptionKeys[IntegrityFailurePolicy], states, cmd).value_or(Policy::IntegrityFailure::Abort); + } + + Policy::TestSharding ParseTestShardingPolicy(const AZ::CommandLine& cmd) + { + const BinaryStateValue states = + { + Policy::TestSharding::Never, + Policy::TestSharding::Always + }; + + return ParseOnOffOption("shard", states, cmd).value_or(Policy::TestSharding::Never); + } + + Policy::TargetOutputCapture ParseTargetOutputCapture(const AZ::CommandLine& cmd) + { + const auto numSwitchValues = cmd.GetNumSwitchValues(OptionKeys[TargetOutputCapture]); + if (numSwitchValues) + { + AZ_TestImpact_Eval( + numSwitchValues <= 2, CommandLineOptionsException, "Unexpected parameters for target output capture option"); + + Policy::TargetOutputCapture targetOutputCapture = Policy::TargetOutputCapture::None; + for (auto i = 0; i < numSwitchValues; i++) + { + const auto option = cmd.GetSwitchValue(OptionKeys[TargetOutputCapture], i); + if (option == OptionKeys[StdOut]) + { + if (targetOutputCapture == Policy::TargetOutputCapture::File) + { + targetOutputCapture = Policy::TargetOutputCapture::StdOutAndFile; + } + else + { + targetOutputCapture = Policy::TargetOutputCapture::StdOut; + } + } + else if (option == OptionKeys[File]) + { + if (targetOutputCapture == Policy::TargetOutputCapture::StdOut) + { + targetOutputCapture = Policy::TargetOutputCapture::StdOutAndFile; + } + else + { + targetOutputCapture = Policy::TargetOutputCapture::File; + } + } + else + { + throw CommandLineOptionsException( + AZStd::string::format("Unexpected value for target output capture option: %s", option.c_str())); + } + } + + return targetOutputCapture; + } + + return Policy::TargetOutputCapture::None; + } + + AZStd::optional ParseMaxConcurrency(const AZ::CommandLine& cmd) + { + return ParseUnsignedIntegerOption(OptionKeys[MaxConcurrency], cmd); + } + + AZStd::optional ParseTestTargetTimeout(const AZ::CommandLine& cmd) + { + return ParseSecondsOption(OptionKeys[TestTargetTimeout], cmd); + } + + AZStd::optional ParseGlobalTimeout(const AZ::CommandLine& cmd) + { + return ParseSecondsOption(OptionKeys[GlobalTimeout], cmd); + } + + bool ParseSafeMode(const AZ::CommandLine& cmd) + { + const BinaryStateValue states = { false, true }; + return ParseOnOffOption(OptionKeys[SafeMode], states, cmd).value_or(false); + } + + AZStd::unordered_set ParseSuitesFilter(const AZ::CommandLine& cmd) + { + AZStd::unordered_set suitesFilter; + const auto numSwitchValues = cmd.GetNumSwitchValues(OptionKeys[SuitesFilter]); + if (numSwitchValues) + { + for (auto i = 0; i < numSwitchValues; i++) + { + const auto value = cmd.GetSwitchValue(OptionKeys[SuitesFilter], i); + AZ_TestImpact_Eval(!value.empty(), CommandLineOptionsException, "Suites option value is empty"); + if (value == OptionKeys[AllSuites]) + { + AZ_TestImpact_Eval( + suitesFilter.empty(), CommandLineOptionsException, "The * suite cannot be used with other suites"); + } + + suitesFilter.insert(value); + } + } + + if (suitesFilter.find(OptionKeys[AllSuites]) != suitesFilter.end()) + { + return {}; + } + + return suitesFilter; + } + } + + CommandLineOptions::CommandLineOptions(int argc, char** argv) + { + AZ::CommandLine cmd; + cmd.Parse(argc, argv); + + m_configurationFile = ParseConfigurationFile(cmd); + m_changeListFile = ParseChangeListFile(cmd); + m_outputChangeList = ParseOutputChangeList(cmd); + m_testSequenceType = ParseTestSequenceType(cmd); + m_testPrioritizationPolicy = ParseTestPrioritizationPolicy(cmd); + m_executionFailurePolicy = ParseExecutionFailurePolicy(cmd); + m_executionFailureDraftingPolicy = ParseExecutionFailureDraftingPolicy(cmd); + m_testFailurePolicy = ParseTestFailurePolicy(cmd); + m_integrityFailurePolicy = ParseIntegrityFailurePolicy(cmd); + m_testShardingPolicy = ParseTestShardingPolicy(cmd); + m_targetOutputCapture = ParseTargetOutputCapture(cmd); + m_maxConcurrency = ParseMaxConcurrency(cmd); + m_testTargetTimeout = ParseTestTargetTimeout(cmd); + m_globalTimeout = ParseGlobalTimeout(cmd); + m_safeMode = ParseSafeMode(cmd); + m_suitesFilter = ParseSuitesFilter(cmd); + } + + bool CommandLineOptions::HasChangeListFile() const + { + return m_changeListFile.has_value(); + } + + bool CommandLineOptions::HasSafeMode() const + { + return m_safeMode; + } + + const AZStd::optional& CommandLineOptions::GetChangeListFile() const + { + return m_changeListFile; + } + + bool CommandLineOptions::HasOutputChangeList() const + { + return m_outputChangeList; + } + + const RepoPath& CommandLineOptions::GetConfigurationFile() const + { + return m_configurationFile; + } + + TestSequenceType CommandLineOptions::GetTestSequenceType() const + { + return m_testSequenceType; + } + + Policy::TestPrioritization CommandLineOptions::GetTestPrioritizationPolicy() const + { + return m_testPrioritizationPolicy; + } + + Policy::ExecutionFailure CommandLineOptions::GetExecutionFailurePolicy() const + { + return m_executionFailurePolicy; + } + + Policy::ExecutionFailureDrafting CommandLineOptions::GetExecutionFailureDraftingPolicy() const + { + return m_executionFailureDraftingPolicy; + } + + Policy::TestFailure CommandLineOptions::GetTestFailurePolicy() const + { + return m_testFailurePolicy; + } + + Policy::IntegrityFailure CommandLineOptions::GetIntegrityFailurePolicy() const + { + return m_integrityFailurePolicy; + } + + Policy::TestSharding CommandLineOptions::GetTestShardingPolicy() const + { + return m_testShardingPolicy; + } + + Policy::TargetOutputCapture CommandLineOptions::GetTargetOutputCapture() const + { + return m_targetOutputCapture; + } + + const AZStd::optional& CommandLineOptions::GetMaxConcurrency() const + { + return m_maxConcurrency; + } + + const AZStd::optional& CommandLineOptions::GetTestTargetTimeout() const + { + return m_testTargetTimeout; + } + + const AZStd::optional& CommandLineOptions::GetGlobalTimeout() const + { + return m_globalTimeout; + } + + const AZStd::unordered_set& CommandLineOptions::GetSuitesFilter() const + { + return m_suitesFilter; + } + + AZStd::string CommandLineOptions::GetCommandLineUsageString() + { + AZStd::string help = + "usage: tiaf [options]\n" + " options:\n" + " -config= Path to the configuration file for the TIAF runtime (default: \n" + " ..json).\n" + " -changelist= Path to the JSON of source file changes to perform test impact \n" + " analysis on.\n" + " -gtimeout= Global timeout value to terminate the entire test sequence should it \n" + " be exceeded.\n" + " -ttimeout= Timeout value to terminate individual test targets should it be \n" + " exceeded.\n" + " -sequence= The type of test sequence to perform, where none runs no tests and\n" + " will report a all tests successful, seed removes any prior coverage \n" + " data and runs all test targets with instrumentation to reseed the \n" + " data from scratch, regular runs all of the test targets without any \n" + " instrumentation to generate coverage data(any prior coverage data is \n" + " left intact), tia uses any prior coverage data to run the instrumented \n" + " subset of selected tests(if no prior coverage data a regular run is \n" + " performed instead) and tiaorseed uses any prior coverage data to run \n" + " the instrumented subset of selected tests(if no prior coverage data a \n" + " seed run is performed instead).\n" + " -safemode= Flag to specify a safe mode sequence where the set of unselected \n" + " tests is run without instrumentation after the set of selected \n" + " instrumented tests is run (this has the effect of ensuring all \n" + " tests are run regardless).\n" + " -shard= Break any test targets with a sharding policy into the number of \n" + " shards according to the maximum concurrency value.\n" + " -rexecfailures= Attempt to execute test targets that previously failed to execute.\n" + " -targetout= Capture of individual test run stdout, where stdout will capture \n" + " each individual test target's stdout and output each one to stdout \n" + " and file will capture each individual test target's stdout and output \n" + " each one individually to a file (multiple values are accepted).\n" + " -epolicy= Policy for handling test execution failure (test targets could not be \n" + " launched due to the binary not being built, incorrect paths, etc.), \n" + " where abort will abort the entire test sequence upon the first test\n" + " target execution failureand report a failure(along with the return \n" + " code of the test target that failed to launch), continue will continue \n" + " with the test sequence in the event of test target execution failures\n" + " and treat the test targets that failed to launch as as test failures\n" + " (along with the return codes of the test targets that failed to \n" + " launch), ignore will continue with the test sequence in the event of \n" + " test target execution failuresand treat the test targets that failed\n" + " to launch as as test passes(along with the return codes of the test \n" + " targets that failed to launch).\n" + " -fpolicy Policy for handling test failures (test targets report failing tests), \n" + " where abort will abort the entire test sequenceupon the first test \n" + " failureand report a failure and continue will continue with the test\n" + " sequence in the event of test failuresand report the test failures.\n" + " -ipolicy= Policy for handling coverage data integrity failures, where abort will \n" + " abort the test sequenceand report a failure, seed will attempt another \n" + " sequence using the seed sequence type, otherwise will abort and report \n" + " a failure (this option has no effect for regularand seed sequence \n" + " types) and rerun will attempt another sequence using the regular \n" + " sequence type, otherwise will abortand report a failure(this option has \n" + " no effect for regular sequence type).\n" + " -ppolicy= Policy for prioritizing selected test targets, where none will not \n" + " attempt any test target prioritization and locality will attempt to \n" + " prioritize test targets according to the locality of their covering \n" + " production targets in the dependency graph(if no dependency graph data \n" + " available, no prioritization will occur).\n" + " -maxconcurrency= The maximum number of concurrent test targets/shards to be in flight at \n" + " any given moment.\n" + " -ochangelist= Outputs the change list used for test selection.\n" + " -suites= The test suites to select from for this test sequence (multiple values are \n" + " allowed). The suite all has special significance and will allow tests from \n" + " any suite to be selected, however this particular suite is mutually exclusive\n" + " with other suite. Note: this option is only applicable to the regular sequence\n" + " and, if safe mode is enables, the tia and tiaorseed sequences."; + + return help; + } +} diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.h b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.h new file mode 100644 index 0000000000..6958f03b80 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.h @@ -0,0 +1,111 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include +#include + +#include +#include +#include +#include + +namespace TestImpact +{ + //! The type of test sequence to run. + enum class TestSequenceType + { + None, //!< Runs no tests and will report a all tests successful. + Seed, //!< Removes any prior coverage data and runs all test targets with instrumentation to reseed the data from scratch. + Regular, //!< Runs all of the test targets without any instrumentation to generate coverage data (any prior coverage data is left intact). + ImpactAnalysis, //!< Uses any prior coverage data to run the instrumented subset of selected tests (if no prior coverage data a regular run is performed instead). + ImpactAnalysisOrSeed //!< Uses any prior coverage data to run the instrumented subset of selected tests (if no prior coverage data a seed run is performed instead). + }; + + //! Representation of the command line options supplied to the console frontend application. + class CommandLineOptions + { + public: + CommandLineOptions(int argc, char** argv); + static AZStd::string GetCommandLineUsageString(); + + //! Returns true if a change list file path has been supplied, otherwise false. + bool HasChangeListFile() const; + + //! Returns true if the safe mode option has been enabled, otherwise false. + bool HasSafeMode() const; + + //! Returns true if the output change list option has been enabled, otherwise false. + bool HasOutputChangeList() const; + + //! Returns the path to the runtime configuration file. + const RepoPath& GetConfigurationFile() const; + + //! Returns the path to the change list file (if any). + const AZStd::optional& GetChangeListFile() const; + + //! Returns the test sequence type to run. + TestSequenceType GetTestSequenceType() const; + + //! Returns the test prioritization policy to use. + Policy::TestPrioritization GetTestPrioritizationPolicy() const; + + //! Returns the test execution failure policy to use. + Policy::ExecutionFailure GetExecutionFailurePolicy() const; + + //! Returns the test historic test execution failure drafting policy to use. + Policy::ExecutionFailureDrafting GetExecutionFailureDraftingPolicy() const; + + //! Returns the test failure policy to use. + Policy::TestFailure GetTestFailurePolicy() const; + + //! Returns the integration failure policy to use. + Policy::IntegrityFailure GetIntegrityFailurePolicy() const; + + //! Returns the test sharding policy to use. + Policy::TestSharding GetTestShardingPolicy() const; + + //! Returns the test target standard output capture policy to use. + Policy::TargetOutputCapture GetTargetOutputCapture() const; + + //! Returns the maximum number of test targets to be in flight at any given time. + const AZStd::optional& GetMaxConcurrency() const; + + //! Returns the individual test target timeout to use (if any). + const AZStd::optional& GetTestTargetTimeout() const; + + //! Returns the global test sequence timeout to use (if any). + const AZStd::optional& GetGlobalTimeout() const; + + //! Returns the filter for test suites that will be allowed to be run. + const AZStd::unordered_set& GetSuitesFilter() const; + + private: + RepoPath m_configurationFile; + AZStd::optional m_changeListFile; + bool m_outputChangeList = false; + TestSequenceType m_testSequenceType; + Policy::TestPrioritization m_testPrioritizationPolicy = Policy::TestPrioritization::None; + Policy::ExecutionFailure m_executionFailurePolicy = Policy::ExecutionFailure::Continue; + Policy::ExecutionFailureDrafting m_executionFailureDraftingPolicy = Policy::ExecutionFailureDrafting::Always; + Policy::TestFailure m_testFailurePolicy = Policy::TestFailure::Abort; + Policy::IntegrityFailure m_integrityFailurePolicy = Policy::IntegrityFailure::Abort; + Policy::TestSharding m_testShardingPolicy = Policy::TestSharding::Never; + Policy::TargetOutputCapture m_targetOutputCapture = Policy::TargetOutputCapture::None; + AZStd::optional m_maxConcurrency; + AZStd::optional m_testTargetTimeout; + AZStd::optional m_globalTimeout; + AZStd::unordered_set m_suitesFilter; + bool m_safeMode = false; + }; +} diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactConsole.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsException.h similarity index 65% rename from Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactConsole.cpp rename to Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsException.h index 12743f4b38..fd0a58b0f5 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactConsole.cpp +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsException.h @@ -10,8 +10,17 @@ * */ -int main([[maybe_unused]] int argc, [[maybe_unused]] char** argv) -{ - return 0; -} +#pragma once +#include + +namespace TestImpact +{ + //! Exception for command line options. + class CommandLineOptionsException + : public Exception + { + public: + using Exception::Exception; + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.cpp new file mode 100644 index 0000000000..3315f68ae5 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.cpp @@ -0,0 +1,77 @@ +/* +* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +* its licensors. +* +* For complete copyright and license terms please see the LICENSE at the root of this +* distribution (the "License"). All use of this software is governed by the License, +* or, if provided, by the license below or the license accompanying this file. Do not +* remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* +*/ + +#include + +namespace TestImpact +{ + //! Attempts to parse a path option value. + AZStd::optional ParsePathOption(const AZStd::string& optionName, const AZ::CommandLine& cmd) + { + const auto numSwitchValues = cmd.GetNumSwitchValues(optionName); + if (numSwitchValues) + { + AZ_TestImpact_Eval( + numSwitchValues == 1, + CommandLineOptionsException, + AZStd::string::format("Unexpected number of parameters for %s option", optionName.c_str())); + + const auto value = cmd.GetSwitchValue(optionName, 0); + AZ_TestImpact_Eval( + !value.empty(), + CommandLineOptionsException, + AZStd::string::format("%s file option value is empty", optionName.c_str())); + + return value; + } + + return AZStd::nullopt; + } + + //! Attempts to pass an unsigned integer option value. + AZStd::optional ParseUnsignedIntegerOption(const AZStd::string& optionName, const AZ::CommandLine& cmd) + { + const auto numSwitchValues = cmd.GetNumSwitchValues(optionName); + if (numSwitchValues) + { + AZ_TestImpact_Eval( + numSwitchValues == 1, + CommandLineOptionsException, + AZStd::string::format("Unexpected number of parameters for %s option", optionName.c_str())); + + const auto str = cmd.GetSwitchValue(optionName, 0); + char* end = nullptr; + auto value = strtoul(str.c_str(), &end, 0); + + AZ_TestImpact_Eval( + end != str, + CommandLineOptionsException, + AZStd::string::format("Couldn't parse unsigned integer option value: %s", str.c_str())); + + return aznumeric_caster(value); + } + + return AZStd::nullopt; + } + + //! Attempts to parse an option value in seconds. + AZStd::optional ParseSecondsOption(const AZStd::string& optionName, const AZ::CommandLine& cmd) + { + if (const auto option = ParseUnsignedIntegerOption(optionName, cmd); + option.has_value()) + { + return AZStd::chrono::seconds(option.value()); + } + + return AZStd::nullopt; + } +} diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.h b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.h new file mode 100644 index 0000000000..0da7f1f16e --- /dev/null +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.h @@ -0,0 +1,119 @@ +/* +* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +* its licensors. +* +* For complete copyright and license terms please see the LICENSE at the root of this +* distribution (the "License"). All use of this software is governed by the License, +* or, if provided, by the license below or the license accompanying this file. Do not +* remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* +*/ + +#pragma once + +#include +#include + +#include + +namespace TestImpact +{ + //! Representation of a command line option value name and its typed value. + template + using OptionValue = AZStd::pair; + + //! Representation of a binary state command line option with its two values. + template + using BinaryStateOption = AZStd::pair, OptionValue>; + + //! Representation of the values for a binary state option. + template + using BinaryStateValue = AZStd::pair; + + //! Attempts to parse the specified binary state option. + template + AZStd::optional ParseBinaryStateOption( + const AZStd::string& optionName, + const AZStd::pair, + OptionValue>& state, const AZ::CommandLine& cmd) + { + const auto numSwitchValues = cmd.GetNumSwitchValues(optionName); + if (numSwitchValues) + { + AZ_TestImpact_Eval( + numSwitchValues == 1, + CommandLineOptionsException, + AZStd::string::format("Unexpected number of parameters for %s option", optionName.c_str())); + + const auto option = cmd.GetSwitchValue(optionName, 0); + if (option == state.first.first) + { + return state.first.second; + } + else if (option == state.second.first) + { + return state.second.second; + } + + throw CommandLineOptionsException( + AZStd::string::format("Unexpected value for %s option: %s", optionName.c_str(), option.c_str())); + } + + return AZStd::nullopt; + } + + //! Attempts to pass an arbitrarily sized state option. + template + AZStd::optional ParseMultiStateOption( + const AZStd::string& optionName, + const AZStd::vector>& states, + const AZ::CommandLine& cmd) + { + const auto numSwitchValues = cmd.GetNumSwitchValues(optionName); + if (numSwitchValues) + { + AZ_TestImpact_Eval( + numSwitchValues == 1, + CommandLineOptionsException, + AZStd::string::format("Unexpected number of parameters for %s option", optionName.c_str())); + + const auto option = cmd.GetSwitchValue(optionName, 0); + for (const auto& state : states) + { + if (option == state.first) + { + return state.second; + } + } + + throw CommandLineOptionsException( + AZStd::string::format("Unexpected value for %s option: %s", optionName.c_str(), option.c_str())); + } + + return AZStd::nullopt; + } + + //! Attempts to pass a specialization of the binary state option where the command line values are "on" and "off". + template + AZStd::optional ParseOnOffOption(const AZStd::string& optionName, const AZStd::pair& states, const AZ::CommandLine& cmd) + { + return ParseBinaryStateOption(optionName, BinaryStateOption{ {"off", states.first}, { "on", states.second } }, cmd); + } + + //! Attempts to pass a specialization of the binary state option where the command line values are "abort" and "continue". + template + AZStd::optional ParseAbortContinueOption(const AZStd::string& optionName, const AZStd::pair& states, const AZ::CommandLine& cmd) + { + return ParseBinaryStateOption(optionName, BinaryStateOption{ {"abort", states.first}, { "continue", states.second } }, cmd); + } + + //! Attempts to parse a path option value. + AZStd::optional ParsePathOption(const AZStd::string& optionName, const AZ::CommandLine& cmd); + + //! Attempts to pass an unsigned integer option value. + AZStd::optional ParseUnsignedIntegerOption(const AZStd::string& optionName, const AZ::CommandLine& cmd); + + //! Attempts to parse an option value in seconds. + AZStd::optional ParseSecondsOption(const AZStd::string& optionName, const AZ::CommandLine& cmd); +} From f50f02eb4a86951b896b3c82e7d431d2284052e9 Mon Sep 17 00:00:00 2001 From: jonawals Date: Fri, 28 May 2021 11:38:02 +0100 Subject: [PATCH 053/102] Add console entry points --- .../Code/Source/TestImpactConsole.cpp | 29 ++ .../TestImpactConsoleApplication.h | 34 +++ .../Source/TestImpactConsoleApplication.cpp | 276 ++++++++++++++++++ 3 files changed, 339 insertions(+) create mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/Executable/Code/Source/TestImpactConsole.cpp create mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Include/TestImpactFramework/TestImpactConsoleApplication.h create mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleApplication.cpp diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Executable/Code/Source/TestImpactConsole.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Executable/Code/Source/TestImpactConsole.cpp new file mode 100644 index 0000000000..079519576f --- /dev/null +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Executable/Code/Source/TestImpactConsole.cpp @@ -0,0 +1,29 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include + +#include +#include + +int main(int argc, char** argv) +{ + AZ::AllocatorInstance::Create(); + AZ::AllocatorInstance::Create(); + + TestImpact::Console::ReturnCode returnCode = TestImpact::Console::Main(argc, argv); + + AZ::AllocatorInstance::Destroy(); + AZ::AllocatorInstance::Destroy(); + + return static_cast(returnCode); +} diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Include/TestImpactFramework/TestImpactConsoleApplication.h b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Include/TestImpactFramework/TestImpactConsoleApplication.h new file mode 100644 index 0000000000..aabc4f70fc --- /dev/null +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Include/TestImpactFramework/TestImpactConsoleApplication.h @@ -0,0 +1,34 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +namespace TestImpact +{ + namespace Console + { + enum class ReturnCode : int + { + Success = 0, //!< The instigation operation(s) returned without error. + InvalidArgs, //!< The specified command line arguments were incorrect. + InvalidUnifiedDiff, //!< The specified unified diff could not be transformed into a valid change list. + InvalidConfiguration, //!< The runtime configuration is malformed. + RuntimeError, //!< The runtime encountered an error that it could not recover from. + UnhandledError, //!< The framework encountered an error that it anticipated but did not handle and could not recover from. + UnknownError, //!< An error of unknown origin was encountered that the console or runtime could not recover from. + TestFailure, //!< The test sequence had one or more test failures. + Timeout //!< The test sequence runtime exceeded the global timeout value. + }; + + [[nodiscard]] ReturnCode Main(int argc, char** argv); + } +} diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleApplication.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleApplication.cpp new file mode 100644 index 0000000000..80c08142e9 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleApplication.cpp @@ -0,0 +1,276 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include + +#include + +namespace TestImpact +{ + namespace Console + { + //! Generates a string to be used for printing to the console for the specified change list. + AZStd::string GenerateChangeListString(const ChangeList& changeList) + { + AZStd::string output; + + const auto& outputFiles = [&output](const AZStd::vector& files) + { + for (const auto& file : files) + { + output += AZStd::string::format("\t%s\n", file.c_str()); + } + }; + + output += AZStd::string::format("Created files (%u):\n", changeList.m_createdFiles.size()); + outputFiles(changeList.m_createdFiles); + + output += AZStd::string::format("Updated files (%u):\n", changeList.m_updatedFiles.size()); + outputFiles(changeList.m_updatedFiles); + + output += AZStd::string::format("Deleted files (%u):\n", changeList.m_deletedFiles.size()); + outputFiles(changeList.m_deletedFiles); + + return output; + } + + //! Gets the appropriate console return code for the specified test sequence result. + ReturnCode GetReturnCodeForTestSequenceResult(TestSequenceResult result) + { + switch (result) + { + case TestSequenceResult::Success: + return ReturnCode::Success; + case TestSequenceResult::Failure: + return ReturnCode::TestFailure; + case TestSequenceResult::Timeout: + return ReturnCode::Timeout; + default: + std::cout << "Unexpected TestSequenceResult value: " << aznumeric_cast(result) << std::endl; + return ReturnCode::UnknownError; + } + } + + //! Entry point for the test impact analysis framework console front end application. + ReturnCode Main(int argc, char** argv) + { + try + { + CommandLineOptions options(argc, argv); + AZStd::optional changeList; + + // If we have a change list, check to see whether or not the client has requested the printing of said change list + if (options.HasChangeListFile()) + { + changeList = DeserializeChangeList(ReadFileContents(*options.GetChangeListFile())); + if (options.HasOutputChangeList()) + { + std::cout << "Change List:\n"; + std::cout << GenerateChangeListString(*changeList).c_str(); + + if (options.GetTestSequenceType() == TestSequenceType::None) + { + return ReturnCode::Success; + } + } + } + + // As of now, there are no other non-test operations other than printing a change list so getting this far is considered an error + AZ_TestImpact_Eval(options.GetTestSequenceType() != TestSequenceType::None, CommandLineOptionsException, "No action specified"); + + std::cout << "Constructing in-memory model of source tree and test coverage, this may take a moment...\n"; + Runtime runtime( + ConfigurationFactory(ReadFileContents(options.GetConfigurationFile())), + options.GetExecutionFailurePolicy(), + options.GetExecutionFailureDraftingPolicy(), + options.GetTestFailurePolicy(), + options.GetIntegrityFailurePolicy(), + options.GetTestShardingPolicy(), + options.GetTargetOutputCapture(), + options.GetMaxConcurrency()); + + if (runtime.HasImpactAnalysisData()) + { + std::cout << "Test impact analysis data for this repository was found.\n"; + } + else + { + + std::cout << "Test impact analysis data for this repository was not found, seed or regular sequence fallbacks will be used.\n"; + } + + TestSequence sequence(&options.GetSuitesFilter()); + + // Wrapper around impact analysis sequences to handle the case where the safe mode option is active + const auto impactAnalysisTestSequence = [&sequence, &options, &runtime, &changeList]() + { + AZ_TestImpact_Eval( + changeList.has_value(), + CommandLineOptionsException, + "Expected a change list for impact analysis but none was provided"); + + TestSequenceResult result = TestSequenceResult::Failure; + if (options.HasSafeMode()) + { + auto [selectedResult, discardedResult] = runtime.SafeImpactAnalysisTestSequence( + changeList.value(), + options.GetSuitesFilter(), + options.GetTestPrioritizationPolicy(), + options.GetTestTargetTimeout(), + options.GetGlobalTimeout(), + AZStd::ref(sequence), + AZStd::ref(sequence), + AZStd::ref(sequence)); + + // Handling the possible timeout and failure permutations of the selected and discarded test results is splitting hairs + // so apply the following, admittedly arbitrary, rules to determine what the composite test sequence result should be + if (selectedResult == TestSequenceResult::Success && discardedResult == TestSequenceResult::Success) + { + // Trivial case: both sequences succeeded + result = TestSequenceResult::Success; + } + else if (selectedResult == TestSequenceResult::Failure || discardedResult == TestSequenceResult::Failure) + { + // One sequence failed whilst the other sequence either succeeded or timed out + result = TestSequenceResult::Failure; + } + else + { + // One sequence timed out whilst the other sequence succeeded or both sequences timed out + result = TestSequenceResult::Timeout; + } + } + else + { + result = runtime.ImpactAnalysisTestSequence( + changeList.value(), + options.GetTestPrioritizationPolicy(), + options.GetTestTargetTimeout(), + options.GetGlobalTimeout(), + AZStd::ref(sequence), + AZStd::ref(sequence), + AZStd::ref(sequence)); + } + + return GetReturnCodeForTestSequenceResult(result); + }; + + switch (const auto type = options.GetTestSequenceType()) + { + case TestSequenceType::Regular: + { + const auto result = runtime.RegularTestSequence( + options.GetSuitesFilter(), + options.GetTestTargetTimeout(), + options.GetGlobalTimeout(), + AZStd::ref(sequence), + AZStd::ref(sequence), + AZStd::ref(sequence)); + + return GetReturnCodeForTestSequenceResult(result); + } + case TestSequenceType::Seed: + { + const auto result = runtime.SeededTestSequence( + options.GetTestTargetTimeout(), + options.GetGlobalTimeout(), + AZStd::ref(sequence), + AZStd::ref(sequence), + AZStd::ref(sequence)); + + return GetReturnCodeForTestSequenceResult(result); + } + case TestSequenceType::ImpactAnalysis: + { + return impactAnalysisTestSequence(); + } + case TestSequenceType::ImpactAnalysisOrSeed: + { + if (runtime.HasImpactAnalysisData()) + { + return impactAnalysisTestSequence(); + } + else + { + const auto result = runtime.SeededTestSequence( + options.GetTestTargetTimeout(), + options.GetGlobalTimeout(), + AZStd::ref(sequence), + AZStd::ref(sequence), + AZStd::ref(sequence)); + + return GetReturnCodeForTestSequenceResult(result); + } + } + default: + std::cout << "Unexpected TestSequenceType value: " << static_cast(type) << std::endl; + return ReturnCode::UnknownError; + } + } + catch (const CommandLineOptionsException& e) + { + std::cout << e.what() << std::endl; + std::cout << CommandLineOptions::GetCommandLineUsageString().c_str() << std::endl; + return ReturnCode::InvalidArgs; + } + catch (const ChangeListException& e) + { + std::cout << e.what() << std::endl; + return ReturnCode::InvalidUnifiedDiff; + } + catch (const ConfigurationException& e) + { + std::cout << e.what() << std::endl; + return ReturnCode::InvalidConfiguration; + } + catch (const RuntimeException& e) + { + std::cout << e.what() << std::endl; + return ReturnCode::RuntimeError; + } + catch (const Exception& e) + { + std::cout << e.what() << std::endl; + return ReturnCode::UnhandledError; + } + catch (const std::exception& e) + { + std::cout << e.what() << std::endl; + return ReturnCode::UnknownError; + } + catch (...) + { + std::cout << "An unknown error occurred" << std::endl; + return ReturnCode::UnknownError; + } + } + } +} From 728d3b4b980d29637d9b528548284e5772b45dd5 Mon Sep 17 00:00:00 2001 From: jonawals Date: Fri, 28 May 2021 12:33:27 +0100 Subject: [PATCH 054/102] Address PR comments --- .../Source/TestImpactCommandLineOptions.cpp | 10 +++++----- .../Source/TestImpactCommandLineOptions.h | 4 ++-- .../TestImpactCommandLineOptionsUtils.cpp | 20 ++++++++++--------- .../TestImpactCommandLineOptionsUtils.h | 10 +++++----- 4 files changed, 23 insertions(+), 21 deletions(-) diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp index e4083641e5..a6eb2e886e 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp @@ -184,8 +184,8 @@ namespace TestImpact Policy::TargetOutputCapture ParseTargetOutputCapture(const AZ::CommandLine& cmd) { - const auto numSwitchValues = cmd.GetNumSwitchValues(OptionKeys[TargetOutputCapture]); - if (numSwitchValues) + if (const auto numSwitchValues = cmd.GetNumSwitchValues(OptionKeys[TargetOutputCapture]); + numSwitchValues) { AZ_TestImpact_Eval( numSwitchValues <= 2, CommandLineOptionsException, "Unexpected parameters for target output capture option"); @@ -253,8 +253,8 @@ namespace TestImpact AZStd::unordered_set ParseSuitesFilter(const AZ::CommandLine& cmd) { AZStd::unordered_set suitesFilter; - const auto numSwitchValues = cmd.GetNumSwitchValues(OptionKeys[SuitesFilter]); - if (numSwitchValues) + if (const auto numSwitchValues = cmd.GetNumSwitchValues(OptionKeys[SuitesFilter]); + numSwitchValues) { for (auto i = 0; i < numSwitchValues; i++) { @@ -460,4 +460,4 @@ namespace TestImpact return help; } -} +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.h b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.h index 6958f03b80..73961b5fcc 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.h +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.h @@ -25,7 +25,7 @@ namespace TestImpact //! The type of test sequence to run. enum class TestSequenceType { - None, //!< Runs no tests and will report a all tests successful. + None, //!< Runs no tests and will report all tests successful. Seed, //!< Removes any prior coverage data and runs all test targets with instrumentation to reseed the data from scratch. Regular, //!< Runs all of the test targets without any instrumentation to generate coverage data (any prior coverage data is left intact). ImpactAnalysis, //!< Uses any prior coverage data to run the instrumented subset of selected tests (if no prior coverage data a regular run is performed instead). @@ -108,4 +108,4 @@ namespace TestImpact AZStd::unordered_set m_suitesFilter; bool m_safeMode = false; }; -} +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.cpp index 3315f68ae5..c03ba98303 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.cpp +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.cpp @@ -12,13 +12,15 @@ #include +#include + namespace TestImpact { //! Attempts to parse a path option value. AZStd::optional ParsePathOption(const AZStd::string& optionName, const AZ::CommandLine& cmd) - { - const auto numSwitchValues = cmd.GetNumSwitchValues(optionName); - if (numSwitchValues) + { + if (const auto numSwitchValues = cmd.GetNumSwitchValues(optionName); + numSwitchValues) { AZ_TestImpact_Eval( numSwitchValues == 1, @@ -40,8 +42,8 @@ namespace TestImpact //! Attempts to pass an unsigned integer option value. AZStd::optional ParseUnsignedIntegerOption(const AZStd::string& optionName, const AZ::CommandLine& cmd) { - const auto numSwitchValues = cmd.GetNumSwitchValues(optionName); - if (numSwitchValues) + if (const auto numSwitchValues = cmd.GetNumSwitchValues(optionName); + numSwitchValues) { AZ_TestImpact_Eval( numSwitchValues == 1, @@ -49,11 +51,11 @@ namespace TestImpact AZStd::string::format("Unexpected number of parameters for %s option", optionName.c_str())); const auto str = cmd.GetSwitchValue(optionName, 0); - char* end = nullptr; - auto value = strtoul(str.c_str(), &end, 0); + size_t end = 0; + auto value = AZStd::stoul(str, &end, 0); AZ_TestImpact_Eval( - end != str, + end, CommandLineOptionsException, AZStd::string::format("Couldn't parse unsigned integer option value: %s", str.c_str())); @@ -74,4 +76,4 @@ namespace TestImpact return AZStd::nullopt; } -} +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.h b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.h index 0da7f1f16e..e635d05710 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.h +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.h @@ -38,8 +38,8 @@ namespace TestImpact const AZStd::pair, OptionValue>& state, const AZ::CommandLine& cmd) { - const auto numSwitchValues = cmd.GetNumSwitchValues(optionName); - if (numSwitchValues) + if (const auto numSwitchValues = cmd.GetNumSwitchValues(optionName); + numSwitchValues) { AZ_TestImpact_Eval( numSwitchValues == 1, @@ -70,8 +70,8 @@ namespace TestImpact const AZStd::vector>& states, const AZ::CommandLine& cmd) { - const auto numSwitchValues = cmd.GetNumSwitchValues(optionName); - if (numSwitchValues) + if (const auto numSwitchValues = cmd.GetNumSwitchValues(optionName); + numSwitchValues) { AZ_TestImpact_Eval( numSwitchValues == 1, @@ -116,4 +116,4 @@ namespace TestImpact //! Attempts to parse an option value in seconds. AZStd::optional ParseSecondsOption(const AZStd::string& optionName, const AZ::CommandLine& cmd); -} +} // namespace TestImpact From c63de35b688aa4479133b8f456339148cd4476b3 Mon Sep 17 00:00:00 2001 From: jonawals Date: Fri, 28 May 2021 15:39:28 +0100 Subject: [PATCH 055/102] Address PR feedback --- .../Source/TestImpactCommandLineOptionsUtils.cpp | 10 +++++----- .../Source/TestImpactCommandLineOptionsUtils.h | 15 +++++++++------ 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.cpp index c03ba98303..b96716059f 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.cpp +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.cpp @@ -50,14 +50,14 @@ namespace TestImpact CommandLineOptionsException, AZStd::string::format("Unexpected number of parameters for %s option", optionName.c_str())); - const auto str = cmd.GetSwitchValue(optionName, 0); - size_t end = 0; - auto value = AZStd::stoul(str, &end, 0); + const auto strValue = cmd.GetSwitchValue(optionName, 0); + size_t successfulParse = 0; // Will be non-zero if the parse was successful + auto value = AZStd::stoul(strValue, &successfulParse, 0); AZ_TestImpact_Eval( - end, + successfulParse, CommandLineOptionsException, - AZStd::string::format("Couldn't parse unsigned integer option value: %s", str.c_str())); + AZStd::string::format("Couldn't parse unsigned integer option value: %s", strValue.c_str())); return aznumeric_caster(value); } diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.h b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.h index e635d05710..1708efadd9 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.h +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.h @@ -47,13 +47,15 @@ namespace TestImpact AZStd::string::format("Unexpected number of parameters for %s option", optionName.c_str())); const auto option = cmd.GetSwitchValue(optionName, 0); - if (option == state.first.first) + if (const auto& [optionValueText, optionValue] = state.first; + option == optionValueText) { - return state.first.second; + return optionValue; } - else if (option == state.second.first) + if (const auto& [optionValueText, optionValue] = state.second; + option == optionValueText) { - return state.second.second; + return optionValue; } throw CommandLineOptionsException( @@ -81,9 +83,10 @@ namespace TestImpact const auto option = cmd.GetSwitchValue(optionName, 0); for (const auto& state : states) { - if (option == state.first) + if (const auto& [optionValueText, optionValue] = state; + option == optionValueText) { - return state.second; + return optionValue; } } From f21794b193b314c63a5404fd5df75569266d5af0 Mon Sep 17 00:00:00 2001 From: jonawals Date: Fri, 28 May 2021 15:55:13 +0100 Subject: [PATCH 056/102] Add missing namespace comments --- .../TestImpactFramework/TestImpactConsoleApplication.h | 5 +++-- .../Static/Code/Source/TestImpactConsoleApplication.cpp | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Include/TestImpactFramework/TestImpactConsoleApplication.h b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Include/TestImpactFramework/TestImpactConsoleApplication.h index aabc4f70fc..c6ad90a36f 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Include/TestImpactFramework/TestImpactConsoleApplication.h +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Include/TestImpactFramework/TestImpactConsoleApplication.h @@ -29,6 +29,7 @@ namespace TestImpact Timeout //!< The test sequence runtime exceeded the global timeout value. }; + //! Entry point for the console front end application. [[nodiscard]] ReturnCode Main(int argc, char** argv); - } -} + } // namespace Console +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleApplication.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleApplication.cpp index 80c08142e9..b3bdf6ec25 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleApplication.cpp +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleApplication.cpp @@ -272,5 +272,5 @@ namespace TestImpact return ReturnCode::UnknownError; } } - } -} + } // namespace Console +} // namespace TestImpact From 9fe91ec7056b27dca039dbf80e43e23278c3402b Mon Sep 17 00:00:00 2001 From: jonawals Date: Tue, 1 Jun 2021 12:46:00 +0100 Subject: [PATCH 057/102] Address PR comments --- .../Code/Source/TestImpactConsole.cpp | 4 +- ...eApplication.h => TestImpactConsoleMain.h} | 0 ...lication.cpp => TestImpactConsoleMain.cpp} | 141 +++++++++--------- .../Windows/platform_windows_files.cmake | 1 + .../Code/Source/TestImpactRepoPath.cpp | 22 +-- .../Runtime/Code/Source/TestImpactRuntime.cpp | 10 +- .../Code/Source/TestImpactRuntimeUtils.h | 2 +- 7 files changed, 92 insertions(+), 88 deletions(-) rename Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Include/TestImpactFramework/{TestImpactConsoleApplication.h => TestImpactConsoleMain.h} (100%) rename Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/{TestImpactConsoleApplication.cpp => TestImpactConsoleMain.cpp} (69%) diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Executable/Code/Source/TestImpactConsole.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Executable/Code/Source/TestImpactConsole.cpp index 079519576f..631eb17b32 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Executable/Code/Source/TestImpactConsole.cpp +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Executable/Code/Source/TestImpactConsole.cpp @@ -10,7 +10,7 @@ * */ -#include +#include #include #include @@ -22,8 +22,8 @@ int main(int argc, char** argv) TestImpact::Console::ReturnCode returnCode = TestImpact::Console::Main(argc, argv); - AZ::AllocatorInstance::Destroy(); AZ::AllocatorInstance::Destroy(); + AZ::AllocatorInstance::Destroy(); return static_cast(returnCode); } diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Include/TestImpactFramework/TestImpactConsoleApplication.h b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Include/TestImpactFramework/TestImpactConsoleMain.h similarity index 100% rename from Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Include/TestImpactFramework/TestImpactConsoleApplication.h rename to Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Include/TestImpactFramework/TestImpactConsoleMain.h diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleApplication.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleMain.cpp similarity index 69% rename from Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleApplication.cpp rename to Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleMain.cpp index b3bdf6ec25..0d4f08406e 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleApplication.cpp +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleMain.cpp @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include #include @@ -22,7 +22,7 @@ #include #include -#include +#include #include #include #include @@ -79,6 +79,64 @@ namespace TestImpact } } + //! Wrapper around impact analysis sequences to handle the case where the safe mode option is active. + ReturnCode WrappedImpactAnalysisTestSequence( + TestSequenceEventHandler& sequenceEventHandler, + const CommandLineOptions& options, + Runtime& runtime, + const AZStd::optional& changeList) + { + AZ_TestImpact_Eval( + changeList.has_value(), + CommandLineOptionsException, + "Expected a change list for impact analysis but none was provided"); + + TestSequenceResult result = TestSequenceResult::Failure; + if (options.HasSafeMode()) + { + auto [selectedResult, discardedResult] = runtime.SafeImpactAnalysisTestSequence( + changeList.value(), + options.GetSuitesFilter(), + options.GetTestPrioritizationPolicy(), + options.GetTestTargetTimeout(), + options.GetGlobalTimeout(), + AZStd::ref(sequenceEventHandler), + AZStd::ref(sequenceEventHandler), + AZStd::ref(sequenceEventHandler)); + + // Handling the possible timeout and failure permutations of the selected and discarded test results is splitting hairs + // so apply the following, admittedly arbitrary, rules to determine what the composite test sequence result should be + if (selectedResult == TestSequenceResult::Success && discardedResult == TestSequenceResult::Success) + { + // Trivial case: both sequences succeeded + result = TestSequenceResult::Success; + } + else if (selectedResult == TestSequenceResult::Failure || discardedResult == TestSequenceResult::Failure) + { + // One sequence failed whilst the other sequence either succeeded or timed out + result = TestSequenceResult::Failure; + } + else + { + // One sequence timed out whilst the other sequence succeeded or both sequences timed out + result = TestSequenceResult::Timeout; + } + } + else + { + result = runtime.ImpactAnalysisTestSequence( + changeList.value(), + options.GetTestPrioritizationPolicy(), + options.GetTestTargetTimeout(), + options.GetGlobalTimeout(), + AZStd::ref(sequenceEventHandler), + AZStd::ref(sequenceEventHandler), + AZStd::ref(sequenceEventHandler)); + } + + return GetReturnCodeForTestSequenceResult(result); + }; + //! Entry point for the test impact analysis framework console front end application. ReturnCode Main(int argc, char** argv) { @@ -123,65 +181,10 @@ namespace TestImpact } else { - std::cout << "Test impact analysis data for this repository was not found, seed or regular sequence fallbacks will be used.\n"; } - TestSequence sequence(&options.GetSuitesFilter()); - - // Wrapper around impact analysis sequences to handle the case where the safe mode option is active - const auto impactAnalysisTestSequence = [&sequence, &options, &runtime, &changeList]() - { - AZ_TestImpact_Eval( - changeList.has_value(), - CommandLineOptionsException, - "Expected a change list for impact analysis but none was provided"); - - TestSequenceResult result = TestSequenceResult::Failure; - if (options.HasSafeMode()) - { - auto [selectedResult, discardedResult] = runtime.SafeImpactAnalysisTestSequence( - changeList.value(), - options.GetSuitesFilter(), - options.GetTestPrioritizationPolicy(), - options.GetTestTargetTimeout(), - options.GetGlobalTimeout(), - AZStd::ref(sequence), - AZStd::ref(sequence), - AZStd::ref(sequence)); - - // Handling the possible timeout and failure permutations of the selected and discarded test results is splitting hairs - // so apply the following, admittedly arbitrary, rules to determine what the composite test sequence result should be - if (selectedResult == TestSequenceResult::Success && discardedResult == TestSequenceResult::Success) - { - // Trivial case: both sequences succeeded - result = TestSequenceResult::Success; - } - else if (selectedResult == TestSequenceResult::Failure || discardedResult == TestSequenceResult::Failure) - { - // One sequence failed whilst the other sequence either succeeded or timed out - result = TestSequenceResult::Failure; - } - else - { - // One sequence timed out whilst the other sequence succeeded or both sequences timed out - result = TestSequenceResult::Timeout; - } - } - else - { - result = runtime.ImpactAnalysisTestSequence( - changeList.value(), - options.GetTestPrioritizationPolicy(), - options.GetTestTargetTimeout(), - options.GetGlobalTimeout(), - AZStd::ref(sequence), - AZStd::ref(sequence), - AZStd::ref(sequence)); - } - - return GetReturnCodeForTestSequenceResult(result); - }; + TestSequenceEventHandler sequenceEventHandler(&options.GetSuitesFilter()); switch (const auto type = options.GetTestSequenceType()) { @@ -191,9 +194,9 @@ namespace TestImpact options.GetSuitesFilter(), options.GetTestTargetTimeout(), options.GetGlobalTimeout(), - AZStd::ref(sequence), - AZStd::ref(sequence), - AZStd::ref(sequence)); + AZStd::ref(sequenceEventHandler), + AZStd::ref(sequenceEventHandler), + AZStd::ref(sequenceEventHandler)); return GetReturnCodeForTestSequenceResult(result); } @@ -202,30 +205,30 @@ namespace TestImpact const auto result = runtime.SeededTestSequence( options.GetTestTargetTimeout(), options.GetGlobalTimeout(), - AZStd::ref(sequence), - AZStd::ref(sequence), - AZStd::ref(sequence)); + AZStd::ref(sequenceEventHandler), + AZStd::ref(sequenceEventHandler), + AZStd::ref(sequenceEventHandler)); return GetReturnCodeForTestSequenceResult(result); } case TestSequenceType::ImpactAnalysis: { - return impactAnalysisTestSequence(); + return WrappedImpactAnalysisTestSequence(sequenceEventHandler, options, runtime, changeList); } case TestSequenceType::ImpactAnalysisOrSeed: { if (runtime.HasImpactAnalysisData()) { - return impactAnalysisTestSequence(); + return WrappedImpactAnalysisTestSequence(sequenceEventHandler, options, runtime, changeList); } else { const auto result = runtime.SeededTestSequence( options.GetTestTargetTimeout(), options.GetGlobalTimeout(), - AZStd::ref(sequence), - AZStd::ref(sequence), - AZStd::ref(sequence)); + AZStd::ref(sequenceEventHandler), + AZStd::ref(sequenceEventHandler), + AZStd::ref(sequenceEventHandler)); return GetReturnCodeForTestSequenceResult(result); } diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/platform_windows_files.cmake b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/platform_windows_files.cmake index 5f5e8fc3d1..e994bcdbbd 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/platform_windows_files.cmake +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/platform_windows_files.cmake @@ -17,4 +17,5 @@ set(FILES Process/TestImpactWin32_Pipe.cpp Process/TestImpactWin32_Pipe.h TestEngine/JobRunner/TestImpactWin32_TestTargetExtension.cpp + TestEngine/TestImpactWin32_TestEngineJobFailure.cpp ) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRepoPath.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRepoPath.cpp index b2ffed2caf..86b9cbbed5 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRepoPath.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRepoPath.cpp @@ -32,70 +32,70 @@ namespace TestImpact return *this; } - inline RepoPath operator/(const RepoPath& lhs, const AZ::IO::PathView& rhs) + RepoPath operator/(const RepoPath& lhs, const AZ::IO::PathView& rhs) { RepoPath result(lhs); result.m_path /= RepoPath(rhs).m_path; return result; } - inline RepoPath operator/(const RepoPath& lhs, AZStd::string_view rhs) + RepoPath operator/(const RepoPath& lhs, AZStd::string_view rhs) { RepoPath result(lhs); result.m_path /= RepoPath(rhs).m_path; return result; } - inline RepoPath operator/(const RepoPath& lhs, const RepoPath::value_type* rhs) + RepoPath operator/(const RepoPath& lhs, const RepoPath::value_type* rhs) { RepoPath result(lhs); result.m_path /= RepoPath(rhs).m_path; return result; } - inline RepoPath operator/(const RepoPath& lhs, const RepoPath& rhs) + RepoPath operator/(const RepoPath& lhs, const RepoPath& rhs) { RepoPath result(lhs); result.m_path /= rhs.m_path; return result; } - inline RepoPath& RepoPath::operator/=(const AZ::IO::PathView& rhs) + RepoPath& RepoPath::operator/=(const AZ::IO::PathView& rhs) { m_path /= RepoPath(rhs).m_path; return *this; } - inline RepoPath& RepoPath::operator/=(AZStd::string_view rhs) + RepoPath& RepoPath::operator/=(AZStd::string_view rhs) { m_path /= RepoPath(rhs).m_path; return *this; } - inline RepoPath& RepoPath::operator/=(const RepoPath::value_type* rhs) + RepoPath& RepoPath::operator/=(const RepoPath::value_type* rhs) { m_path /= RepoPath(rhs).m_path; return *this; } - inline RepoPath& RepoPath::operator/=(const RepoPath& rhs) + RepoPath& RepoPath::operator/=(const RepoPath& rhs) { m_path /= rhs.m_path; return *this; } - inline bool operator==(const RepoPath& lhs, const RepoPath& rhs) noexcept + bool operator==(const RepoPath& lhs, const RepoPath& rhs) noexcept { return lhs.m_path.Compare(rhs.m_path) == 0; } - inline bool operator!=(const RepoPath& lhs, const RepoPath& rhs) noexcept + bool operator!=(const RepoPath& lhs, const RepoPath& rhs) noexcept { return lhs.m_path.Compare(rhs.m_path) != 0; } - inline bool operator<([[maybe_unused]] const RepoPath& lhs, [[maybe_unused]] const RepoPath& rhs) noexcept + bool operator<([[maybe_unused]] const RepoPath& lhs, [[maybe_unused]] const RepoPath& rhs) noexcept { return lhs.m_path.String() < rhs.m_path.String(); } diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp index 9c9351244f..e02bee03c6 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp @@ -309,7 +309,7 @@ namespace TestImpact if (testSequenceEndCallback.has_value()) { - (*testSequenceEndCallback)(CreateSequenceFailureReport(testJobs), timer.Elapsed()); + (*testSequenceEndCallback)(GenerateSequenceFailureReport(testJobs), timer.Elapsed()); } return result; @@ -353,7 +353,7 @@ namespace TestImpact if (testSequenceEndCallback.has_value()) { - (*testSequenceEndCallback)(CreateSequenceFailureReport(testJobs), timer.Elapsed()); + (*testSequenceEndCallback)(GenerateSequenceFailureReport(testJobs), timer.Elapsed()); } return result; @@ -419,8 +419,8 @@ namespace TestImpact if (testSequenceEndCallback.has_value()) { (*testSequenceEndCallback)( - CreateSequenceFailureReport(selectedTestJobs), - CreateSequenceFailureReport(discardedTestJobs), + GenerateSequenceFailureReport(selectedTestJobs), + GenerateSequenceFailureReport(discardedTestJobs), timer.Elapsed()); } @@ -471,7 +471,7 @@ namespace TestImpact if (testSequenceEndCallback.has_value()) { - (*testSequenceEndCallback)(CreateSequenceFailureReport(testJobs), timer.Elapsed()); + (*testSequenceEndCallback)(GenerateSequenceFailureReport(testJobs), timer.Elapsed()); } return result; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.h index e9d1c5425b..56775b1c58 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.h @@ -114,7 +114,7 @@ namespace TestImpact } case Client::TestRunResult::TestFailures: { - testRunFailures.push_back(ExtractTestRunFailure(testJob)); + testRunFailures.push_back(GenerateTestRunFailure(testJob)); break; } default: From 6432582b7f228147b22b864d96db0a7efcb51761 Mon Sep 17 00:00:00 2001 From: jonawals Date: Tue, 1 Jun 2021 15:58:02 +0100 Subject: [PATCH 058/102] Add configuration factory --- .../Source/TestImpactConfigurationFactory.cpp | 210 ++++++++++++++++++ .../Source/TestImpactConfigurationFactory.h | 18 ++ 2 files changed, 228 insertions(+) create mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConfigurationFactory.cpp create mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConfigurationFactory.h diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConfigurationFactory.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConfigurationFactory.cpp new file mode 100644 index 0000000000..a6e8ee9639 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConfigurationFactory.cpp @@ -0,0 +1,210 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include + +#include + +#include +#include +#include + + +namespace TestImpact +{ + + //! Returns an absolute path for a path relative to the specified root. + RepoPath GetAbsPathFromRelPath(const RepoPath& root, const RepoPath& rel) + { + return root / rel; + } + + ConfigMeta ParseConfigMeta(const rapidjson::Value& meta) + { + ConfigMeta configMeta; + configMeta.m_platform = meta["platform"].GetString(); + return configMeta; + } + + RepoConfig ParseRepoConfig(const rapidjson::Value& repo) + { + RepoConfig repoConfig; + repoConfig.m_root = repo["root"].GetString(); + return repoConfig; + } + + WorkspaceConfig::Temp ParseTempWorkspaceConfig(const rapidjson::Value& tempWorkspace) + { + WorkspaceConfig::Temp tempWorkspaceConfig; + tempWorkspaceConfig.m_root = tempWorkspace["root"].GetString(); + tempWorkspaceConfig.m_artifactDirectory = + GetAbsPathFromRelPath(tempWorkspaceConfig.m_root, tempWorkspace["relative_paths"]["artifact_dir"].GetString()); + return tempWorkspaceConfig; + } + + WorkspaceConfig::Active ParseActiveWorkspaceConfig(const rapidjson::Value& activeWorkspace) + { + WorkspaceConfig::Active activeWorkspaceConfig; + const auto& relativePaths = activeWorkspace["relative_paths"]; + activeWorkspaceConfig.m_root = activeWorkspace["root"].GetString(); + activeWorkspaceConfig.m_enumerationCacheDirectory + = GetAbsPathFromRelPath(activeWorkspaceConfig.m_root, relativePaths["enumeration_cache_dir"].GetString()); + activeWorkspaceConfig.m_sparTIAFile + = GetAbsPathFromRelPath(activeWorkspaceConfig.m_root, relativePaths["test_impact_data_file"].GetString()); + return activeWorkspaceConfig; + } + + WorkspaceConfig ParseWorkspaceConfig(const rapidjson::Value& workspace) + { + WorkspaceConfig workspaceConfig; + workspaceConfig.m_temp = ParseTempWorkspaceConfig(workspace["temp"]); + workspaceConfig.m_active = ParseActiveWorkspaceConfig(workspace["active"]); + return workspaceConfig; + } + + BuildTargetDescriptorConfig ParseBuildTargetDescriptorConfig(const rapidjson::Value& buildTargetDescriptor) + { + BuildTargetDescriptorConfig buildTargetDescriptorConfig; + const auto& targetSources = buildTargetDescriptor["target_sources"]; + const auto& staticTargetSources = targetSources["static"]; + const auto& autogenTargetSources = targetSources["autogen"]; + buildTargetDescriptorConfig.m_mappingDirectory = buildTargetDescriptor["dir"].GetString(); + const auto& staticInclusionFilters = staticTargetSources["include_filters"].GetArray(); + if (!staticInclusionFilters.Empty()) + { + buildTargetDescriptorConfig.m_staticInclusionFilters.reserve(staticInclusionFilters.Size()); + for (const auto& staticInclusionFilter : staticInclusionFilters) + { + buildTargetDescriptorConfig.m_staticInclusionFilters.push_back(staticInclusionFilter.GetString()); + } + } + + buildTargetDescriptorConfig.m_inputOutputPairer = autogenTargetSources["input_output_pairer"].GetString(); + const auto& inputInclusionFilters = autogenTargetSources["input"]["include_filters"].GetArray(); + if (!inputInclusionFilters.Empty()) + { + buildTargetDescriptorConfig.m_inputInclusionFilters.reserve(inputInclusionFilters.Size()); + for (const auto& inputInclusionFilter : inputInclusionFilters) + { + buildTargetDescriptorConfig.m_inputInclusionFilters.push_back(inputInclusionFilter.GetString()); + } + } + + return buildTargetDescriptorConfig; + } + + DependencyGraphDataConfig ParseDependencyGraphDataConfig(const rapidjson::Value& dependencyGraphData) + { + DependencyGraphDataConfig dependencyGraphDataConfig; + const auto& matchers = dependencyGraphData["matchers"]; + dependencyGraphDataConfig.m_graphDirectory = dependencyGraphData["dir"].GetString(); + dependencyGraphDataConfig.m_targetDependencyFileMatcher = matchers["target_dependency_file"].GetString(); + dependencyGraphDataConfig.m_targetVertexMatcher = matchers["target_vertex"].GetString(); + return dependencyGraphDataConfig; + } + + TestTargetMetaConfig ParseTestTargetMetaConfig(const rapidjson::Value& testTargetMeta) + { + TestTargetMetaConfig testTargetMetaConfig; + testTargetMetaConfig.m_metaFile = testTargetMeta["file"].GetString(); + return testTargetMetaConfig; + } + + TestEngineConfig ParseTestEngineConfig(const rapidjson::Value& testEngine) + { + TestEngineConfig testEngineConfig; + testEngineConfig.m_testRunner.m_binary = testEngine["test_runner"]["bin"].GetString(); + testEngineConfig.m_instrumentation.m_binary = testEngine["instrumentation"]["bin"].GetString(); + return testEngineConfig; + } + TargetConfig ParseTargetConfig(const rapidjson::Value& target) + { + TargetConfig targetConfig; + targetConfig.m_outputDirectory = target["dir"].GetString(); + const auto& testExcludes = + target["exclude"].GetArray(); + if (!testExcludes.Empty()) + { + targetConfig.m_excludedTestTargets.reserve(testExcludes.Size()); + for (const auto& testExclude : testExcludes) + { + targetConfig.m_excludedTestTargets.push_back(testExclude.GetString()); + } + } + const auto& testShards = + target["shard"].GetArray(); + if (!testShards.Empty()) + { + targetConfig.m_shardedTestTargets.reserve(testShards.Size()); + for (const auto& testShard : testShards) + { + const auto getShardingConfiguration = [](const AZStd::string& config) + { + if (config == "fixture_contiguous") + { + return ShardConfiguration::FixtureContiguous; + } + else if (config == "fixture_interleaved") + { + return ShardConfiguration::FixtureInterleaved; + } + else if (config == "test_contiguous") + { + return ShardConfiguration::TestContiguous; + } + else if (config == "test_interleaved") + { + return ShardConfiguration::TestInterleaved; + } + else if (config == "never") + { + return ShardConfiguration::Never; + } + else + { + throw ConfigurationException(AZStd::string::format("Unexpected sharding configuration: %s", config.c_str())); + } + }; + + TargetConfig::ShardedTarget shard; + shard.m_name = testShard["target"].GetString(); + shard.m_configuration = getShardingConfiguration(testShard["policy"].GetString()); + targetConfig.m_shardedTestTargets.push_back(AZStd::move(shard)); + } + } + + return targetConfig; + } + + RuntimeConfig ConfigurationFactory(const AZStd::string& configurationData) + { + RuntimeConfig runtimeConfig; + rapidjson::Document configurationFile; + + if (configurationFile.Parse(configurationData.c_str()).HasParseError()) + { + throw TestImpact::ConfigurationException("Could not parse runtimeConfig data, JSON has errors"); + } + + const auto& staticArtifacts = configurationFile["artifacts"]["static"]; + runtimeConfig.m_meta = ParseConfigMeta(configurationFile["meta"]); + runtimeConfig.m_repo = ParseRepoConfig(configurationFile["repo"]); + runtimeConfig.m_workspace = ParseWorkspaceConfig(configurationFile["workspace"]); + runtimeConfig.m_buildTargetDescriptor = ParseBuildTargetDescriptorConfig(staticArtifacts["build_target_descriptor"]); + runtimeConfig.m_dependencyGraphData = ParseDependencyGraphDataConfig(staticArtifacts["dependency_graph_data"]); + runtimeConfig.m_testTargetMeta = ParseTestTargetMetaConfig(staticArtifacts["static"]["test_target_meta"]); + runtimeConfig.m_testEngine = ParseTestEngineConfig(configurationFile["test_engine"]); + runtimeConfig.m_target = ParseTargetConfig(configurationFile["target"]); + + return runtimeConfig; + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConfigurationFactory.h b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConfigurationFactory.h new file mode 100644 index 0000000000..2cc75ab0fa --- /dev/null +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConfigurationFactory.h @@ -0,0 +1,18 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include + +namespace TestImpact +{ + RuntimeConfig ConfigurationFactory(const AZStd::string& configurationData); +} // namespace TestImpact From c7753f472bbd7e248689a273125b3b75d7b63ead Mon Sep 17 00:00:00 2001 From: jonawals Date: Tue, 1 Jun 2021 16:11:25 +0100 Subject: [PATCH 059/102] Add API comment and rename factory --- .../Console/Static/Code/Source/TestImpactConsoleMain.cpp | 4 ++-- ...nFactory.cpp => TestImpactRuntimeConfigurationFactory.cpp} | 4 ++-- ...ationFactory.h => TestImpactRuntimeConfigurationFactory.h} | 3 ++- 3 files changed, 6 insertions(+), 5 deletions(-) rename Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/{TestImpactConfigurationFactory.cpp => TestImpactRuntimeConfigurationFactory.cpp} (98%) rename Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/{TestImpactConfigurationFactory.h => TestImpactRuntimeConfigurationFactory.h} (76%) diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleMain.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleMain.cpp index 0d4f08406e..84e16c2332 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleMain.cpp +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleMain.cpp @@ -24,7 +24,7 @@ #include #include -#include +#include #include #include @@ -166,7 +166,7 @@ namespace TestImpact std::cout << "Constructing in-memory model of source tree and test coverage, this may take a moment...\n"; Runtime runtime( - ConfigurationFactory(ReadFileContents(options.GetConfigurationFile())), + RuntimeConfigurationFactory(ReadFileContents(options.GetConfigurationFile())), options.GetExecutionFailurePolicy(), options.GetExecutionFailureDraftingPolicy(), options.GetTestFailurePolicy(), diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConfigurationFactory.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactRuntimeConfigurationFactory.cpp similarity index 98% rename from Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConfigurationFactory.cpp rename to Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactRuntimeConfigurationFactory.cpp index a6e8ee9639..4ac97bef49 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConfigurationFactory.cpp +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactRuntimeConfigurationFactory.cpp @@ -12,7 +12,7 @@ #include -#include +#include #include #include @@ -185,7 +185,7 @@ namespace TestImpact return targetConfig; } - RuntimeConfig ConfigurationFactory(const AZStd::string& configurationData) + RuntimeConfig RuntimeConfigurationFactory(const AZStd::string& configurationData) { RuntimeConfig runtimeConfig; rapidjson::Document configurationFile; diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConfigurationFactory.h b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactRuntimeConfigurationFactory.h similarity index 76% rename from Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConfigurationFactory.h rename to Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactRuntimeConfigurationFactory.h index 2cc75ab0fa..ac7dbbac94 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConfigurationFactory.h +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactRuntimeConfigurationFactory.h @@ -14,5 +14,6 @@ namespace TestImpact { - RuntimeConfig ConfigurationFactory(const AZStd::string& configurationData); + //! Parses the configuration data (in JSON format) and returns the constructed runtime configuration. + RuntimeConfig RuntimeConfigurationFactory(const AZStd::string& configurationData); } // namespace TestImpact From 9be54d4e5b595484c60097529cdeda2d6880ed12 Mon Sep 17 00:00:00 2001 From: jonawals Date: Tue, 1 Jun 2021 16:31:23 +0100 Subject: [PATCH 060/102] Address PR comments --- .../TestImpactRuntimeConfigurationFactory.cpp | 109 ++++++++---------- 1 file changed, 49 insertions(+), 60 deletions(-) diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactRuntimeConfigurationFactory.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactRuntimeConfigurationFactory.cpp index 4ac97bef49..4b473b7190 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactRuntimeConfigurationFactory.cpp +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactRuntimeConfigurationFactory.cpp @@ -79,24 +79,19 @@ namespace TestImpact const auto& autogenTargetSources = targetSources["autogen"]; buildTargetDescriptorConfig.m_mappingDirectory = buildTargetDescriptor["dir"].GetString(); const auto& staticInclusionFilters = staticTargetSources["include_filters"].GetArray(); - if (!staticInclusionFilters.Empty()) + + buildTargetDescriptorConfig.m_staticInclusionFilters.reserve(staticInclusionFilters.Size()); + for (const auto& staticInclusionFilter : staticInclusionFilters) { - buildTargetDescriptorConfig.m_staticInclusionFilters.reserve(staticInclusionFilters.Size()); - for (const auto& staticInclusionFilter : staticInclusionFilters) - { - buildTargetDescriptorConfig.m_staticInclusionFilters.push_back(staticInclusionFilter.GetString()); - } + buildTargetDescriptorConfig.m_staticInclusionFilters.push_back(staticInclusionFilter.GetString()); } buildTargetDescriptorConfig.m_inputOutputPairer = autogenTargetSources["input_output_pairer"].GetString(); const auto& inputInclusionFilters = autogenTargetSources["input"]["include_filters"].GetArray(); - if (!inputInclusionFilters.Empty()) + buildTargetDescriptorConfig.m_inputInclusionFilters.reserve(inputInclusionFilters.Size()); + for (const auto& inputInclusionFilter : inputInclusionFilters) { - buildTargetDescriptorConfig.m_inputInclusionFilters.reserve(inputInclusionFilters.Size()); - for (const auto& inputInclusionFilter : inputInclusionFilters) - { - buildTargetDescriptorConfig.m_inputInclusionFilters.push_back(inputInclusionFilter.GetString()); - } + buildTargetDescriptorConfig.m_inputInclusionFilters.push_back(inputInclusionFilter.GetString()); } return buildTargetDescriptorConfig; @@ -126,60 +121,54 @@ namespace TestImpact testEngineConfig.m_instrumentation.m_binary = testEngine["instrumentation"]["bin"].GetString(); return testEngineConfig; } + TargetConfig ParseTargetConfig(const rapidjson::Value& target) { TargetConfig targetConfig; targetConfig.m_outputDirectory = target["dir"].GetString(); - const auto& testExcludes = - target["exclude"].GetArray(); - if (!testExcludes.Empty()) + const auto& testExcludes = target["exclude"].GetArray(); + targetConfig.m_excludedTestTargets.reserve(testExcludes.Size()); + for (const auto& testExclude : testExcludes) { - targetConfig.m_excludedTestTargets.reserve(testExcludes.Size()); - for (const auto& testExclude : testExcludes) - { - targetConfig.m_excludedTestTargets.push_back(testExclude.GetString()); - } + targetConfig.m_excludedTestTargets.push_back(testExclude.GetString()); } - const auto& testShards = - target["shard"].GetArray(); - if (!testShards.Empty()) - { - targetConfig.m_shardedTestTargets.reserve(testShards.Size()); - for (const auto& testShard : testShards) - { - const auto getShardingConfiguration = [](const AZStd::string& config) - { - if (config == "fixture_contiguous") - { - return ShardConfiguration::FixtureContiguous; - } - else if (config == "fixture_interleaved") - { - return ShardConfiguration::FixtureInterleaved; - } - else if (config == "test_contiguous") - { - return ShardConfiguration::TestContiguous; - } - else if (config == "test_interleaved") - { - return ShardConfiguration::TestInterleaved; - } - else if (config == "never") - { - return ShardConfiguration::Never; - } - else - { - throw ConfigurationException(AZStd::string::format("Unexpected sharding configuration: %s", config.c_str())); - } - }; - TargetConfig::ShardedTarget shard; - shard.m_name = testShard["target"].GetString(); - shard.m_configuration = getShardingConfiguration(testShard["policy"].GetString()); - targetConfig.m_shardedTestTargets.push_back(AZStd::move(shard)); - } + const auto& testShards = target["shard"].GetArray(); + targetConfig.m_shardedTestTargets.reserve(testShards.Size()); + for (const auto& testShard : testShards) + { + const auto getShardingConfiguration = [](const AZStd::string& config) + { + if (config == "fixture_contiguous") + { + return ShardConfiguration::FixtureContiguous; + } + else if (config == "fixture_interleaved") + { + return ShardConfiguration::FixtureInterleaved; + } + else if (config == "test_contiguous") + { + return ShardConfiguration::TestContiguous; + } + else if (config == "test_interleaved") + { + return ShardConfiguration::TestInterleaved; + } + else if (config == "never") + { + return ShardConfiguration::Never; + } + else + { + throw ConfigurationException(AZStd::string::format("Unexpected sharding configuration: %s", config.c_str())); + } + }; + + TargetConfig::ShardedTarget shard; + shard.m_name = testShard["target"].GetString(); + shard.m_configuration = getShardingConfiguration(testShard["policy"].GetString()); + targetConfig.m_shardedTestTargets.push_back(AZStd::move(shard)); } return targetConfig; @@ -187,7 +176,6 @@ namespace TestImpact RuntimeConfig RuntimeConfigurationFactory(const AZStd::string& configurationData) { - RuntimeConfig runtimeConfig; rapidjson::Document configurationFile; if (configurationFile.Parse(configurationData.c_str()).HasParseError()) @@ -195,6 +183,7 @@ namespace TestImpact throw TestImpact::ConfigurationException("Could not parse runtimeConfig data, JSON has errors"); } + RuntimeConfig runtimeConfig; const auto& staticArtifacts = configurationFile["artifacts"]["static"]; runtimeConfig.m_meta = ParseConfigMeta(configurationFile["meta"]); runtimeConfig.m_repo = ParseRepoConfig(configurationFile["repo"]); From 105eca52d7f3b06fb72f2f9905246f5c449d073a Mon Sep 17 00:00:00 2001 From: jonawals Date: Wed, 2 Jun 2021 11:16:36 +0100 Subject: [PATCH 061/102] Address PR comments --- .../TestImpactRuntimeConfigurationFactory.cpp | 182 +++++++++++++----- .../TestImpactConfiguration.h | 18 +- .../Runtime/Code/Source/TestImpactRuntime.cpp | 12 +- 3 files changed, 147 insertions(+), 65 deletions(-) diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactRuntimeConfigurationFactory.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactRuntimeConfigurationFactory.cpp index 4b473b7190..9db99d3c2e 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactRuntimeConfigurationFactory.cpp +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactRuntimeConfigurationFactory.cpp @@ -18,9 +18,101 @@ #include #include - namespace TestImpact { + namespace Config + { + // Keys for pertinent JSON elements + constexpr const char* Keys[] = + { + "root", + "platform", + "relative_paths", + "artifact_dir", + "enumeration_cache_dir", + "test_impact_data_file", + "temp", + "active", + "target_sources", + "static", + "autogen", + "static", + "include_filters", + "input_output_pairer", + "input", + "dir", + "matchers", + "target_dependency_file", + "target_vertex", + "file", + "test_runner", + "instrumentation", + "bin", + "exclude", + "shard", + "fixture_contiguous", + "fixture_interleaved", + "test_contiguous", + "test_interleaved", + "never", + "target", + "policy", + "artifacts", + "meta", + "repo", + "workspace", + "build_target_descriptor", + "dependency_graph_data", + "test_target_meta", + "test_engine", + "target" + }; + + enum + { + Root = 0, + PlatformName, + RelativePaths, + ArtifactDir, + EnumerationCacheDir, + TestImpactDataFile, + TempWorkspace, + ActiveWorkspace, + TargetSources, + StaticSources, + AutogenSources, + StaticArtifacts, + SourceIncludeFilters, + AutogenInputOutputPairer, + AutogenInputSources, + Directory, + DependencyGraphMatchers, + TargetDependencyFileMatcher, + TargetVertexMatcher, + TestTargetMetaFile, + TestRunner, + TestInstrumentation, + BinaryFile, + TargetExcludeFilter, + TestSharding, + ContinuousFixtureSharding, + InterleavedFixtureSharding, + ContinuousTestSharding, + InterleavedTestSharding, + NeverShard, + TargetName, + TestShardingPolicy, + Artifacts, + Meta, + Repository, + Workspace, + BuildTargetDescriptor, + DependencyGraphData, + TestTargetMeta, + TestEngine, + TargetConfig + }; + } //! Returns an absolute path for a path relative to the specified root. RepoPath GetAbsPathFromRelPath(const RepoPath& root, const RepoPath& rel) @@ -31,54 +123,55 @@ namespace TestImpact ConfigMeta ParseConfigMeta(const rapidjson::Value& meta) { ConfigMeta configMeta; - configMeta.m_platform = meta["platform"].GetString(); + configMeta.m_platform = meta[Config::Keys[Config::PlatformName]].GetString(); return configMeta; } RepoConfig ParseRepoConfig(const rapidjson::Value& repo) { RepoConfig repoConfig; - repoConfig.m_root = repo["root"].GetString(); + repoConfig.m_root = repo[Config::Keys[Config::Root]].GetString(); return repoConfig; } WorkspaceConfig::Temp ParseTempWorkspaceConfig(const rapidjson::Value& tempWorkspace) { WorkspaceConfig::Temp tempWorkspaceConfig; - tempWorkspaceConfig.m_root = tempWorkspace["root"].GetString(); + tempWorkspaceConfig.m_root = tempWorkspace[Config::Keys[Config::Root]].GetString(); tempWorkspaceConfig.m_artifactDirectory = - GetAbsPathFromRelPath(tempWorkspaceConfig.m_root, tempWorkspace["relative_paths"]["artifact_dir"].GetString()); + GetAbsPathFromRelPath( + tempWorkspaceConfig.m_root, tempWorkspace[Config::Keys[Config::RelativePaths]][Config::Keys[Config::ArtifactDir]].GetString()); return tempWorkspaceConfig; } WorkspaceConfig::Active ParseActiveWorkspaceConfig(const rapidjson::Value& activeWorkspace) { WorkspaceConfig::Active activeWorkspaceConfig; - const auto& relativePaths = activeWorkspace["relative_paths"]; - activeWorkspaceConfig.m_root = activeWorkspace["root"].GetString(); + const auto& relativePaths = activeWorkspace[Config::Keys[Config::RelativePaths]]; + activeWorkspaceConfig.m_root = activeWorkspace[Config::Keys[Config::Root]].GetString(); activeWorkspaceConfig.m_enumerationCacheDirectory - = GetAbsPathFromRelPath(activeWorkspaceConfig.m_root, relativePaths["enumeration_cache_dir"].GetString()); + = GetAbsPathFromRelPath(activeWorkspaceConfig.m_root, relativePaths[Config::Keys[Config::EnumerationCacheDir]].GetString()); activeWorkspaceConfig.m_sparTIAFile - = GetAbsPathFromRelPath(activeWorkspaceConfig.m_root, relativePaths["test_impact_data_file"].GetString()); + = GetAbsPathFromRelPath(activeWorkspaceConfig.m_root, relativePaths[Config::Keys[Config::TestImpactDataFile]].GetString()); return activeWorkspaceConfig; } WorkspaceConfig ParseWorkspaceConfig(const rapidjson::Value& workspace) { WorkspaceConfig workspaceConfig; - workspaceConfig.m_temp = ParseTempWorkspaceConfig(workspace["temp"]); - workspaceConfig.m_active = ParseActiveWorkspaceConfig(workspace["active"]); + workspaceConfig.m_temp = ParseTempWorkspaceConfig(workspace[Config::Keys[Config::TempWorkspace]]); + workspaceConfig.m_active = ParseActiveWorkspaceConfig(workspace[Config::Keys[Config::ActiveWorkspace]]); return workspaceConfig; } BuildTargetDescriptorConfig ParseBuildTargetDescriptorConfig(const rapidjson::Value& buildTargetDescriptor) { BuildTargetDescriptorConfig buildTargetDescriptorConfig; - const auto& targetSources = buildTargetDescriptor["target_sources"]; - const auto& staticTargetSources = targetSources["static"]; - const auto& autogenTargetSources = targetSources["autogen"]; - buildTargetDescriptorConfig.m_mappingDirectory = buildTargetDescriptor["dir"].GetString(); - const auto& staticInclusionFilters = staticTargetSources["include_filters"].GetArray(); + const auto& targetSources = buildTargetDescriptor[Config::Keys[Config::TargetSources]]; + const auto& staticTargetSources = targetSources[Config::Keys[Config::StaticSources]]; + const auto& autogenTargetSources = targetSources[Config::Keys[Config::AutogenSources]]; + buildTargetDescriptorConfig.m_mappingDirectory = buildTargetDescriptor[Config::Keys[Config::Directory]].GetString(); + const auto& staticInclusionFilters = staticTargetSources[Config::Keys[Config::SourceIncludeFilters]].GetArray(); buildTargetDescriptorConfig.m_staticInclusionFilters.reserve(staticInclusionFilters.Size()); for (const auto& staticInclusionFilter : staticInclusionFilters) @@ -86,8 +179,9 @@ namespace TestImpact buildTargetDescriptorConfig.m_staticInclusionFilters.push_back(staticInclusionFilter.GetString()); } - buildTargetDescriptorConfig.m_inputOutputPairer = autogenTargetSources["input_output_pairer"].GetString(); - const auto& inputInclusionFilters = autogenTargetSources["input"]["include_filters"].GetArray(); + buildTargetDescriptorConfig.m_inputOutputPairer = autogenTargetSources[Config::Keys[Config::AutogenInputOutputPairer]].GetString(); + const auto& inputInclusionFilters = + autogenTargetSources[Config::Keys[Config::AutogenInputSources]][Config::Keys[Config::SourceIncludeFilters]].GetArray(); buildTargetDescriptorConfig.m_inputInclusionFilters.reserve(inputInclusionFilters.Size()); for (const auto& inputInclusionFilter : inputInclusionFilters) { @@ -100,62 +194,62 @@ namespace TestImpact DependencyGraphDataConfig ParseDependencyGraphDataConfig(const rapidjson::Value& dependencyGraphData) { DependencyGraphDataConfig dependencyGraphDataConfig; - const auto& matchers = dependencyGraphData["matchers"]; - dependencyGraphDataConfig.m_graphDirectory = dependencyGraphData["dir"].GetString(); - dependencyGraphDataConfig.m_targetDependencyFileMatcher = matchers["target_dependency_file"].GetString(); - dependencyGraphDataConfig.m_targetVertexMatcher = matchers["target_vertex"].GetString(); + const auto& matchers = dependencyGraphData[Config::Keys[Config::DependencyGraphMatchers]]; + dependencyGraphDataConfig.m_graphDirectory = dependencyGraphData[Config::Keys[Config::Directory]].GetString(); + dependencyGraphDataConfig.m_targetDependencyFileMatcher = matchers[Config::Keys[Config::TargetDependencyFileMatcher]].GetString(); + dependencyGraphDataConfig.m_targetVertexMatcher = matchers[Config::Keys[Config::TargetVertexMatcher]].GetString(); return dependencyGraphDataConfig; } TestTargetMetaConfig ParseTestTargetMetaConfig(const rapidjson::Value& testTargetMeta) { TestTargetMetaConfig testTargetMetaConfig; - testTargetMetaConfig.m_metaFile = testTargetMeta["file"].GetString(); + testTargetMetaConfig.m_metaFile = testTargetMeta[Config::Keys[Config::TestTargetMetaFile]].GetString(); return testTargetMetaConfig; } TestEngineConfig ParseTestEngineConfig(const rapidjson::Value& testEngine) { TestEngineConfig testEngineConfig; - testEngineConfig.m_testRunner.m_binary = testEngine["test_runner"]["bin"].GetString(); - testEngineConfig.m_instrumentation.m_binary = testEngine["instrumentation"]["bin"].GetString(); + testEngineConfig.m_testRunner.m_binary = testEngine[Config::Keys[Config::TestRunner]][Config::Keys[Config::BinaryFile]].GetString(); + testEngineConfig.m_instrumentation.m_binary = testEngine[Config::Keys[Config::TestInstrumentation]][Config::Keys[Config::BinaryFile]].GetString(); return testEngineConfig; } TargetConfig ParseTargetConfig(const rapidjson::Value& target) { TargetConfig targetConfig; - targetConfig.m_outputDirectory = target["dir"].GetString(); - const auto& testExcludes = target["exclude"].GetArray(); + targetConfig.m_outputDirectory = target[Config::Keys[Config::Directory]].GetString(); + const auto& testExcludes = target[Config::Keys[Config::TargetExcludeFilter]].GetArray(); targetConfig.m_excludedTestTargets.reserve(testExcludes.Size()); for (const auto& testExclude : testExcludes) { targetConfig.m_excludedTestTargets.push_back(testExclude.GetString()); } - const auto& testShards = target["shard"].GetArray(); + const auto& testShards = target[Config::Keys[Config::TestSharding]].GetArray(); targetConfig.m_shardedTestTargets.reserve(testShards.Size()); for (const auto& testShard : testShards) { const auto getShardingConfiguration = [](const AZStd::string& config) { - if (config == "fixture_contiguous") + if (config == Config::Keys[Config::ContinuousFixtureSharding]) { return ShardConfiguration::FixtureContiguous; } - else if (config == "fixture_interleaved") + else if (config == Config::Keys[Config::InterleavedFixtureSharding]) { return ShardConfiguration::FixtureInterleaved; } - else if (config == "test_contiguous") + else if (config == Config::Keys[Config::ContinuousTestSharding]) { return ShardConfiguration::TestContiguous; } - else if (config == "test_interleaved") + else if (config == Config::Keys[Config::InterleavedTestSharding]) { return ShardConfiguration::TestInterleaved; } - else if (config == "never") + else if (config == Config::Keys[Config::NeverShard]) { return ShardConfiguration::Never; } @@ -166,8 +260,8 @@ namespace TestImpact }; TargetConfig::ShardedTarget shard; - shard.m_name = testShard["target"].GetString(); - shard.m_configuration = getShardingConfiguration(testShard["policy"].GetString()); + shard.m_name = testShard[Config::Keys[Config::TargetName]].GetString(); + shard.m_configuration = getShardingConfiguration(testShard[Config::Keys[Config::TestShardingPolicy]].GetString()); targetConfig.m_shardedTestTargets.push_back(AZStd::move(shard)); } @@ -184,15 +278,15 @@ namespace TestImpact } RuntimeConfig runtimeConfig; - const auto& staticArtifacts = configurationFile["artifacts"]["static"]; - runtimeConfig.m_meta = ParseConfigMeta(configurationFile["meta"]); - runtimeConfig.m_repo = ParseRepoConfig(configurationFile["repo"]); - runtimeConfig.m_workspace = ParseWorkspaceConfig(configurationFile["workspace"]); - runtimeConfig.m_buildTargetDescriptor = ParseBuildTargetDescriptorConfig(staticArtifacts["build_target_descriptor"]); - runtimeConfig.m_dependencyGraphData = ParseDependencyGraphDataConfig(staticArtifacts["dependency_graph_data"]); - runtimeConfig.m_testTargetMeta = ParseTestTargetMetaConfig(staticArtifacts["static"]["test_target_meta"]); - runtimeConfig.m_testEngine = ParseTestEngineConfig(configurationFile["test_engine"]); - runtimeConfig.m_target = ParseTargetConfig(configurationFile["target"]); + const auto& staticArtifacts = configurationFile[Config::Keys[Config::Artifacts]][Config::Keys[Config::StaticArtifacts]]; + runtimeConfig.m_meta = ParseConfigMeta(configurationFile[Config::Keys[Config::Meta]]); + runtimeConfig.m_repo = ParseRepoConfig(configurationFile[Config::Keys[Config::Repository]]); + runtimeConfig.m_workspace = ParseWorkspaceConfig(configurationFile[Config::Keys[Config::Workspace]]); + runtimeConfig.m_buildTargetDescriptor = ParseBuildTargetDescriptorConfig(staticArtifacts[Config::Keys[Config::BuildTargetDescriptor]]); + runtimeConfig.m_dependencyGraphData = ParseDependencyGraphDataConfig(staticArtifacts[Config::Keys[Config::DependencyGraphData]]); + runtimeConfig.m_testTargetMeta = ParseTestTargetMetaConfig(staticArtifacts[Config::Keys[Config::TestTargetMeta]]); + runtimeConfig.m_testEngine = ParseTestEngineConfig(configurationFile[Config::Keys[Config::TestEngine]]); + runtimeConfig.m_target = ParseTargetConfig(configurationFile[Config::Keys[Config::TargetConfig]]); return runtimeConfig; } diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactConfiguration.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactConfiguration.h index 3406789c9e..91d748edfe 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactConfiguration.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactConfiguration.h @@ -38,28 +38,16 @@ namespace TestImpact //! Temporary workspace configuration. struct Temp { - //! Paths relative to root. - struct RelativePaths - { - RepoPath m_artifactDirectory; //!< Path to read and write runtime artifacts to and from. - }; - RepoPath m_root; //!< Path to the temporary workspace (cleaned prior to use). - RelativePaths m_relativePaths; + RepoPath m_artifactDirectory; //!< Path to read and write runtime artifacts to and from. }; //! Active persistent data workspace configuration. struct Active { - //! Paths relative to root. - struct RelativePaths - { - RepoPath m_sparTIAFile; //!< Path to the test impact analysis data. - RepoPath m_enumerationCacheDirectory; //!< Path to the test enumerations cache. - }; - RepoPath m_root; //!< Path to the persistent workspace tracked by the repository. - RelativePaths m_relativePaths; + RepoPath m_sparTIAFile; //!< Path to the test impact analysis data. + RepoPath m_enumerationCacheDirectory; //!< Path to the test enumerations cache. }; Temp m_temp; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp index e02bee03c6..83808f4ed2 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp @@ -101,8 +101,8 @@ namespace TestImpact m_testEngine = AZStd::make_unique( m_config.m_repo.m_root, m_config.m_target.m_outputDirectory, - m_config.m_workspace.m_active.m_relativePaths.m_enumerationCacheDirectory, - m_config.m_workspace.m_temp.m_relativePaths.m_artifactDirectory, + m_config.m_workspace.m_active.m_enumerationCacheDirectory, + m_config.m_workspace.m_temp.m_artifactDirectory, m_config.m_testEngine.m_testRunner.m_binary, m_config.m_testEngine.m_instrumentation.m_binary, m_maxConcurrency); @@ -110,7 +110,7 @@ namespace TestImpact try { // Populate the dynamic dependency map with the existing source coverage data (if any) - const auto tiaDataRaw = ReadFileContents(m_config.m_workspace.m_active.m_relativePaths.m_sparTIAFile); + const auto tiaDataRaw = ReadFileContents(m_config.m_workspace.m_active.m_sparTIAFile); const auto tiaData = DeserializeSourceCoveringTestsList(tiaDataRaw); if (tiaData.GetNumSources()) { @@ -136,7 +136,7 @@ namespace TestImpact } catch ([[maybe_unused]]const Exception& e) { - AZ_Printf("No test impact analysis data found at %s", m_config.m_workspace.m_active.m_relativePaths.m_sparTIAFile.c_str()); + AZ_Printf("No test impact analysis data found at %s", m_config.m_workspace.m_active.m_sparTIAFile.c_str()); } } @@ -233,7 +233,7 @@ namespace TestImpact void Runtime::ClearDynamicDependencyMapAndRemoveExistingFile() { - DeleteFile(m_config.m_workspace.m_active.m_relativePaths.m_sparTIAFile); + DeleteFile(m_config.m_workspace.m_active.m_sparTIAFile); m_dynamicDependencyMap->ClearAllSourceCoverage(); } @@ -247,7 +247,7 @@ namespace TestImpact m_dynamicDependencyMap->ReplaceSourceCoverage(sourceCoverageTestsList); const auto sparTIA = m_dynamicDependencyMap->ExportSourceCoverage(); const auto sparTIAData = SerializeSourceCoveringTestsList(sparTIA); - WriteFileContents(sparTIAData, m_config.m_workspace.m_active.m_relativePaths.m_sparTIAFile); + WriteFileContents(sparTIAData, m_config.m_workspace.m_active.m_sparTIAFile); m_hasImpactAnalysisData = true; } From d72bc3a4489c6c1bc3b5a5bedd6ad41760a00a00 Mon Sep 17 00:00:00 2001 From: jonawals Date: Wed, 2 Jun 2021 14:30:09 +0100 Subject: [PATCH 062/102] Add provisional Jenkins job --- scripts/build/TestImpactAnalysis/git_utils.py | 42 ++++ scripts/build/TestImpactAnalysis/tiaf.py | 202 ++++++++++++++++++ .../build/TestImpactAnalysis/tiaf_driver.py | 64 ++++++ 3 files changed, 308 insertions(+) create mode 100644 scripts/build/TestImpactAnalysis/git_utils.py create mode 100644 scripts/build/TestImpactAnalysis/tiaf.py create mode 100644 scripts/build/TestImpactAnalysis/tiaf_driver.py diff --git a/scripts/build/TestImpactAnalysis/git_utils.py b/scripts/build/TestImpactAnalysis/git_utils.py new file mode 100644 index 0000000000..5abedd16f0 --- /dev/null +++ b/scripts/build/TestImpactAnalysis/git_utils.py @@ -0,0 +1,42 @@ +# +# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +# its licensors. +# +# For complete copyright and license terms please see the LICENSE at the root of this +# distribution (the "License"). All use of this software is governed by the License, +# or, if provided, by the license below or the license accompanying this file. Do not +# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# + +import os +import subprocess +import git + +# Returns True if the dst commit descends from the src commit, otherwise False +def is_descendent(src_commit_hash, dst_commit_hash): + if src_commit_hash is None or dst_commit_hash is None: + return False + result = subprocess.run(["git", "merge-base", "--is-ancestor", src_commit_hash, dst_commit_hash]) + return result.returncode == 0 + +# Attempts to create a diff from the src and dst commits and write to the specified output file +def create_diff_file(src_commit_hash, dst_commit_hash, output_path): + if os.path.isfile(output_path): + os.remove(output_path) + os.makedirs(os.path.dirname(output_path), exist_ok=True) + # git diff will only write to the output file if both commit hashes are valid + subprocess.run(["git", "diff", "--name-status", f"--output={output_path}", src_commit_hash, dst_commit_hash]) + if not os.path.isfile(output_path): + raise FileNotFoundError(f"Source commit '{src_commit_hash}' and/or destination commit '{dst_commit_hash}' are invalid") + +# Basic representation of a repository +class Repo: + def __init__(self, repo_path): + self.__repo = git.Repo(repo_path) + + # Returns the current branch + @property + def current_branch(self): + branch = self.__repo.active_branch + return branch.name diff --git a/scripts/build/TestImpactAnalysis/tiaf.py b/scripts/build/TestImpactAnalysis/tiaf.py new file mode 100644 index 0000000000..4686c08f93 --- /dev/null +++ b/scripts/build/TestImpactAnalysis/tiaf.py @@ -0,0 +1,202 @@ +# +# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +# its licensors. +# +# For complete copyright and license terms please see the LICENSE at the root of this +# distribution (the "License"). All use of this software is governed by the License, +# or, if provided, by the license below or the license accompanying this file. Do not +# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# + +import os +import json +import subprocess +import re +import git_utils +from git_utils import Repo +from enum import Enum + +# Returns True if the specified child path is a child of the specified parent path, otherwise False +def is_child_path(parent_path, child_path): + parent_path = os.path.abspath(parent_path) + child_path = os.path.abspath(child_path) + return os.path.commonpath([os.path.abspath(parent_path)]) == os.path.commonpath([os.path.abspath(parent_path), os.path.abspath(child_path)]) + +# Enumerations for test sequence types +class SequenceType(Enum): + REGULAR = 1 + TEST_IMPACT_ANALYSIS = 2 + +class TestImpact: + def __init__(self, config_file, dst_commit): + self.__parse_config_file(config_file) + self.__init_repo(dst_commit) + self.__generate_change_list() + + # Parse the configuration file and retrieve the data needed for launching the test impact analysis runtime + def __parse_config_file(self, config_file): + print(f"Attempting to parse configuration file '{config_file}'...") + with open(config_file, "r") as config_data: + config = json.load(config_data) + self.__repo_dir = config["repo"]["root"] + self.__tiaf_bin = config["repo"]["tiaf_bin"] + if not os.path.isfile(self.__tiaf_bin): + raise FileNotFoundError("Could not find tiaf binary") + self.__source_of_truth = config["repo"]["source_of_truth"] + self.__active_workspace = config["workspace"]["active"]["root"] + self.__historic_workspace = config["workspace"]["historic"]["root"] + self.__temp_workspace = config["workspace"]["temp"]["root"] + last_commit_hash_path_rel = config["workspace"]["historic"]["relative_paths"]["last_run_hash_file"] + self.__last_commit_hash_path = os.path.join(self.__historic_workspace, last_commit_hash_path_rel) + print("The configuration file was parsed successfully.") + + # Initializes the internal representation of the repository being targeted for test impact analysis + def __init_repo(self, dst_commit): + self.__repo = Repo(self.__repo_dir) + self.__branch = self.__repo.current_branch + self.__dst_commit = dst_commit + self.__src_commit = None + self.__has_src_commit = False + if self.__repo.current_branch == self.__source_of_truth: + self.__is_source_of_truth = True + else: + self.__is_source_of_truth = False + print(f"The repository is located at '{self.__repo_dir}' and the current branch is '{self.__branch}'.") + print(f"The source of truth branch is '{self.__source_of_truth}'.") + if self.__is_source_of_truth: + print("I am the source of truth.") + else: + print("I am *not* the source of truth.") + + # Restricts change lists from checking in test impact analysis files + def __check_for_restricted_files(self, file_path): + if is_child_path(self.__active_workspace, file_path) or is_child_path(self.__historic_workspace, file_path) or is_child_path(self.__temp_workspace, file_path): + raise ValueError(f"Checking in test impact analysis framework files is illegal: '{file_path}''.") + + def __read_last_run_hash(self): + self.__has_src_commit = False + if os.path.isfile(self.__last_commit_hash_path): + print(f"Previous commit hash found at 'self.__last_commit_hash_path'") + with open(self.__last_commit_hash_path) as file: + self.__src_commit = file.read() + self.__has_src_commit = True + + def __write_last_run_hash(self, last_run_hash): + f = open(self.__last_commit_hash_path, "w") + f.write(last_run_hash) + f.close() + + # Determines the change list bewteen now and the last tiaf run (if any) + def __generate_change_list(self): + self.__has_change_list = False + self.__change_list_path = None + # Check whether or not a previous commit hash exists (no hash is not a failure) + self.__read_last_run_hash() + if self.__has_src_commit == True: + if git_utils.is_descendent(self.__src_commit, self.__dst_commit) == False: + print(f"Source commit '{self.__src_commit}' and destination commit '{self.__dst_commit}' are not related.") + return + diff_path = os.path.join(self.__temp_workspace, "changelist.diff") + try: + git_utils.create_diff_file(self.__src_commit, self.__dst_commit, diff_path) + except FileNotFoundError as e: + print(e) + return + # A diff was generated, attempt to parse the diff and construct the change list + print(f"Generated diff between commits {self.__src_commit} and {self.__dst_commit}.") + change_list = {} + change_list["createdFiles"] = [] + change_list["updatedFiles"] = [] + change_list["deletedFiles"] = [] + with open(diff_path, "r") as diff_data: + lines = diff_data.readlines() + for line in lines: + match = re.split("^R[0-9]+\\s(\\S+)\\s(\\S+)", line) + if len(match) > 1: + # File rename + self.__check_for_restricted_files(match[1]) + self.__check_for_restricted_files(match[2]) + # Treat renames as a deletion and an addition + change_list["deletedFiles"].append(match[1]) + change_list["createdFiles"].append(match[2]) + else: + match = re.split("^[AMD]\\s(\\S+)", line) + self.__check_for_restricted_files(match[1]) + if len(match) > 1: + if line[0] == 'A': + # File addition + change_list["createdFiles"].append(match[1]) + elif line[0] == 'M': + # File modification + change_list["updatedFiles"].append(match[1]) + elif line[0] == 'D': + # File Deletion + change_list["deletedFiles"].append(match[1]) + # Serialize the change list to the JSON format the test impact analysis runtime expects + change_list_json = json.dumps(change_list, indent = 4) + change_list_path = os.path.join(self.__temp_workspace, "changelist.json") + f = open(change_list_path, "w") + f.write(change_list_json) + f.close() + print(f"Change list constructed successfully: '{change_list_path}'.") + print(f"{len(change_list['createdFiles'])} created files, {len(change_list['updatedFiles'])} updated files and {len(change_list['deletedFiles'])} deleted files.") + # Note: an empty change list generated due to no changes between last and current commit is valid + self.__has_change_list = True + self.__change_list_path = change_list_path + else: + print("No previous commit hash found, regular or seeded sequences only will be run.") + self.__has_change_list = False + return + + # Runs the specified test sequence + def run(self, sequence_type, safe_mode, test_timeout, global_timeout): + args = [] + if self.__has_change_list: + args.append(f"-changelist={self.__change_list_path}") + if sequence_type == SequenceType.REGULAR: + print("Sequence type: regular.") + args.append("--sequence=regular") + elif sequence_type == SequenceType.TEST_IMPACT_ANALYSIS: + print("Sequence type: test impact analysis.") + if self.__is_source_of_truth: + # Source of truth branch will allways attempt a seed if no test impact analysis data is available + print("This branch is the source of truth, a seed sequence will be run if there is no test impact analysis data.") + args.append("--sequence=tiaorseed") + args.append("--fpolicy=continue") + else: + # Non source of truth branches will fall back to a regular test run if no test impact analysis data is available + print("This branch is not the source of truth, a regular sequence will be run if there is no test impact analysis data.") + args.append("--sequence=tia") + args.append("--fpolicy=abort") + + if safe_mode == True: + print("Safe mode is on, the discarded test targets will be run after the selected test targets.") + args.append("--safemode=on") + else: + print("Safe mode is off, the discarded test targets will not be run.") + else: + raise ValueError(sequence_type) + else: + print(f"No change list was generated, this will cause test impact analysis sequences on branches other than the source of truth to fall back to a regular sequence.") + print("Sequence type: Regular.") + args.append("--sequence=regular") + + if test_timeout != None: + args.append(f"--ttimeout={test_timeout}") + print(f"Test target timeout is set to {test_timeout} seconds.") + if global_timeout != None: + args.append(f"--gtimeout={global_timeout}") + print(f"Global sequence timeout is set to {test_timeout} seconds.") + + print("Args: ", end='') + print(*args) + result = subprocess.run([self.__tiaf_bin] + args) + if result.returncode == 0: + print("Test impact analysis runtime returned successfully. Updating historical artifacts...") + self.__write_last_run_hash(self.__dst_commit) + print("Complete!") + else: + print(f"The test impact analysis runtime returned with error: '{result.returncode}'.") + return result.returncode + diff --git a/scripts/build/TestImpactAnalysis/tiaf_driver.py b/scripts/build/TestImpactAnalysis/tiaf_driver.py new file mode 100644 index 0000000000..4f20fa1070 --- /dev/null +++ b/scripts/build/TestImpactAnalysis/tiaf_driver.py @@ -0,0 +1,64 @@ +# +# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +# its licensors. +# +# For complete copyright and license terms please see the LICENSE at the root of this +# distribution (the "License"). All use of this software is governed by the License, +# or, if provided, by the license below or the license accompanying this file. Do not +# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# + +import argparse +from tiaf import TestImpact +from tiaf import SequenceType + +import sys +import os +import datetime +import json +import socket + +def parse_args(): + def file_path(value): + if os.path.isfile(value): + return value + else: + raise FileNotFoundError(value) + + def sequence_type(value): + if value == "regular": + return SequenceType.REGULAR + elif value == "tia": + return SequenceType.TEST_IMPACT_ANALYSIS + else: + raise ValueError(value) + + def timout_type(value): + value = int(value) + if value <= 0: + raise ValueError("Timer values must be positive integers") + return value + + parser = argparse.ArgumentParser() + parser.add_argument('--config', dest="config", type=file_path, help="Path to the test impact analysis framework configuration file", required=True) + parser.add_argument('--destCommit', dest="dst_commit", help="Commit to run test impact analysis on (if empty, HEAD^ will be used)") + parser.add_argument('--sequenceType', dest="sequence_type", type=sequence_type, help="Test sequence type to run ('regular' or 'tia')", required=True) + parser.add_argument('--suites', dest="suites", nargs='*', help="Suites to include for regular tes sequences (use '*' for all suites)") + parser.add_argument('--safeMode', dest='safe_mode', help="If set, will run any test impact analysis runs in safe mode (unselected tests will still be run)") + parser.add_argument('--testTimeout', dest="test_timeout", type=timout_type, help="Maximum flight time (in seconds) of any test target before being terminated", required=False) + parser.add_argument('--globalTimeout', dest="global_timeout", type=timout_type, help="Maximum tun time of the sequence before being terminated", required=False) + parser.set_defaults(dst_commit="HEAD^") + parser.set_defaults(suites="*") + parser.set_defaults(safe_mode=False) + parser.set_defaults(test_timeout=None) + parser.set_defaults(global_timeout=None) + args = parser.parse_args() + + return args + +if __name__ == "__main__": + args = parse_args() + tiaf = TestImpact(args.config, args.dst_commit) + return_code = tiaf.run(args.sequence_type, args.safe_mode, args.test_timeout, args.global_timeout) + sys.exit(return_code) \ No newline at end of file From 786b406d9824d09bbbbe82e27e8df75f6a5956f7 Mon Sep 17 00:00:00 2001 From: jonawals Date: Thu, 3 Jun 2021 11:34:33 +0100 Subject: [PATCH 063/102] Add seed and tia jobs --- .../build/Platform/Windows/build_config.json | 24 ++++++- scripts/build/TestImpactAnalysis/tiaf.py | 72 ++++++++----------- .../build/TestImpactAnalysis/tiaf_driver.py | 13 ++-- 3 files changed, 60 insertions(+), 49 deletions(-) diff --git a/scripts/build/Platform/Windows/build_config.json b/scripts/build/Platform/Windows/build_config.json index bd10c9e59a..081855b05a 100644 --- a/scripts/build/Platform/Windows/build_config.json +++ b/scripts/build/Platform/Windows/build_config.json @@ -21,6 +21,7 @@ ], "steps": [ "debug_vs2019", + "test_impact_seed", "test_debug_vs2019" ] }, @@ -30,6 +31,7 @@ ], "steps": [ "profile_vs2019", + "test_impact_analysis", "asset_profile_vs2019", "test_cpu_profile_vs2019" ] @@ -78,6 +80,24 @@ "SCRIPT_PARAMETERS": "--platform 3rdParty --type 3rdParty_all" } }, + "test_impact_seed": { + "TAGS": [ + ], + "COMMAND": "python_windows.cmd", + "PARAMETERS": { + "SCRIPT_PATH": "scripts/build/TestImpactAnalysis/tiaf_driver.py", + "SCRIPT_PARAMETERS": "--sequenceType seed --config \"build\\windows_vs2019\\bin\\TestImpactFramework\\persistent\\tiaf.debug.json\"" + } + }, + "test_impact_analysis": { + "TAGS": [ + ], + "COMMAND": "python_windows.cmd", + "PARAMETERS": { + "SCRIPT_PATH": "scripts/build/TestImpactAnalysis/tiaf_driver.py", + "SCRIPT_PARAMETERS": "--sequenceType tia --destCommit !CHANGE_ID! --config \"build\\windows_vs2019\\bin\\TestImpactFramework\\persistent\\tiaf.profile.json\"" + } + }, "debug_vs2019": { "TAGS": [ "weekly-build-metrics" @@ -86,7 +106,7 @@ "PARAMETERS": { "CONFIGURATION": "debug", "OUTPUT_DIRECTORY": "build\\windows_vs2019", - "CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE -DLY_BUILD_WITH_INCREMENTAL_LINKING_DEBUG=FALSE", + "CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE -DLY_BUILD_WITH_INCREMENTAL_LINKING_DEBUG=FALSE -DLY_TEST_IMPACT_ACTIVE=1 -DLY_TEST_IMPACT_INSTRUMENTATION_BIN=\"c:\\ly\\3rdParty\\ackages\\OpenCppCoverage\\Binary\\windows-x64\\OpenCppCoverage.exe\"", "CMAKE_LY_PROJECTS": "AutomatedTesting", "CMAKE_TARGET": "ALL_BUILD", "CMAKE_NATIVE_BUILD_ARGS": "/m /nologo" @@ -117,7 +137,7 @@ "PARAMETERS": { "CONFIGURATION": "profile", "OUTPUT_DIRECTORY": "build\\windows_vs2019", - "CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE", + "CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE -DLY_TEST_IMPACT_ACTIVE=1 -DLY_TEST_IMPACT_INSTRUMENTATION_BIN=\"c:\\ly\\3rdParty\\ackages\\OpenCppCoverage\\Binary\\windows-x64\\OpenCppCoverage.exe\"", "CMAKE_LY_PROJECTS": "AutomatedTesting", "CMAKE_TARGET": "ALL_BUILD", "CMAKE_NATIVE_BUILD_ARGS": "/m /nologo" diff --git a/scripts/build/TestImpactAnalysis/tiaf.py b/scripts/build/TestImpactAnalysis/tiaf.py index 4686c08f93..01e2cde8ba 100644 --- a/scripts/build/TestImpactAnalysis/tiaf.py +++ b/scripts/build/TestImpactAnalysis/tiaf.py @@ -25,8 +25,12 @@ def is_child_path(parent_path, child_path): # Enumerations for test sequence types class SequenceType(Enum): + # Regular sequence as-per the tiaf regular sequence REGULAR = 1 + # TIA sequence as-per the tiaf read-only impact analysis sequence TEST_IMPACT_ANALYSIS = 2 + # Seed sequence as-per the tiaf seed sequence + SEED = 3 class TestImpact: def __init__(self, config_file, dst_commit): @@ -43,7 +47,6 @@ class TestImpact: self.__tiaf_bin = config["repo"]["tiaf_bin"] if not os.path.isfile(self.__tiaf_bin): raise FileNotFoundError("Could not find tiaf binary") - self.__source_of_truth = config["repo"]["source_of_truth"] self.__active_workspace = config["workspace"]["active"]["root"] self.__historic_workspace = config["workspace"]["historic"]["root"] self.__temp_workspace = config["workspace"]["temp"]["root"] @@ -58,16 +61,7 @@ class TestImpact: self.__dst_commit = dst_commit self.__src_commit = None self.__has_src_commit = False - if self.__repo.current_branch == self.__source_of_truth: - self.__is_source_of_truth = True - else: - self.__is_source_of_truth = False print(f"The repository is located at '{self.__repo_dir}' and the current branch is '{self.__branch}'.") - print(f"The source of truth branch is '{self.__source_of_truth}'.") - if self.__is_source_of_truth: - print("I am the source of truth.") - else: - print("I am *not* the source of truth.") # Restricts change lists from checking in test impact analysis files def __check_for_restricted_files(self, file_path): @@ -77,7 +71,7 @@ class TestImpact: def __read_last_run_hash(self): self.__has_src_commit = False if os.path.isfile(self.__last_commit_hash_path): - print(f"Previous commit hash found at 'self.__last_commit_hash_path'") + print(f"Previous commit hash found at '{self.__last_commit_hash_path}'") with open(self.__last_commit_hash_path) as file: self.__src_commit = file.read() self.__has_src_commit = True @@ -104,7 +98,7 @@ class TestImpact: print(e) return # A diff was generated, attempt to parse the diff and construct the change list - print(f"Generated diff between commits {self.__src_commit} and {self.__dst_commit}.") + print(f"Generated diff between commits '{self.__src_commit}' and '{self.__dst_commit}': '{diff_path}'.") change_list = {} change_list["createdFiles"] = [] change_list["updatedFiles"] = [] @@ -152,35 +146,27 @@ class TestImpact: # Runs the specified test sequence def run(self, sequence_type, safe_mode, test_timeout, global_timeout): args = [] - if self.__has_change_list: - args.append(f"-changelist={self.__change_list_path}") - if sequence_type == SequenceType.REGULAR: - print("Sequence type: regular.") - args.append("--sequence=regular") - elif sequence_type == SequenceType.TEST_IMPACT_ANALYSIS: - print("Sequence type: test impact analysis.") - if self.__is_source_of_truth: - # Source of truth branch will allways attempt a seed if no test impact analysis data is available - print("This branch is the source of truth, a seed sequence will be run if there is no test impact analysis data.") - args.append("--sequence=tiaorseed") - args.append("--fpolicy=continue") - else: - # Non source of truth branches will fall back to a regular test run if no test impact analysis data is available - print("This branch is not the source of truth, a regular sequence will be run if there is no test impact analysis data.") - args.append("--sequence=tia") - args.append("--fpolicy=abort") - - if safe_mode == True: - print("Safe mode is on, the discarded test targets will be run after the selected test targets.") - args.append("--safemode=on") - else: - print("Safe mode is off, the discarded test targets will not be run.") - else: - raise ValueError(sequence_type) - else: - print(f"No change list was generated, this will cause test impact analysis sequences on branches other than the source of truth to fall back to a regular sequence.") - print("Sequence type: Regular.") + print("Please note: test impact analysis sequences will be run in read-only mode (seed sequences are unaffected).") + if sequence_type == SequenceType.REGULAR: + print("Sequence type: regular.") args.append("--sequence=regular") + args.append("--fpolicy=abort") + elif sequence_type == SequenceType.SEED: + print("Sequence type: seed.") + args.append("--sequence=seed") + args.append("--fpolicy=continue") + elif sequence_type == SequenceType.TEST_IMPACT_ANALYSIS: + print("Sequence type: test impact analysis (no write).") + args.append("--fpolicy=abort") + if self.__has_change_list: + args.append(f"-changelist={self.__change_list_path}") + args.append("--sequence=tianowrite") + else: + print(f"No change list was generated, falling back to a regular sequence.") + print("Sequence type: Regular.") + args.append("--sequence=regular") + else: + raise ValueError(sequence_type) if test_timeout != None: args.append(f"--ttimeout={test_timeout}") @@ -193,8 +179,10 @@ class TestImpact: print(*args) result = subprocess.run([self.__tiaf_bin] + args) if result.returncode == 0: - print("Test impact analysis runtime returned successfully. Updating historical artifacts...") - self.__write_last_run_hash(self.__dst_commit) + print("Test impact analysis runtime returned successfully.") + if sequence_type == SequenceType.SEED: + print("Writing historical meta-data...") + self.__write_last_run_hash(self.__dst_commit) print("Complete!") else: print(f"The test impact analysis runtime returned with error: '{result.returncode}'.") diff --git a/scripts/build/TestImpactAnalysis/tiaf_driver.py b/scripts/build/TestImpactAnalysis/tiaf_driver.py index 4f20fa1070..1cc7ecb630 100644 --- a/scripts/build/TestImpactAnalysis/tiaf_driver.py +++ b/scripts/build/TestImpactAnalysis/tiaf_driver.py @@ -31,6 +31,8 @@ def parse_args(): return SequenceType.REGULAR elif value == "tia": return SequenceType.TEST_IMPACT_ANALYSIS + elif value == "seed": + return SequenceType.SEED else: raise ValueError(value) @@ -42,23 +44,24 @@ def parse_args(): parser = argparse.ArgumentParser() parser.add_argument('--config', dest="config", type=file_path, help="Path to the test impact analysis framework configuration file", required=True) - parser.add_argument('--destCommit', dest="dst_commit", help="Commit to run test impact analysis on (if empty, HEAD^ will be used)") - parser.add_argument('--sequenceType', dest="sequence_type", type=sequence_type, help="Test sequence type to run ('regular' or 'tia')", required=True) + parser.add_argument('--destCommit', dest="dst_commit", help="Commit to run test impact analysis on (not required for seed)") + parser.add_argument('--sequenceType', dest="sequence_type", type=sequence_type, help="Test sequence type to run ('regular', 'seed' or 'tia')", required=True) parser.add_argument('--suites', dest="suites", nargs='*', help="Suites to include for regular tes sequences (use '*' for all suites)") - parser.add_argument('--safeMode', dest='safe_mode', help="If set, will run any test impact analysis runs in safe mode (unselected tests will still be run)") parser.add_argument('--testTimeout', dest="test_timeout", type=timout_type, help="Maximum flight time (in seconds) of any test target before being terminated", required=False) parser.add_argument('--globalTimeout', dest="global_timeout", type=timout_type, help="Maximum tun time of the sequence before being terminated", required=False) - parser.set_defaults(dst_commit="HEAD^") parser.set_defaults(suites="*") parser.set_defaults(safe_mode=False) parser.set_defaults(test_timeout=None) parser.set_defaults(global_timeout=None) args = parser.parse_args() + + if args.sequence_type == SequenceType.TEST_IMPACT_ANALYSIS and args.dst_commit == None: + raise ValueError("Test impact analysis sequence must have a change list") return args if __name__ == "__main__": args = parse_args() tiaf = TestImpact(args.config, args.dst_commit) - return_code = tiaf.run(args.sequence_type, args.safe_mode, args.test_timeout, args.global_timeout) + return_code = tiaf.run(args.sequence_type, args.test_timeout, args.global_timeout) sys.exit(return_code) \ No newline at end of file From ac42a9a748fdb889136e5da378273711caa51f4a Mon Sep 17 00:00:00 2001 From: jonawals Date: Fri, 4 Jun 2021 19:33:26 +0100 Subject: [PATCH 064/102] Changes for read-only runs and new test suites --- .../Source/TestImpactCommandLineOptions.cpp | 55 +++----- .../Source/TestImpactCommandLineOptions.h | 9 +- .../Code/Source/TestImpactConsoleMain.cpp | 83 ++++++++---- .../TestImpactRuntimeConfigurationFactory.cpp | 21 ++- .../TestImpactConfiguration.h | 3 +- .../TestImpactFramework/TestImpactRuntime.h | 10 +- .../TestImpactTestSequence.h | 35 +++++ .../TestImpactTestTargetMetaMapFactory.cpp | 49 ++++--- .../TestImpactTestTargetMetaMapFactory.h | 6 +- .../TestImpactDynamicDependencyMap.cpp | 6 +- .../Runtime/Code/Source/TestImpactRuntime.cpp | 127 ++++++++++-------- .../Code/Source/TestImpactRuntimeUtils.cpp | 7 +- .../Code/Source/TestImpactRuntimeUtils.h | 1 + .../testimpactframework_runtime_files.cmake | 92 ++++++++----- ...timpactframework_runtime_tests_files.cmake | 38 ------ cmake/LYTestWrappers.cmake | 50 +++++-- .../ConsoleFrontendConfig.in | 23 +++- .../LYTestImpactFramework.cmake | 115 ++++++++++------ 18 files changed, 449 insertions(+), 281 deletions(-) delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_tests_files.cmake diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp index a6eb2e886e..5ff070601a 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp @@ -36,21 +36,21 @@ namespace TestImpact MaxConcurrency, TestTargetTimeout, GlobalTimeout, - SuitesFilter, + SuiteFilter, SafeMode, // Values None, Seed, Regular, ImpactAnalysis, + ImpactAnalysisNoWrite, ImpactAnalysisOrSeed, Locality, Abort, Continue, Ignore, StdOut, - File, - AllSuites + File }; constexpr const char* OptionKeys[] = @@ -70,21 +70,21 @@ namespace TestImpact "maxconcurrency", "ttimeout", "gtimeout", - "suites", + "suite", "safemode", // Values "none", "seed", "regular", "tia", + "tianowrite", "tiaorseed", "locality", "abort", "continue", "ignore", "stdout", - "file", - "*" + "file" }; RepoPath ParseConfigurationFile(const AZ::CommandLine& cmd) @@ -110,6 +110,7 @@ namespace TestImpact {OptionKeys[Seed], TestSequenceType::Seed}, {OptionKeys[Regular], TestSequenceType::Regular}, {OptionKeys[ImpactAnalysis], TestSequenceType::ImpactAnalysis}, + {OptionKeys[ImpactAnalysisNoWrite], TestSequenceType::ImpactAnalysisNoWrite}, {OptionKeys[ImpactAnalysisOrSeed], TestSequenceType::ImpactAnalysisOrSeed} }; @@ -250,32 +251,16 @@ namespace TestImpact return ParseOnOffOption(OptionKeys[SafeMode], states, cmd).value_or(false); } - AZStd::unordered_set ParseSuitesFilter(const AZ::CommandLine& cmd) + SuiteType ParseSuiteFilter(const AZ::CommandLine& cmd) { - AZStd::unordered_set suitesFilter; - if (const auto numSwitchValues = cmd.GetNumSwitchValues(OptionKeys[SuitesFilter]); - numSwitchValues) + const AZStd::vector> states = { - for (auto i = 0; i < numSwitchValues; i++) - { - const auto value = cmd.GetSwitchValue(OptionKeys[SuitesFilter], i); - AZ_TestImpact_Eval(!value.empty(), CommandLineOptionsException, "Suites option value is empty"); - if (value == OptionKeys[AllSuites]) - { - AZ_TestImpact_Eval( - suitesFilter.empty(), CommandLineOptionsException, "The * suite cannot be used with other suites"); - } + {GetSuiteTypeName(SuiteType::Main), SuiteType::Main}, + {GetSuiteTypeName(SuiteType::Periodic), SuiteType::Periodic}, + {GetSuiteTypeName(SuiteType::Sandbox), SuiteType::Sandbox} + }; - suitesFilter.insert(value); - } - } - - if (suitesFilter.find(OptionKeys[AllSuites]) != suitesFilter.end()) - { - return {}; - } - - return suitesFilter; + return ParseMultiStateOption(OptionKeys[SuiteFilter], states, cmd).value_or(SuiteType::Main); } } @@ -299,7 +284,7 @@ namespace TestImpact m_testTargetTimeout = ParseTestTargetTimeout(cmd); m_globalTimeout = ParseGlobalTimeout(cmd); m_safeMode = ParseSafeMode(cmd); - m_suitesFilter = ParseSuitesFilter(cmd); + m_suiteFilter = ParseSuiteFilter(cmd); } bool CommandLineOptions::HasChangeListFile() const @@ -382,9 +367,9 @@ namespace TestImpact return m_globalTimeout; } - const AZStd::unordered_set& CommandLineOptions::GetSuitesFilter() const + SuiteType CommandLineOptions::GetSuiteFilter() const { - return m_suitesFilter; + return m_suiteFilter; } AZStd::string CommandLineOptions::GetCommandLineUsageString() @@ -452,11 +437,7 @@ namespace TestImpact " -maxconcurrency= The maximum number of concurrent test targets/shards to be in flight at \n" " any given moment.\n" " -ochangelist= Outputs the change list used for test selection.\n" - " -suites= The test suites to select from for this test sequence (multiple values are \n" - " allowed). The suite all has special significance and will allow tests from \n" - " any suite to be selected, however this particular suite is mutually exclusive\n" - " with other suite. Note: this option is only applicable to the regular sequence\n" - " and, if safe mode is enables, the tia and tiaorseed sequences."; + " -suite= The test suite to select from for this test sequence."; return help; } diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.h b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.h index 73961b5fcc..99b1b98dba 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.h +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.h @@ -18,7 +18,6 @@ #include #include #include -#include namespace TestImpact { @@ -29,6 +28,8 @@ namespace TestImpact Seed, //!< Removes any prior coverage data and runs all test targets with instrumentation to reseed the data from scratch. Regular, //!< Runs all of the test targets without any instrumentation to generate coverage data (any prior coverage data is left intact). ImpactAnalysis, //!< Uses any prior coverage data to run the instrumented subset of selected tests (if no prior coverage data a regular run is performed instead). + ImpactAnalysisNoWrite, //!< Uses any prior coverage data to run the uninstrumented subset of selected tests (if no prior coverage data a regular run is performed instead). + //!< The coverage data is not updated with the subset of selected tests. ImpactAnalysisOrSeed //!< Uses any prior coverage data to run the instrumented subset of selected tests (if no prior coverage data a seed run is performed instead). }; @@ -87,8 +88,8 @@ namespace TestImpact //! Returns the global test sequence timeout to use (if any). const AZStd::optional& GetGlobalTimeout() const; - //! Returns the filter for test suites that will be allowed to be run. - const AZStd::unordered_set& GetSuitesFilter() const; + //! Returns the filter for test suite that will be allowed to be run. + SuiteType GetSuiteFilter() const; private: RepoPath m_configurationFile; @@ -105,7 +106,7 @@ namespace TestImpact AZStd::optional m_maxConcurrency; AZStd::optional m_testTargetTimeout; AZStd::optional m_globalTimeout; - AZStd::unordered_set m_suitesFilter; + SuiteType m_suiteFilter; bool m_safeMode = false; }; } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleMain.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleMain.cpp index 84e16c2332..fab6509244 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleMain.cpp +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleMain.cpp @@ -86,6 +86,8 @@ namespace TestImpact Runtime& runtime, const AZStd::optional& changeList) { + // Even though it is possible for a regular run to be selected (see below) which does not actually require a change list, + // consider any impact analysis sequence type without a change list to be an error AZ_TestImpact_Eval( changeList.has_value(), CommandLineOptionsException, @@ -94,39 +96,72 @@ namespace TestImpact TestSequenceResult result = TestSequenceResult::Failure; if (options.HasSafeMode()) { - auto [selectedResult, discardedResult] = runtime.SafeImpactAnalysisTestSequence( - changeList.value(), - options.GetSuitesFilter(), - options.GetTestPrioritizationPolicy(), - options.GetTestTargetTimeout(), - options.GetGlobalTimeout(), - AZStd::ref(sequenceEventHandler), - AZStd::ref(sequenceEventHandler), - AZStd::ref(sequenceEventHandler)); + if (options.GetTestSequenceType() == TestSequenceType::ImpactAnalysis) + { + auto [selectedResult, discardedResult] = runtime.SafeImpactAnalysisTestSequence( + changeList.value(), + options.GetTestPrioritizationPolicy(), + options.GetTestTargetTimeout(), + options.GetGlobalTimeout(), + AZStd::ref(sequenceEventHandler), + AZStd::ref(sequenceEventHandler), + AZStd::ref(sequenceEventHandler)); - // Handling the possible timeout and failure permutations of the selected and discarded test results is splitting hairs - // so apply the following, admittedly arbitrary, rules to determine what the composite test sequence result should be - if (selectedResult == TestSequenceResult::Success && discardedResult == TestSequenceResult::Success) - { - // Trivial case: both sequences succeeded - result = TestSequenceResult::Success; + // Handling the possible timeout and failure permutations of the selected and discarded test results is splitting hairs + // so apply the following, admittedly arbitrary, rules to determine what the composite test sequence result should be + if (selectedResult == TestSequenceResult::Success && discardedResult == TestSequenceResult::Success) + { + // Trivial case: both sequences succeeded + result = TestSequenceResult::Success; + } + else if (selectedResult == TestSequenceResult::Failure || discardedResult == TestSequenceResult::Failure) + { + // One sequence failed whilst the other sequence either succeeded or timed out + result = TestSequenceResult::Failure; + } + else + { + // One sequence timed out whilst the other sequence succeeded or both sequences timed out + result = TestSequenceResult::Timeout; + } } - else if (selectedResult == TestSequenceResult::Failure || discardedResult == TestSequenceResult::Failure) + else if (options.GetTestSequenceType() == TestSequenceType::ImpactAnalysisNoWrite) { - // One sequence failed whilst the other sequence either succeeded or timed out - result = TestSequenceResult::Failure; + // A no-write impact analysis sequence with safe mode enabled is functionally identical to a regular sequence type + // due to a) the selected tests being run without instrumentation and b) the discarded tests also being run without + // instrumentation + result = runtime.RegularTestSequence( + options.GetTestTargetTimeout(), + options.GetGlobalTimeout(), + AZStd::ref(sequenceEventHandler), + AZStd::ref(sequenceEventHandler), + AZStd::ref(sequenceEventHandler)); } else { - // One sequence timed out whilst the other sequence succeeded or both sequences timed out - result = TestSequenceResult::Timeout; + throw(Exception("Unexpected sequence type")); } } else { + Policy::DynamicDependencyMap dynamicDependencyMapPolicy; + if (options.GetTestSequenceType() == TestSequenceType::ImpactAnalysis) + { + dynamicDependencyMapPolicy = Policy::DynamicDependencyMap::Update; + } + else if (options.GetTestSequenceType() == TestSequenceType::ImpactAnalysisNoWrite) + { + dynamicDependencyMapPolicy = Policy::DynamicDependencyMap::Discard; + } + else + { + throw(Exception("Unexpected sequence type")); + } + result = runtime.ImpactAnalysisTestSequence( changeList.value(), options.GetTestPrioritizationPolicy(), + dynamicDependencyMapPolicy, options.GetTestTargetTimeout(), options.GetGlobalTimeout(), AZStd::ref(sequenceEventHandler), @@ -164,9 +199,11 @@ namespace TestImpact // As of now, there are no other non-test operations other than printing a change list so getting this far is considered an error AZ_TestImpact_Eval(options.GetTestSequenceType() != TestSequenceType::None, CommandLineOptionsException, "No action specified"); - std::cout << "Constructing in-memory model of source tree and test coverage, this may take a moment...\n"; + std::cout << "Constructing in-memory model of source tree and test coverage for test suite "; + std::cout << GetSuiteTypeName(options.GetSuiteFilter()).c_str() << ", this may take a moment...\n"; Runtime runtime( RuntimeConfigurationFactory(ReadFileContents(options.GetConfigurationFile())), + options.GetSuiteFilter(), options.GetExecutionFailurePolicy(), options.GetExecutionFailureDraftingPolicy(), options.GetTestFailurePolicy(), @@ -184,14 +221,13 @@ namespace TestImpact std::cout << "Test impact analysis data for this repository was not found, seed or regular sequence fallbacks will be used.\n"; } - TestSequenceEventHandler sequenceEventHandler(&options.GetSuitesFilter()); + TestSequenceEventHandler sequenceEventHandler(options.GetSuiteFilter()); switch (const auto type = options.GetTestSequenceType()) { case TestSequenceType::Regular: { const auto result = runtime.RegularTestSequence( - options.GetSuitesFilter(), options.GetTestTargetTimeout(), options.GetGlobalTimeout(), AZStd::ref(sequenceEventHandler), @@ -211,6 +247,7 @@ namespace TestImpact return GetReturnCodeForTestSequenceResult(result); } + case TestSequenceType::ImpactAnalysisNoWrite: case TestSequenceType::ImpactAnalysis: { return WrappedImpactAnalysisTestSequence(sequenceEventHandler, options, runtime, changeList); diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactRuntimeConfigurationFactory.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactRuntimeConfigurationFactory.cpp index 9db99d3c2e..6ef1ce4172 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactRuntimeConfigurationFactory.cpp +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactRuntimeConfigurationFactory.cpp @@ -30,7 +30,7 @@ namespace TestImpact "relative_paths", "artifact_dir", "enumeration_cache_dir", - "test_impact_data_file", + "test_impact_data_files", "temp", "active", "target_sources", @@ -75,7 +75,7 @@ namespace TestImpact RelativePaths, ArtifactDir, EnumerationCacheDir, - TestImpactDataFile, + TestImpactDataFiles, TempWorkspace, ActiveWorkspace, TargetSources, @@ -144,6 +144,19 @@ namespace TestImpact return tempWorkspaceConfig; } + AZStd::array ParseTestImpactAnalysisDataFiles(const RepoPath& root, const rapidjson::Value& sparTIAFile) + { + AZStd::array sparTIAFiles; + sparTIAFiles[static_cast(SuiteType::Main)] = + GetAbsPathFromRelPath(root, sparTIAFile[GetSuiteTypeName(SuiteType::Main).c_str()].GetString()); + sparTIAFiles[static_cast(SuiteType::Periodic)] = + GetAbsPathFromRelPath(root, sparTIAFile[GetSuiteTypeName(SuiteType::Periodic).c_str()].GetString()); + sparTIAFiles[static_cast(SuiteType::Sandbox)] = + GetAbsPathFromRelPath(root, sparTIAFile[GetSuiteTypeName(SuiteType::Sandbox).c_str()].GetString()); + + return sparTIAFiles; + } + WorkspaceConfig::Active ParseActiveWorkspaceConfig(const rapidjson::Value& activeWorkspace) { WorkspaceConfig::Active activeWorkspaceConfig; @@ -151,8 +164,8 @@ namespace TestImpact activeWorkspaceConfig.m_root = activeWorkspace[Config::Keys[Config::Root]].GetString(); activeWorkspaceConfig.m_enumerationCacheDirectory = GetAbsPathFromRelPath(activeWorkspaceConfig.m_root, relativePaths[Config::Keys[Config::EnumerationCacheDir]].GetString()); - activeWorkspaceConfig.m_sparTIAFile - = GetAbsPathFromRelPath(activeWorkspaceConfig.m_root, relativePaths[Config::Keys[Config::TestImpactDataFile]].GetString()); + activeWorkspaceConfig.m_sparTIAFiles = + ParseTestImpactAnalysisDataFiles(activeWorkspaceConfig.m_root, relativePaths[Config::Keys[Config::TestImpactDataFiles]]); return activeWorkspaceConfig; } diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactConfiguration.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactConfiguration.h index 91d748edfe..ee73ac572d 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactConfiguration.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactConfiguration.h @@ -16,6 +16,7 @@ #include #include +#include #include namespace TestImpact @@ -46,8 +47,8 @@ namespace TestImpact struct Active { RepoPath m_root; //!< Path to the persistent workspace tracked by the repository. - RepoPath m_sparTIAFile; //!< Path to the test impact analysis data. RepoPath m_enumerationCacheDirectory; //!< Path to the test enumerations cache. + AZStd::array m_sparTIAFiles; //!< Paths to the test impact analysis data files for each test suite. }; Temp m_temp; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h index 266bd5ef7a..ae1f78c5c7 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h @@ -92,6 +92,7 @@ namespace TestImpact public: //! Constructs a runtime with the specified configuration and policies. //! @param config The configuration used for this runtime instance. + //! @param suiteFilter The test suite for which the coverage data and test selection will draw from. //! @param executionFailurePolicy Determines how to handle test targets that fail to execute. //! @param executionFailureDraftingPolicy Determines how test targets that previously failed to execute are drafted into subsequent test sequences. //! @param testFailurePolicy Determines how to handle test targets that report test failures. @@ -99,6 +100,7 @@ namespace TestImpact //! @param testShardingPolicy Determines how to handle test targets that have opted in to test sharding. Runtime( RuntimeConfig&& config, + SuiteType suiteFilter, Policy::ExecutionFailure executionFailurePolicy, Policy::ExecutionFailureDrafting executionFailureDraftingPolicy, Policy::TestFailure testFailurePolicy, @@ -110,7 +112,6 @@ namespace TestImpact ~Runtime(); //! Runs a test sequence where all tests with a matching suite in the suite filter and also not on the excluded list are selected. - //! @param suitesFilter The test suites that will be included in the test selection. //! @param testTargetTimeout The maximum duration individual test targets may be in flight for (infinite if empty). //! @param globalTimeout The maximum duration the entire test sequence may run for (infinite if empty). //! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests. @@ -118,7 +119,6 @@ namespace TestImpact //! @param testRunCompleteCallback The client function to be called after an individual test run has completed. //! @returns TestSequenceResult RegularTestSequence( - const AZStd::unordered_set suitesFilter, AZStd::optional testTargetTimeout, AZStd::optional globalTimeout, AZStd::optional testSequenceStartCallback, @@ -128,6 +128,7 @@ namespace TestImpact //! Runs a test sequence where tests are selected according to test impact analysis so long as they are not on the excluded list. //! @param changeList The change list used to determine the tests to select. //! @param testPrioritizationPolicy Determines how selected tests will be prioritized. + //! @param dynamicDependencyMapPolicy The policy to determine how the coverage data of produced by test sequences is used to update the dynamic dependency map. //! @param testTargetTimeout The maximum duration individual test targets may be in flight for (infinite if empty). //! @param globalTimeout The maximum duration the entire test sequence may run for (infinite if empty). //! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests. @@ -137,6 +138,7 @@ namespace TestImpact TestSequenceResult ImpactAnalysisTestSequence( const ChangeList& changeList, Policy::TestPrioritization testPrioritizationPolicy, + Policy::DynamicDependencyMap dynamicDependencyMapPolicy, AZStd::optional testTargetTimeout, AZStd::optional globalTimeout, AZStd::optional testSequenceStartCallback, @@ -145,7 +147,6 @@ namespace TestImpact //! Runs a test sequence as per the ImpactAnalysisTestSequence where the tests not selected are also run (albeit without instrumentation). //! @param changeList The change list used to determine the tests to select. - //! @param suitesFilter The test suites that will be included in the test selection. //! @param testPrioritizationPolicy Determines how selected tests will be prioritized. //! @param testTargetTimeout The maximum duration individual test targets may be in flight for (infinite if empty). //! @param globalTimeout The maximum duration the entire test sequence may run for (infinite if empty). @@ -155,7 +156,6 @@ namespace TestImpact //! @returns AZStd::pair SafeImpactAnalysisTestSequence( const ChangeList& changeList, - const AZStd::unordered_set suitesFilter, Policy::TestPrioritization testPrioritizationPolicy, AZStd::optional testTargetTimeout, AZStd::optional globalTimeout, @@ -207,6 +207,8 @@ namespace TestImpact void UpdateAndSerializeDynamicDependencyMap(const SourceCoveringTestsList& sourceCoverageTestsList); RuntimeConfig m_config; + SuiteType m_suiteFilter; + RepoPath m_sparTIAFile; Policy::ExecutionFailure m_executionFailurePolicy; Policy::ExecutionFailureDrafting m_executionFailureDraftingPolicy; Policy::TestFailure m_testFailurePolicy; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactTestSequence.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactTestSequence.h index 594c997c45..4e8908cf3f 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactTestSequence.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactTestSequence.h @@ -12,6 +12,10 @@ #pragma once +#include + +#include + namespace TestImpact { namespace Policy @@ -55,6 +59,13 @@ namespace TestImpact Continue //!< Continue the test sequence and report the test failures after the run. }; + //! Policy for updating the dynamic dependency map with the coverage data of produced by test sequences. + enum class DynamicDependencyMap + { + Discard, //!< Discard the coverage data produced by test sequences. + Update //!< Update the dynamic dependency map with the coverage data produced by test sequences. + }; + //! Policy for sharding test targets that have been marked for test sharding. enum class TestSharding { @@ -82,6 +93,30 @@ namespace TestImpact TestInterleaved //!< Tests are interlaced across shards agnostic of fixtures (fastest but prone to inter-test dependency problems). }; + //! Test suite types to select from. + enum class SuiteType : AZ::u8 + { + Main = 0, + Periodic, + Sandbox + }; + + //! User-friendly names for the test suite types. + inline AZStd::string GetSuiteTypeName(SuiteType suiteType) + { + switch (suiteType) + { + case SuiteType::Main: + return "main"; + case SuiteType::Periodic: + return "periodic"; + case SuiteType::Sandbox: + return "sandbox"; + default: + throw(RuntimeException("Unexpected suite type")); + } + } + //! Result of a test sequence that was run. enum class TestSequenceResult { diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestTargetMetaMapFactory.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestTargetMetaMapFactory.cpp index 8bbb5200bb..4b247086ed 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestTargetMetaMapFactory.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestTargetMetaMapFactory.cpp @@ -19,7 +19,7 @@ namespace TestImpact { - TestTargetMetaMap TestTargetMetaMapFactory(const AZStd::string& masterTestListData) + TestTargetMetaMap TestTargetMetaMapFactory(const AZStd::string& masterTestListData, SuiteType suiteType) { // Keys for pertinent JSON node and attribute names constexpr const char* Keys[] = @@ -27,6 +27,7 @@ namespace TestImpact "google", "test", "tests", + "suites", "suite", "launch_method", "test_runner", @@ -41,6 +42,7 @@ namespace TestImpact GoogleKey, TestKey, TestsKey, + TestSuitesKey, SuiteKey, LaunchMethodKey, TestRunnerKey, @@ -64,26 +66,35 @@ namespace TestImpact for (const auto& test : tests) { TestTargetMeta testMeta; - testMeta.m_suite = test[Keys[SuiteKey]].GetString(); - testMeta.m_customArgs = test[Keys[CommandKey]].GetString(); - testMeta.m_timeout = AZStd::chrono::seconds{ test[Keys[TimeoutKey]].GetUint() }; + const auto testSuites = test[Keys[TestSuitesKey]].GetArray(); + for (const auto& suite : testSuites) + { + // Check to see if this test target has the suite we're looking for + if (const auto suiteName = suite[Keys[SuiteKey]].GetString(); + strcmp(GetSuiteTypeName(suiteType).c_str(), suiteName) == 0) + { + testMeta.m_suite = suiteName; + testMeta.m_customArgs = suite[Keys[CommandKey]].GetString(); + testMeta.m_timeout = AZStd::chrono::seconds{ suite[Keys[TimeoutKey]].GetUint() }; + if (const auto buildTypeString = test[Keys[LaunchMethodKey]].GetString(); strcmp(buildTypeString, Keys[TestRunnerKey]) == 0) + { + testMeta.m_launchMethod = LaunchMethod::TestRunner; + } + else if (strcmp(buildTypeString, Keys[StandAloneKey]) == 0) + { + testMeta.m_launchMethod = LaunchMethod::StandAlone; + } + else + { + throw(ArtifactException("Unexpected test build type")); + } - if (const auto buildTypeString = test[Keys[LaunchMethodKey]].GetString(); strcmp(buildTypeString, Keys[TestRunnerKey]) == 0) - { - testMeta.m_launchMethod = LaunchMethod::TestRunner; + AZStd::string name = test[Keys[NameKey]].GetString(); + AZ_TestImpact_Eval(!name.empty(), ArtifactException, "Test name field cannot be empty"); + testMetas.emplace(AZStd::move(name), AZStd::move(testMeta)); + break; + } } - else if (strcmp(buildTypeString, Keys[StandAloneKey]) == 0) - { - testMeta.m_launchMethod = LaunchMethod::StandAlone; - } - else - { - throw(ArtifactException("Unexpected test build type")); - } - - AZStd::string name = test[Keys[NameKey]].GetString(); - AZ_TestImpact_Eval(!name.empty(), ArtifactException, "Test name field cannot be empty"); - testMetas.emplace(AZStd::move(name), AZStd::move(testMeta)); } // If there's no tests in the repo then something is seriously wrong diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestTargetMetaMapFactory.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestTargetMetaMapFactory.h index 039a9e89e2..b08babf528 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestTargetMetaMapFactory.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestTargetMetaMapFactory.h @@ -12,14 +12,16 @@ #pragma once +#include #include #include namespace TestImpact { - //! Constructs a list of test target meta-data artifacts from the specified master test list data. + //! Constructs a list of test target meta-data artifacts of the specified suite type from the specified master test list data. //! @param masterTestListData The raw master test list data in JSON format. + //! @param suiteType The suite type to select the target meta-data artifacts from. //! @return The constructed list of test target meta-data artifacts. - TestTargetMetaMap TestTargetMetaMapFactory(const AZStd::string& masterTestListData); + TestTargetMetaMap TestTargetMetaMapFactory(const AZStd::string& masterTestListData, SuiteType suiteType); } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp index 826fa0a87d..ece5a7c13b 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp @@ -371,11 +371,11 @@ namespace TestImpact { if (sourceDependency->GetNumCoveringTestTargets()) { - AZ_Warning( - "File Update", false, AZStd::string::format("Source file %s is potentially an orphan (used by build targets " + AZ_Printf( + "File Update", AZStd::string::format("Source file '%s' is potentially an orphan (used by build targets " "without explicitly being added to the build system, e.g. an include directive pulling in a header from the " "repository). Running the covering tests for this file with instrumentation will confirm whether or nor this " - "is the case", updatedFile.c_str()).c_str()); + "is the case.\n", updatedFile.c_str()).c_str()); updateDependencies.emplace_back(AZStd::move(*sourceDependency)); coverageToDelete.push_back(updatedFile); diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp index 83808f4ed2..2331c76880 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp @@ -72,6 +72,7 @@ namespace TestImpact Runtime::Runtime( RuntimeConfig&& config, + SuiteType suiteFilter, Policy::ExecutionFailure executionFailurePolicy, Policy::ExecutionFailureDrafting executionFailureDraftingPolicy, Policy::TestFailure testFailurePolicy, @@ -80,6 +81,7 @@ namespace TestImpact Policy::TargetOutputCapture targetOutputCapture, AZStd::optional maxConcurrency) : m_config(AZStd::move(config)) + , m_suiteFilter(suiteFilter) , m_executionFailurePolicy(executionFailurePolicy) , m_executionFailureDraftingPolicy(executionFailureDraftingPolicy) , m_testFailurePolicy(testFailurePolicy) @@ -89,7 +91,7 @@ namespace TestImpact , m_maxConcurrency(maxConcurrency.value_or(AZStd::thread::hardware_concurrency())) { // Construct the dynamic dependency map from the build target descriptors - m_dynamicDependencyMap = ConstructDynamicDependencyMap(m_config.m_buildTargetDescriptor, m_config.m_testTargetMeta); + m_dynamicDependencyMap = ConstructDynamicDependencyMap(suiteFilter, m_config.m_buildTargetDescriptor, m_config.m_testTargetMeta); // Construct the test selector and prioritizer from the dependency graph data (NOTE: currently not implemented) m_testSelectorAndPrioritizer = AZStd::make_unique(m_dynamicDependencyMap.get(), DependencyGraphDataMap{}); @@ -110,7 +112,8 @@ namespace TestImpact try { // Populate the dynamic dependency map with the existing source coverage data (if any) - const auto tiaDataRaw = ReadFileContents(m_config.m_workspace.m_active.m_sparTIAFile); + m_sparTIAFile = m_config.m_workspace.m_active.m_sparTIAFiles[static_cast(m_suiteFilter)].String(); + const auto tiaDataRaw = ReadFileContents(m_sparTIAFile); const auto tiaData = DeserializeSourceCoveringTestsList(tiaDataRaw); if (tiaData.GetNumSources()) { @@ -118,13 +121,17 @@ namespace TestImpact m_hasImpactAnalysisData = true; // Enumerate new test targets - m_testEngine->UpdateEnumerationCache( - m_dynamicDependencyMap->GetNotCoveringTests(), - Policy::ExecutionFailure::Ignore, - Policy::TestFailure::Continue, - AZStd::nullopt, - AZStd::nullopt, - AZStd::nullopt); + const auto testTargetsWithNoEnumeration = m_dynamicDependencyMap->GetNotCoveringTests(); + if (!testTargetsWithNoEnumeration.empty()) + { + m_testEngine->UpdateEnumerationCache( + testTargetsWithNoEnumeration, + Policy::ExecutionFailure::Ignore, + Policy::TestFailure::Continue, + AZStd::nullopt, + AZStd::nullopt, + AZStd::nullopt); + } } } catch (const DependencyException& e) @@ -136,8 +143,10 @@ namespace TestImpact } catch ([[maybe_unused]]const Exception& e) { - AZ_Printf("No test impact analysis data found at %s", m_config.m_workspace.m_active.m_sparTIAFile.c_str()); - } + AZ_Printf("TestImpactRuntime", + AZStd::string::format( + "No test impact analysis data found for suite '%s' at %s", GetSuiteTypeName(m_suiteFilter).c_str(), m_sparTIAFile.c_str()).c_str()); + } } Runtime::~Runtime() = default; @@ -168,13 +177,16 @@ namespace TestImpact addMutatedTestTargetsToEnumerationList(changeDependencyList.GetDeleteSourceDependencies()); // Enumerate the mutated test targets to ensure their enumeration caches are up to date - m_testEngine->UpdateEnumerationCache( - testTargets, - Policy::ExecutionFailure::Ignore, - Policy::TestFailure::Continue, - AZStd::nullopt, - AZStd::nullopt, - AZStd::nullopt); + if (!testTargets.empty()) + { + m_testEngine->UpdateEnumerationCache( + testTargets, + Policy::ExecutionFailure::Ignore, + Policy::TestFailure::Continue, + AZStd::nullopt, + AZStd::nullopt, + AZStd::nullopt); + } } AZStd::pair, AZStd::vector> Runtime::SelectCoveringTestTargetsAndUpdateEnumerationCache( @@ -233,7 +245,7 @@ namespace TestImpact void Runtime::ClearDynamicDependencyMapAndRemoveExistingFile() { - DeleteFile(m_config.m_workspace.m_active.m_sparTIAFile); + DeleteFile(m_sparTIAFile); m_dynamicDependencyMap->ClearAllSourceCoverage(); } @@ -247,12 +259,11 @@ namespace TestImpact m_dynamicDependencyMap->ReplaceSourceCoverage(sourceCoverageTestsList); const auto sparTIA = m_dynamicDependencyMap->ExportSourceCoverage(); const auto sparTIAData = SerializeSourceCoveringTestsList(sparTIA); - WriteFileContents(sparTIAData, m_config.m_workspace.m_active.m_sparTIAFile); + WriteFileContents(sparTIAData, m_sparTIAFile); m_hasImpactAnalysisData = true; } TestSequenceResult Runtime::RegularTestSequence( - const AZStd::unordered_set suitesFilter, AZStd::optional testTargetTimeout, AZStd::optional globalTimeout, AZStd::optional testSequenceStartCallback, @@ -268,21 +279,7 @@ namespace TestImpact { if (!m_testTargetExcludeList.contains(&testTarget)) { - if (suitesFilter.empty()) - { - // Suite filter is empty, all tests that are not on the excluded list are included - includedTestTargets.push_back(&testTarget); - } - else if(suitesFilter.contains(testTarget.GetSuite())) - { - // Test target belonging to a suite in the suite filter are included, provided that are not on the exclude list - includedTestTargets.push_back(&testTarget); - } - else - { - // Test target not belonging to a suite in the suite filter are excluded - excludedTestTargets.push_back(&testTarget); - } + includedTestTargets.push_back(&testTarget); } else { @@ -318,6 +315,7 @@ namespace TestImpact TestSequenceResult Runtime::ImpactAnalysisTestSequence( const ChangeList& changeList, Policy::TestPrioritization testPrioritizationPolicy, + Policy::DynamicDependencyMap dynamicDependencyMapPolicy, AZStd::optional testTargetTimeout, AZStd::optional globalTimeout, AZStd::optional testSequenceStartCallback, @@ -338,30 +336,51 @@ namespace TestImpact ExtractTestTargetNames(draftedTestTargets)); } - const auto [result, testJobs] = m_testEngine->InstrumentedRun( - includedSelectedTestTargets, - m_testShardingPolicy, - m_executionFailurePolicy, - Policy::IntegrityFailure::Continue, - m_testFailurePolicy, - m_targetOutputCapture, - testTargetTimeout, - globalTimeout, - TestRunCompleteCallbackHandler(testCompleteCallback)); - - UpdateAndSerializeDynamicDependencyMap(CreateSourceCoveringTestFromTestCoverages(testJobs, m_config.m_repo.m_root)); - - if (testSequenceEndCallback.has_value()) + if (dynamicDependencyMapPolicy == Policy::DynamicDependencyMap::Update) { - (*testSequenceEndCallback)(GenerateSequenceFailureReport(testJobs), timer.Elapsed()); - } + const auto [result, testJobs] = m_testEngine->InstrumentedRun( + includedSelectedTestTargets, + m_testShardingPolicy, + m_executionFailurePolicy, + Policy::IntegrityFailure::Continue, + m_testFailurePolicy, + m_targetOutputCapture, + testTargetTimeout, + globalTimeout, + TestRunCompleteCallbackHandler(testCompleteCallback)); - return result; + UpdateAndSerializeDynamicDependencyMap(CreateSourceCoveringTestFromTestCoverages(testJobs, m_config.m_repo.m_root)); + + if (testSequenceEndCallback.has_value()) + { + (*testSequenceEndCallback)(GenerateSequenceFailureReport(testJobs), timer.Elapsed()); + } + + return result; + } + else + { + const auto [result, testJobs] = m_testEngine->RegularRun( + includedSelectedTestTargets, + m_testShardingPolicy, + m_executionFailurePolicy, + m_testFailurePolicy, + m_targetOutputCapture, + testTargetTimeout, + globalTimeout, + TestRunCompleteCallbackHandler(testCompleteCallback)); + + if (testSequenceEndCallback.has_value()) + { + (*testSequenceEndCallback)(GenerateSequenceFailureReport(testJobs), timer.Elapsed()); + } + + return result; + } } AZStd::pair Runtime::SafeImpactAnalysisTestSequence( const ChangeList& changeList, - const AZStd::unordered_set suitesFilter, Policy::TestPrioritization testPrioritizationPolicy, AZStd::optional testTargetTimeout, AZStd::optional globalTimeout, diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.cpp index 87a402bd6a..47c0e83233 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.cpp @@ -22,10 +22,10 @@ namespace TestImpact { - TestTargetMetaMap ReadTestTargetMetaMapFile(const RepoPath& testTargetMetaConfigFile) + TestTargetMetaMap ReadTestTargetMetaMapFile(SuiteType suiteFilter, const RepoPath& testTargetMetaConfigFile) { const auto masterTestListData = ReadFileContents(testTargetMetaConfigFile); - return TestTargetMetaMapFactory(masterTestListData); + return TestTargetMetaMapFactory(masterTestListData, suiteFilter); } AZStd::vector ReadBuildTargetDescriptorFiles(const BuildTargetDescriptorConfig& buildTargetDescriptorConfig) @@ -46,10 +46,11 @@ namespace TestImpact } AZStd::unique_ptr ConstructDynamicDependencyMap( + SuiteType suiteFilter, const BuildTargetDescriptorConfig& buildTargetDescriptorConfig, const TestTargetMetaConfig& testTargetMetaConfig) { - auto testTargetmetaMap = ReadTestTargetMetaMapFile(testTargetMetaConfig.m_metaFile); + auto testTargetmetaMap = ReadTestTargetMetaMapFile(suiteFilter, testTargetMetaConfig.m_metaFile); auto buildTargetDescriptors = ReadBuildTargetDescriptorFiles(buildTargetDescriptorConfig); auto buildTargets = CompileTargetDescriptors(AZStd::move(buildTargetDescriptors), AZStd::move(testTargetmetaMap)); auto&& [productionTargets, testTargets] = buildTargets; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.h index 56775b1c58..943b0cbec4 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.h @@ -30,6 +30,7 @@ namespace TestImpact { //! Construct a dynamic dependency map from the build target descriptors and test target metas. AZStd::unique_ptr ConstructDynamicDependencyMap( + SuiteType suiteFilter, const BuildTargetDescriptorConfig& buildTargetDescriptorConfig, const TestTargetMetaConfig& testTargetMetaConfig); diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_files.cmake b/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_files.cmake index da3693433e..2053a1ea5e 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_files.cmake +++ b/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_files.cmake @@ -10,15 +10,23 @@ # set(FILES - Include/TestImpactFramework/TestImpactBitwise.h - Include/TestImpactFramework/TestImpactCallback.h Include/TestImpactFramework/TestImpactException.h - Include/TestImpactFramework/TestImpactFrameworkPath.h + Include/TestImpactFramework/TestImpactRepoPath.h + Include/TestImpactFramework/TestImpactRuntime.h + Include/TestImpactFramework/TestImpactRuntimeException.h + Include/TestImpactFramework/TestImpactConfiguration.h + Include/TestImpactFramework/TestImpactConfigurationException.h + Include/TestImpactFramework/TestImpactChangelist.h + Include/TestImpactFramework/TestImpactChangelistSerializer.h + Include/TestImpactFramework/TestImpactChangelistException.h + Include/TestImpactFramework/TestImpactTestSequence.h + Include/TestImpactFramework/TestImpactClientTestSelection.h + Include/TestImpactFramework/TestImpactClientTestRun.h + Include/TestImpactFramework/TestImpactClientFailureReport.h + Include/TestImpactFramework/TestImpactFileUtils.h Source/Artifact/TestImpactArtifactException.h Source/Artifact/Factory/TestImpactBuildTargetDescriptorFactory.cpp Source/Artifact/Factory/TestImpactBuildTargetDescriptorFactory.h - Source/Artifact/Factory/TestImpactChangeListFactory.cpp - Source/Artifact/Factory/TestImpactChangeListFactory.h Source/Artifact/Factory/TestImpactTestEnumerationSuiteFactory.cpp Source/Artifact/Factory/TestImpactTestEnumerationSuiteFactory.h Source/Artifact/Factory/TestImpactTestRunSuiteFactory.cpp @@ -27,6 +35,8 @@ set(FILES Source/Artifact/Factory/TestImpactTestTargetMetaMapFactory.h Source/Artifact/Factory/TestImpactModuleCoverageFactory.cpp Source/Artifact/Factory/TestImpactModuleCoverageFactory.h + Source/Artifact/Factory/TestImpactDependencyGraphDataFactory.cpp + Source/Artifact/Factory/TestImpactDependencyGraphDataFactory.h Source/Artifact/Static/TestImpactBuildTargetDescriptor.cpp Source/Artifact/Static/TestImpactBuildTargetDescriptor.h Source/Artifact/Static/TestImpactTargetDescriptorCompiler.cpp @@ -37,7 +47,6 @@ set(FILES Source/Artifact/Static/TestImpactTestTargetDescriptor.cpp Source/Artifact/Static/TestImpactTestTargetDescriptor.h Source/Artifact/Static/TestImpactDependencyGraphData.h - Source/Artifact/Dynamic/TestImpactChangelist.h Source/Artifact/Dynamic/TestImpactTestEnumerationSuite.h Source/Artifact/Dynamic/TestImpactTestRunSuite.h Source/Artifact/Dynamic/TestImpactTestSuite.h @@ -50,6 +59,8 @@ set(FILES Source/Process/TestImpactProcessLauncher.h Source/Process/JobRunner/TestImpactProcessJob.h Source/Process/JobRunner/TestImpactProcessJobInfo.h + Source/Process/JobRunner/TestImpactProcessJobMeta.cpp + Source/Process/JobRunner/TestImpactProcessJobMeta.h Source/Process/JobRunner/TestImpactProcessJobRunner.h Source/Process/Scheduler/TestImpactProcessScheduler.cpp Source/Process/Scheduler/TestImpactProcessScheduler.h @@ -64,6 +75,8 @@ set(FILES Source/Dependency/TestImpactTestSelectorAndPrioritizer.cpp Source/Dependency/TestImpactSourceCoveringTestsList.h Source/Dependency/TestImpactSourceCoveringTestsList.cpp + Source/Dependency/TestImpactSourceCoveringTestsSerializer.cpp + Source/Dependency/TestImpactSourceCoveringTestsSerializer.h Source/Target/TestImpactBuildTarget.cpp Source/Target/TestImpactBuildTarget.h Source/Target/TestImpactBuildTargetList.h @@ -74,29 +87,48 @@ set(FILES Source/Target/TestImpactTestTarget.cpp Source/Target/TestImpactTestTarget.h Source/Target/TestImpactTestTargetList.h - Source/Test/Enumeration/TestImpactTestEnumeration.h - Source/Test/Enumeration/TestImpactTestEnumerationException.h - Source/Test/Enumeration/TestImpactTestEnumerationSerializer.cpp - Source/Test/Enumeration/TestImpactTestEnumerationSerializer.h - Source/Test/Enumeration/TestImpactTestEnumerator.cpp - Source/Test/Enumeration/TestImpactTestEnumerator.h - Source/Test/Run/TestImpactTestRunSerializer.cpp - Source/Test/Run/TestImpactTestRunSerializer.h - Source/Test/Run/TestImpactTestRunner.cpp - Source/Test/Run/TestImpactTestRunner.h - Source/Test/Run/TestImpactInstrumentedTestRunner.cpp - Source/Test/Run/TestImpactInstrumentedTestRunner.h - Source/Test/Run/TestImpactTestRun.cpp - Source/Test/Run/TestImpactTestRun.h - Source/Test/Run/TestImpactTestRunJobData.cpp - Source/Test/Run/TestImpactTestRunJobData.h - Source/Test/Run/TestImpactTestCoverage.cpp - Source/Test/Run/TestImpactTestCoverage.h - Source/Test/Run/TestImpactTestRunException.h - Source/Test/Job/TestImpactTestJobRunner.h - Source/Test/Job/TestImpactTestJobException.h - Source/Test/Job/TestImpactTestJobCommon.h - Source/Test/TestImpactTestSuiteContainer.h + Source/TestEngine/Enumeration/TestImpactTestEnumeration.h + Source/TestEngine/Enumeration/TestImpactTestEnumerationSerializer.cpp + Source/TestEngine/Enumeration/TestImpactTestEnumerationSerializer.h + Source/TestEngine/Enumeration/TestImpactTestEnumerator.cpp + Source/TestEngine/Enumeration/TestImpactTestEnumerator.h + Source/TestEngine/Run/TestImpactTestRunSerializer.cpp + Source/TestEngine/Run/TestImpactTestRunSerializer.h + Source/TestEngine/Run/TestImpactTestRunner.cpp + Source/TestEngine/Run/TestImpactTestRunner.h + Source/TestEngine/Run/TestImpactInstrumentedTestRunner.cpp + Source/TestEngine/Run/TestImpactInstrumentedTestRunner.h + Source/TestEngine/Run/TestImpactTestRun.cpp + Source/TestEngine/Run/TestImpactTestRun.h + Source/TestEngine/Run/TestImpactTestRunJobData.cpp + Source/TestEngine/Run/TestImpactTestRunJobData.h + Source/TestEngine/Run/TestImpactTestCoverage.cpp + Source/TestEngine/Run/TestImpactTestCoverage.h + Source/TestEngine/JobRunner/TestImpactTestJobRunner.h + Source/TestEngine/JobRunner/TestImpactTestJobInfoGenerator.cpp + Source/TestEngine/JobRunner/TestImpactTestJobInfoGenerator.h + Source/TestEngine/JobRunner/TestImpactTestTargetExtension.h + Source/TestEngine/TestImpactTestEngineJobFailure.cpp + Source/TestEngine/TestImpactTestEngineJobFailure.h + Source/TestEngine/TestImpactTestSuiteContainer.h + Source/TestEngine/TestImpactTestEngine.cpp + Source/TestEngine/TestImpactTestEngine.h + Source/TestEngine/TestImpactTestEngineJob.cpp + Source/TestEngine/TestImpactTestEngineJob.h + Source/TestEngine/TestImpactTestEngineEnumeration.cpp + Source/TestEngine/TestImpactTestEngineEnumeration.h + Source/TestEngine/TestImpactTestEngineRegularRun.cpp + Source/TestEngine/TestImpactTestEngineRegularRun.h + Source/TestEngine/TestImpactTestEngineInstrumentedRun.cpp + Source/TestEngine/TestImpactTestEngineInstrumentedRun.h + Source/TestEngine/TestImpactTestEngineException.h Source/TestImpactException.cpp - Source/TestImpactFrameworkPath.cpp + Source/TestImpactRuntime.cpp + Source/TestImpactRuntimeUtils.cpp + Source/TestImpactRuntimeUtils.h + Source/TestImpactClientTestSelection.cpp + Source/TestImpactClientTestRun.cpp + Source/TestImpactClientFailureReport.cpp + Source/TestImpactChangeListSerializer.cpp + Source/TestImpactRepoPath.cpp ) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_tests_files.cmake b/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_tests_files.cmake deleted file mode 100644 index 13f788c37b..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_tests_files.cmake +++ /dev/null @@ -1,38 +0,0 @@ -# -# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or -# its licensors. -# -# For complete copyright and license terms please see the LICENSE at the root of this -# distribution (the "License"). All use of this software is governed by the License, -# or, if provided, by the license below or the license accompanying this file. Do not -# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# - -set(FILES - Tests/Artifact/TestImpactTargetDescriptorCompilerTest.cpp - Tests/Artifact/TestImpactBuildTargetDescriptorFactoryTest.cpp - Tests/Artifact/TestImpactModuleCoverageFactoryTest.cpp - Tests/Artifact/TestImpactChangeListFactoryTest.cpp - Tests/Artifact/TestImpactTestEnumerationSuiteFactoryTest.cpp - Tests/Artifact/TestImpactTestRunSuiteFactoryTest.cpp - Tests/Artifact/TestImpactTestTargetMetaMapFactoryTest.cpp - - Tests/Process/TestImpactProcessSchedulerTest.cpp - - Tests/Process/TestImpactProcessTest.cpp - Tests/Target/TestImpactBuildTargetTest.cpp - Tests/TestImpactExceptionTest.cpp - Tests/TestImpactFrameworkPathTest.cpp - Tests/Test/TestImpactTestEnumeratorTest.cpp - Tests/Test/TestImpactTestEumerationSerializerTest.cpp - Tests/Test/TestImpactTestRunSerializerTest.cpp - Tests/Test/TestImpactTestRunnerTest.cpp - Tests/Test/TestImpactInstrumentedTestRunnerTest.cpp - Tests/Test/TestImpactTestCoverageTest.cpp - Tests/TestImpactTestJobRunnerCommon.h - Tests/TestImpactTestMain.cpp - Tests/TestImpactTestUtils.cpp - Tests/TestImpactTestUtils.h - -) diff --git a/cmake/LYTestWrappers.cmake b/cmake/LYTestWrappers.cmake index 5b7e1c766b..3308c4411a 100644 --- a/cmake/LYTestWrappers.cmake +++ b/cmake/LYTestWrappers.cmake @@ -241,12 +241,18 @@ function(ly_add_test) endif() - # Store the test so we can walk through all of them in LYTestImpactFramework.cmake - set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS ${ly_add_test_NAME}) - set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_test_NAME}_TEST_SUITE ${ly_add_test_TEST_SUITE}) - set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_test_NAME}_TEST_LIBRARY ${ly_add_test_TEST_LIBRARY}) - set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_test_NAME}_TEST_TIMEOUT ${ly_add_test_TIMEOUT}) - + # Check to see whether or not this test target has been stored in the global list for walking by the test impact analysis framework + get_property(all_tests GLOBAL PROPERTY LY_ALL_TESTS) + if(NOT "${LY_ALL_TESTS_TARGET_NAME}" IN_LIST all_tests) + # This is the first reference to this test target so add it to the global list + set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS ${LY_ALL_TESTS_TARGET_NAME}) + set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${LY_ALL_TESTS_TARGET_NAME}_TEST_LIBRARY ${ly_add_test_TEST_LIBRARY}) + endif() + # Add the test suite and timeout value to the test target params + set(LY_TEST_PARAMS "${LY_TEST_PARAMS}#${ly_add_test_TEST_SUITE}") + set(LY_TEST_PARAMS "${LY_TEST_PARAMS}#${ly_add_test_TIMEOUT}") + # Store the params for this test target + set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${LY_ALL_TESTS_TARGET_NAME}_PARAMS ${LY_TEST_PARAMS}) endfunction() #! ly_add_pytest: registers target PyTest-based test with CTest @@ -288,6 +294,11 @@ function(ly_add_pytest) string(REPLACE "::" "_" pytest_report_directory "${PYTEST_XML_OUTPUT_DIR}/${ly_add_pytest_NAME}.xml") + # Set the name of the current test target for storage in the global list + set(LY_ALL_TESTS_TARGET_NAME ${ly_add_pytest_NAME}) + # Add the script path to the test target params + set(LY_TEST_PARAMS "${ly_add_pytest_PATH}") + ly_add_test( NAME ${ly_add_pytest_NAME} TEST_SUITE ${ly_add_pytest_TEST_SUITE} @@ -298,7 +309,6 @@ function(ly_add_pytest) ${ly_add_pytest_UNPARSED_ARGUMENTS} ) - set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_pytest_NAME}_SCRIPT_PATH ${ly_add_pytest_PATH}) set_tests_properties(${LY_ADDED_TEST_NAME} PROPERTIES RUN_SERIAL "${ly_add_pytest_TEST_SERIAL}") endfunction() @@ -341,6 +351,10 @@ function(ly_add_editor_python_test) file(REAL_PATH ${ly_add_editor_python_test_TEST_PROJECT} project_real_path BASE_DIRECTORY ${LY_ROOT_FOLDER}) + # Set the name of the current test target for storage in the global list + set(LY_ALL_TESTS_TARGET_NAME ${ly_add_editor_python_test_NAME}) + # Add the script path to the test target params + set(LY_TEST_PARAMS "${ly_add_editor_python_test_PATH}") # Run test via the run_epbtest.cmake script. # Parameters used are explained in run_epbtest.cmake. ly_add_test( @@ -362,8 +376,6 @@ function(ly_add_editor_python_test) TIMEOUT ${ly_add_editor_python_test_TIMEOUT} COMPONENT ${ly_add_editor_python_test_COMPONENT} ) - - set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_editor_python_test_NAME}_SCRIPT_PATH ${ly_add_editor_python_test_PATH}) set_tests_properties(${LY_ADDED_TEST_NAME} PROPERTIES RUN_SERIAL "${ly_add_editor_python_test_TEST_SERIAL}") endfunction() @@ -431,18 +443,23 @@ function(ly_add_googletest) set(full_test_command $ $ AzRunUnitTests) # Add AzTestRunner as a build dependency ly_add_dependencies(${build_target} AZ::AzTestRunner) + # Start the test target params and dd the command runner command # Ideally, we would populate the full command procedurally but the generator expressions won't be expanded by the time we need this data - set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_googletest_NAME}_TEST_COMMAND "AzRunUnitTests") + set(LY_TEST_PARAMS "AzRunUnitTests") else() set(full_test_command ${ly_add_googletest_TEST_COMMAND}) # Remove the generator expressions so we are left with the argument(s) required to run unit tests for executable targets string(REPLACE ";" "" stripped_test_command ${full_test_command}) string(GENEX_STRIP ${stripped_test_command} stripped_test_command) - set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_googletest_NAME}_TEST_COMMAND ${stripped_test_command}) + # Start the test target params and dd the command runner command + set(LY_TEST_PARAMS "${stripped_test_command}") endif() string(REPLACE "::" "_" report_directory "${GTEST_XML_OUTPUT_DIR}/${ly_add_googletest_NAME}.xml") + # Set the name of the current test target for storage in the global list + set(LY_ALL_TESTS_TARGET_NAME ${target_name}) + # Invoke the lower level ly_add_test command to add the actual ctest and setup the test labels to add_dependencies on the target ly_add_test( NAME ${ly_add_googletest_NAME} @@ -520,16 +537,21 @@ function(ly_add_googlebenchmark) # If command is not supplied attempts, uses the AzTestRunner to run googlebenchmarks on the supplied TARGET set(full_test_command $ $ AzRunBenchmarks ${output_format_args}) + # Start the test target params and dd the command runner command # Ideally, we would populate the full command procedurally but the generator expressions won't be expanded by the time we need this data - set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_googlebenchmark_NAME}_TEST_COMMAND "AzRunUnitTests") + set(LY_TEST_PARAMS "AzRunUnitTests") else() set(full_test_command ${ly_add_googlebenchmark_TEST_COMMAND}) # Remove the generator expressions so we are left with the argument(s) required to run unit tests for executable targets string(REPLACE ";" "" stripped_test_command ${full_test_command}) string(GENEX_STRIP ${stripped_test_command} stripped_test_command) - set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_googletest_NAME}_TEST_COMMAND ${stripped_test_command}) + # Start the test target params and dd the command runner command + set(LY_TEST_PARAMS "${stripped_test_command}") endif() - + + # Set the name of the current test target for storage in the global list + set(LY_ALL_TESTS_TARGET_NAME ${ly_add_googlebenchmark_NAME}) + # Set the name of the current test target for storage in the global list ly_add_test( NAME ${ly_add_googlebenchmark_NAME} TEST_REQUIRES ${ly_add_googlebenchmark_TEST_REQUIRES} diff --git a/cmake/TestImpactFramework/ConsoleFrontendConfig.in b/cmake/TestImpactFramework/ConsoleFrontendConfig.in index f6af3d801d..357499771c 100644 --- a/cmake/TestImpactFramework/ConsoleFrontendConfig.in +++ b/cmake/TestImpactFramework/ConsoleFrontendConfig.in @@ -4,7 +4,8 @@ "timestamp": "${timestamp}" }, "repo": { - "root": "${repo_dir}" + "root": "${repo_dir}", + "tiaf_bin": "${tiaf_bin}" }, "workspace": { "temp": { @@ -13,11 +14,23 @@ "artifact_dir": "RuntimeArtifact" } }, - "persistent": { - "root": "${persistent_dir}", + "active": { + "root": "${active_dir}", "relative_paths": { - "test_impact_data_file": "TestImpactData.spartia", - "enumeration_cache_dir": "EnumerationCache" + "test_impact_data_files": { + "main": "TestImpactData.main.spartia", + "periodic": "TestImpactData.periodic.spartia", + "sandbox": "TestImpactData.sandbox.spartia" + }, + "enumeration_cache_dir": "EnumerationCache", + "last_build_target_list_file": "LastRunBuildTargets.json" + } + }, + "historic": { + "root": "${historic_dir}", + "relative_paths": { + "last_run_hash_file": "last_run.hash", + "last_build_target_list_file": "LastRunBuildTargets.json" } } }, diff --git a/cmake/TestImpactFramework/LYTestImpactFramework.cmake b/cmake/TestImpactFramework/LYTestImpactFramework.cmake index 3081a4c89c..54dadbd8ad 100644 --- a/cmake/TestImpactFramework/LYTestImpactFramework.cmake +++ b/cmake/TestImpactFramework/LYTestImpactFramework.cmake @@ -89,7 +89,7 @@ function(ly_test_impact_get_test_launch_method TARGET_NAME LAUNCH_METHOD) elseif("${target_type}" STREQUAL "EXECUTABLE") set(${LAUNCH_METHOD} "stand_alone" PARENT_SCOPE) else() - message(FATAL_ERROR "Cannot deduce test target launch method for the target type ${target_type}") + message(FATAL_ERROR "Cannot deduce test target launch method for the target ${TARGET_NAME} with type ${target_type}") endif() endfunction() @@ -131,33 +131,50 @@ function(ly_test_impact_extract_python_test COMPOSITE_TEST TEST_NAME) set(${TEST_NAME} ${test_name} PARENT_SCOPE) endfunction() -#! ly_test_impact_extract_google_test_params: extracts the google test name and command parameters. +#! ly_test_impact_extract_google_test_params: extracts the suites for the given google test. # # \arg:COMPOSITE_TEST test in the form 'namespace::test' +# \arg:COMPOSITE_SUITES composite list of suites for this target # \arg:TEST_NAME name of test -# \arg:TEST_COMMAND optional command arguments to run the test -function(ly_test_impact_extract_google_test_params COMPOSITE_TEST TEST_NAME TEST_COMMAND) - get_property(test_command GLOBAL PROPERTY LY_ALL_TESTS_${COMPOSITE_TEST}_TEST_COMMAND) +# \arg:TEST_SUITES extracted list of suites for this target in JSON format +function(ly_test_impact_extract_google_test_params COMPOSITE_TEST COMPOSITE_SUITES TEST_NAME TEST_SUITES) # Namespace and test are mandatory string(REPLACE "::" ";" test_components ${COMPOSITE_TEST}) list(LENGTH test_components num_test_components) if(num_test_components LESS 2) message(FATAL_ERROR "The test ${test_components} appears to have been specified without a namespace, i.e.:\ly_add_googletest/benchmark(NAME ${test_components})\nInstead of (perhaps):\ly_add_googletest/benchmark(NAME Gem::${test_components})\nPlease add the missing namespace before proceeding.") endif() - + list(GET test_components 0 test_namespace) list(GET test_components 1 test_name) set(${TEST_NAMESPACE} ${test_namespace} PARENT_SCOPE) set(${TEST_NAME} ${test_name} PARENT_SCOPE) - set(${TEST_COMMAND} ${test_command} PARENT_SCOPE) + + set(test_suites "") + foreach(composite_suite ${COMPOSITE_SUITES}) + # Command, suite, timeout + string(REPLACE "#" ";" suite_components ${composite_suite}) + list(LENGTH suite_components num_suite_components) + if(num_suite_components LESS 3) + message(FATAL_ERROR "The suite components ${composite_suite} are required to be in the following format: command#suite#string.") + endif() + list(GET suite_components 0 test_command) + list(GET suite_components 1 test_suite) + list(GET suite_components 2 test_timeout) + set(suite_params "{ \"suite\": \"${test_suite}\", \"command\": \"${test_command}\", \"timeout\": ${test_timeout} }") + list(APPEND test_suites "${suite_params}") + endforeach() + string(REPLACE ";" ", " test_suites "${test_suites}") + set(${TEST_SUITES} ${test_suites} PARENT_SCOPE) endfunction() #! ly_test_impact_extract_python_test_params: extracts the python test name and relative script path parameters. # # \arg:COMPOSITE_TEST test in form 'namespace::test' or 'test' +# \arg:COMPOSITE_SUITES composite list of suites for this target # \arg:TEST_NAME name of test -# \arg:SCRIPT_PATH name of test -function(ly_test_impact_extract_python_test_params COMPOSITE_TEST TEST_NAME SCRIPT_PATH) +# \arg:TEST_SUITES extracted list of suites for this target in JSON format +function(ly_test_impact_extract_python_test_params COMPOSITE_TEST COMPOSITE_SUITES TEST_NAME TEST_SUITES) get_property(script_path GLOBAL PROPERTY LY_ALL_TESTS_${COMPOSITE_TEST}_SCRIPT_PATH) # namespace is optional, in which case this component will be simply the test name @@ -169,15 +186,30 @@ function(ly_test_impact_extract_python_test_params COMPOSITE_TEST TEST_NAME SCRI set(test_name ${test_components}) endif() - # Get python script path relative to repo root - ly_test_impact_rebase_file_to_repo_root( - ${script_path} - script_path - ${LY_ROOT_FOLDER} - ) - set(${TEST_NAME} ${test_name} PARENT_SCOPE) - set(${SCRIPT_PATH} ${script_path} PARENT_SCOPE) + + set(test_suites "") + foreach(composite_suite ${COMPOSITE_SUITES}) + # Script path, suite, timeout + string(REPLACE "#" ";" suite_components ${composite_suite}) + list(LENGTH suite_components num_suite_components) + if(num_suite_components LESS 3) + message(FATAL_ERROR "The suite components ${composite_suite} are required to be in the following format: script_path#suite#string.") + endif() + list(GET suite_components 0 script_path) + list(GET suite_components 1 test_suite) + list(GET suite_components 2 test_timeout) + # Get python script path relative to repo root + ly_test_impact_rebase_file_to_repo_root( + ${script_path} + script_path + ${LY_ROOT_FOLDER} + ) + set(suite_params "{ \"suite\": \"${test_suite}\", \"script\": \"${script_path}\", \"timeout\": ${test_timeout} }") + list(APPEND test_suites "${suite_params}") + endforeach() + string(REPLACE ";" ", " test_suites "${test_suites}") + set(${TEST_SUITES} ${test_suites} PARENT_SCOPE) endfunction() #! ly_test_impact_write_test_enumeration_file: exports the master test lists to file. @@ -185,7 +217,6 @@ endfunction() # \arg:TEST_ENUMERATION_TEMPLATE_FILE path to test enumeration template file function(ly_test_impact_write_test_enumeration_file TEST_ENUMERATION_TEMPLATE_FILE) get_property(LY_ALL_TESTS GLOBAL PROPERTY LY_ALL_TESTS) - # Enumerated tests for each type set(google_tests "") set(google_benchmarks "") @@ -196,29 +227,28 @@ function(ly_test_impact_write_test_enumeration_file TEST_ENUMERATION_TEMPLATE_FI # Walk the test list foreach(test ${LY_ALL_TESTS}) message(TRACE "Parsing ${test}") + get_property(test_params GLOBAL PROPERTY LY_ALL_TESTS_${test}_PARAMS) get_property(test_type GLOBAL PROPERTY LY_ALL_TESTS_${test}_TEST_LIBRARY) - get_property(test_suite GLOBAL PROPERTY LY_ALL_TESTS_${test}_TEST_SUITE) - get_property(test_timeout GLOBAL PROPERTY LY_ALL_TESTS_${test}_TEST_TIMEOUT) if("${test_type}" STREQUAL "pytest") # Python tests - ly_test_impact_extract_python_test_params(${test} test_name script_path) - list(APPEND python_tests " { \"name\": \"${test_name}\", \"suite\": \"${test_suite}\", \"script\": \"${script_path}\", \"timeout\":${test_timeout} }") + ly_test_impact_extract_python_test_params(${test} "${test_params}" test_name test_suites) + list(APPEND python_tests " { \"name\": \"${test_name}\", \"suites\": [${test_suites}] }") elseif("${test_type}" STREQUAL "pytest_editor") - # Python editor tests - ly_test_impact_extract_python_test_params(${test} test_name script_path) - list(APPEND python_editor_tests " { \"name\": \"${test_name}\", \"suite\": \"${test_suite}\", \"script\": \"${script_path}\", \"timeout\":${test_timeout} }") + # Python editor tests + ly_test_impact_extract_python_test_params(${test} "${test_params}" test_name test_suites) + list(APPEND python_editor_tests " { \"name\": \"${test_name}\", \"suites\": [${test_suites}] }") elseif("${test_type}" STREQUAL "googletest") # Google tests - ly_test_impact_extract_google_test_params(${test} test_name test_command) - ly_test_impact_get_test_launch_method(${test_name} launch_method) - list(APPEND google_tests " { \"name\": \"${test_name}\", \"suite\": \"${test_suite}\", \"command\": \"${test_command}\", \"timeout\":${test_timeout}, \"launch_method\": \"${launch_method}\" }") + ly_test_impact_extract_google_test_params(${test} "${test_params}" test_name test_suites) + ly_test_impact_get_test_launch_method(${test} launch_method) + list(APPEND google_tests " { \"name\": \"${test_name}\", \"launch_method\": \"${launch_method}\", \"suites\": [${test_suites}] }") elseif("${test_type}" STREQUAL "googlebenchmark") # Google benchmarks - ly_test_impact_extract_google_test_params(${test} test_name test_command) - list(APPEND google_benchmarks " { \"name\": \"${test_name}\", \"suite\": \"${test_suite}\", \"command\": \"${test_command}\", \"timeout\":${test_timeout} }") + ly_test_impact_extract_google_test_params(${test} "${test_params}" test_name test_suites) + list(APPEND google_benchmarks " { \"name\": \"${test_name}\", \"launch_method\": \"${launch_method}\", \"suites\": [${test_suites}] }") else() message("${test_name} is of unknown type (TEST_LIBRARY property is empty)") - list(APPEND unknown_tests " { \"name\": \"${test}\" }") + list(APPEND unknown_tests " { \"name\": \"${test}\", \"type\": \"${test_type}\" }") endif() endforeach() @@ -330,8 +360,11 @@ function(ly_test_impact_write_config_file CONFIG_TEMPLATE_FILE PERSISTENT_DATA_D # Temp dir set(temp_dir "${LY_TEST_IMPACT_TEMP_DIR}") - # Persistent dir - set(persistent_dir "${PERSISTENT_DATA_DIR}") + # Active persistent data dir + set(active_dir "${PERSISTENT_DATA_DIR}/active") + + # Historic persistent data dir + set(historic_dir "${PERSISTENT_DATA_DIR}/historic") # Source to target mappings dir set(source_target_mapping_dir "${LY_TEST_IMPACT_SOURCE_TARGET_MAPPING_DIR}") @@ -341,6 +374,9 @@ function(ly_test_impact_write_config_file CONFIG_TEMPLATE_FILE PERSISTENT_DATA_D # Build dependency artifact dir set(target_dependency_dir "${LY_TEST_IMPACT_TARGET_DEPENDENCY_DIR}") + + # Test impact analysis framework binary + set(tiaf_bin "$") # Substitute config file template with above vars file(READ "${CONFIG_TEMPLATE_FILE}" config_file) @@ -348,9 +384,13 @@ function(ly_test_impact_write_config_file CONFIG_TEMPLATE_FILE PERSISTENT_DATA_D # Write out entire config contents to a file in the build directory of the test impact framework console target file(GENERATE - OUTPUT "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/$/$.$.json" + OUTPUT "${PERSISTENT_DATA_DIR}/$.$.json" CONTENT ${config_file} ) + + # Set the above config file as the default config file to use for the test impact framework console target + target_compile_definitions(${LY_TEST_IMPACT_CONSOLE_STATIC_TARGET} PUBLIC "LY_TEST_IMPACT_DEFAULT_CONFIG_FILE=\"${PERSISTENT_DATA_DIR}/$.$.json\"") + message(DEBUG "Test impact framework post steps complete") endfunction() #! ly_test_impact_post_step: runs the post steps to be executed after all other cmake scripts have been executed. @@ -360,8 +400,7 @@ function(ly_test_impact_post_step) endif() # Directory per build config for persistent test impact data (to be checked in) - set(persistent_data_dir "${LY_ROOT_FOLDER}/Tests/test_impact_framework/${CMAKE_SYSTEM_NAME}/$") - + set(persistent_data_dir "${LY_TEST_IMPACT_WORKING_DIR}/persistent") # Directory for binaries built for this profile set(bin_dir "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/$") @@ -388,8 +427,4 @@ function(ly_test_impact_post_step) # Copy over the graphviz options file for the build dependency graphs message(DEBUG "Test impact framework config file written") file(COPY "cmake/TestImpactFramework/CMakeGraphVizOptions.cmake" DESTINATION ${CMAKE_BINARY_DIR}) - - # Set the above config file as the default config file to use for the test impact framework console target - target_compile_definitions(${LY_TEST_IMPACT_CONSOLE_STATIC_TARGET} PUBLIC "LY_TEST_IMPACT_DEFAULT_CONFIG_FILE=\"${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/$/$.$.json\"") - message(DEBUG "Test impact framework post steps complete") endfunction() From b00f56909027667fe99fc6e713175f91f401c128 Mon Sep 17 00:00:00 2001 From: jonawals Date: Sun, 6 Jun 2021 19:09:24 +0100 Subject: [PATCH 065/102] Address PR comments --- .../Code/Source/TestImpactConsoleMain.cpp | 2 +- cmake/LYTestWrappers.cmake | 32 ++++++++++--------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleMain.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleMain.cpp index fab6509244..4853c6da15 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleMain.cpp +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleMain.cpp @@ -121,7 +121,7 @@ namespace TestImpact } else { - // One sequence timed out whilst the other sequence succeeded or both sequences timed out + // One or both sequences timed out or failed result = TestSequenceResult::Timeout; } } diff --git a/cmake/LYTestWrappers.cmake b/cmake/LYTestWrappers.cmake index 3308c4411a..eebc281d06 100644 --- a/cmake/LYTestWrappers.cmake +++ b/cmake/LYTestWrappers.cmake @@ -71,6 +71,7 @@ endfunction() #! ly_add_test: Adds a new RUN_TEST using for the specified target using the supplied command # # \arg:NAME - Name to for the test run target +# \arg:PARENT_NAME(optional) - Name of the parent test run target (if this is a subsequent call to specify a suite) # \arg:TEST_REQUIRES(optional) - List of system resources that are required to run this test. # Only available option is "gpu" # \arg:TEST_SUITE(optional) - "smoke" or "periodic" or "sandbox" - prevents the test from running normally @@ -94,7 +95,7 @@ endfunction() # sets LY_ADDED_TEST_NAME to the fully qualified name of the test, in parent scope function(ly_add_test) set(options EXCLUDE_TEST_RUN_TARGET_FROM_IDE) - set(one_value_args NAME TEST_LIBRARY TEST_SUITE TIMEOUT) + set(one_value_args NAME PARENT_NAME TEST_LIBRARY TEST_SUITE TIMEOUT) set(multi_value_args TEST_REQUIRES TEST_COMMAND NON_IDE_PARAMS RUNTIME_DEPENDENCIES COMPONENT LABELS) # note that we dont use TEST_LIBRARY here, but PAL files might so do not remove! @@ -241,18 +242,24 @@ function(ly_add_test) endif() + if(NOT ly_add_test_PARENT_NAME) + set(test_target ${ly_add_test_NAME}) + else() + set(test_target ${ly_add_test_PARENT_NAME}) + endif() + # Check to see whether or not this test target has been stored in the global list for walking by the test impact analysis framework get_property(all_tests GLOBAL PROPERTY LY_ALL_TESTS) - if(NOT "${LY_ALL_TESTS_TARGET_NAME}" IN_LIST all_tests) + if(NOT "${test_target}" IN_LIST all_tests) # This is the first reference to this test target so add it to the global list - set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS ${LY_ALL_TESTS_TARGET_NAME}) - set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${LY_ALL_TESTS_TARGET_NAME}_TEST_LIBRARY ${ly_add_test_TEST_LIBRARY}) + set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS ${test_target}) + set_property(GLOBAL PROPERTY LY_ALL_TESTS_${test_target}_TEST_LIBRARY ${ly_add_test_TEST_LIBRARY}) endif() # Add the test suite and timeout value to the test target params set(LY_TEST_PARAMS "${LY_TEST_PARAMS}#${ly_add_test_TEST_SUITE}") set(LY_TEST_PARAMS "${LY_TEST_PARAMS}#${ly_add_test_TIMEOUT}") # Store the params for this test target - set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${LY_ALL_TESTS_TARGET_NAME}_PARAMS ${LY_TEST_PARAMS}) + set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${test_target}_PARAMS ${LY_TEST_PARAMS}) endfunction() #! ly_add_pytest: registers target PyTest-based test with CTest @@ -294,13 +301,12 @@ function(ly_add_pytest) string(REPLACE "::" "_" pytest_report_directory "${PYTEST_XML_OUTPUT_DIR}/${ly_add_pytest_NAME}.xml") - # Set the name of the current test target for storage in the global list - set(LY_ALL_TESTS_TARGET_NAME ${ly_add_pytest_NAME}) # Add the script path to the test target params set(LY_TEST_PARAMS "${ly_add_pytest_PATH}") ly_add_test( NAME ${ly_add_pytest_NAME} + PARENT_NAME ${ly_add_pytest_NAME} TEST_SUITE ${ly_add_pytest_TEST_SUITE} LABELS FRAMEWORK_pytest TEST_COMMAND ${LY_PYTEST_EXECUTABLE} ${ly_add_pytest_PATH} ${ly_add_pytest_EXTRA_ARGS} --junitxml=${pytest_report_directory} ${custom_marks_args} @@ -351,14 +357,13 @@ function(ly_add_editor_python_test) file(REAL_PATH ${ly_add_editor_python_test_TEST_PROJECT} project_real_path BASE_DIRECTORY ${LY_ROOT_FOLDER}) - # Set the name of the current test target for storage in the global list - set(LY_ALL_TESTS_TARGET_NAME ${ly_add_editor_python_test_NAME}) # Add the script path to the test target params set(LY_TEST_PARAMS "${ly_add_editor_python_test_PATH}") # Run test via the run_epbtest.cmake script. # Parameters used are explained in run_epbtest.cmake. ly_add_test( NAME ${ly_add_editor_python_test_NAME} + PARENT_NAME ${ly_add_editor_python_test_NAME} TEST_REQUIRES ${ly_add_editor_python_test_TEST_REQUIRES} TEST_COMMAND ${CMAKE_COMMAND} -DCMD_ARG_TEST_PROJECT=${project_real_path} @@ -457,12 +462,10 @@ function(ly_add_googletest) string(REPLACE "::" "_" report_directory "${GTEST_XML_OUTPUT_DIR}/${ly_add_googletest_NAME}.xml") - # Set the name of the current test target for storage in the global list - set(LY_ALL_TESTS_TARGET_NAME ${target_name}) - # Invoke the lower level ly_add_test command to add the actual ctest and setup the test labels to add_dependencies on the target ly_add_test( NAME ${ly_add_googletest_NAME} + PARENT_NAME ${target_name} TEST_SUITE ${ly_add_googletest_TEST_SUITE} LABELS FRAMEWORK_googletest TEST_COMMAND ${full_test_command} --gtest_output=xml:${report_directory} ${LY_GOOGLETEST_EXTRA_PARAMS} @@ -548,12 +551,11 @@ function(ly_add_googlebenchmark) # Start the test target params and dd the command runner command set(LY_TEST_PARAMS "${stripped_test_command}") endif() - - # Set the name of the current test target for storage in the global list - set(LY_ALL_TESTS_TARGET_NAME ${ly_add_googlebenchmark_NAME}) + # Set the name of the current test target for storage in the global list ly_add_test( NAME ${ly_add_googlebenchmark_NAME} + PARENT_NAME ${ly_add_googlebenchmark_NAME} TEST_REQUIRES ${ly_add_googlebenchmark_TEST_REQUIRES} TEST_COMMAND ${full_test_command} ${LY_GOOGLETEST_EXTRA_PARAMS} TEST_SUITE "benchmark" From 4ed0c7b1d8ccaf70bec71674763dc0dd65ad5f9f Mon Sep 17 00:00:00 2001 From: jonawals Date: Mon, 7 Jun 2021 08:52:20 +0100 Subject: [PATCH 066/102] Add drafting of failed tests --- .../Source/TestImpactCommandLineOptions.cpp | 50 ++++++++++-------- .../Source/TestImpactCommandLineOptions.h | 6 +-- .../Code/Source/TestImpactConsoleMain.cpp | 2 +- .../TestImpactFramework/TestImpactRuntime.h | 7 +-- .../TestImpactTestSequence.h | 8 +-- .../TestImpactDynamicDependencyMap.cpp | 48 +++++++++++------ .../TestImpactDynamicDependencyMap.h | 9 ++-- .../Runtime/Code/Source/TestImpactRuntime.cpp | 51 ++++++++++++++----- 8 files changed, 119 insertions(+), 62 deletions(-) diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp index 5ff070601a..aad9319cf9 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp @@ -28,7 +28,7 @@ namespace TestImpact Sequence, TestPrioritizationPolicy, ExecutionFailurePolicy, - ExecutionFailureDraftingPolicy, + FailedTestCoveragePolicy, TestFailurePolicy, IntegrityFailurePolicy, TestShardingPolicy, @@ -50,7 +50,9 @@ namespace TestImpact Continue, Ignore, StdOut, - File + File, + Remove, + Keep }; constexpr const char* OptionKeys[] = @@ -62,7 +64,7 @@ namespace TestImpact "sequence", "ppolicy", "epolicy", - "rexecfailures", + "cpolicy", "fpolicy", "ipolicy", "shard", @@ -84,7 +86,9 @@ namespace TestImpact "continue", "ignore", "stdout", - "file" + "file", + "remove", + "keep" }; RepoPath ParseConfigurationFile(const AZ::CommandLine& cmd) @@ -139,15 +143,15 @@ namespace TestImpact return ParseMultiStateOption(OptionKeys[ExecutionFailurePolicy], states, cmd).value_or(Policy::ExecutionFailure::Continue); } - Policy::ExecutionFailureDrafting ParseExecutionFailureDraftingPolicy(const AZ::CommandLine& cmd) + Policy::FailedTestCoverage ParseFailedTestCoveragePolicy(const AZ::CommandLine& cmd) { - const BinaryStateValue states = + const AZStd::vector> states = { - Policy::ExecutionFailureDrafting::Never, - Policy::ExecutionFailureDrafting::Always + {OptionKeys[Remove], Policy::FailedTestCoverage::Remove}, + {OptionKeys[Keep], Policy::FailedTestCoverage::Keep} }; - return ParseOnOffOption(OptionKeys[ExecutionFailureDraftingPolicy], states, cmd).value_or(Policy::ExecutionFailureDrafting::Always); + return ParseMultiStateOption(OptionKeys[FailedTestCoveragePolicy], states, cmd).value_or(Policy::FailedTestCoverage::Keep); } Policy::TestFailure ParseTestFailurePolicy(const AZ::CommandLine& cmd) @@ -275,7 +279,7 @@ namespace TestImpact m_testSequenceType = ParseTestSequenceType(cmd); m_testPrioritizationPolicy = ParseTestPrioritizationPolicy(cmd); m_executionFailurePolicy = ParseExecutionFailurePolicy(cmd); - m_executionFailureDraftingPolicy = ParseExecutionFailureDraftingPolicy(cmd); + m_failedTestCoveragePolicy = ParseFailedTestCoveragePolicy(cmd); m_testFailurePolicy = ParseTestFailurePolicy(cmd); m_integrityFailurePolicy = ParseIntegrityFailurePolicy(cmd); m_testShardingPolicy = ParseTestShardingPolicy(cmd); @@ -327,9 +331,9 @@ namespace TestImpact return m_executionFailurePolicy; } - Policy::ExecutionFailureDrafting CommandLineOptions::GetExecutionFailureDraftingPolicy() const + Policy::FailedTestCoverage CommandLineOptions::GetFailedTestCoveragePolicy() const { - return m_executionFailureDraftingPolicy; + return m_failedTestCoveragePolicy; } Policy::TestFailure CommandLineOptions::GetTestFailurePolicy() const @@ -401,7 +405,11 @@ namespace TestImpact " tests are run regardless).\n" " -shard= Break any test targets with a sharding policy into the number of \n" " shards according to the maximum concurrency value.\n" - " -rexecfailures= Attempt to execute test targets that previously failed to execute.\n" + " -cpolicy= Policy for handling the coverage data of failed tests (both test that \n" + " failed to execute and tests that ran but failed), where remove will \n" + " remove the failed tests from the all coverage data(causing them to be \n" + " drafted into future test runs) and keep will keep any existing coverage \n" + " data and update the coverage data for failed tests that produce coverage.\n" " -targetout= Capture of individual test run stdout, where stdout will capture \n" " each individual test target's stdout and output each one to stdout \n" " and file will capture each individual test target's stdout and output \n" @@ -409,25 +417,25 @@ namespace TestImpact " -epolicy= Policy for handling test execution failure (test targets could not be \n" " launched due to the binary not being built, incorrect paths, etc.), \n" " where abort will abort the entire test sequence upon the first test\n" - " target execution failureand report a failure(along with the return \n" + " target execution failure and report a failure(along with the return \n" " code of the test target that failed to launch), continue will continue \n" " with the test sequence in the event of test target execution failures\n" " and treat the test targets that failed to launch as as test failures\n" " (along with the return codes of the test targets that failed to \n" " launch), ignore will continue with the test sequence in the event of \n" - " test target execution failuresand treat the test targets that failed\n" - " to launch as as test passes(along with the return codes of the test \n" + " test target execution failures and treat the test targets that failed\n" + " to launch as test passes(along with the return codes of the test \n" " targets that failed to launch).\n" " -fpolicy Policy for handling test failures (test targets report failing tests), \n" - " where abort will abort the entire test sequenceupon the first test \n" - " failureand report a failure and continue will continue with the test\n" - " sequence in the event of test failuresand report the test failures.\n" + " where abort will abort the entire test sequence upon the first test \n" + " failure and report a failure and continue will continue with the test\n" + " sequence in the event of test failures and report the test failures.\n" " -ipolicy= Policy for handling coverage data integrity failures, where abort will \n" " abort the test sequenceand report a failure, seed will attempt another \n" " sequence using the seed sequence type, otherwise will abort and report \n" - " a failure (this option has no effect for regularand seed sequence \n" + " a failure (this option has no effect for regular and seed sequence \n" " types) and rerun will attempt another sequence using the regular \n" - " sequence type, otherwise will abortand report a failure(this option has \n" + " sequence type, otherwise will abort and report a failure(this option has \n" " no effect for regular sequence type).\n" " -ppolicy= Policy for prioritizing selected test targets, where none will not \n" " attempt any test target prioritization and locality will attempt to \n" diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.h b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.h index 99b1b98dba..b215261a87 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.h +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.h @@ -64,8 +64,8 @@ namespace TestImpact //! Returns the test execution failure policy to use. Policy::ExecutionFailure GetExecutionFailurePolicy() const; - //! Returns the test historic test execution failure drafting policy to use. - Policy::ExecutionFailureDrafting GetExecutionFailureDraftingPolicy() const; + //! Returns failed test coverage drafting policy to use. + Policy::FailedTestCoverage GetFailedTestCoveragePolicy() const; //! Returns the test failure policy to use. Policy::TestFailure GetTestFailurePolicy() const; @@ -98,7 +98,7 @@ namespace TestImpact TestSequenceType m_testSequenceType; Policy::TestPrioritization m_testPrioritizationPolicy = Policy::TestPrioritization::None; Policy::ExecutionFailure m_executionFailurePolicy = Policy::ExecutionFailure::Continue; - Policy::ExecutionFailureDrafting m_executionFailureDraftingPolicy = Policy::ExecutionFailureDrafting::Always; + Policy::FailedTestCoverage m_failedTestCoveragePolicy = Policy::FailedTestCoverage::Keep; Policy::TestFailure m_testFailurePolicy = Policy::TestFailure::Abort; Policy::IntegrityFailure m_integrityFailurePolicy = Policy::IntegrityFailure::Abort; Policy::TestSharding m_testShardingPolicy = Policy::TestSharding::Never; diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleMain.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleMain.cpp index 4853c6da15..77b1d98b3c 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleMain.cpp +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleMain.cpp @@ -205,7 +205,7 @@ namespace TestImpact RuntimeConfigurationFactory(ReadFileContents(options.GetConfigurationFile())), options.GetSuiteFilter(), options.GetExecutionFailurePolicy(), - options.GetExecutionFailureDraftingPolicy(), + options.GetFailedTestCoveragePolicy(), options.GetTestFailurePolicy(), options.GetIntegrityFailurePolicy(), options.GetTestShardingPolicy(), diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h index ae1f78c5c7..86f2907ab7 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h @@ -35,6 +35,7 @@ namespace TestImpact class TestEngine; class TestTarget; class SourceCoveringTestsList; + class TestEngineInstrumentedRun; //! Callback for a test sequence that isn't using test impact analysis to determine selected tests. //! @param tests The tests that will be run for this sequence. @@ -102,7 +103,7 @@ namespace TestImpact RuntimeConfig&& config, SuiteType suiteFilter, Policy::ExecutionFailure executionFailurePolicy, - Policy::ExecutionFailureDrafting executionFailureDraftingPolicy, + Policy::FailedTestCoverage failedTestCoveragePolicy, Policy::TestFailure testFailurePolicy, Policy::IntegrityFailure integrationFailurePolicy, Policy::TestSharding testShardingPolicy, @@ -204,13 +205,13 @@ namespace TestImpact void ClearDynamicDependencyMapAndRemoveExistingFile(); //! Updates the dynamic dependency map and serializes the entire map to disk. - void UpdateAndSerializeDynamicDependencyMap(const SourceCoveringTestsList& sourceCoverageTestsList); + void UpdateAndSerializeDynamicDependencyMap(const AZStd::vector& jobs); RuntimeConfig m_config; SuiteType m_suiteFilter; RepoPath m_sparTIAFile; Policy::ExecutionFailure m_executionFailurePolicy; - Policy::ExecutionFailureDrafting m_executionFailureDraftingPolicy; + Policy::FailedTestCoverage m_failedTestCoveragePolicy; Policy::TestFailure m_testFailurePolicy; Policy::IntegrityFailure m_integrationFailurePolicy; Policy::TestSharding m_testShardingPolicy; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactTestSequence.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactTestSequence.h index 4e8908cf3f..5f9bff952f 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactTestSequence.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactTestSequence.h @@ -31,11 +31,11 @@ namespace TestImpact Ignore //!< Continue the test sequence and ignore the execution failures. }; - //! Policy for reattempting the execution of test targets that failed to execute in previous runs. - enum class ExecutionFailureDrafting + //! Policy for handling the coverage data of failed tests targets (both test that failed to execute and tests that ran but failed). + enum class FailedTestCoverage { - Never, //!< Do not attempt to execute historic execution failures. - Always //!< Reattempt the exectution of historic execution failures. + Remove, //!< Remove the failed test targets from the all coverage data (causing them to be drafted into future test runs). + Keep //!< Keep any existing coverage data and update the coverage data for failed test targetss that produce coverage. }; //! Policy for prioritizing selected tests. diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp index ece5a7c13b..22bcc97c2a 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp @@ -56,7 +56,7 @@ namespace TestImpact for (const auto& target : m_testTargets.GetTargets()) { mapBuildTargetSources(&target); - m_testTargetSourceCoverageCount[&target] = 0; + m_testTargetSourceCoverage[&target] = {}; } } @@ -144,18 +144,15 @@ namespace TestImpact sourceCoverage.GetPath().c_str()).c_str()); auto [sourceDependencyIt, inserted] = m_sourceDependencyMap.insert(sourceCoverage.GetPath().String()); - auto& [key, sourceDependency] = *sourceDependencyIt; + auto& [source, sourceDependency] = *sourceDependencyIt; - // Knock down the source coverage count for the test targets and clear any existing coverage for the delta + // Remove the source from the test target covering sources map and clear any existing coverage for the delta for (const auto& testTarget : sourceDependency.m_coveringTestTargets) { - if (auto coveringTestTargetIt = m_testTargetSourceCoverageCount.find(testTarget); - coveringTestTargetIt != m_testTargetSourceCoverageCount.end()) + if (auto coveringTestTargetIt = m_testTargetSourceCoverage.find(testTarget); + coveringTestTargetIt != m_testTargetSourceCoverage.end()) { - if (coveringTestTargetIt->second > 0) - { - coveringTestTargetIt->second--; - } + coveringTestTargetIt->second.erase(source); } } sourceDependency.m_coveringTestTargets.clear(); @@ -169,8 +166,8 @@ namespace TestImpact // Source to covering test target mapping sourceDependency.m_coveringTestTargets.insert(testTarget); - // Test target covering sources count - m_testTargetSourceCoverageCount[testTarget]++; + // Add the source to the test target covering sources map + m_testTargetSourceCoverage[testTarget].insert(source); // Build target to covering test target mapping for (const auto& parentTarget : sourceDependency.m_parentTargets) @@ -429,12 +426,33 @@ namespace TestImpact return ChangeDependencyList(AZStd::move(createDependencies), AZStd::move(updateDependencies), AZStd::move(deleteDependencies)); } + void DynamicDependencyMap::RemoveTestTargetFromSourceCoverage(const TestTarget* testTarget) + { + if (const auto& it = m_testTargetSourceCoverage.find(testTarget); + it != m_testTargetSourceCoverage.end()) + { + for (const auto& source : it->second) + { + const auto sourceDependency = m_sourceDependencyMap.find(source); + AZ_TestImpact_Eval( + sourceDependency != m_sourceDependencyMap.end(), + DependencyException, + AZStd::string::format("Test target '%s' has covering source '%s' yet cannot be found in the dependency map", + testTarget->GetName().c_str(), source.c_str())); + + sourceDependency->second.m_coveringTestTargets.erase(testTarget); + } + + m_testTargetSourceCoverage.erase(testTarget); + } + } + AZStd::vector DynamicDependencyMap::GetCoveringTests() const { AZStd::vector covering; - for (const auto& [testTarget, coveringSources] : m_testTargetSourceCoverageCount) + for (const auto& [testTarget, coveringSources] : m_testTargetSourceCoverage) { - if (coveringSources > 0) + if (!coveringSources.empty()) { covering.push_back(testTarget); } @@ -446,9 +464,9 @@ namespace TestImpact AZStd::vector DynamicDependencyMap::GetNotCoveringTests() const { AZStd::vector notCovering; - for(const auto& [testTarget, coveringSources] : m_testTargetSourceCoverageCount) + for(const auto& [testTarget, coveringSources] : m_testTargetSourceCoverage) { - if(coveringSources == 0) + if (coveringSources.empty()) { notCovering.push_back(testTarget); } diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.h index f209e22823..cd01a1a728 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.h @@ -100,6 +100,9 @@ namespace TestImpact //! @returns The change list as resolved to the appropriate source dependencies. [[nodiscard]] ChangeDependencyList ApplyAndResoveChangeList(const ChangeList& changeList); + //! Removes the specified test target from all source coverage. + void RemoveTestTargetFromSourceCoverage(const TestTarget* testTarget); + //! Returns the test targets that cover one or more sources in the repository. AZStd::vector GetCoveringTests() const; @@ -120,13 +123,13 @@ namespace TestImpact //! The dependency map of sources to their parent build targets and covering test targets. AZStd::unordered_map m_sourceDependencyMap; + //! Map of all test targets and the sources they cover. + AZStd::unordered_map> m_testTargetSourceCoverage; + //! The map of build targets and their covering test targets. AZStd::unordered_map> m_buildTargetCoverage; //! Mapping of autogen input sources to their generated output sources. AZStd::unordered_map> m_autogenInputToOutputMap; - - //! Number of sources that each test target in the repository covers. - AZStd::unordered_map m_testTargetSourceCoverageCount; }; } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp index 2331c76880..44657dad21 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp @@ -70,11 +70,20 @@ namespace TestImpact }; } + template + AZStd::vector ConcatenateVectors(const AZStd::vector& v1, const AZStd::vector& v2) + { + AZStd::vector result; + result.reserve(v1.size() + v2.size()); + result.insert(result.end(), v1.begin(), v1.end()); + result.insert(result.end(), v2.begin(), v2.end()); + return result; + } Runtime::Runtime( RuntimeConfig&& config, SuiteType suiteFilter, Policy::ExecutionFailure executionFailurePolicy, - Policy::ExecutionFailureDrafting executionFailureDraftingPolicy, + Policy::FailedTestCoverage failedTestCoveragePolicy, Policy::TestFailure testFailurePolicy, Policy::IntegrityFailure integrationFailurePolicy, Policy::TestSharding testShardingPolicy, @@ -83,7 +92,7 @@ namespace TestImpact : m_config(AZStd::move(config)) , m_suiteFilter(suiteFilter) , m_executionFailurePolicy(executionFailurePolicy) - , m_executionFailureDraftingPolicy(executionFailureDraftingPolicy) + , m_failedTestCoveragePolicy(failedTestCoveragePolicy) , m_testFailurePolicy(testFailurePolicy) , m_integrationFailurePolicy(integrationFailurePolicy) , m_testShardingPolicy(testShardingPolicy) @@ -145,7 +154,7 @@ namespace TestImpact { AZ_Printf("TestImpactRuntime", AZStd::string::format( - "No test impact analysis data found for suite '%s' at %s", GetSuiteTypeName(m_suiteFilter).c_str(), m_sparTIAFile.c_str()).c_str()); + "No test impact analysis data found for suite '%s' at %s\n", GetSuiteTypeName(m_suiteFilter).c_str(), m_sparTIAFile.c_str()).c_str()); } } @@ -249,14 +258,28 @@ namespace TestImpact m_dynamicDependencyMap->ClearAllSourceCoverage(); } - void Runtime::UpdateAndSerializeDynamicDependencyMap(const SourceCoveringTestsList& sourceCoverageTestsList) + void Runtime::UpdateAndSerializeDynamicDependencyMap(const AZStd::vector& jobs) { + const auto sourceCoverageTestsList = CreateSourceCoveringTestFromTestCoverages(jobs, m_config.m_repo.m_root); if (!sourceCoverageTestsList.GetNumSources()) { return; } m_dynamicDependencyMap->ReplaceSourceCoverage(sourceCoverageTestsList); + + if (m_failedTestCoveragePolicy == Policy::FailedTestCoverage::Remove) + { + for (const auto& job : jobs) + { + if (job.GetTestResult() != Client::TestRunResult::AllTestsPass || + !job.GetTestCoverge().has_value()) + { + m_dynamicDependencyMap->RemoveTestTargetFromSourceCoverage(job.GetTestTarget()); + } + } + } + const auto sparTIA = m_dynamicDependencyMap->ExportSourceCoverage(); const auto sparTIAData = SerializeSourceCoveringTestsList(sparTIA); WriteFileContents(sparTIAData, m_sparTIAFile); @@ -323,11 +346,12 @@ namespace TestImpact AZStd::optional testCompleteCallback) { Timer timer; - AZStd::vector draftedTestTargets; + AZStd::vector draftedTestTargets = m_dynamicDependencyMap->GetNotCoveringTests(); auto [selectedTestTargets, discardedTestTargets] = SelectCoveringTestTargetsAndUpdateEnumerationCache(changeList, testPrioritizationPolicy); auto [includedSelectedTestTargets, excludedSelectedTestTargets] = SelectTestTargetsByExcludeList(selectedTestTargets); + AZStd::vector testTargetsToRun = ConcatenateVectors(includedSelectedTestTargets, draftedTestTargets); if (testSequenceStartCallback.has_value()) { (*testSequenceStartCallback)( @@ -336,10 +360,11 @@ namespace TestImpact ExtractTestTargetNames(draftedTestTargets)); } + if (dynamicDependencyMapPolicy == Policy::DynamicDependencyMap::Update) { const auto [result, testJobs] = m_testEngine->InstrumentedRun( - includedSelectedTestTargets, + testTargetsToRun, m_testShardingPolicy, m_executionFailurePolicy, Policy::IntegrityFailure::Continue, @@ -349,7 +374,7 @@ namespace TestImpact globalTimeout, TestRunCompleteCallbackHandler(testCompleteCallback)); - UpdateAndSerializeDynamicDependencyMap(CreateSourceCoveringTestFromTestCoverages(testJobs, m_config.m_repo.m_root)); + UpdateAndSerializeDynamicDependencyMap(testJobs); if (testSequenceEndCallback.has_value()) { @@ -361,7 +386,7 @@ namespace TestImpact else { const auto [result, testJobs] = m_testEngine->RegularRun( - includedSelectedTestTargets, + testTargetsToRun, m_testShardingPolicy, m_executionFailurePolicy, m_testFailurePolicy, @@ -389,12 +414,13 @@ namespace TestImpact AZStd::optional testCompleteCallback) { Timer timer; - AZStd::vector draftedTestTargets; + AZStd::vector draftedTestTargets = m_dynamicDependencyMap->GetNotCoveringTests(); auto [selectedTestTargets, discardedTestTargets] = SelectCoveringTestTargetsAndUpdateEnumerationCache(changeList, testPrioritizationPolicy); auto [includedSelectedTestTargets, excludedSelectedTestTargets] = SelectTestTargetsByExcludeList(selectedTestTargets); auto [includedDiscardedTestTargets, excludedDiscardedTestTargets] = SelectTestTargetsByExcludeList(discardedTestTargets); + AZStd::vector testTargetsToRun = ConcatenateVectors(includedSelectedTestTargets, draftedTestTargets); if (testSequenceStartCallback.has_value()) { (*testSequenceStartCallback)( @@ -403,9 +429,10 @@ namespace TestImpact ExtractTestTargetNames(draftedTestTargets)); } + // Impact analysis run of the selected test targets const auto [selectedResult, selectedTestJobs] = m_testEngine->InstrumentedRun( - includedSelectedTestTargets, + testTargetsToRun, m_testShardingPolicy, m_executionFailurePolicy, Policy::IntegrityFailure::Continue, @@ -433,7 +460,7 @@ namespace TestImpact globalTimeout, TestRunCompleteCallbackHandler(testCompleteCallback)); - UpdateAndSerializeDynamicDependencyMap(CreateSourceCoveringTestFromTestCoverages(selectedTestJobs, m_config.m_repo.m_root)); + UpdateAndSerializeDynamicDependencyMap(selectedTestJobs); if (testSequenceEndCallback.has_value()) { @@ -486,7 +513,7 @@ namespace TestImpact TestRunCompleteCallbackHandler(testCompleteCallback)); ClearDynamicDependencyMapAndRemoveExistingFile(); - UpdateAndSerializeDynamicDependencyMap(CreateSourceCoveringTestFromTestCoverages(testJobs, m_config.m_repo.m_root)); + UpdateAndSerializeDynamicDependencyMap(testJobs); if (testSequenceEndCallback.has_value()) { From 0da739ef18c41ec1ecb29cb563b69d676ef5d7cd Mon Sep 17 00:00:00 2001 From: jonawals Date: Mon, 7 Jun 2021 09:05:29 +0100 Subject: [PATCH 067/102] Add curiously missing comments --- .../Runtime/Code/Source/TestImpactRuntime.cpp | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp index 44657dad21..d83976b488 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp @@ -70,6 +70,7 @@ namespace TestImpact }; } + //! Utility for concatenating two vectors. template AZStd::vector ConcatenateVectors(const AZStd::vector& v1, const AZStd::vector& v2) { @@ -79,6 +80,7 @@ namespace TestImpact result.insert(result.end(), v2.begin(), v2.end()); return result; } + Runtime::Runtime( RuntimeConfig&& config, SuiteType suiteFilter, @@ -346,12 +348,20 @@ namespace TestImpact AZStd::optional testCompleteCallback) { Timer timer; + + // Draft in the test targets that have no coverage entries in the dynamic dependency map AZStd::vector draftedTestTargets = m_dynamicDependencyMap->GetNotCoveringTests(); + // The test targets that were selected for the change list by the dynamic dependency map and the test targets that were not auto [selectedTestTargets, discardedTestTargets] = SelectCoveringTestTargetsAndUpdateEnumerationCache(changeList, testPrioritizationPolicy); + + // The subset of selected test targets that are not on the configuration's exclude list and those that are auto [includedSelectedTestTargets, excludedSelectedTestTargets] = SelectTestTargetsByExcludeList(selectedTestTargets); + // We present to the client the included selected test targets and the drafted test targets as distinct sets but internally + // we consider the concatenated set of the two the actual set of tests to run AZStd::vector testTargetsToRun = ConcatenateVectors(includedSelectedTestTargets, draftedTestTargets); + if (testSequenceStartCallback.has_value()) { (*testSequenceStartCallback)( @@ -360,7 +370,6 @@ namespace TestImpact ExtractTestTargetNames(draftedTestTargets)); } - if (dynamicDependencyMapPolicy == Policy::DynamicDependencyMap::Update) { const auto [result, testJobs] = m_testEngine->InstrumentedRun( @@ -413,14 +422,24 @@ namespace TestImpact AZStd::optional testSequenceEndCallback, AZStd::optional testCompleteCallback) { - Timer timer; + Timer timer; + + // Draft in the test targets that have no coverage entries in the dynamic dependency map AZStd::vector draftedTestTargets = m_dynamicDependencyMap->GetNotCoveringTests(); + // The test targets that were selected for the change list by the dynamic dependency map and the test targets that were not auto [selectedTestTargets, discardedTestTargets] = SelectCoveringTestTargetsAndUpdateEnumerationCache(changeList, testPrioritizationPolicy); + + // The subset of selected test targets that are not on the configuration's exclude list and those that are auto [includedSelectedTestTargets, excludedSelectedTestTargets] = SelectTestTargetsByExcludeList(selectedTestTargets); + + // The subset of discarded test targets that are not on the configuration's exclude list and those that are auto [includedDiscardedTestTargets, excludedDiscardedTestTargets] = SelectTestTargetsByExcludeList(discardedTestTargets); + // We present to the client the included selected test targets and the drafted test targets as distinct sets but internally + // we consider the concatenated set of the two the actual set of tests to run AZStd::vector testTargetsToRun = ConcatenateVectors(includedSelectedTestTargets, draftedTestTargets); + if (testSequenceStartCallback.has_value()) { (*testSequenceStartCallback)( @@ -429,7 +448,6 @@ namespace TestImpact ExtractTestTargetNames(draftedTestTargets)); } - // Impact analysis run of the selected test targets const auto [selectedResult, selectedTestJobs] = m_testEngine->InstrumentedRun( testTargetsToRun, From d341dab7f54d866849a2c1f96190808e7e0f622b Mon Sep 17 00:00:00 2001 From: jonawals Date: Mon, 7 Jun 2021 10:39:58 +0100 Subject: [PATCH 068/102] Address PR comments --- .../Static/Code/Source/TestImpactCommandLineOptions.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp index aad9319cf9..01005718bc 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp @@ -405,9 +405,9 @@ namespace TestImpact " tests are run regardless).\n" " -shard= Break any test targets with a sharding policy into the number of \n" " shards according to the maximum concurrency value.\n" - " -cpolicy= Policy for handling the coverage data of failed tests (both test that \n" + " -cpolicy= Policy for handling the coverage data of failed tests (both tests that \n" " failed to execute and tests that ran but failed), where remove will \n" - " remove the failed tests from the all coverage data(causing them to be \n" + " remove the failed tests from the all coverage data (causing them to be \n" " drafted into future test runs) and keep will keep any existing coverage \n" " data and update the coverage data for failed tests that produce coverage.\n" " -targetout= Capture of individual test run stdout, where stdout will capture \n" From a0feeec608c694c24367b129a61a74749525a32f Mon Sep 17 00:00:00 2001 From: jonawals Date: Mon, 7 Jun 2021 10:56:07 +0100 Subject: [PATCH 069/102] Add console output to runtime --- ...tImpactConsoleTestSequenceEventHandler.cpp | 234 ++++++++++++++++++ ...estImpactConsoleTestSequenceEventHandler.h | 72 ++++++ .../Code/Source/TestImpactConsoleUtils.cpp | 34 +++ .../Code/Source/TestImpactConsoleUtils.h | 56 +++++ 4 files changed, 396 insertions(+) create mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleTestSequenceEventHandler.cpp create mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleTestSequenceEventHandler.h create mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleUtils.cpp create mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleUtils.h diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleTestSequenceEventHandler.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleTestSequenceEventHandler.cpp new file mode 100644 index 0000000000..8595fc0683 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleTestSequenceEventHandler.cpp @@ -0,0 +1,234 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include + +#include + +#include + +namespace TestImpact +{ + namespace Console + { + namespace Output + { + void TestSuiteFilter(SuiteType filter) + { + std::cout << "Test suite filter: " << GetSuiteTypeName(filter).c_str() << "\n"; + } + + void ImpactAnalysisTestSelection(size_t numSelectedTests, size_t numDiscardedTests, size_t numExcludedTests, size_t numDraftedTests) + { + const float totalTests = numSelectedTests + numDiscardedTests; + const float saving = (1.0 - (numSelectedTests / totalTests)) * 100.0f; + + std::cout << numSelectedTests << " tests selected, " << numDiscardedTests << " tests discarded (" << saving << "% test saving)\n"; + std::cout << "Of which " << numExcludedTests << " tests have been excluded and " << numDraftedTests << " tests have been drafted.\n"; + } + + void FailureReport(const Client::SequenceFailure& failureReport, AZStd::chrono::milliseconds duration) + { + std::cout << "Sequence completed in " << (duration.count() / 1000.f) << "s with"; + + if (failureReport.GetExecutionFailures().size() || + failureReport.GetTestRunFailures().size() || + failureReport.GetTimedOutTests().size() || + failureReport.GetUnexecutedTests().size()) + { + std::cout << ":\n"; + std::cout << SetColor(Foreground::White, Background::Red).c_str() + << failureReport.GetTestRunFailures().size() + << ResetColor().c_str() << " test failures\n"; + + std::cout << SetColor(Foreground::White, Background::Red).c_str() + << failureReport.GetExecutionFailures().size() + << ResetColor().c_str() << " execution failures\n"; + + std::cout << SetColor(Foreground::White, Background::Red).c_str() + << failureReport.GetTimedOutTests().size() + << ResetColor().c_str() << " test timeouts\n"; + + std::cout << SetColor(Foreground::White, Background::Red).c_str() + << failureReport.GetUnexecutedTests().size() + << ResetColor().c_str() << " unexecuted tests\n"; + + if (!failureReport.GetTestRunFailures().empty()) + { + std::cout << "\nTest failures:\n"; + for (const auto& testRunFailure : failureReport.GetTestRunFailures()) + { + std::cout << " " << testRunFailure.GetTargetName().c_str(); + for (const auto& testCaseFailure : testRunFailure.GetTestCaseFailures()) + { + std::cout << "." << testCaseFailure.GetName().c_str(); + for (const auto& testFailure : testCaseFailure.GetTestFailures()) + { + std::cout << "." << testFailure.GetName().c_str() << "\n"; + } + } + } + } + + if (!failureReport.GetExecutionFailures().empty()) + { + std::cout << "\nExecution failures:\n"; + for (const auto& executionFailure : failureReport.GetExecutionFailures()) + { + std::cout << " " << executionFailure.GetTargetName().c_str() << "\n"; + std::cout << executionFailure.GetCommandString().c_str() << "\n"; + } + } + + if (!failureReport.GetTimedOutTests().empty()) + { + std::cout << "\nTimed out tests:\n"; + for (const auto& testTimeout : failureReport.GetTimedOutTests()) + { + std::cout << " " << testTimeout.GetTargetName().c_str() << "\n"; + } + } + + if (!failureReport.GetUnexecutedTests().empty()) + { + std::cout << "\nUnexecuted tests:\n"; + for (const auto& unexecutedTest : failureReport.GetUnexecutedTests()) + { + std::cout << " " << unexecutedTest.GetTargetName().c_str() << "\n"; + } + } + } + else + { + std::cout << SetColor(Foreground::White, Background::Green).c_str() << " \100% passes!\n" << ResetColor().c_str(); + } + } + } + + TestSequenceEventHandler::TestSequenceEventHandler(SuiteType suiteFilter) + : m_suiteFilter(suiteFilter) + { + } + + // TestSequenceStartCallback + void TestSequenceEventHandler::operator()(Client::TestRunSelection&& selectedTests) + { + ClearState(); + m_numTests = selectedTests.GetNumIncludedTestRuns(); + + Output::TestSuiteFilter(m_suiteFilter); + std::cout << selectedTests.GetNumIncludedTestRuns() << " tests selected, " << selectedTests.GetNumExcludedTestRuns() << " excluded.\n"; + } + + // ImpactAnalysisTestSequenceStartCallback + void TestSequenceEventHandler::operator()( + Client::TestRunSelection&& selectedTests, + AZStd::vector&& discardedTests, + AZStd::vector&& draftedTests) + { + ClearState(); + m_numTests = selectedTests.GetNumIncludedTestRuns() + draftedTests.size(); + + Output::TestSuiteFilter(m_suiteFilter); + Output::ImpactAnalysisTestSelection( + selectedTests.GetTotalNumTests(), discardedTests.size(), selectedTests.GetNumExcludedTestRuns(), draftedTests.size()); + } + + // SafeImpactAnalysisTestSequenceStartCallback + void TestSequenceEventHandler::operator()( + Client::TestRunSelection&& selectedTests, + Client::TestRunSelection&& discardedTests, + AZStd::vector&& draftedTests) + { + ClearState(); + m_numTests = selectedTests.GetNumIncludedTestRuns() + draftedTests.size(); + + Output::TestSuiteFilter(m_suiteFilter); + Output::ImpactAnalysisTestSelection( + selectedTests.GetTotalNumTests(), + discardedTests.GetTotalNumTests(), + selectedTests.GetNumExcludedTestRuns() + discardedTests.GetNumExcludedTestRuns(), + draftedTests.size()); + } + + // TestSequenceCompleteCallback + void TestSequenceEventHandler::operator()( + Client::SequenceFailure&& failureReport, + AZStd::chrono::milliseconds duration) + { + + Output::FailureReport(failureReport, duration); + std::cout << "Updating and serializing the test impact analysis data, this may take a moment...\n"; + } + + // SafeTestSequenceCompleteCallback + void TestSequenceEventHandler::operator()( + Client::SequenceFailure&& selectedFailureReport, + Client::SequenceFailure&& discardedFailureReport, + AZStd::chrono::milliseconds selectedDuration, + AZStd::chrono::milliseconds discaredDuration) + { + std::cout << "Selected test run:\n"; + Output::FailureReport(selectedFailureReport, selectedDuration); + + std::cout << "Discarded test run:\n"; + Output::FailureReport(discardedFailureReport, discaredDuration); + + std::cout << "Updating and serializing the test impact analysis data, this may take a moment...\n"; + } + + // TestRunCompleteCallback + void TestSequenceEventHandler::operator()([[maybe_unused]] Client::TestRun&& test) + { + m_numTestsComplete++; + const auto progress = AZStd::string::format("(%03u/%03u)", m_numTestsComplete, m_numTests, test.GetTargetName().c_str()); + + AZStd::string result; + switch (test.GetResult()) + { + case Client::TestRunResult::AllTestsPass: + { + result = SetColorForString(Foreground::White, Background::Green, "PASS"); + break; + } + case Client::TestRunResult::FailedToExecute: + { + result = SetColorForString(Foreground::White, Background::Red, "EXEC"); + break; + } + case Client::TestRunResult::NotRun: + { + result = SetColorForString(Foreground::White, Background::Yellow, "SKIP"); + break; + } + case Client::TestRunResult::TestFailures: + { + result = SetColorForString(Foreground::White, Background::Red, "FAIL"); + break; + } + case Client::TestRunResult::Timeout: + { + result = SetColorForString(Foreground::White, Background::Magenta, "TIME"); + break; + } + } + + std::cout << progress.c_str() << " " << result.c_str() << " " << test.GetTargetName().c_str() << " (" << (test.GetDuration().count() / 1000.f) << "s)\n"; + } + + void TestSequenceEventHandler::ClearState() + { + m_numTests = 0; + m_numTestsComplete = 0; + } + } // namespace Console +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleTestSequenceEventHandler.h b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleTestSequenceEventHandler.h new file mode 100644 index 0000000000..01f59b2c64 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleTestSequenceEventHandler.h @@ -0,0 +1,72 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include +#include +#include + +#include +#include +#include + +#pragma once + +namespace TestImpact +{ + namespace Console + { + //! Event handler for all test sequence types. + class TestSequenceEventHandler + { + public: + TestSequenceEventHandler(SuiteType suiteFilter); + + //! TestSequenceStartCallback. + void operator()(Client::TestRunSelection&& selectedTests); + + //! ImpactAnalysisTestSequenceStartCallback. + void operator()( + Client::TestRunSelection&& selectedTests, + AZStd::vector&& discardedTests, + AZStd::vector&& draftedTests); + + //! SafeImpactAnalysisTestSequenceStartCallback. + void operator()( + Client::TestRunSelection&& selectedTests, + Client::TestRunSelection&& discardedTests, + AZStd::vector&& draftedTests); + + //! TestSequenceCompleteCallback. + void operator()( + Client::SequenceFailure&& failureReport, + AZStd::chrono::milliseconds duration); + + //! SafeTestSequenceCompleteCallback. + void operator()( + Client::SequenceFailure&& selectedFailureReport, + Client::SequenceFailure&& discardedFailureReport, + AZStd::chrono::milliseconds selectedDuration, + AZStd::chrono::milliseconds discaredDuration); + + //! TestRunCompleteCallback. + void operator()(Client::TestRun&& test); + + private: + void ClearState(); + + SuiteType m_suiteFilter; + size_t m_numTests = 0; + size_t m_numTestsComplete = 0; + }; + } // namespace Console +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleUtils.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleUtils.cpp new file mode 100644 index 0000000000..e00a766eb5 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleUtils.cpp @@ -0,0 +1,34 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include + +namespace TestImpact +{ + namespace Console + { + AZStd::string SetColor(Foreground fgd, Background bgd) + { + return AZStd::string::format("\033[%u;%um", static_cast(fgd), static_cast(bgd)); + } + + AZStd::string SetColorForString(Foreground fgd, Background bgd, const AZStd::string& str) + { + return AZStd::string::format("%s%s%s", SetColor(fgd, bgd).c_str(), str.c_str(), ResetColor().c_str()); + } + + AZStd::string ResetColor() + { + return "\033[0m"; + } + } // namespace Console +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleUtils.h b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleUtils.h new file mode 100644 index 0000000000..d0fc2495d3 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleUtils.h @@ -0,0 +1,56 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +namespace TestImpact +{ + namespace Console + { + //! The set of available foreground colors. + enum class Foreground + { + Black = 30, + Red, + Green, + Yellow, + Blue, + Magenta, + Cyan, + White + }; + + //! The set of available background colors. + enum class Background + { + Black = 40, + Red, + Green, + Yellow, + Blue, + Magenta, + Cyan, + White + }; + + //! Returns a string to be used to set the specified foreground and background color. + AZStd::string SetColor(Foreground fgd, Background bgd); + + //! Returns a string with the specified string set to the specified foreground and background color followed by a color reset. + AZStd::string SetColorForString(Foreground fgd, Background bgd, const AZStd::string& str); + + //! Returns a string to be used to reset the color back to white foreground on black background. + AZStd::string ResetColor(); + } // namespace Console +} // namespace TestImpact From 796b5be286a713d9e42f2363a3cdfcf0bdf5dafd Mon Sep 17 00:00:00 2001 From: jonawals Date: Mon, 7 Jun 2021 15:32:17 +0100 Subject: [PATCH 070/102] Address PR comments --- .../Source/TestImpactConsoleTestSequenceEventHandler.cpp | 8 ++++---- .../Source/TestImpactConsoleTestSequenceEventHandler.h | 2 +- .../Console/Static/Code/Source/TestImpactConsoleUtils.cpp | 8 ++++---- .../Console/Static/Code/Source/TestImpactConsoleUtils.h | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleTestSequenceEventHandler.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleTestSequenceEventHandler.cpp index 8595fc0683..0a745514a5 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleTestSequenceEventHandler.cpp +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleTestSequenceEventHandler.cpp @@ -40,10 +40,10 @@ namespace TestImpact { std::cout << "Sequence completed in " << (duration.count() / 1000.f) << "s with"; - if (failureReport.GetExecutionFailures().size() || - failureReport.GetTestRunFailures().size() || - failureReport.GetTimedOutTests().size() || - failureReport.GetUnexecutedTests().size()) + if (!failureReport.GetExecutionFailures().empty() || + !failureReport.GetTestRunFailures().empty() || + !failureReport.GetTimedOutTests().empty() || + !failureReport.GetUnexecutedTests().empty()) { std::cout << ":\n"; std::cout << SetColor(Foreground::White, Background::Red).c_str() diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleTestSequenceEventHandler.h b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleTestSequenceEventHandler.h index 01f59b2c64..ee5eee0bf8 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleTestSequenceEventHandler.h +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleTestSequenceEventHandler.h @@ -29,7 +29,7 @@ namespace TestImpact class TestSequenceEventHandler { public: - TestSequenceEventHandler(SuiteType suiteFilter); + explicit TestSequenceEventHandler(SuiteType suiteFilter); //! TestSequenceStartCallback. void operator()(Client::TestRunSelection&& selectedTests); diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleUtils.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleUtils.cpp index e00a766eb5..76ce830371 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleUtils.cpp +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleUtils.cpp @@ -16,14 +16,14 @@ namespace TestImpact { namespace Console { - AZStd::string SetColor(Foreground fgd, Background bgd) + AZStd::string SetColor(Foreground foreground, Background background) { - return AZStd::string::format("\033[%u;%um", static_cast(fgd), static_cast(bgd)); + return AZStd::string::format("\033[%u;%um", aznumeric_cast(foreground), aznumeric_cast(background)); } - AZStd::string SetColorForString(Foreground fgd, Background bgd, const AZStd::string& str) + AZStd::string SetColorForString(Foreground foreground, Background background, const AZStd::string& str) { - return AZStd::string::format("%s%s%s", SetColor(fgd, bgd).c_str(), str.c_str(), ResetColor().c_str()); + return AZStd::string::format("%s%s%s", SetColor(foreground, background).c_str(), str.c_str(), ResetColor().c_str()); } AZStd::string ResetColor() diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleUtils.h b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleUtils.h index d0fc2495d3..a003e2db26 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleUtils.h +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleUtils.h @@ -45,10 +45,10 @@ namespace TestImpact }; //! Returns a string to be used to set the specified foreground and background color. - AZStd::string SetColor(Foreground fgd, Background bgd); + AZStd::string SetColor(Foreground foreground, Background background); //! Returns a string with the specified string set to the specified foreground and background color followed by a color reset. - AZStd::string SetColorForString(Foreground fgd, Background bgd, const AZStd::string& str); + AZStd::string SetColorForString(Foreground foreground, Background background, const AZStd::string& str); //! Returns a string to be used to reset the color back to white foreground on black background. AZStd::string ResetColor(); From cc9e8a72e447a65dd67f9b252a79dacfcce7b9c6 Mon Sep 17 00:00:00 2001 From: jonawals Date: Mon, 7 Jun 2021 17:45:48 +0100 Subject: [PATCH 071/102] Fix coverage clear bug and other minor refactors --- .../Source/TestImpactCommandLineOptions.cpp | 16 ++-- .../TestImpactFramework/TestImpactRuntime.h | 7 +- .../TestImpactTestSequence.h | 2 +- .../TestImpactDynamicDependencyMap.cpp | 39 +++++++- .../TestImpactDynamicDependencyMap.h | 9 +- .../Enumeration/TestImpactTestEnumerator.cpp | 2 +- .../Run/TestImpactInstrumentedTestRunner.cpp | 2 +- .../TestEngine/Run/TestImpactTestRunner.cpp | 2 +- .../Runtime/Code/Source/TestImpactRuntime.cpp | 91 ++++++++++++++----- .../Code/Source/TestImpactRuntimeUtils.cpp | 52 +---------- .../Code/Source/TestImpactRuntimeUtils.h | 7 +- 11 files changed, 132 insertions(+), 97 deletions(-) diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp index 01005718bc..992b466a54 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp @@ -51,7 +51,7 @@ namespace TestImpact Ignore, StdOut, File, - Remove, + Discard, Keep }; @@ -87,7 +87,7 @@ namespace TestImpact "ignore", "stdout", "file", - "remove", + "discard", "keep" }; @@ -147,7 +147,7 @@ namespace TestImpact { const AZStd::vector> states = { - {OptionKeys[Remove], Policy::FailedTestCoverage::Remove}, + {OptionKeys[Discard], Policy::FailedTestCoverage::Discard}, {OptionKeys[Keep], Policy::FailedTestCoverage::Keep} }; @@ -405,11 +405,11 @@ namespace TestImpact " tests are run regardless).\n" " -shard= Break any test targets with a sharding policy into the number of \n" " shards according to the maximum concurrency value.\n" - " -cpolicy= Policy for handling the coverage data of failed tests (both tests that \n" - " failed to execute and tests that ran but failed), where remove will \n" - " remove the failed tests from the all coverage data (causing them to be \n" - " drafted into future test runs) and keep will keep any existing coverage \n" - " data and update the coverage data for failed tests that produce coverage.\n" + " -cpolicy= Policy for handling the coverage data of failing tests, where discard \n" + " will discard the coverage data produced by the failing tests, causing \n" + " them to be drafted into future test runs and keep will keep any existing \n" + " coverage data and update the coverage data for failed tests that produce \n" + " coverage.\n" " -targetout= Capture of individual test run stdout, where stdout will capture \n" " each individual test target's stdout and output each one to stdout \n" " and file will capture each individual test target's stdout and output \n" diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h index 86f2907ab7..2a7720ee55 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h @@ -81,7 +81,8 @@ namespace TestImpact using SafeTestSequenceCompleteCallback = AZStd::function; + AZStd::chrono::milliseconds selectedDuration, + AZStd::chrono::milliseconds discardedDuration)>; //! Callback for test runs that have completed for any reason. //! @param selectedTests The test that has completed. @@ -201,6 +202,10 @@ namespace TestImpact AZStd::pair, AZStd::vector> SelectTestTargetsByExcludeList( AZStd::vector testTargets) const; + //! Prunes the existing coverage for the specified jobs and creates the consolidates source covering tests list from the + //! test engine instrumented run jobs. + SourceCoveringTestsList CreateSourceCoveringTestFromTestCoverages(const AZStd::vector& jobs); + //! Prepares the dynamic dependency map for a seed update by clearing all existing data and deleting the file that will be serialized. void ClearDynamicDependencyMapAndRemoveExistingFile(); diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactTestSequence.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactTestSequence.h index 5f9bff952f..5a5a6196e2 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactTestSequence.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactTestSequence.h @@ -34,7 +34,7 @@ namespace TestImpact //! Policy for handling the coverage data of failed tests targets (both test that failed to execute and tests that ran but failed). enum class FailedTestCoverage { - Remove, //!< Remove the failed test targets from the all coverage data (causing them to be drafted into future test runs). + Discard, //!< Discard the coverage data produced by the failing tests, causing them to be drafted into future test runs. Keep //!< Keep any existing coverage data and update the coverage data for failed test targetss that produce coverage. }; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp index 22bcc97c2a..28b817e32c 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp @@ -133,8 +133,9 @@ namespace TestImpact return buildTarget; } - void DynamicDependencyMap::ReplaceSourceCoverage(const SourceCoveringTestsList& sourceCoverageDelta) + void DynamicDependencyMap::ReplaceSourceCoverageInternal(const SourceCoveringTestsList& sourceCoverageDelta, bool pruneIfNoParentsOrCoverage) { + AZStd::vector killList; for (const auto& sourceCoverage : sourceCoverageDelta.GetCoverage()) { // Autogen input files are not compiled sources and thus supplying coverage data for them makes no sense @@ -146,7 +147,12 @@ namespace TestImpact auto [sourceDependencyIt, inserted] = m_sourceDependencyMap.insert(sourceCoverage.GetPath().String()); auto& [source, sourceDependency] = *sourceDependencyIt; - // Remove the source from the test target covering sources map and clear any existing coverage for the delta + // Before we can replace the coverage for this source dependency, we must: + // 1. Remove the source from the test target covering sources map + // 2. Prune the covered targets for the parent test target(s) of this source dependency + // 3. Clear any existing coverage for the delta + + // 1. for (const auto& testTarget : sourceDependency.m_coveringTestTargets) { if (auto coveringTestTargetIt = m_testTargetSourceCoverage.find(testTarget); @@ -154,7 +160,20 @@ namespace TestImpact { coveringTestTargetIt->second.erase(source); } + + } + + // 2. + // This step is prohibitively expensive as it requires iterating over all of the sources of the build targets covered by + // the parent test targets of this source dependency to ensure that this is in fact the last source being cleared and thus + // it can be determined that the parent test target is no longer covering the given build target + // + // The implications of this are that multiple calls to the test selector and priritizor's SelectTestTargets method will end + // up pulling in more test targets than needed for newly-created production sources until the next time the dynamic dependency + // map reconstructed, however until this use case materializes the implications described will not be addressed + + // 3. sourceDependency.m_coveringTestTargets.clear(); // Update the dependency with any new coverage data @@ -183,13 +202,18 @@ namespace TestImpact } // If the new coverage data results in a parentless and coverageless entry, consider it a dead entry and remove accordingly - if (sourceDependency.m_coveringTestTargets.empty() && sourceDependency.m_parentTargets.empty()) + if (sourceDependency.m_coveringTestTargets.empty() && sourceDependency.m_parentTargets.empty() && pruneIfNoParentsOrCoverage) { m_sourceDependencyMap.erase(sourceDependencyIt); } } } + void DynamicDependencyMap::ReplaceSourceCoverage(const SourceCoveringTestsList& sourceCoverageDelta) + { + ReplaceSourceCoverageInternal(sourceCoverageDelta, true); + } + void DynamicDependencyMap::ClearSourceCoverage(const AZStd::vector& paths) { for (const auto& path : paths) @@ -212,9 +236,14 @@ namespace TestImpact void DynamicDependencyMap::ClearAllSourceCoverage() { - for (const auto& [path, coverage] : m_sourceDependencyMap) + for (auto it = m_sourceDependencyMap.begin(); it != m_sourceDependencyMap.end(); ++it) { - ReplaceSourceCoverage(SourceCoveringTestsList(AZStd::vector{ SourceCoveringTests(RepoPath(path)) })); + const auto& [path, coverage] = *it; + ReplaceSourceCoverageInternal(SourceCoveringTestsList(AZStd::vector{ SourceCoveringTests(RepoPath(path)) }), false); + if (coverage.m_coveringTestTargets.empty() && coverage.m_parentTargets.empty()) + { + it = m_sourceDependencyMap.erase(it); + } } } diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.h index cd01a1a728..701f69e68d 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.h @@ -81,7 +81,6 @@ namespace TestImpact SourceDependency GetSourceDependencyOrThrow(const RepoPath& path) const; //! Replaces the source coverage of the specified sources with the specified source coverage. - //! @note The covering targets for the parent test target(s) will not be pruned if those covering targets are removed. //! @param sourceCoverageDelta The source coverage delta to replace in the dependency map. void ReplaceSourceCoverage(const SourceCoveringTestsList& sourceCoverageDelta); @@ -110,6 +109,13 @@ namespace TestImpact AZStd::vector GetNotCoveringTests() const; private: + //! Internal handler for ReplaceSourceCoverage where the pruning of parentless and coverageless source depenencies after the + //! source coverage has been replaced must be explicitly stated. + //! @note The covered targets for the source dependency's parent test target(s) will not be pruned if those covering targets are removed. + //! @param sourceCoverageDelta The source coverage delta to replace in the dependency map. + //! @param pruneIfNoParentsOrCoverage Flag to specify whether or not newly parentless and coverageless dependencies will be removed. + void ReplaceSourceCoverageInternal(const SourceCoveringTestsList& sourceCoverageDelta, bool pruneIfNoParentsOrCoverage); + //! Clears the source coverage of the specified sources. //! @note The covering targets for the parent test target(s) will not be pruned if those covering targets are removed. void ClearSourceCoverage(const AZStd::vector& paths); @@ -127,6 +133,7 @@ namespace TestImpact AZStd::unordered_map> m_testTargetSourceCoverage; //! The map of build targets and their covering test targets. + //! @note As per the note for ReplaceSourceCoverageInternal, this map is currently not pruned when source coverage is replaced. AZStd::unordered_map> m_buildTargetCoverage; //! Mapping of autogen input sources to their generated output sources. diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.cpp index 58ffb85cb3..17784dd937 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.cpp @@ -73,7 +73,7 @@ namespace TestImpact } catch (const TestEngineException& e) { - AZ_Printf("Enumerate", "Enumeration cache error: %s", e.what()); + AZ_Printf("Enumerate", AZStd::string::format("Enumeration cache error: %s\n", e.what()).c_str()); DeleteFile(jobInfo->GetCache()->m_file); } diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.cpp index aa8ff04657..5a0ef74472 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.cpp @@ -72,7 +72,7 @@ namespace TestImpact } catch (const Exception& e) { - AZ_Printf("RunInstrumentedTests", e.what()); + AZ_Printf("RunInstrumentedTests", AZStd::string::format("%s\n", e.what()).c_str()); runs[jobId] = AZStd::nullopt; } } diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunner.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunner.cpp index 7ac13f85b2..16717a1fe5 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunner.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunner.cpp @@ -46,7 +46,7 @@ namespace TestImpact } catch (const Exception& e) { - AZ_Printf("RunTests", e.what()); + AZ_Printf("RunTests", AZStd::string::format("%s\n", e.what()).c_str()); runs[jobId] = AZStd::nullopt; } } diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp index d83976b488..79f5269188 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp @@ -256,32 +256,77 @@ namespace TestImpact void Runtime::ClearDynamicDependencyMapAndRemoveExistingFile() { - DeleteFile(m_sparTIAFile); m_dynamicDependencyMap->ClearAllSourceCoverage(); + DeleteFile(m_sparTIAFile); + } + + SourceCoveringTestsList Runtime::CreateSourceCoveringTestFromTestCoverages(const AZStd::vector& jobs) + { + AZStd::unordered_map> coverage; + for (const auto& job : jobs) + { + // First we must remove any existing coverage for the test target so as to not end up with source remnants from previous + // coverage that is no longer covered by this revision of the test target + m_dynamicDependencyMap->RemoveTestTargetFromSourceCoverage(job.GetTestTarget()); + + // Next we will update the coverage of test targets that completed (with or without failures), unless the failed test coverage + // policy dictates we should instead discard the coverage of test targets with failing tests + if (const auto testResult = job.GetTestResult(); + testResult == Client::TestRunResult::AllTestsPass || + (m_failedTestCoveragePolicy == Policy::FailedTestCoverage::Keep && testResult == Client::TestRunResult::TestFailures)) + { + if (testResult == Client::TestRunResult::AllTestsPass) + { + // Passing tests should have coverage data, otherwise something is very wrong + AZ_TestImpact_Eval( + job.GetTestCoverge().has_value(), + RuntimeException, + AZStd::string::format( + "Test target '%s' completed its test run successfully but produced no coverage data", + job.GetTestTarget()->GetName().c_str())); + } + else if (!job.GetTestCoverge().has_value()) + { + // When a test run completes with failing tests but produces no coverage artifact that's typically a sign of the + // test aborting due to an unhandled exception, in which case ignore it and let it be picked up in the failure report + continue; + } + + for (const auto& source : job.GetTestCoverge().value().GetSourcesCovered()) + { + coverage[source.String()].insert(job.GetTestTarget()->GetName()); + } + } + } + + AZStd::vector sourceCoveringTests; + sourceCoveringTests.reserve(coverage.size()); + for (auto&& [source, testTargets] : coverage) + { + if (const auto sourcePath = RepoPath(source); + sourcePath.IsRelativeTo(m_config.m_repo.m_root)) + { + sourceCoveringTests.push_back( + SourceCoveringTests(RepoPath(sourcePath.LexicallyRelative(m_config.m_repo.m_root)), AZStd::move(testTargets))); + } + else + { + AZ_Warning("TestImpact", false, "Ignoring source, source it outside of repo: '%s'", sourcePath.c_str()); + } + } + + return SourceCoveringTestsList(AZStd::move(sourceCoveringTests)); } void Runtime::UpdateAndSerializeDynamicDependencyMap(const AZStd::vector& jobs) { - const auto sourceCoverageTestsList = CreateSourceCoveringTestFromTestCoverages(jobs, m_config.m_repo.m_root); + const auto sourceCoverageTestsList = CreateSourceCoveringTestFromTestCoverages(jobs); if (!sourceCoverageTestsList.GetNumSources()) { return; } m_dynamicDependencyMap->ReplaceSourceCoverage(sourceCoverageTestsList); - - if (m_failedTestCoveragePolicy == Policy::FailedTestCoverage::Remove) - { - for (const auto& job : jobs) - { - if (job.GetTestResult() != Client::TestRunResult::AllTestsPass || - !job.GetTestCoverge().has_value()) - { - m_dynamicDependencyMap->RemoveTestTargetFromSourceCoverage(job.GetTestTarget()); - } - } - } - const auto sparTIA = m_dynamicDependencyMap->ExportSourceCoverage(); const auto sparTIAData = SerializeSourceCoveringTestsList(sparTIA); WriteFileContents(sparTIAData, m_sparTIAFile); @@ -422,7 +467,7 @@ namespace TestImpact AZStd::optional testSequenceEndCallback, AZStd::optional testCompleteCallback) { - Timer timer; + Timer timer; // Draft in the test targets that have no coverage entries in the dynamic dependency map AZStd::vector draftedTestTargets = m_dynamicDependencyMap->GetNotCoveringTests(); @@ -459,6 +504,8 @@ namespace TestImpact testTargetTimeout, globalTimeout, TestRunCompleteCallbackHandler(testCompleteCallback)); + + const auto selectedDuraton = timer.Elapsed(); // Carry the remaining global sequence time over to the discarded test run if (globalTimeout.has_value()) @@ -478,16 +525,18 @@ namespace TestImpact globalTimeout, TestRunCompleteCallbackHandler(testCompleteCallback)); - UpdateAndSerializeDynamicDependencyMap(selectedTestJobs); + const auto discardedDuraton = timer.Elapsed(); if (testSequenceEndCallback.has_value()) { (*testSequenceEndCallback)( GenerateSequenceFailureReport(selectedTestJobs), GenerateSequenceFailureReport(discardedTestJobs), - timer.Elapsed()); + selectedDuraton, + discardedDuraton); } + UpdateAndSerializeDynamicDependencyMap(selectedTestJobs); return { selectedResult, discardedResult }; } @@ -530,14 +579,14 @@ namespace TestImpact globalTimeout, TestRunCompleteCallbackHandler(testCompleteCallback)); - ClearDynamicDependencyMapAndRemoveExistingFile(); - UpdateAndSerializeDynamicDependencyMap(testJobs); - if (testSequenceEndCallback.has_value()) { (*testSequenceEndCallback)(GenerateSequenceFailureReport(testJobs), timer.Elapsed()); } + ClearDynamicDependencyMapAndRemoveExistingFile(); + UpdateAndSerializeDynamicDependencyMap(testJobs); + return result; } diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.cpp index 47c0e83233..68482ab721 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.cpp @@ -82,54 +82,4 @@ namespace TestImpact return testNames; } - - SourceCoveringTestsList CreateSourceCoveringTestFromTestCoverages(const AZStd::vector& jobs, const RepoPath& root) - { - AZStd::unordered_map> coverage; - for (const auto& job : jobs) - { - if (const auto testResult = job.GetTestResult(); - testResult == Client::TestRunResult::AllTestsPass || testResult == Client::TestRunResult::TestFailures) - { - if (testResult == Client::TestRunResult::AllTestsPass) - { - // Passing tests should have coverage data, otherwise something is very wrong - AZ_TestImpact_Eval( - job.GetTestCoverge().has_value(), - RuntimeException, - AZStd::string::format( - "Test target '%s' completed its test run successfully but produced no coverage data", - job.GetTestTarget()->GetName().c_str())); - } - else if (!job.GetTestCoverge().has_value()) - { - // When a test run completes with failing tests but produces no coverage artifact that's typically a sign of the - // test aborting due to an unhandled exception, in which case ignore it and let it be picked up in the failure report - continue; - } - - for (const auto& source : job.GetTestCoverge().value().GetSourcesCovered()) - { - coverage[source.String()].insert(job.GetTestTarget()->GetName()); - } - } - } - - AZStd::vector sourceCoveringTests; - sourceCoveringTests.reserve(coverage.size()); - for (auto&& [source, testTargets] : coverage) - { - if (const auto sourcePath = RepoPath(source); - sourcePath.IsRelativeTo(root)) - { - sourceCoveringTests.push_back(SourceCoveringTests(RepoPath(sourcePath.LexicallyRelative(root)), AZStd::move(testTargets))); - } - else - { - AZ_Warning("TestImpact", false, "Ignoring source, source it outside of repo: %s", sourcePath.c_str()); - } - } - - return SourceCoveringTestsList(AZStd::move(sourceCoveringTests)); - } -} +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.h index 943b0cbec4..807911d854 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.h @@ -40,12 +40,7 @@ namespace TestImpact const AZStd::vector& excludedTestTargets); //! Extracts the name information from the specified test targets. - AZStd::vector ExtractTestTargetNames(const AZStd::vector testTargets); - - //! Creates the consolidates source covering tests list from the test engine instrumented run jobs. - SourceCoveringTestsList CreateSourceCoveringTestFromTestCoverages( - const AZStd::vector& jobs, - const RepoPath& root); + AZStd::vector ExtractTestTargetNames(const AZStd::vector testTargets); //! Generates a test run failure report from the specified test engine job information. //! @tparam TestJob The test engine job type. From f603b08d0542983c18c1b76cbed72aae88851f7f Mon Sep 17 00:00:00 2001 From: jonawals Date: Mon, 7 Jun 2021 18:37:20 +0100 Subject: [PATCH 072/102] Address PR comments --- .../Runtime/Code/Source/TestImpactRuntime.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp index 79f5269188..ef4bf5cd03 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp @@ -271,9 +271,15 @@ namespace TestImpact // Next we will update the coverage of test targets that completed (with or without failures), unless the failed test coverage // policy dictates we should instead discard the coverage of test targets with failing tests - if (const auto testResult = job.GetTestResult(); - testResult == Client::TestRunResult::AllTestsPass || - (m_failedTestCoveragePolicy == Policy::FailedTestCoverage::Keep && testResult == Client::TestRunResult::TestFailures)) + const auto testResult = job.GetTestResult(); + + if (m_failedTestCoveragePolicy == Policy::FailedTestCoverage::Discard && testResult == Client::TestRunResult::TestFailures) + { + // Discard the coverage for this job + continue; + } + + if (testResult == Client::TestRunResult::AllTestsPass || testResult == Client::TestRunResult::TestFailures) { if (testResult == Client::TestRunResult::AllTestsPass) { From a56591fa126862e58ec6ab5b384282d2acaea538 Mon Sep 17 00:00:00 2001 From: jonawals Date: Mon, 7 Jun 2021 18:38:54 +0100 Subject: [PATCH 073/102] Address PR comments --- .../Runtime/Code/Source/TestImpactRuntime.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp index ef4bf5cd03..784a46ffa0 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp @@ -291,7 +291,8 @@ namespace TestImpact "Test target '%s' completed its test run successfully but produced no coverage data", job.GetTestTarget()->GetName().c_str())); } - else if (!job.GetTestCoverge().has_value()) + + if (!job.GetTestCoverge().has_value()) { // When a test run completes with failing tests but produces no coverage artifact that's typically a sign of the // test aborting due to an unhandled exception, in which case ignore it and let it be picked up in the failure report From e0f953cf6bf787b979c59bf5607978b91c504c11 Mon Sep 17 00:00:00 2001 From: jonawals Date: Tue, 8 Jun 2021 11:59:06 +0100 Subject: [PATCH 074/102] Address PR comments --- .../Source/TestImpactCommandLineOptions.cpp | 133 +++++++++--------- .../TestImpactFramework/TestImpactRuntime.h | 2 +- 2 files changed, 69 insertions(+), 66 deletions(-) diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp index 992b466a54..0a4c80afa4 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp @@ -381,71 +381,74 @@ namespace TestImpact AZStd::string help = "usage: tiaf [options]\n" " options:\n" - " -config= Path to the configuration file for the TIAF runtime (default: \n" - " ..json).\n" - " -changelist= Path to the JSON of source file changes to perform test impact \n" - " analysis on.\n" - " -gtimeout= Global timeout value to terminate the entire test sequence should it \n" - " be exceeded.\n" - " -ttimeout= Timeout value to terminate individual test targets should it be \n" - " exceeded.\n" - " -sequence= The type of test sequence to perform, where none runs no tests and\n" - " will report a all tests successful, seed removes any prior coverage \n" - " data and runs all test targets with instrumentation to reseed the \n" - " data from scratch, regular runs all of the test targets without any \n" - " instrumentation to generate coverage data(any prior coverage data is \n" - " left intact), tia uses any prior coverage data to run the instrumented \n" - " subset of selected tests(if no prior coverage data a regular run is \n" - " performed instead) and tiaorseed uses any prior coverage data to run \n" - " the instrumented subset of selected tests(if no prior coverage data a \n" - " seed run is performed instead).\n" - " -safemode= Flag to specify a safe mode sequence where the set of unselected \n" - " tests is run without instrumentation after the set of selected \n" - " instrumented tests is run (this has the effect of ensuring all \n" - " tests are run regardless).\n" - " -shard= Break any test targets with a sharding policy into the number of \n" - " shards according to the maximum concurrency value.\n" - " -cpolicy= Policy for handling the coverage data of failing tests, where discard \n" - " will discard the coverage data produced by the failing tests, causing \n" - " them to be drafted into future test runs and keep will keep any existing \n" - " coverage data and update the coverage data for failed tests that produce \n" - " coverage.\n" - " -targetout= Capture of individual test run stdout, where stdout will capture \n" - " each individual test target's stdout and output each one to stdout \n" - " and file will capture each individual test target's stdout and output \n" - " each one individually to a file (multiple values are accepted).\n" - " -epolicy= Policy for handling test execution failure (test targets could not be \n" - " launched due to the binary not being built, incorrect paths, etc.), \n" - " where abort will abort the entire test sequence upon the first test\n" - " target execution failure and report a failure(along with the return \n" - " code of the test target that failed to launch), continue will continue \n" - " with the test sequence in the event of test target execution failures\n" - " and treat the test targets that failed to launch as as test failures\n" - " (along with the return codes of the test targets that failed to \n" - " launch), ignore will continue with the test sequence in the event of \n" - " test target execution failures and treat the test targets that failed\n" - " to launch as test passes(along with the return codes of the test \n" - " targets that failed to launch).\n" - " -fpolicy Policy for handling test failures (test targets report failing tests), \n" - " where abort will abort the entire test sequence upon the first test \n" - " failure and report a failure and continue will continue with the test\n" - " sequence in the event of test failures and report the test failures.\n" - " -ipolicy= Policy for handling coverage data integrity failures, where abort will \n" - " abort the test sequenceand report a failure, seed will attempt another \n" - " sequence using the seed sequence type, otherwise will abort and report \n" - " a failure (this option has no effect for regular and seed sequence \n" - " types) and rerun will attempt another sequence using the regular \n" - " sequence type, otherwise will abort and report a failure(this option has \n" - " no effect for regular sequence type).\n" - " -ppolicy= Policy for prioritizing selected test targets, where none will not \n" - " attempt any test target prioritization and locality will attempt to \n" - " prioritize test targets according to the locality of their covering \n" - " production targets in the dependency graph(if no dependency graph data \n" - " available, no prioritization will occur).\n" - " -maxconcurrency= The maximum number of concurrent test targets/shards to be in flight at \n" - " any given moment.\n" - " -ochangelist= Outputs the change list used for test selection.\n" - " -suite= The test suite to select from for this test sequence."; + " -config= Path to the configuration file for the TIAF runtime (default: \n" + " ..json).\n" + " -changelist= Path to the JSON of source file changes to perform test impact \n" + " analysis on.\n" + " -gtimeout= Global timeout value to terminate the entire test sequence should it \n" + " be exceeded.\n" + " -ttimeout= Timeout value to terminate individual test targets should it be \n" + " exceeded.\n" + " -sequence= The type of test sequence to perform, where 'none' runs no tests and\n" + " will report a all tests successful, 'seed' removes any prior coverage \n" + " data and runs all test targets with instrumentation to reseed the \n" + " data from scratch, 'regular' runs all of the test targets without any \n" + " instrumentation to generate coverage data(any prior coverage data is \n" + " left intact), 'tia' uses any prior coverage data to run the instrumented \n" + " subset of selected tests(if no prior coverage data a regular run is \n" + " performed instead), 'tianowrite' uses any prior coverage data to run the \n" + " uninstrumented subset of selected tests (if no prior coverage data a \n" + " regular run is performed instead). The coverage data is not updated with \n" + " the subset of selected tests and 'tiaorseed' uses any prior coverage data \n" + " to run the instrumented subset of selected tests (if no prior coverage \n" + " data a seed run is performed instead).\n" + " -safemode= Flag to specify a safe mode sequence where the set of unselected \n" + " tests is run without instrumentation after the set of selected \n" + " instrumented tests is run (this has the effect of ensuring all \n" + " tests are run regardless).\n" + " -shard= Break any test targets with a sharding policy into the number of \n" + " shards according to the maximum concurrency value.\n" + " -cpolicy= Policy for handling the coverage data of failing tests, where 'discard' \n" + " will discard the coverage data produced by the failing tests, causing \n" + " them to be drafted into future test runs and 'keep' will keep any existing \n" + " coverage data and update the coverage data for failed tests that produce \n" + " coverage.\n" + " -targetout= Capture of individual test run stdout, where 'stdout' will capture \n" + " each individual test target's stdout and output each one to stdout \n" + " and 'file' will capture each individual test target's stdout and output \n" + " each one individually to a file (multiple values are accepted).\n" + " -epolicy= Policy for handling test execution failure (test targets could not be \n" + " launched due to the binary not being built, incorrect paths, etc.), \n" + " where 'abort' will abort the entire test sequence upon the first test\n" + " target execution failure and report a failure(along with the return \n" + " code of the test target that failed to launch), 'continue' will continue \n" + " with the test sequence in the event of test target execution failures\n" + " and treat the test targets that failed to launch as test failures\n" + " (along with the return codes of the test targets that failed to \n" + " launch), 'ignore' will continue with the test sequence in the event of \n" + " test target execution failures and treat the test targets that failed\n" + " to launch as test passes(along with the return codes of the test \n" + " targets that failed to launch).\n" + " -fpolicy Policy for handling test failures (test targets report failing tests), \n" + " where 'abort' will abort the entire test sequence upon the first test \n" + " failure and report a failure and 'continue' will continue with the test\n" + " sequence in the event of test failures and report the test failures.\n" + " -ipolicy= Policy for handling coverage data integrity failures, where 'abort' will \n" + " abort the test sequence and report a failure, 'seed' will attempt another \n" + " sequence using the seed sequence type, otherwise will abort and report \n" + " a failure (this option has no effect for regular and seed sequence \n" + " types) and 'rerun' will attempt another sequence using the regular \n" + " sequence type, otherwise will abort and report a failure(this option has \n" + " no effect for regular sequence type).\n" + " -ppolicy= Policy for prioritizing selected test targets, where 'none' will not \n" + " attempt any test target prioritization and 'locality' will attempt to \n" + " prioritize test targets according to the locality of their covering \n" + " production targets in the dependency graph(if no dependency graph data \n" + " available, no prioritization will occur).\n" + " -maxconcurrency= The maximum number of concurrent test targets/shards to be in flight at \n" + " any given moment.\n" + " -ochangelist= Outputs the change list used for test selection.\n" + " -suite= The test suite to select from for this test sequence."; return help; } diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h index 2a7720ee55..485b966a1e 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h @@ -202,7 +202,7 @@ namespace TestImpact AZStd::pair, AZStd::vector> SelectTestTargetsByExcludeList( AZStd::vector testTargets) const; - //! Prunes the existing coverage for the specified jobs and creates the consolidates source covering tests list from the + //! Prunes the existing coverage for the specified jobs and creates the consolidated source covering tests list from the //! test engine instrumented run jobs. SourceCoveringTestsList CreateSourceCoveringTestFromTestCoverages(const AZStd::vector& jobs); From c9351f4d45714bbb6b8432ed26fb3e4286d84f58 Mon Sep 17 00:00:00 2001 From: jonawals Date: Tue, 8 Jun 2021 12:05:38 +0100 Subject: [PATCH 075/102] Make TestEngine clean up artifact dir before runs --- .../Code/Source/TestEngine/TestImpactTestEngine.cpp | 12 ++++++++++++ .../Code/Source/TestEngine/TestImpactTestEngine.h | 4 ++++ 2 files changed, 16 insertions(+) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.cpp index b563c1846b..f3ce777938 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.cpp @@ -10,6 +10,8 @@ * */ +#include + #include #include #include @@ -247,11 +249,17 @@ namespace TestImpact , m_testEnumerator(AZStd::make_unique(maxConcurrentRuns)) , m_instrumentedTestRunner(AZStd::make_unique(maxConcurrentRuns)) , m_testRunner(AZStd::make_unique(maxConcurrentRuns)) + , m_artifactDir(artifactDir) { } TestEngine::~TestEngine() = default; + void TestEngine::CleanArtifactDir() const + { + DeleteFiles(m_artifactDir, "*xml*"); + } + AZStd::pair> TestEngine::UpdateEnumerationCache( const AZStd::vector& testTargets, Policy::ExecutionFailure executionFailurePolicy, @@ -283,6 +291,8 @@ namespace TestImpact AZStd::optional globalTimeout, AZStd::optional callback) { + CleanArtifactDir(); + TestEngineJobMap engineJobs; const auto jobInfos = m_testJobInfoGenerator->GenerateRegularTestRunJobInfos(testTargets); @@ -308,6 +318,8 @@ namespace TestImpact AZStd::optional globalTimeout, AZStd::optional callback) { + CleanArtifactDir(); + TestEngineJobMap engineJobs; const auto jobInfos = m_testJobInfoGenerator->GenerateInstrumentedTestRunJobInfos(testTargets, CoverageLevel::Source); diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.h index 83f57f00ef..5192a1d3b7 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.h @@ -118,10 +118,14 @@ namespace TestImpact AZStd::optional callback); private: + //! Cleans up the artifacts directory of any artifacts from previous runs. + void CleanArtifactDir() const; + size_t m_maxConcurrentRuns = 0; AZStd::unique_ptr m_testJobInfoGenerator; AZStd::unique_ptr m_testEnumerator; AZStd::unique_ptr m_instrumentedTestRunner; AZStd::unique_ptr m_testRunner; + RepoPath m_artifactDir; }; } // namespace TestImpact From 9e187d67a4f272bd4e9bdb928fc02c48ab172a53 Mon Sep 17 00:00:00 2001 From: jonawals Date: Tue, 8 Jun 2021 12:08:42 +0100 Subject: [PATCH 076/102] Fix typo with delete files filter --- .../Runtime/Code/Source/TestEngine/TestImpactTestEngine.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.cpp index f3ce777938..4c7d660004 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.cpp @@ -257,7 +257,7 @@ namespace TestImpact void TestEngine::CleanArtifactDir() const { - DeleteFiles(m_artifactDir, "*xml*"); + DeleteFiles(m_artifactDir, "*.xml"); } AZStd::pair> TestEngine::UpdateEnumerationCache( From 7a884c85b37d7bfa810b53d7a9720fcc5e3f5d9c Mon Sep 17 00:00:00 2001 From: jonawals Date: Tue, 8 Jun 2021 12:19:07 +0100 Subject: [PATCH 077/102] Address PR comments --- .../Runtime/Code/Source/TestEngine/TestImpactTestEngine.cpp | 6 +++--- .../Runtime/Code/Source/TestEngine/TestImpactTestEngine.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.cpp index 4c7d660004..a9d0e15781 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.cpp @@ -255,7 +255,7 @@ namespace TestImpact TestEngine::~TestEngine() = default; - void TestEngine::CleanArtifactDir() const + void TestEngine::DeleteArtifactXmls() const { DeleteFiles(m_artifactDir, "*.xml"); } @@ -291,7 +291,7 @@ namespace TestImpact AZStd::optional globalTimeout, AZStd::optional callback) { - CleanArtifactDir(); + DeleteArtifactXmls(); TestEngineJobMap engineJobs; const auto jobInfos = m_testJobInfoGenerator->GenerateRegularTestRunJobInfos(testTargets); @@ -318,7 +318,7 @@ namespace TestImpact AZStd::optional globalTimeout, AZStd::optional callback) { - CleanArtifactDir(); + DeleteArtifactXmls(); TestEngineJobMap engineJobs; const auto jobInfos = m_testJobInfoGenerator->GenerateInstrumentedTestRunJobInfos(testTargets, CoverageLevel::Source); diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.h index 5192a1d3b7..7d16f352f3 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.h @@ -119,7 +119,7 @@ namespace TestImpact private: //! Cleans up the artifacts directory of any artifacts from previous runs. - void CleanArtifactDir() const; + void DeleteArtifactXmls() const; size_t m_maxConcurrentRuns = 0; AZStd::unique_ptr m_testJobInfoGenerator; From 656aa528d86d8a561d12b22d048ced7b1a40273e Mon Sep 17 00:00:00 2001 From: jonawals Date: Tue, 8 Jun 2021 13:39:37 +0100 Subject: [PATCH 078/102] Implement new handling of tiaf seed data --- .../Frontend/Console/CMakeLists.txt | 13 +++++ .../Console/Executable/CMakeLists.txt | 12 +++++ .../Console/Executable/Code/CMakeLists.txt | 25 ++++++++++ ...pactframework_frontend_console_files.cmake | 14 ++++++ ...amework_frontend_console_tests_files.cmake | 0 .../Frontend/Console/Static/CMakeLists.txt | 12 +++++ .../Console/Static/Code/CMakeLists.txt | 50 +++++++++++++++++++ ...mework_frontend_console_static_files.cmake | 26 ++++++++++ ..._frontend_console_static_tests_files.cmake | 14 ++++++ .../testimpactframework_runtime_files.cmake | 2 - ...timpactframework_runtime_tests_files.cmake | 13 +++++ .../ConsoleFrontendConfig.in | 7 +++ .../LYTestImpactFramework.cmake | 9 +++- scripts/build/Jenkins/Jenkinsfile | 2 +- .../build/Platform/Windows/build_config.json | 22 +++----- 15 files changed, 201 insertions(+), 20 deletions(-) create mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/CMakeLists.txt create mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/Executable/CMakeLists.txt create mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/Executable/Code/CMakeLists.txt create mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/Executable/Code/testimpactframework_frontend_console_files.cmake create mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/Executable/Code/testimpactframework_frontend_console_tests_files.cmake create mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/Static/CMakeLists.txt create mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/CMakeLists.txt create mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/testimpactframework_frontend_console_static_files.cmake create mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/testimpactframework_frontend_console_static_tests_files.cmake create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_tests_files.cmake diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/CMakeLists.txt b/Code/Tools/TestImpactFramework/Frontend/Console/CMakeLists.txt new file mode 100644 index 0000000000..a25d506156 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Frontend/Console/CMakeLists.txt @@ -0,0 +1,13 @@ +# +# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +# its licensors. +# +# For complete copyright and license terms please see the LICENSE at the root of this +# distribution (the "License"). All use of this software is governed by the License, +# or, if provided, by the license below or the license accompanying this file. Do not +# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# + +add_subdirectory(Executable) +add_subdirectory(Static) \ No newline at end of file diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Executable/CMakeLists.txt b/Code/Tools/TestImpactFramework/Frontend/Console/Executable/CMakeLists.txt new file mode 100644 index 0000000000..8298bb7123 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Executable/CMakeLists.txt @@ -0,0 +1,12 @@ +# +# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +# its licensors. +# +# For complete copyright and license terms please see the LICENSE at the root of this +# distribution (the "License"). All use of this software is governed by the License, +# or, if provided, by the license below or the license accompanying this file. Do not +# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# + +add_subdirectory(Code) \ No newline at end of file diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Executable/Code/CMakeLists.txt b/Code/Tools/TestImpactFramework/Frontend/Console/Executable/Code/CMakeLists.txt new file mode 100644 index 0000000000..88da472e1e --- /dev/null +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Executable/Code/CMakeLists.txt @@ -0,0 +1,25 @@ +# +# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +# its licensors. +# +# For complete copyright and license terms please see the LICENSE at the root of this +# distribution (the "License"). All use of this software is governed by the License, +# or, if provided, by the license below or the license accompanying this file. Do not +# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# + +ly_add_target( + NAME TestImpact.Frontend.Console EXECUTABLE + OUTPUT_NAME tiaf + NAMESPACE AZ + FILES_CMAKE + testimpactframework_frontend_console_files.cmake + INCLUDE_DIRECTORIES + PRIVATE + Source + BUILD_DEPENDENCIES + PRIVATE + AZ::TestImpact.Frontend.Console.Static +) + diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Executable/Code/testimpactframework_frontend_console_files.cmake b/Code/Tools/TestImpactFramework/Frontend/Console/Executable/Code/testimpactframework_frontend_console_files.cmake new file mode 100644 index 0000000000..19e99649a7 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Executable/Code/testimpactframework_frontend_console_files.cmake @@ -0,0 +1,14 @@ +# +# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +# its licensors. +# +# For complete copyright and license terms please see the LICENSE at the root of this +# distribution (the "License"). All use of this software is governed by the License, +# or, if provided, by the license below or the license accompanying this file. Do not +# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# + +set(FILES + Source/TestImpactConsole.cpp +) diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Executable/Code/testimpactframework_frontend_console_tests_files.cmake b/Code/Tools/TestImpactFramework/Frontend/Console/Executable/Code/testimpactframework_frontend_console_tests_files.cmake new file mode 100644 index 0000000000..e69de29bb2 diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/CMakeLists.txt b/Code/Tools/TestImpactFramework/Frontend/Console/Static/CMakeLists.txt new file mode 100644 index 0000000000..8298bb7123 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/CMakeLists.txt @@ -0,0 +1,12 @@ +# +# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +# its licensors. +# +# For complete copyright and license terms please see the LICENSE at the root of this +# distribution (the "License"). All use of this software is governed by the License, +# or, if provided, by the license below or the license accompanying this file. Do not +# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# + +add_subdirectory(Code) \ No newline at end of file diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/CMakeLists.txt b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/CMakeLists.txt new file mode 100644 index 0000000000..2302cb5484 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/CMakeLists.txt @@ -0,0 +1,50 @@ +# +# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +# its licensors. +# +# For complete copyright and license terms please see the LICENSE at the root of this +# distribution (the "License"). All use of this software is governed by the License, +# or, if provided, by the license below or the license accompanying this file. Do not +# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# + +ly_add_target( + NAME TestImpact.Frontend.Console.Static STATIC + NAMESPACE AZ + FILES_CMAKE + testimpactframework_frontend_console_static_files.cmake + INCLUDE_DIRECTORIES + PUBLIC + Include + PRIVATE + Source + BUILD_DEPENDENCIES + PUBLIC + AZ::TestImpact.Runtime.Static +) + +################################################################################ +# Tests +################################################################################ + +ly_add_target( + NAME TestImpact.Frontend.Console.Static.Tests ${PAL_TRAIT_TEST_TARGET_TYPE} + NAMESPACE AZ + FILES_CMAKE + testimpactframework_frontend_console_static_tests_files.cmake + INCLUDE_DIRECTORIES + PRIVATE + Include + Source + Tests + BUILD_DEPENDENCIES + PRIVATE + AZ::AzTestShared + AZ::AzTest + AZ::TestImpact.Frontend.Console.Static +) + +ly_add_googletest( + NAME AZ::TestImpact.Frontend.Console.Static.Tests +) diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/testimpactframework_frontend_console_static_files.cmake b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/testimpactframework_frontend_console_static_files.cmake new file mode 100644 index 0000000000..97081dc08c --- /dev/null +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/testimpactframework_frontend_console_static_files.cmake @@ -0,0 +1,26 @@ +# +# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +# its licensors. +# +# For complete copyright and license terms please see the LICENSE at the root of this +# distribution (the "License"). All use of this software is governed by the License, +# or, if provided, by the license below or the license accompanying this file. Do not +# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# + +set(FILES + Include/TestImpactFramework/TestImpactConsoleMain.h + Source/TestImpactCommandLineOptions.h + Source/TestImpactCommandLineOptions.cpp + Source/TestImpactCommandLineOptionsUtils.cpp + Source/TestImpactCommandLineOptionsUtils.h + Source/TestImpactCommandLineOptionsException.h + Source/TestImpactRuntimeConfigurationFactory.h + Source/TestImpactRuntimeConfigurationFactory.cpp + Source/TestImpactConsoleMain.cpp + Source/TestImpactConsoleTestSequenceEventHandler.cpp + Source/TestImpactConsoleTestSequenceEventHandler.h + Source/TestImpactConsoleUtils.cpp + Source/TestImpactConsoleUtils.h +) diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/testimpactframework_frontend_console_static_tests_files.cmake b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/testimpactframework_frontend_console_static_tests_files.cmake new file mode 100644 index 0000000000..4e45f540ad --- /dev/null +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/testimpactframework_frontend_console_static_tests_files.cmake @@ -0,0 +1,14 @@ +# +# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +# its licensors. +# +# For complete copyright and license terms please see the LICENSE at the root of this +# distribution (the "License"). All use of this software is governed by the License, +# or, if provided, by the license below or the license accompanying this file. Do not +# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# + +set(FILES + +) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_files.cmake b/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_files.cmake index 2053a1ea5e..250894464b 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_files.cmake +++ b/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_files.cmake @@ -35,8 +35,6 @@ set(FILES Source/Artifact/Factory/TestImpactTestTargetMetaMapFactory.h Source/Artifact/Factory/TestImpactModuleCoverageFactory.cpp Source/Artifact/Factory/TestImpactModuleCoverageFactory.h - Source/Artifact/Factory/TestImpactDependencyGraphDataFactory.cpp - Source/Artifact/Factory/TestImpactDependencyGraphDataFactory.h Source/Artifact/Static/TestImpactBuildTargetDescriptor.cpp Source/Artifact/Static/TestImpactBuildTargetDescriptor.h Source/Artifact/Static/TestImpactTargetDescriptorCompiler.cpp diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_tests_files.cmake b/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_tests_files.cmake new file mode 100644 index 0000000000..5714be5dfb --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_tests_files.cmake @@ -0,0 +1,13 @@ +# +# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +# its licensors. +# +# For complete copyright and license terms please see the LICENSE at the root of this +# distribution (the "License"). All use of this software is governed by the License, +# or, if provided, by the license below or the license accompanying this file. Do not +# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# + +set(FILES +) diff --git a/cmake/TestImpactFramework/ConsoleFrontendConfig.in b/cmake/TestImpactFramework/ConsoleFrontendConfig.in index 357499771c..9338672fbd 100644 --- a/cmake/TestImpactFramework/ConsoleFrontendConfig.in +++ b/cmake/TestImpactFramework/ConsoleFrontendConfig.in @@ -3,6 +3,13 @@ "platform": "${platform}", "timestamp": "${timestamp}" }, + "jenkins": { + "pipeline_of_truth" : [ + "nightly-incremental", + "nightly-clean" + ], + "use_test_impact_analysis": ${use_tiaf} + }, "repo": { "root": "${repo_dir}", "tiaf_bin": "${tiaf_bin}" diff --git a/cmake/TestImpactFramework/LYTestImpactFramework.cmake b/cmake/TestImpactFramework/LYTestImpactFramework.cmake index 54dadbd8ad..661a066477 100644 --- a/cmake/TestImpactFramework/LYTestImpactFramework.cmake +++ b/cmake/TestImpactFramework/LYTestImpactFramework.cmake @@ -344,9 +344,14 @@ function(ly_test_impact_write_config_file CONFIG_TEMPLATE_FILE PERSISTENT_DATA_D # Instrumentation binary if(NOT LY_TEST_IMPACT_INSTRUMENTATION_BIN) - message(FATAL_ERROR "No test impact framework instrumentation binary was specified, please provide the path with option LY_TEST_IMPACT_INSTRUMENTATION_BIN") + # No binary specified is not an error, it just means that the test impact analysis part of the framework is disabled + message("No test impact framework instrumentation binary was specified, test impact analysis framework will fall back to regular test sequences instead") + set(use_tiaf false) + set(instrumentation_bin "") + else() + set(use_tiaf true) + file(TO_CMAKE_PATH ${LY_TEST_IMPACT_INSTRUMENTATION_BIN} instrumentation_bin) endif() - file(TO_CMAKE_PATH ${LY_TEST_IMPACT_INSTRUMENTATION_BIN} instrumentation_bin) # Testrunner binary set(test_runner_bin $) diff --git a/scripts/build/Jenkins/Jenkinsfile b/scripts/build/Jenkins/Jenkinsfile index 2f6be2b060..5c11efe5a8 100644 --- a/scripts/build/Jenkins/Jenkinsfile +++ b/scripts/build/Jenkins/Jenkinsfile @@ -447,7 +447,7 @@ try { // repositoryName is the full repository name repositoryName = (repositoryUrl =~ /https:\/\/github.com\/(.*)\.git/)[0][1] (projectName, pipelineName) = GetRunningPipelineName(env.JOB_NAME) // env.JOB_NAME is the name of the job given by Jenkins - + env.PIPELINE_NAME = pipelineName if(env.BRANCH_NAME) { branchName = env.BRANCH_NAME } else { diff --git a/scripts/build/Platform/Windows/build_config.json b/scripts/build/Platform/Windows/build_config.json index 5dede68d92..be729c2b7f 100644 --- a/scripts/build/Platform/Windows/build_config.json +++ b/scripts/build/Platform/Windows/build_config.json @@ -22,16 +22,17 @@ ], "steps": [ "debug_vs2019", - "test_impact_seed", "test_debug_vs2019" ] }, "profile_vs2019_pipe": { "TAGS": [ - "default" + "default", + "nightly-incremental", + "nightly-clean" ], "steps": [ - "profile_vs2019", + "profile_vs2019", "test_impact_analysis", "asset_profile_vs2019", "test_cpu_profile_vs2019" @@ -81,22 +82,13 @@ "SCRIPT_PARAMETERS": "--platform 3rdParty --type 3rdParty_all" } }, - "test_impact_seed": { - "TAGS": [ - ], - "COMMAND": "python_windows.cmd", - "PARAMETERS": { - "SCRIPT_PATH": "scripts/build/TestImpactAnalysis/tiaf_driver.py", - "SCRIPT_PARAMETERS": "--sequenceType seed --config \"build\\windows_vs2019\\bin\\TestImpactFramework\\persistent\\tiaf.debug.json\"" - } - }, "test_impact_analysis": { "TAGS": [ ], "COMMAND": "python_windows.cmd", "PARAMETERS": { "SCRIPT_PATH": "scripts/build/TestImpactAnalysis/tiaf_driver.py", - "SCRIPT_PARAMETERS": "--sequenceType tia --destCommit !CHANGE_ID! --config \"build\\windows_vs2019\\bin\\TestImpactFramework\\persistent\\tiaf.profile.json\"" + "SCRIPT_PARAMETERS": "--safeMode --suite main --pipeline !PIPELINE_NAME! --destCommit !CHANGE_ID! --config \"build\\windows_vs2019\\bin\\TestImpactFramework\\persistent\\tiaf.profile.json\"" } }, "debug_vs2019": { @@ -107,7 +99,7 @@ "PARAMETERS": { "CONFIGURATION": "debug", "OUTPUT_DIRECTORY": "build\\windows_vs2019", - "CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE -DLY_BUILD_WITH_INCREMENTAL_LINKING_DEBUG=FALSE -DLY_TEST_IMPACT_ACTIVE=1 -DLY_TEST_IMPACT_INSTRUMENTATION_BIN=\"c:\\ly\\3rdParty\\ackages\\OpenCppCoverage\\Binary\\windows-x64\\OpenCppCoverage.exe\"", + "CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE -DLY_BUILD_WITH_INCREMENTAL_LINKING_DEBUG=FALSE", "CMAKE_LY_PROJECTS": "AutomatedTesting", "CMAKE_TARGET": "ALL_BUILD", "CMAKE_NATIVE_BUILD_ARGS": "/m /nologo" @@ -139,7 +131,7 @@ "PARAMETERS": { "CONFIGURATION": "profile", "OUTPUT_DIRECTORY": "build\\windows_vs2019", - "CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE -DLY_TEST_IMPACT_ACTIVE=1 -DLY_TEST_IMPACT_INSTRUMENTATION_BIN=\"c:\\ly\\3rdParty\\ackages\\OpenCppCoverage\\Binary\\windows-x64\\OpenCppCoverage.exe\"", + "CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE -DLY_TEST_IMPACT_ACTIVE=1 -DLY_TEST_IMPACT_INSTRUMENTATION_BIN=!TEST_IMPACT_WIN_BINARY!", "CMAKE_LY_PROJECTS": "AutomatedTesting", "CMAKE_TARGET": "ALL_BUILD", "CMAKE_NATIVE_BUILD_ARGS": "/m /nologo" From 4b819953d65763f7e7e9fd07276945c455c6981f Mon Sep 17 00:00:00 2001 From: jonawals Date: Tue, 8 Jun 2021 14:41:18 +0100 Subject: [PATCH 079/102] Fix build script --- scripts/build/Platform/Windows/build_config.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/build/Platform/Windows/build_config.json b/scripts/build/Platform/Windows/build_config.json index d45bac0363..d94f34d9e0 100644 --- a/scripts/build/Platform/Windows/build_config.json +++ b/scripts/build/Platform/Windows/build_config.json @@ -84,6 +84,9 @@ }, "test_impact_analysis": { "TAGS": [ + "nightly-incremental", + "nightly-clean", + "default" ], "COMMAND": "python_windows.cmd", "PARAMETERS": { From 7a38a93b9c127c41b36fe7ead4d4521b1122ea89 Mon Sep 17 00:00:00 2001 From: jonawals Date: Tue, 8 Jun 2021 14:47:57 +0100 Subject: [PATCH 080/102] Remove test_impact_analysis --- scripts/build/Platform/Windows/build_config.json | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/scripts/build/Platform/Windows/build_config.json b/scripts/build/Platform/Windows/build_config.json index d94f34d9e0..bb4a0decbf 100644 --- a/scripts/build/Platform/Windows/build_config.json +++ b/scripts/build/Platform/Windows/build_config.json @@ -32,8 +32,7 @@ "nightly-clean" ], "steps": [ - "profile_vs2019", - "test_impact_analysis", + "profile_vs2019", "asset_profile_vs2019", "test_cpu_profile_vs2019" ] @@ -84,9 +83,6 @@ }, "test_impact_analysis": { "TAGS": [ - "nightly-incremental", - "nightly-clean", - "default" ], "COMMAND": "python_windows.cmd", "PARAMETERS": { From 98670a21bc8de75e6fc5f406f8e40032aa8afd9c Mon Sep 17 00:00:00 2001 From: jonawals Date: Tue, 8 Jun 2021 15:42:49 +0100 Subject: [PATCH 081/102] Fix missing copyright header --- .../Code/testimpactframework_frontend_console_tests_files.cmake | 0 ...testimpactframework_frontend_console_static_tests_files.cmake | 1 - 2 files changed, 1 deletion(-) delete mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/Executable/Code/testimpactframework_frontend_console_tests_files.cmake diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Executable/Code/testimpactframework_frontend_console_tests_files.cmake b/Code/Tools/TestImpactFramework/Frontend/Console/Executable/Code/testimpactframework_frontend_console_tests_files.cmake deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/testimpactframework_frontend_console_static_tests_files.cmake b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/testimpactframework_frontend_console_static_tests_files.cmake index 4e45f540ad..5714be5dfb 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/testimpactframework_frontend_console_static_tests_files.cmake +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/testimpactframework_frontend_console_static_tests_files.cmake @@ -10,5 +10,4 @@ # set(FILES - ) From a9e6eaa57a7cdfc05b096a646b5cf2540cc2981a Mon Sep 17 00:00:00 2001 From: jonawals Date: Tue, 8 Jun 2021 16:43:02 +0100 Subject: [PATCH 082/102] Fix anon enum clash with uber files --- .../Source/TestImpactCommandLineOptions.cpp | 62 +++++++++---------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp index 0a4c80afa4..9a05fc6fc9 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp @@ -22,22 +22,22 @@ namespace TestImpact enum { // Options - Config, - ChangeList, - OutputChangeList, - Sequence, - TestPrioritizationPolicy, - ExecutionFailurePolicy, - FailedTestCoveragePolicy, - TestFailurePolicy, - IntegrityFailurePolicy, - TestShardingPolicy, - TargetOutputCapture, - MaxConcurrency, - TestTargetTimeout, - GlobalTimeout, - SuiteFilter, - SafeMode, + ConfigKey, + ChangeListKey, + OutputChangeListKey, + SequenceKey, + TestPrioritizationPolicyKey, + ExecutionFailurePolicyKey, + FailedTestCoveragePolicyKey, + TestFailurePolicyKey, + IntegrityFailurePolicyKey, + TestShardingPolicyKey, + TargetOutputCaptureKey, + MaxConcurrencyKey, + TestTargetTimeoutKey, + GlobalTimeoutKey, + SuiteFilterKey, + SafeModeKey, // Values None, Seed, @@ -93,17 +93,17 @@ namespace TestImpact RepoPath ParseConfigurationFile(const AZ::CommandLine& cmd) { - return ParsePathOption(OptionKeys[Config], cmd).value_or(LY_TEST_IMPACT_DEFAULT_CONFIG_FILE); + return ParsePathOption(OptionKeys[ConfigKey], cmd).value_or(LY_TEST_IMPACT_DEFAULT_CONFIG_FILE); } AZStd::optional ParseChangeListFile(const AZ::CommandLine& cmd) { - return ParsePathOption(OptionKeys[ChangeList], cmd); + return ParsePathOption(OptionKeys[ChangeListKey], cmd); } bool ParseOutputChangeList(const AZ::CommandLine& cmd) { - return ParseOnOffOption(OptionKeys[OutputChangeList], BinaryStateValue{ false, true }, cmd).value_or(false); + return ParseOnOffOption(OptionKeys[OutputChangeListKey], BinaryStateValue{ false, true }, cmd).value_or(false); } TestSequenceType ParseTestSequenceType(const AZ::CommandLine& cmd) @@ -118,7 +118,7 @@ namespace TestImpact {OptionKeys[ImpactAnalysisOrSeed], TestSequenceType::ImpactAnalysisOrSeed} }; - return ParseMultiStateOption(OptionKeys[Sequence], states, cmd).value_or(TestSequenceType::None); + return ParseMultiStateOption(OptionKeys[SequenceKey], states, cmd).value_or(TestSequenceType::None); } Policy::TestPrioritization ParseTestPrioritizationPolicy(const AZ::CommandLine& cmd) @@ -129,7 +129,7 @@ namespace TestImpact {OptionKeys[Locality], Policy::TestPrioritization::DependencyLocality} }; - return ParseBinaryStateOption(OptionKeys[TestPrioritizationPolicy], states, cmd).value_or(Policy::TestPrioritization::None); + return ParseBinaryStateOption(OptionKeys[TestPrioritizationPolicyKey], states, cmd).value_or(Policy::TestPrioritization::None); } Policy::ExecutionFailure ParseExecutionFailurePolicy(const AZ::CommandLine& cmd) @@ -140,7 +140,7 @@ namespace TestImpact {OptionKeys[Continue], Policy::ExecutionFailure::Continue}, {OptionKeys[Ignore], Policy::ExecutionFailure::Ignore} }; - return ParseMultiStateOption(OptionKeys[ExecutionFailurePolicy], states, cmd).value_or(Policy::ExecutionFailure::Continue); + return ParseMultiStateOption(OptionKeys[ExecutionFailurePolicyKey], states, cmd).value_or(Policy::ExecutionFailure::Continue); } Policy::FailedTestCoverage ParseFailedTestCoveragePolicy(const AZ::CommandLine& cmd) @@ -151,7 +151,7 @@ namespace TestImpact {OptionKeys[Keep], Policy::FailedTestCoverage::Keep} }; - return ParseMultiStateOption(OptionKeys[FailedTestCoveragePolicy], states, cmd).value_or(Policy::FailedTestCoverage::Keep); + return ParseMultiStateOption(OptionKeys[FailedTestCoveragePolicyKey], states, cmd).value_or(Policy::FailedTestCoverage::Keep); } Policy::TestFailure ParseTestFailurePolicy(const AZ::CommandLine& cmd) @@ -162,7 +162,7 @@ namespace TestImpact Policy::TestFailure::Continue }; - return ParseAbortContinueOption(OptionKeys[TestFailurePolicy], states, cmd).value_or(Policy::TestFailure::Abort); + return ParseAbortContinueOption(OptionKeys[TestFailurePolicyKey], states, cmd).value_or(Policy::TestFailure::Abort); } Policy::IntegrityFailure ParseIntegrityFailurePolicy(const AZ::CommandLine& cmd) @@ -173,7 +173,7 @@ namespace TestImpact Policy::IntegrityFailure::Continue }; - return ParseAbortContinueOption(OptionKeys[IntegrityFailurePolicy], states, cmd).value_or(Policy::IntegrityFailure::Abort); + return ParseAbortContinueOption(OptionKeys[IntegrityFailurePolicyKey], states, cmd).value_or(Policy::IntegrityFailure::Abort); } Policy::TestSharding ParseTestShardingPolicy(const AZ::CommandLine& cmd) @@ -189,7 +189,7 @@ namespace TestImpact Policy::TargetOutputCapture ParseTargetOutputCapture(const AZ::CommandLine& cmd) { - if (const auto numSwitchValues = cmd.GetNumSwitchValues(OptionKeys[TargetOutputCapture]); + if (const auto numSwitchValues = cmd.GetNumSwitchValues(OptionKeys[TargetOutputCaptureKey]); numSwitchValues) { AZ_TestImpact_Eval( @@ -236,23 +236,23 @@ namespace TestImpact AZStd::optional ParseMaxConcurrency(const AZ::CommandLine& cmd) { - return ParseUnsignedIntegerOption(OptionKeys[MaxConcurrency], cmd); + return ParseUnsignedIntegerOption(OptionKeys[MaxConcurrencyKey], cmd); } AZStd::optional ParseTestTargetTimeout(const AZ::CommandLine& cmd) { - return ParseSecondsOption(OptionKeys[TestTargetTimeout], cmd); + return ParseSecondsOption(OptionKeys[TestTargetTimeoutKey], cmd); } AZStd::optional ParseGlobalTimeout(const AZ::CommandLine& cmd) { - return ParseSecondsOption(OptionKeys[GlobalTimeout], cmd); + return ParseSecondsOption(OptionKeys[GlobalTimeoutKey], cmd); } bool ParseSafeMode(const AZ::CommandLine& cmd) { const BinaryStateValue states = { false, true }; - return ParseOnOffOption(OptionKeys[SafeMode], states, cmd).value_or(false); + return ParseOnOffOption(OptionKeys[SafeModeKey], states, cmd).value_or(false); } SuiteType ParseSuiteFilter(const AZ::CommandLine& cmd) @@ -264,7 +264,7 @@ namespace TestImpact {GetSuiteTypeName(SuiteType::Sandbox), SuiteType::Sandbox} }; - return ParseMultiStateOption(OptionKeys[SuiteFilter], states, cmd).value_or(SuiteType::Main); + return ParseMultiStateOption(OptionKeys[SuiteFilterKey], states, cmd).value_or(SuiteType::Main); } } From dd93c42342352d3c9445e18f362a983f5db44d32 Mon Sep 17 00:00:00 2001 From: jonawals Date: Tue, 8 Jun 2021 16:49:49 +0100 Subject: [PATCH 083/102] Fix missing keys --- .../Static/Code/Source/TestImpactCommandLineOptions.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp index 9a05fc6fc9..7eff998dda 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp @@ -184,7 +184,7 @@ namespace TestImpact Policy::TestSharding::Always }; - return ParseOnOffOption("shard", states, cmd).value_or(Policy::TestSharding::Never); + return ParseOnOffOption(OptionKeys[TestShardingPolicyKey], states, cmd).value_or(Policy::TestSharding::Never); } Policy::TargetOutputCapture ParseTargetOutputCapture(const AZ::CommandLine& cmd) @@ -198,7 +198,7 @@ namespace TestImpact Policy::TargetOutputCapture targetOutputCapture = Policy::TargetOutputCapture::None; for (auto i = 0; i < numSwitchValues; i++) { - const auto option = cmd.GetSwitchValue(OptionKeys[TargetOutputCapture], i); + const auto option = cmd.GetSwitchValue(OptionKeys[TargetOutputCaptureKey], i); if (option == OptionKeys[StdOut]) { if (targetOutputCapture == Policy::TargetOutputCapture::File) From f78ad7f3770ad95392c2f582d72af52fb28a07fe Mon Sep 17 00:00:00 2001 From: jonawals Date: Tue, 8 Jun 2021 16:58:25 +0100 Subject: [PATCH 084/102] Add missing tiaf scripts --- scripts/build/TestImpactAnalysis/tiaf.py | 166 ++++++++++++++---- .../build/TestImpactAnalysis/tiaf_driver.py | 31 +--- 2 files changed, 135 insertions(+), 62 deletions(-) diff --git a/scripts/build/TestImpactAnalysis/tiaf.py b/scripts/build/TestImpactAnalysis/tiaf.py index 01e2cde8ba..1523d39b4a 100644 --- a/scripts/build/TestImpactAnalysis/tiaf.py +++ b/scripts/build/TestImpactAnalysis/tiaf.py @@ -23,33 +23,40 @@ def is_child_path(parent_path, child_path): child_path = os.path.abspath(child_path) return os.path.commonpath([os.path.abspath(parent_path)]) == os.path.commonpath([os.path.abspath(parent_path), os.path.abspath(child_path)]) -# Enumerations for test sequence types -class SequenceType(Enum): - # Regular sequence as-per the tiaf regular sequence - REGULAR = 1 - # TIA sequence as-per the tiaf read-only impact analysis sequence - TEST_IMPACT_ANALYSIS = 2 - # Seed sequence as-per the tiaf seed sequence - SEED = 3 - class TestImpact: - def __init__(self, config_file, dst_commit): + def __init__(self, config_file, pipeline, dst_commit): + self.__pipeline = pipeline self.__parse_config_file(config_file) self.__init_repo(dst_commit) - self.__generate_change_list() + if self.__use_test_impact_analysis and not self.__is_pipeline_of_truth: + self.__generate_change_list() # Parse the configuration file and retrieve the data needed for launching the test impact analysis runtime def __parse_config_file(self, config_file): print(f"Attempting to parse configuration file '{config_file}'...") with open(config_file, "r") as config_data: config = json.load(config_data) + # Repository self.__repo_dir = config["repo"]["root"] + # Jenkins + self.__use_test_impact_analysis = config["jenkins"]["use_test_impact_analysis"] + self.__pipeline_of_truth = config["jenkins"]["pipeline_of_truth"] + print(f"Pipeline of truth: '{self.__pipeline_of_truth}'.") + print(f"This pipeline: '{self.__pipeline}'.") + if self.__pipeline in self.__pipeline_of_truth: + self.__is_pipeline_of_truth = True + else: + self.__is_pipeline_of_truth = False + print(f"Is pipeline of truth: '{self.__is_pipeline_of_truth}'.") + # TIAF binary self.__tiaf_bin = config["repo"]["tiaf_bin"] - if not os.path.isfile(self.__tiaf_bin): + if self.__use_test_impact_analysis and not os.path.isfile(self.__tiaf_bin): raise FileNotFoundError("Could not find tiaf binary") + # Workspaces self.__active_workspace = config["workspace"]["active"]["root"] self.__historic_workspace = config["workspace"]["historic"]["root"] self.__temp_workspace = config["workspace"]["temp"]["root"] + # Last commit hash last_commit_hash_path_rel = config["workspace"]["historic"]["relative_paths"]["last_run_hash_file"] self.__last_commit_hash_path = os.path.join(self.__historic_workspace, last_commit_hash_path_rel) print("The configuration file was parsed successfully.") @@ -71,7 +78,7 @@ class TestImpact: def __read_last_run_hash(self): self.__has_src_commit = False if os.path.isfile(self.__last_commit_hash_path): - print(f"Previous commit hash found at '{self.__last_commit_hash_path}'") + print(f"Previous commit hash found at '{self.__last_commit_hash_path}'.") with open(self.__last_commit_hash_path) as file: self.__src_commit = file.read() self.__has_src_commit = True @@ -144,43 +151,79 @@ class TestImpact: return # Runs the specified test sequence - def run(self, sequence_type, safe_mode, test_timeout, global_timeout): + def run(self, suite, safe_mode, test_timeout, global_timeout): args = [] - print("Please note: test impact analysis sequences will be run in read-only mode (seed sequences are unaffected).") - if sequence_type == SequenceType.REGULAR: - print("Sequence type: regular.") - args.append("--sequence=regular") - args.append("--fpolicy=abort") - elif sequence_type == SequenceType.SEED: - print("Sequence type: seed.") - args.append("--sequence=seed") - args.append("--fpolicy=continue") - elif sequence_type == SequenceType.TEST_IMPACT_ANALYSIS: - print("Sequence type: test impact analysis (no write).") - args.append("--fpolicy=abort") - if self.__has_change_list: - args.append(f"-changelist={self.__change_list_path}") - args.append("--sequence=tianowrite") - else: - print(f"No change list was generated, falling back to a regular sequence.") - print("Sequence type: Regular.") - args.append("--sequence=regular") - else: - raise ValueError(sequence_type) - + # Suite + args.append(f"--suite={suite}") + print(f"Test suite is set to '{suite}'.") + # Timeouts if test_timeout != None: args.append(f"--ttimeout={test_timeout}") print(f"Test target timeout is set to {test_timeout} seconds.") if global_timeout != None: args.append(f"--gtimeout={global_timeout}") print(f"Global sequence timeout is set to {test_timeout} seconds.") - + # If test impact analysis is enabled: + # -> Pipleine of truth will perform a seed that will continue until the sequence is complete regardless of test failues + # -> Non pipline of truth will attempt to perform an impact analysis sequence and exit early upon the fist test failure + # If test impact analysis is disabled: + # -> Pipleine of truth will perform a regular sequence that will continue until the sequence is complete regardless of test failues + # -> Non pipline of truth will perform a regular sequence and exit early upon the fist test failure + if self.__use_test_impact_analysis: + print("Test impact analysis ie enabled.") + # Pipeline of truth sequence + if self.__is_pipeline_of_truth: + # Sequence type + args.append("--sequence=seed") + print("Sequence type is set to 'seed'.") + # Test failure policy + args.append("--fpolicy=continue") + print("Test failure policy is set to 'continue'.") + # Non pipeline of truth sequence + else: + if self.__has_change_list: + # Change list + args.append(f"--changelist={self.__change_list_path}") + print(f"Change list is set to '{self.__change_list_path}'.") + # Sequence type + args.append("--sequence=tianowrite") + print("Sequence type is set to 'tianowrite'.") + # Safe mode + if safe_mode: + args.append("--safemode=on") + print("Safe mode set to 'on'.") + else: + args.append("--safemode=off") + print("Safe mode set to 'off'.") + else: + args.append("--sequence=regular") + print("Sequence type is set to 'regular'.") + # Test failure policy + args.append("--fpolicy=abort") + print("Test failure policy is set to 'abort'.") + else: + print("Test impact analysis ie disabled.") + # Sequence type + args.append("--sequence=regular") + print("Sequence type is set to 'seed'.") + # Pipeline of truth sequence + if self.__is_pipeline_of_truth: + # Test failure policy + args.append("--fpolicy=continue") + print("Test failure policy is set to 'continue'.") + # Non pipeline of truth sequence + else: + # Test failure policy + args.append("--fpolicy=abort") + print("Test failure policy is set to 'abort'.") + print("Args: ", end='') print(*args) result = subprocess.run([self.__tiaf_bin] + args) - if result.returncode == 0: + # If the sequence completed 9with or without failures) we will update the historical meta-data + if result.returncode == 0 or result.returncode == 7: print("Test impact analysis runtime returned successfully.") - if sequence_type == SequenceType.SEED: + if self.__is_pipeline_of_truth: print("Writing historical meta-data...") self.__write_last_run_hash(self.__dst_commit) print("Complete!") @@ -188,3 +231,48 @@ class TestImpact: print(f"The test impact analysis runtime returned with error: '{result.returncode}'.") return result.returncode + + + #args = [] + #print("Please note: test impact analysis sequences will be run in read-only mode (seed sequences are unaffected).") + #if sequence_type == SequenceType.REGULAR: + # print("Sequence type: regular.") + # args.append("--sequence=regular") + # args.append("--fpolicy=abort") + #elif sequence_type == SequenceType.SEED: + # print("Sequence type: seed.") + # args.append("--sequence=seed") + # args.append("--fpolicy=continue") + #elif sequence_type == SequenceType.TEST_IMPACT_ANALYSIS: + # print("Sequence type: test impact analysis (no write).") + # args.append("--fpolicy=abort") + # if self.__has_change_list: + # args.append(f"-changelist={self.__change_list_path}") + # args.append("--sequence=tianowrite") + # else: + # print(f"No change list was generated, falling back to a regular sequence.") + # print("Sequence type: Regular.") + # args.append("--sequence=regular") + #else: + # raise ValueError(sequence_type) + # + #if test_timeout != None: + # args.append(f"--ttimeout={test_timeout}") + # print(f"Test target timeout is set to {test_timeout} seconds.") + #if global_timeout != None: + # args.append(f"--gtimeout={global_timeout}") + # print(f"Global sequence timeout is set to {test_timeout} seconds.") + # + #print("Args: ", end='') + #print(*args) + #result = subprocess.run([self.__tiaf_bin] + args) + #if result.returncode == 0: + # print("Test impact analysis runtime returned successfully.") + # if sequence_type == SequenceType.SEED: + # print("Writing historical meta-data...") + # self.__write_last_run_hash(self.__dst_commit) + # print("Complete!") + #else: + # print(f"The test impact analysis runtime returned with error: '{result.returncode}'.") + #return result.returncode + diff --git a/scripts/build/TestImpactAnalysis/tiaf_driver.py b/scripts/build/TestImpactAnalysis/tiaf_driver.py index 1cc7ecb630..917fd66fc6 100644 --- a/scripts/build/TestImpactAnalysis/tiaf_driver.py +++ b/scripts/build/TestImpactAnalysis/tiaf_driver.py @@ -11,7 +11,6 @@ import argparse from tiaf import TestImpact -from tiaf import SequenceType import sys import os @@ -26,16 +25,6 @@ def parse_args(): else: raise FileNotFoundError(value) - def sequence_type(value): - if value == "regular": - return SequenceType.REGULAR - elif value == "tia": - return SequenceType.TEST_IMPACT_ANALYSIS - elif value == "seed": - return SequenceType.SEED - else: - raise ValueError(value) - def timout_type(value): value = int(value) if value <= 0: @@ -44,24 +33,20 @@ def parse_args(): parser = argparse.ArgumentParser() parser.add_argument('--config', dest="config", type=file_path, help="Path to the test impact analysis framework configuration file", required=True) - parser.add_argument('--destCommit', dest="dst_commit", help="Commit to run test impact analysis on (not required for seed)") - parser.add_argument('--sequenceType', dest="sequence_type", type=sequence_type, help="Test sequence type to run ('regular', 'seed' or 'tia')", required=True) - parser.add_argument('--suites', dest="suites", nargs='*', help="Suites to include for regular tes sequences (use '*' for all suites)") - parser.add_argument('--testTimeout', dest="test_timeout", type=timout_type, help="Maximum flight time (in seconds) of any test target before being terminated", required=False) - parser.add_argument('--globalTimeout', dest="global_timeout", type=timout_type, help="Maximum tun time of the sequence before being terminated", required=False) - parser.set_defaults(suites="*") - parser.set_defaults(safe_mode=False) + parser.add_argument('--pipeline', dest="pipeline", help="Pipeline the test impact analysis framework is running on", required=True) + parser.add_argument('--destCommit', dest="dst_commit", help="Commit to run test impact analysis on (ignored when seeding)", required=True) + parser.add_argument('--suite', dest="suite", help="Test suite to run", required=True) + parser.add_argument('--safeMode', dest="safe_mode", action='store_true', help="Run impact analysis tests in safe mode (ignored when seeding)") + parser.add_argument('--testTimeout', dest="test_timeout", type=timout_type, help="Maximum run time (in seconds) of any test target before being terminated", required=False) + parser.add_argument('--globalTimeout', dest="global_timeout", type=timout_type, help="Maximum run time of the sequence before being terminated", required=False) parser.set_defaults(test_timeout=None) parser.set_defaults(global_timeout=None) args = parser.parse_args() - - if args.sequence_type == SequenceType.TEST_IMPACT_ANALYSIS and args.dst_commit == None: - raise ValueError("Test impact analysis sequence must have a change list") return args if __name__ == "__main__": args = parse_args() - tiaf = TestImpact(args.config, args.dst_commit) - return_code = tiaf.run(args.sequence_type, args.test_timeout, args.global_timeout) + tiaf = TestImpact(args.config, args.pipeline, args.dst_commit) + return_code = tiaf.run(args.suite, args.safe_mode, args.test_timeout, args.global_timeout) sys.exit(return_code) \ No newline at end of file From 46be3efd5de449c7e8c625f05e7674bdba32a0af Mon Sep 17 00:00:00 2001 From: jonawals Date: Tue, 8 Jun 2021 18:47:36 +0100 Subject: [PATCH 085/102] Add missing test impact pipe --- scripts/build/Platform/Windows/build_config.json | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/build/Platform/Windows/build_config.json b/scripts/build/Platform/Windows/build_config.json index bb4a0decbf..98ed04250c 100644 --- a/scripts/build/Platform/Windows/build_config.json +++ b/scripts/build/Platform/Windows/build_config.json @@ -33,6 +33,7 @@ ], "steps": [ "profile_vs2019", + "test_impact_analysis", "asset_profile_vs2019", "test_cpu_profile_vs2019" ] From efd0d8bcc921e89e3db6d146f35f782fede0a7f9 Mon Sep 17 00:00:00 2001 From: jonawals Date: Tue, 8 Jun 2021 19:03:59 +0100 Subject: [PATCH 086/102] Fix detatched head in tiaf driver --- scripts/build/TestImpactAnalysis/tiaf.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/scripts/build/TestImpactAnalysis/tiaf.py b/scripts/build/TestImpactAnalysis/tiaf.py index 1523d39b4a..93b876836e 100644 --- a/scripts/build/TestImpactAnalysis/tiaf.py +++ b/scripts/build/TestImpactAnalysis/tiaf.py @@ -26,8 +26,10 @@ def is_child_path(parent_path, child_path): class TestImpact: def __init__(self, config_file, pipeline, dst_commit): self.__pipeline = pipeline + self.__dst_commit = dst_commit + self.__src_commit = None + self.__has_src_commit = False self.__parse_config_file(config_file) - self.__init_repo(dst_commit) if self.__use_test_impact_analysis and not self.__is_pipeline_of_truth: self.__generate_change_list() @@ -61,15 +63,6 @@ class TestImpact: self.__last_commit_hash_path = os.path.join(self.__historic_workspace, last_commit_hash_path_rel) print("The configuration file was parsed successfully.") - # Initializes the internal representation of the repository being targeted for test impact analysis - def __init_repo(self, dst_commit): - self.__repo = Repo(self.__repo_dir) - self.__branch = self.__repo.current_branch - self.__dst_commit = dst_commit - self.__src_commit = None - self.__has_src_commit = False - print(f"The repository is located at '{self.__repo_dir}' and the current branch is '{self.__branch}'.") - # Restricts change lists from checking in test impact analysis files def __check_for_restricted_files(self, file_path): if is_child_path(self.__active_workspace, file_path) or is_child_path(self.__historic_workspace, file_path) or is_child_path(self.__temp_workspace, file_path): From 112493c26bda15c425833af5f1b14ff3b627190c Mon Sep 17 00:00:00 2001 From: jonawals Date: Tue, 8 Jun 2021 19:27:37 +0100 Subject: [PATCH 087/102] Non pipline of truth no early exit experiment --- scripts/build/TestImpactAnalysis/tiaf.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/scripts/build/TestImpactAnalysis/tiaf.py b/scripts/build/TestImpactAnalysis/tiaf.py index 93b876836e..478322ca17 100644 --- a/scripts/build/TestImpactAnalysis/tiaf.py +++ b/scripts/build/TestImpactAnalysis/tiaf.py @@ -146,6 +146,8 @@ class TestImpact: # Runs the specified test sequence def run(self, suite, safe_mode, test_timeout, global_timeout): args = [] + pipeline_of_truth_test_failure_policy = "continue" + non_pipeline_of_truth_test_failure_policy = "continue" # Suite args.append(f"--suite={suite}") print(f"Test suite is set to '{suite}'.") @@ -170,8 +172,8 @@ class TestImpact: args.append("--sequence=seed") print("Sequence type is set to 'seed'.") # Test failure policy - args.append("--fpolicy=continue") - print("Test failure policy is set to 'continue'.") + args.append(f"--fpolicy={pipeline_of_truth_test_failure_policy}") + print(f"Test failure policy is set to '{pipeline_of_truth_test_failure_policy}'.") # Non pipeline of truth sequence else: if self.__has_change_list: @@ -192,8 +194,8 @@ class TestImpact: args.append("--sequence=regular") print("Sequence type is set to 'regular'.") # Test failure policy - args.append("--fpolicy=abort") - print("Test failure policy is set to 'abort'.") + args.append(f"--fpolicy={non_pipeline_of_truth_test_failure_policy}") + print(f"Test failure policy is set to '{non_pipeline_of_truth_test_failure_policy}'.") else: print("Test impact analysis ie disabled.") # Sequence type @@ -202,13 +204,13 @@ class TestImpact: # Pipeline of truth sequence if self.__is_pipeline_of_truth: # Test failure policy - args.append("--fpolicy=continue") - print("Test failure policy is set to 'continue'.") + args.append(f"--fpolicy={pipeline_of_truth_test_failure_policy}") + print(f"Test failure policy is set to '{pipeline_of_truth_test_failure_policy}'.") # Non pipeline of truth sequence else: # Test failure policy - args.append("--fpolicy=abort") - print("Test failure policy is set to 'abort'.") + args.append(f"--fpolicy={non_pipeline_of_truth_test_failure_policy}") + print(f"Test failure policy is set to '{non_pipeline_of_truth_test_failure_policy}'.") print("Args: ", end='') print(*args) From 351d54e4f603c13c66383c40ed3c15851ccd3ec3 Mon Sep 17 00:00:00 2001 From: jonawals Date: Wed, 9 Jun 2021 08:49:18 +0100 Subject: [PATCH 088/102] Fix non-existent dir for historic data --- scripts/build/TestImpactAnalysis/tiaf.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/build/TestImpactAnalysis/tiaf.py b/scripts/build/TestImpactAnalysis/tiaf.py index 478322ca17..887d5f1085 100644 --- a/scripts/build/TestImpactAnalysis/tiaf.py +++ b/scripts/build/TestImpactAnalysis/tiaf.py @@ -59,8 +59,8 @@ class TestImpact: self.__historic_workspace = config["workspace"]["historic"]["root"] self.__temp_workspace = config["workspace"]["temp"]["root"] # Last commit hash - last_commit_hash_path_rel = config["workspace"]["historic"]["relative_paths"]["last_run_hash_file"] - self.__last_commit_hash_path = os.path.join(self.__historic_workspace, last_commit_hash_path_rel) + last_commit_hash_path_file = config["workspace"]["historic"]["relative_paths"]["last_run_hash_file"] + self.__last_commit_hash_path = os.path.join(self.__historic_workspace, last_commit_hash_path_file) print("The configuration file was parsed successfully.") # Restricts change lists from checking in test impact analysis files @@ -77,6 +77,7 @@ class TestImpact: self.__has_src_commit = True def __write_last_run_hash(self, last_run_hash): + os.mkdir(self.__historic_workspace) f = open(self.__last_commit_hash_path, "w") f.write(last_run_hash) f.close() From d44a20f4702624521c69951fa5bcad82c50c3f95 Mon Sep 17 00:00:00 2001 From: jonawals Date: Wed, 9 Jun 2021 08:59:11 +0100 Subject: [PATCH 089/102] Add flag for non-seed sequence test failure policy --- .../build/Platform/Windows/build_config.json | 2 +- scripts/build/TestImpactAnalysis/tiaf.py | 21 +++++++------------ 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/scripts/build/Platform/Windows/build_config.json b/scripts/build/Platform/Windows/build_config.json index 98ed04250c..9201e13a80 100644 --- a/scripts/build/Platform/Windows/build_config.json +++ b/scripts/build/Platform/Windows/build_config.json @@ -88,7 +88,7 @@ "COMMAND": "python_windows.cmd", "PARAMETERS": { "SCRIPT_PATH": "scripts/build/TestImpactAnalysis/tiaf_driver.py", - "SCRIPT_PARAMETERS": "--safeMode --suite main --pipeline !PIPELINE_NAME! --destCommit !CHANGE_ID! --config \"build\\windows_vs2019\\bin\\TestImpactFramework\\persistent\\tiaf.profile.json\"" + "SCRIPT_PARAMETERS": "--testFailurePolicy abort --safeMode --suite main --pipeline !PIPELINE_NAME! --destCommit !CHANGE_ID! --config \"build\\windows_vs2019\\bin\\TestImpactFramework\\persistent\\tiaf.profile.json\"" } }, "debug_vs2019": { diff --git a/scripts/build/TestImpactAnalysis/tiaf.py b/scripts/build/TestImpactAnalysis/tiaf.py index 887d5f1085..c040ae1147 100644 --- a/scripts/build/TestImpactAnalysis/tiaf.py +++ b/scripts/build/TestImpactAnalysis/tiaf.py @@ -71,7 +71,7 @@ class TestImpact: def __read_last_run_hash(self): self.__has_src_commit = False if os.path.isfile(self.__last_commit_hash_path): - print(f"Previous commit hash found at '{self.__last_commit_hash_path}'.") + print(f"Previous commit hash found at '{self.__last_commit_hash_path}'.") with open(self.__last_commit_hash_path) as file: self.__src_commit = file.read() self.__has_src_commit = True @@ -145,10 +145,9 @@ class TestImpact: return # Runs the specified test sequence - def run(self, suite, safe_mode, test_timeout, global_timeout): + def run(self, suite, test_failure_policy, safe_mode, test_timeout, global_timeout): args = [] pipeline_of_truth_test_failure_policy = "continue" - non_pipeline_of_truth_test_failure_policy = "continue" # Suite args.append(f"--suite={suite}") print(f"Test suite is set to '{suite}'.") @@ -159,14 +158,8 @@ class TestImpact: if global_timeout != None: args.append(f"--gtimeout={global_timeout}") print(f"Global sequence timeout is set to {test_timeout} seconds.") - # If test impact analysis is enabled: - # -> Pipleine of truth will perform a seed that will continue until the sequence is complete regardless of test failues - # -> Non pipline of truth will attempt to perform an impact analysis sequence and exit early upon the fist test failure - # If test impact analysis is disabled: - # -> Pipleine of truth will perform a regular sequence that will continue until the sequence is complete regardless of test failues - # -> Non pipline of truth will perform a regular sequence and exit early upon the fist test failure if self.__use_test_impact_analysis: - print("Test impact analysis ie enabled.") + print("Test impact analysis is enabled.") # Pipeline of truth sequence if self.__is_pipeline_of_truth: # Sequence type @@ -195,8 +188,8 @@ class TestImpact: args.append("--sequence=regular") print("Sequence type is set to 'regular'.") # Test failure policy - args.append(f"--fpolicy={non_pipeline_of_truth_test_failure_policy}") - print(f"Test failure policy is set to '{non_pipeline_of_truth_test_failure_policy}'.") + args.append(f"--fpolicy={test_failure_policy}") + print(f"Test failure policy is set to '{test_failure_policy}'.") else: print("Test impact analysis ie disabled.") # Sequence type @@ -210,8 +203,8 @@ class TestImpact: # Non pipeline of truth sequence else: # Test failure policy - args.append(f"--fpolicy={non_pipeline_of_truth_test_failure_policy}") - print(f"Test failure policy is set to '{non_pipeline_of_truth_test_failure_policy}'.") + args.append(f"--fpolicy={test_failure_policy}") + print(f"Test failure policy is set to '{test_failure_policy}'.") print("Args: ", end='') print(*args) From fdbc15b195037bc64740df42af5406f5f6e13ea9 Mon Sep 17 00:00:00 2001 From: jonawals Date: Wed, 9 Jun 2021 10:00:53 +0100 Subject: [PATCH 090/102] Fix missing args for tiaf driver --- scripts/build/TestImpactAnalysis/tiaf_driver.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/scripts/build/TestImpactAnalysis/tiaf_driver.py b/scripts/build/TestImpactAnalysis/tiaf_driver.py index 917fd66fc6..7ff73e505b 100644 --- a/scripts/build/TestImpactAnalysis/tiaf_driver.py +++ b/scripts/build/TestImpactAnalysis/tiaf_driver.py @@ -30,15 +30,23 @@ def parse_args(): if value <= 0: raise ValueError("Timer values must be positive integers") return value - + + def test_failure_policy(value): + if value == "continue" or value == "abort" or value == "ignore": + return value + else: + raise ValueError("Test failure policy must be 'abort', 'continue' or 'ignore'") + parser = argparse.ArgumentParser() parser.add_argument('--config', dest="config", type=file_path, help="Path to the test impact analysis framework configuration file", required=True) parser.add_argument('--pipeline', dest="pipeline", help="Pipeline the test impact analysis framework is running on", required=True) parser.add_argument('--destCommit', dest="dst_commit", help="Commit to run test impact analysis on (ignored when seeding)", required=True) parser.add_argument('--suite', dest="suite", help="Test suite to run", required=True) + parser.add_argument('--testFailurePolicy', dest="test_failure_policy", type=test_failure_policy, help="Test failure policy for regular and test impact sequences (ignored when seeding)", required=True) parser.add_argument('--safeMode', dest="safe_mode", action='store_true', help="Run impact analysis tests in safe mode (ignored when seeding)") parser.add_argument('--testTimeout', dest="test_timeout", type=timout_type, help="Maximum run time (in seconds) of any test target before being terminated", required=False) parser.add_argument('--globalTimeout', dest="global_timeout", type=timout_type, help="Maximum run time of the sequence before being terminated", required=False) + parser.set_defaults(test_failure_policy="abort") parser.set_defaults(test_timeout=None) parser.set_defaults(global_timeout=None) args = parser.parse_args() @@ -48,5 +56,5 @@ def parse_args(): if __name__ == "__main__": args = parse_args() tiaf = TestImpact(args.config, args.pipeline, args.dst_commit) - return_code = tiaf.run(args.suite, args.safe_mode, args.test_timeout, args.global_timeout) + return_code = tiaf.run(args.suite, args.test_failure_policy, args.safe_mode, args.test_timeout, args.global_timeout) sys.exit(return_code) \ No newline at end of file From 169d4da28848bda599e48f7896ffe0fa4c7aa086 Mon Sep 17 00:00:00 2001 From: John Date: Thu, 10 Jun 2021 14:54:46 +0100 Subject: [PATCH 091/102] Address PR comments --- .../Frontend/Console/CMakeLists.txt | 3 +- .../Console/{Static => }/Code/CMakeLists.txt | 55 +- .../TestImpactConsoleMain.h | 0 .../Source/TestImpactCommandLineOptions.cpp | 0 .../Source/TestImpactCommandLineOptions.h | 0 .../TestImpactCommandLineOptionsException.h | 0 .../TestImpactCommandLineOptionsUtils.cpp | 0 .../TestImpactCommandLineOptionsUtils.h | 0 .../Code/Source/TestImpactConsole.cpp | 0 .../Code/Source/TestImpactConsoleMain.cpp | 0 ...tImpactConsoleTestSequenceEventHandler.cpp | 0 ...estImpactConsoleTestSequenceEventHandler.h | 0 .../Code/Source/TestImpactConsoleUtils.cpp | 0 .../Code/Source/TestImpactConsoleUtils.h | 0 .../TestImpactRuntimeConfigurationFactory.cpp | 0 .../TestImpactRuntimeConfigurationFactory.h | 0 .../TestImpactCommandLineOptionsTest.cpp | 1436 +++++++++++++++++ .../Code/Tests/TestImpactConsoleMainTest.cpp | 0 ...actConsoleTestSequenceEventHandlerTest.cpp | 42 + ...estImpactFrontendConsoleStaticTestMain.cpp | 34 + ...tImpactRuntimeConfigurationFactoryTest.cpp | 27 + ...pactframework_frontend_console_files.cmake | 0 ...mework_frontend_console_static_files.cmake | 0 ..._frontend_console_static_tests_files.cmake | 5 + .../Console/Executable/CMakeLists.txt | 12 - .../Console/Executable/Code/CMakeLists.txt | 25 - .../Frontend/Console/Static/CMakeLists.txt | 12 - .../Runtime/Code/CMakeLists.txt | 91 +- cmake/LYTestWrappers.cmake | 1 + .../build/Platform/Windows/build_config.json | 2 +- scripts/build/TestImpactAnalysis/tiaf.py | 47 +- .../build/TestImpactAnalysis/tiaf_driver.py | 3 +- 32 files changed, 1631 insertions(+), 164 deletions(-) rename Code/Tools/TestImpactFramework/Frontend/Console/{Static => }/Code/CMakeLists.txt (64%) rename Code/Tools/TestImpactFramework/Frontend/Console/{Static => }/Code/Include/TestImpactFramework/TestImpactConsoleMain.h (100%) rename Code/Tools/TestImpactFramework/Frontend/Console/{Static => }/Code/Source/TestImpactCommandLineOptions.cpp (100%) rename Code/Tools/TestImpactFramework/Frontend/Console/{Static => }/Code/Source/TestImpactCommandLineOptions.h (100%) rename Code/Tools/TestImpactFramework/Frontend/Console/{Static => }/Code/Source/TestImpactCommandLineOptionsException.h (100%) rename Code/Tools/TestImpactFramework/Frontend/Console/{Static => }/Code/Source/TestImpactCommandLineOptionsUtils.cpp (100%) rename Code/Tools/TestImpactFramework/Frontend/Console/{Static => }/Code/Source/TestImpactCommandLineOptionsUtils.h (100%) rename Code/Tools/TestImpactFramework/Frontend/Console/{Executable => }/Code/Source/TestImpactConsole.cpp (100%) rename Code/Tools/TestImpactFramework/Frontend/Console/{Static => }/Code/Source/TestImpactConsoleMain.cpp (100%) rename Code/Tools/TestImpactFramework/Frontend/Console/{Static => }/Code/Source/TestImpactConsoleTestSequenceEventHandler.cpp (100%) rename Code/Tools/TestImpactFramework/Frontend/Console/{Static => }/Code/Source/TestImpactConsoleTestSequenceEventHandler.h (100%) rename Code/Tools/TestImpactFramework/Frontend/Console/{Static => }/Code/Source/TestImpactConsoleUtils.cpp (100%) rename Code/Tools/TestImpactFramework/Frontend/Console/{Static => }/Code/Source/TestImpactConsoleUtils.h (100%) rename Code/Tools/TestImpactFramework/Frontend/Console/{Static => }/Code/Source/TestImpactRuntimeConfigurationFactory.cpp (100%) rename Code/Tools/TestImpactFramework/Frontend/Console/{Static => }/Code/Source/TestImpactRuntimeConfigurationFactory.h (100%) create mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/Code/Tests/TestImpactCommandLineOptionsTest.cpp create mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/Code/Tests/TestImpactConsoleMainTest.cpp create mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/Code/Tests/TestImpactConsoleTestSequenceEventHandlerTest.cpp create mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/Code/Tests/TestImpactFrontendConsoleStaticTestMain.cpp create mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/Code/Tests/TestImpactRuntimeConfigurationFactoryTest.cpp rename Code/Tools/TestImpactFramework/Frontend/Console/{Executable => }/Code/testimpactframework_frontend_console_files.cmake (100%) rename Code/Tools/TestImpactFramework/Frontend/Console/{Static => }/Code/testimpactframework_frontend_console_static_files.cmake (100%) rename Code/Tools/TestImpactFramework/Frontend/Console/{Static => }/Code/testimpactframework_frontend_console_static_tests_files.cmake (67%) delete mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/Executable/CMakeLists.txt delete mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/Executable/Code/CMakeLists.txt delete mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/Static/CMakeLists.txt diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/CMakeLists.txt b/Code/Tools/TestImpactFramework/Frontend/Console/CMakeLists.txt index a25d506156..8298bb7123 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/CMakeLists.txt +++ b/Code/Tools/TestImpactFramework/Frontend/Console/CMakeLists.txt @@ -9,5 +9,4 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # -add_subdirectory(Executable) -add_subdirectory(Static) \ No newline at end of file +add_subdirectory(Code) \ No newline at end of file diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/CMakeLists.txt b/Code/Tools/TestImpactFramework/Frontend/Console/Code/CMakeLists.txt similarity index 64% rename from Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/CMakeLists.txt rename to Code/Tools/TestImpactFramework/Frontend/Console/Code/CMakeLists.txt index 2302cb5484..a26814fa51 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/CMakeLists.txt +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Code/CMakeLists.txt @@ -24,27 +24,42 @@ ly_add_target( AZ::TestImpact.Runtime.Static ) +ly_add_target( + NAME TestImpact.Frontend.Console EXECUTABLE + OUTPUT_NAME tiaf + NAMESPACE AZ + FILES_CMAKE + testimpactframework_frontend_console_files.cmake + INCLUDE_DIRECTORIES + PRIVATE + Source + BUILD_DEPENDENCIES + PRIVATE + AZ::TestImpact.Frontend.Console.Static +) + ################################################################################ # Tests ################################################################################ -ly_add_target( - NAME TestImpact.Frontend.Console.Static.Tests ${PAL_TRAIT_TEST_TARGET_TYPE} - NAMESPACE AZ - FILES_CMAKE - testimpactframework_frontend_console_static_tests_files.cmake - INCLUDE_DIRECTORIES - PRIVATE - Include - Source - Tests - BUILD_DEPENDENCIES - PRIVATE - AZ::AzTestShared - AZ::AzTest - AZ::TestImpact.Frontend.Console.Static -) - -ly_add_googletest( - NAME AZ::TestImpact.Frontend.Console.Static.Tests -) +# Disbled:SPEC-7246 +#ly_add_target( +# NAME TestImpact.Frontend.Console.Tests ${PAL_TRAIT_TEST_TARGET_TYPE} +# NAMESPACE AZ +# FILES_CMAKE +# testimpactframework_frontend_console_static_tests_files.cmake +# INCLUDE_DIRECTORIES +# PRIVATE +# Include +# Source +# Tests +# BUILD_DEPENDENCIES +# PRIVATE +# AZ::AzTestShared +# AZ::AzTest +# AZ::TestImpact.Frontend.Console.Static +#) +# +#ly_add_googletest( +# NAME AZ::TestImpact.Frontend.Console.Static.Tests +#) diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Include/TestImpactFramework/TestImpactConsoleMain.h b/Code/Tools/TestImpactFramework/Frontend/Console/Code/Include/TestImpactFramework/TestImpactConsoleMain.h similarity index 100% rename from Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Include/TestImpactFramework/TestImpactConsoleMain.h rename to Code/Tools/TestImpactFramework/Frontend/Console/Code/Include/TestImpactFramework/TestImpactConsoleMain.h diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactCommandLineOptions.cpp similarity index 100% rename from Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp rename to Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactCommandLineOptions.cpp diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.h b/Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactCommandLineOptions.h similarity index 100% rename from Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.h rename to Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactCommandLineOptions.h diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsException.h b/Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactCommandLineOptionsException.h similarity index 100% rename from Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsException.h rename to Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactCommandLineOptionsException.h diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactCommandLineOptionsUtils.cpp similarity index 100% rename from Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.cpp rename to Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactCommandLineOptionsUtils.cpp diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.h b/Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactCommandLineOptionsUtils.h similarity index 100% rename from Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.h rename to Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactCommandLineOptionsUtils.h diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Executable/Code/Source/TestImpactConsole.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactConsole.cpp similarity index 100% rename from Code/Tools/TestImpactFramework/Frontend/Console/Executable/Code/Source/TestImpactConsole.cpp rename to Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactConsole.cpp diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleMain.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactConsoleMain.cpp similarity index 100% rename from Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleMain.cpp rename to Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactConsoleMain.cpp diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleTestSequenceEventHandler.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactConsoleTestSequenceEventHandler.cpp similarity index 100% rename from Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleTestSequenceEventHandler.cpp rename to Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactConsoleTestSequenceEventHandler.cpp diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleTestSequenceEventHandler.h b/Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactConsoleTestSequenceEventHandler.h similarity index 100% rename from Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleTestSequenceEventHandler.h rename to Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactConsoleTestSequenceEventHandler.h diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleUtils.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactConsoleUtils.cpp similarity index 100% rename from Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleUtils.cpp rename to Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactConsoleUtils.cpp diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleUtils.h b/Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactConsoleUtils.h similarity index 100% rename from Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleUtils.h rename to Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactConsoleUtils.h diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactRuntimeConfigurationFactory.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactRuntimeConfigurationFactory.cpp similarity index 100% rename from Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactRuntimeConfigurationFactory.cpp rename to Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactRuntimeConfigurationFactory.cpp diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactRuntimeConfigurationFactory.h b/Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactRuntimeConfigurationFactory.h similarity index 100% rename from Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactRuntimeConfigurationFactory.h rename to Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactRuntimeConfigurationFactory.h diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Code/Tests/TestImpactCommandLineOptionsTest.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Code/Tests/TestImpactCommandLineOptionsTest.cpp new file mode 100644 index 0000000000..cd02f5c5c7 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Code/Tests/TestImpactCommandLineOptionsTest.cpp @@ -0,0 +1,1436 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include + +#include +#include +#include +#include + +namespace UnitTest +{ + class CommandLineOptionsTestFixture + : public AllocatorsTestFixture + { + public: + void SetUp() override + { + AllocatorsTestFixture::SetUp(); + m_args.push_back("program.exe"); + } + protected: + void InitOptions(); + + AZStd::unique_ptr m_options; + AZStd::vector m_args; + }; + + void CommandLineOptionsTestFixture::InitOptions() + { + m_options = AZStd::make_unique(m_args.size(), const_cast(m_args.data())); + } + + TEST_F(CommandLineOptionsTestFixture, CheckEmptyArgs_ExpectDefaultValues) + { + InitOptions(); + EXPECT_EQ(m_options->GetConfigurationFile(), LY_TEST_IMPACT_DEFAULT_CONFIG_FILE); + EXPECT_EQ(m_options->GetFailedTestCoveragePolicy(), TestImpact::Policy::FailedTestCoverage::Keep); + EXPECT_EQ(m_options->GetExecutionFailurePolicy(), TestImpact::Policy::ExecutionFailure::Continue); + EXPECT_FALSE(m_options->GetGlobalTimeout().has_value()); + EXPECT_FALSE(m_options->GetTestTargetTimeout().has_value()); + EXPECT_FALSE(m_options->GetMaxConcurrency().has_value()); + EXPECT_FALSE(m_options->HasOutputChangeList()); + EXPECT_EQ(m_options->GetTargetOutputCapture(), TestImpact::Policy::TargetOutputCapture::None); + EXPECT_EQ(m_options->GetTestFailurePolicy(), TestImpact::Policy::TestFailure::Abort); + EXPECT_EQ(m_options->GetIntegrityFailurePolicy(), TestImpact::Policy::IntegrityFailure::Abort); + EXPECT_EQ(m_options->GetTestPrioritizationPolicy(), TestImpact::Policy::TestPrioritization::None); + EXPECT_EQ(m_options->GetTestSequenceType(), TestImpact::TestSequenceType::None); + EXPECT_EQ(m_options->GetTestShardingPolicy(), TestImpact::Policy::TestSharding::Never); + EXPECT_FALSE(m_options->HasChangeListFile()); + EXPECT_FALSE(m_options->GetChangeListFile().has_value()); + EXPECT_FALSE(m_options->HasSafeMode()); + EXPECT_EQ(m_options->GetSuiteFilter(), TestImpact::SuiteType::Main); + } + + TEST_F(CommandLineOptionsTestFixture, ConfigurationFileHasEmptyPath_ExpectCommandLineOptionsException) + { + m_args.push_back("-config"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(CommandLineOptionsTestFixture, ConfigurationFileHasSpecifiedPath_ExpectPath) + { + m_args.push_back("-config"); + m_args.push_back("Foo\\Bar"); + InitOptions(); + EXPECT_STREQ(m_options->GetConfigurationFile().c_str(), "Foo\\Bar"); + } + + TEST_F(CommandLineOptionsTestFixture, ConfigurationFileHasMultiplePaths_ExpectCommandLineOptionsException) + { + m_args.push_back("-config"); + m_args.push_back("value1,value2"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + // + + TEST_F(CommandLineOptionsTestFixture, UnifiedDiffFileHasEmptyPath_ExpectCommandLineOptionsException) + { + m_args.push_back("-changelist"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(CommandLineOptionsTestFixture, UnifiedDiffFileHasSpecifiedPath_ExpectPath) + { + m_args.push_back("-changelist"); + m_args.push_back("Foo\\Bar"); + InitOptions(); + EXPECT_TRUE(m_options->HasChangeListFile()); + EXPECT_STREQ(m_options->GetChangeListFile()->c_str(), "Foo\\Bar"); + } + + TEST_F(CommandLineOptionsTestFixture, UnifiedDiffFileHasMultiplePaths_ExpectCommandLineOptionsException) + { + m_args.push_back("-changelist"); + m_args.push_back("value1,value2"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + // + + TEST_F(CommandLineOptionsTestFixture, OutputChangeListHasEmptyOption_ExpectCommandLineOptionsException) + { + m_args.push_back("-ochangelist"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(CommandLineOptionsTestFixture, OutputChangeListHasMultipleValues_ExpectCommandLineOptionsException) + { + m_args.push_back("-ochangelist"); + m_args.push_back("value1,value2,value3"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + // + + TEST_F(CommandLineOptionsTestFixture, TestSequenceTypeHasEmptyOption_ExpectCommandLineOptionsException) + { + m_args.push_back("-sequence"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(CommandLineOptionsTestFixture, TestSequenceTypeHasNoneOption_ExpectNoneTestSequenceType) + { + m_args.push_back("-sequence"); + m_args.push_back("none"); + InitOptions(); + EXPECT_EQ(m_options->GetTestSequenceType(), TestImpact::TestSequenceType::None); + } + + TEST_F(CommandLineOptionsTestFixture, TestSequenceTypeHasSeedOption_ExpectSeedTestSequenceType) + { + m_args.push_back("-sequence"); + m_args.push_back("seed"); + InitOptions(); + EXPECT_EQ(m_options->GetTestSequenceType(), TestImpact::TestSequenceType::Seed); + } + + TEST_F(CommandLineOptionsTestFixture, TestSequenceTypeHasRegularOption_ExpectRegularTestSequenceType) + { + m_args.push_back("-sequence"); + m_args.push_back("regular"); + InitOptions(); + EXPECT_EQ(m_options->GetTestSequenceType(), TestImpact::TestSequenceType::Regular); + } + + TEST_F(CommandLineOptionsTestFixture, TestSequenceTypeHasImpactAnalysisOption_ExpectImpactAnalysisTestSequenceType) + { + m_args.push_back("-sequence"); + m_args.push_back("tia"); + InitOptions(); + EXPECT_EQ(m_options->GetTestSequenceType(), TestImpact::TestSequenceType::ImpactAnalysis); + } + + TEST_F(CommandLineOptionsTestFixture, TestSequenceTypeHasImpactAnalysisNoWriteOption_ExpectImpactAnalysisNoWriteTestSequenceType) + { + m_args.push_back("-sequence"); + m_args.push_back("tianowrite"); + InitOptions(); + EXPECT_EQ(m_options->GetTestSequenceType(), TestImpact::TestSequenceType::ImpactAnalysisNoWrite); + } + + TEST_F(CommandLineOptionsTestFixture, TestSequenceTypeHasSafeImpactAnalysisOption_ExpectSafeImpactAnalysisTestSequenceType) + { + m_args.push_back("-sequence"); + m_args.push_back("tiaorseed"); + InitOptions(); + EXPECT_EQ(m_options->GetTestSequenceType(), TestImpact::TestSequenceType::ImpactAnalysisOrSeed); + } + + TEST_F(CommandLineOptionsTestFixture, TestSequenceTypeHasInvalidOption_ExpectCommandLineOptionsException) + { + m_args.push_back("-sequence"); + m_args.push_back("foo"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(CommandLineOptionsTestFixture, TestSequenceTypeHasMultipleValues_ExpectCommandLineOptionsException) + { + m_args.push_back("-sequence"); + m_args.push_back("seed,tia"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + // + + TEST_F(CommandLineOptionsTestFixture, TestPrioritizationPolicyHasEmptyOption_ExpectCommandLineOptionsException) + { + m_args.push_back("-ppolicy"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(CommandLineOptionsTestFixture, TestPrioritizationPolicyHasNoneOption_ExpectNoneTestPrioritizationPolicy) + { + m_args.push_back("-ppolicy"); + m_args.push_back("none"); + InitOptions(); + EXPECT_EQ(m_options->GetTestPrioritizationPolicy(), TestImpact::Policy::TestPrioritization::None); + } + + TEST_F(CommandLineOptionsTestFixture, TestPrioritizationPolicyHasDependencyLocalityOption_ExpectDependencyLocalityTestPrioritizationPolicy) + { + m_args.push_back("-ppolicy"); + m_args.push_back("locality"); + InitOptions(); + EXPECT_EQ(m_options->GetTestPrioritizationPolicy(), TestImpact::Policy::TestPrioritization::DependencyLocality); + } + + TEST_F(CommandLineOptionsTestFixture, TestPrioritizationPolicyInvalidOption_ExpectCommandLineOptionsException) + { + m_args.push_back("-ppolicy"); + m_args.push_back("none,locality"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + // + + TEST_F(CommandLineOptionsTestFixture, ExecutionFailurePolicyHasEmptyOption_ExpectCommandLineOptionsException) + { + m_args.push_back("-ppolicy"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(CommandLineOptionsTestFixture, ExecutionFailurePolicyHasAbortOption_ExpectAbortExecutionFailurePolicy) + { + m_args.push_back("-epolicy"); + m_args.push_back("abort"); + InitOptions(); + EXPECT_EQ(m_options->GetExecutionFailurePolicy(), TestImpact::Policy::ExecutionFailure::Abort); + } + + TEST_F(CommandLineOptionsTestFixture, ExecutionFailurePolicyHasContinueOption_ExpectContinueExecutionFailurePolicy) + { + m_args.push_back("-epolicy"); + m_args.push_back("continue"); + InitOptions(); + EXPECT_EQ(m_options->GetExecutionFailurePolicy(), TestImpact::Policy::ExecutionFailure::Continue); + } + + TEST_F(CommandLineOptionsTestFixture, ExecutionFailurePolicyHasIgnoreOption_ExpectIgnoreExecutionFailurePolicy) + { + m_args.push_back("-epolicy"); + m_args.push_back("ignore"); + InitOptions(); + EXPECT_EQ(m_options->GetExecutionFailurePolicy(), TestImpact::Policy::ExecutionFailure::Ignore); + } + + TEST_F(CommandLineOptionsTestFixture, ExecutionFailurePolicyHasInvalidOption_ExpectCommandLineOptionsException) + { + m_args.push_back("-epolicy"); + m_args.push_back("foo"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(CommandLineOptionsTestFixture, ExecutionFailurePolicyHasMultipleValues_ExpectCommandLineOptionsException) + { + m_args.push_back("-epolicy"); + m_args.push_back("abort,ingore"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + // + + TEST_F(CommandLineOptionsTestFixture, ExecutionFailureDraftingPolicyHasEmptyOption_ExpectCommandLineOptionsException) + { + m_args.push_back("-rexecfailures"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(CommandLineOptionsTestFixture, FailedTestCoveragePolicyHasKeepOption_ExpectKeepFailedTestCoveragePolicy) + { + m_args.push_back("-cpolicy"); + m_args.push_back("keep"); + InitOptions(); + EXPECT_EQ(m_options->GetFailedTestCoveragePolicy(), TestImpact::Policy::FailedTestCoverage::Keep); + } + + TEST_F(CommandLineOptionsTestFixture, FailedTestCoveragePolicyHasDiscardOption_ExpectDiscardFailedTestCoveragePolicy) + { + m_args.push_back("-cpolicy"); + m_args.push_back("discard"); + InitOptions(); + EXPECT_EQ(m_options->GetFailedTestCoveragePolicy(), TestImpact::Policy::FailedTestCoverage::Discard); + } + + TEST_F(CommandLineOptionsTestFixture, FailedTestCoveragePolicyInvalidOption_ExpectCommandLineOptionsException) + { + m_args.push_back("-cpolicy"); + m_args.push_back("foo"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(CommandLineOptionsTestFixture, FailedTestCoveragePolicyPolicyHasMultipleValues_ExpectCommandLineOptionsException) + { + m_args.push_back("--cpolicy"); + m_args.push_back("keep,discard"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + // + + TEST_F(CommandLineOptionsTestFixture, TestFailurePolicyHasEmptyOption_ExpectCommandLineOptionsException) + { + m_args.push_back("-fpolicy"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(CommandLineOptionsTestFixture, TestFailurePolicyHasAbortOption_ExpectAbortTestFailurePolicy) + { + m_args.push_back("-fpolicy"); + m_args.push_back("abort"); + InitOptions(); + EXPECT_EQ(m_options->GetTestFailurePolicy(), TestImpact::Policy::TestFailure::Abort); + } + + TEST_F(CommandLineOptionsTestFixture, TestFailurePolicyHasContinueOption_ExpectContinueTestFailurePolicy) + { + m_args.push_back("-fpolicy"); + m_args.push_back("continue"); + InitOptions(); + EXPECT_EQ(m_options->GetTestFailurePolicy(), TestImpact::Policy::TestFailure::Continue); + } + + TEST_F(CommandLineOptionsTestFixture, TestFailurePolicyInvalidOption_ExpectCommandLineOptionsException) + { + m_args.push_back("-fpolicy"); + m_args.push_back("foo"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(CommandLineOptionsTestFixture, TestFailurePolicyHasMultipeValues_ExpectCommandLineOptionsException) + { + m_args.push_back("-fpolicy"); + m_args.push_back("abort,continue"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + // + + TEST_F(CommandLineOptionsTestFixture, IntegrityFailurePolicyHasEmptyOption_ExpectCommandLineOptionsException) + { + m_args.push_back("-ipolicy"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(CommandLineOptionsTestFixture, IntegrityFailurePolicyHasAbortOption_ExpectAbortIntegrityFailurePolicy) + { + m_args.push_back("-ipolicy"); + m_args.push_back("abort"); + InitOptions(); + EXPECT_EQ(m_options->GetIntegrityFailurePolicy(), TestImpact::Policy::IntegrityFailure::Abort); + } + + TEST_F(CommandLineOptionsTestFixture, IntegrityFailurePolicyHasContinueOption_ExpectContinueIntegrityFailurePolicy) + { + m_args.push_back("-ipolicy"); + m_args.push_back("continue"); + InitOptions(); + EXPECT_EQ(m_options->GetIntegrityFailurePolicy(), TestImpact::Policy::IntegrityFailure::Continue); + } + + TEST_F(CommandLineOptionsTestFixture, IntegrityFailurePolicyInvalidOption_ExpectCommandLineOptionsException) + { + m_args.push_back("-ipolicy"); + m_args.push_back("foo"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(CommandLineOptionsTestFixture, IntegrityFailurePolicyHasMultipeValues_ExpectCommandLineOptionsException) + { + m_args.push_back("-ipolicy"); + m_args.push_back("abort,continue"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + // + + TEST_F(CommandLineOptionsTestFixture, TestShardingHasEmptyOption_ExpectCommandLineOptionsException) + { + m_args.push_back("-shard"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(CommandLineOptionsTestFixture, TestShardingHasOnOption_ExpectOnTestSharding) + { + m_args.push_back("-shard"); + m_args.push_back("on"); + InitOptions(); + EXPECT_EQ(m_options->GetTestShardingPolicy(), TestImpact::Policy::TestSharding::Always); + } + + TEST_F(CommandLineOptionsTestFixture, TestShardingHasOffOption_ExpectOffTestSharding) + { + m_args.push_back("-shard"); + m_args.push_back("off"); + InitOptions(); + EXPECT_EQ(m_options->GetTestShardingPolicy(), TestImpact::Policy::TestSharding::Never); + } + + TEST_F(CommandLineOptionsTestFixture, TestShardingInvalidOption_ExpectCommandLineOptionsException) + { + m_args.push_back("-shard"); + m_args.push_back("foo"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(CommandLineOptionsTestFixture, TestShardingHasMultipeValues_ExpectCommandLineOptionsException) + { + m_args.push_back("-shard"); + m_args.push_back("on,off"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + // + + TEST_F(CommandLineOptionsTestFixture, TargetOutputCaptureHasEmptyOption_ExpectCommandLineOptionsException) + { + m_args.push_back("-targetout"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(CommandLineOptionsTestFixture, TargetOutputCaptureHasStdOutOption_ExpectStdOutTargetOutputCapture) + { + m_args.push_back("-targetout"); + m_args.push_back("stdout"); + InitOptions(); + EXPECT_EQ(m_options->GetTargetOutputCapture(), TestImpact::Policy::TargetOutputCapture::StdOut); + } + + TEST_F(CommandLineOptionsTestFixture, TargetOutputCaptureHasFileOption_ExpectFileTargetOutputCapture) + { + m_args.push_back("-targetout"); + m_args.push_back("file"); + InitOptions(); + EXPECT_EQ(m_options->GetTargetOutputCapture(), TestImpact::Policy::TargetOutputCapture::File); + } + + TEST_F(CommandLineOptionsTestFixture, TargetOutputCaptureHasStdOutAndFileOption_ExpectStdOutAndFileTargetOutputCapture) + { + m_args.push_back("-targetout"); + m_args.push_back("stdout,file"); + InitOptions(); + EXPECT_EQ(m_options->GetTargetOutputCapture(), TestImpact::Policy::TargetOutputCapture::StdOutAndFile); + } + + TEST_F(CommandLineOptionsTestFixture, TargetOutputCaptureInvalidOption_ExpectCommandLineOptionsException) + { + m_args.push_back("-targetout"); + m_args.push_back("foo"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(CommandLineOptionsTestFixture, TargetOutputCaptureHasExcessValues_ExpectCommandLineOptionsException) + { + m_args.push_back("-targetout"); + m_args.push_back("stdout,file,stdout"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + // + + TEST_F(CommandLineOptionsTestFixture, MaxConcurrencyHasEmptyOption_ExpectCommandLineOptionsException) + { + m_args.push_back("-maxconcurrency"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(CommandLineOptionsTestFixture, MaxConcurrencyHasInRangeOptions_ExpectInRangeMaxConcurrency) + { + m_args.push_back("-maxconcurrency"); + m_args.push_back("10"); + InitOptions(); + EXPECT_EQ(m_options->GetMaxConcurrency(), 10); + } + + TEST_F(CommandLineOptionsTestFixture, MaxConcurrencyHasOutOfRangeOption_ExpectCommandLineOptionsException) + { + m_args.push_back("-maxconcurrency"); + m_args.push_back("-1"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(CommandLineOptionsTestFixture, MaxConcurrencyInvalidOption_ExpectCommandLineOptionsException) + { + m_args.push_back("-maxconcurrency"); + m_args.push_back("foo"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(CommandLineOptionsTestFixture, MaxConcurrencyHasMultipeValues_ExpectCommandLineOptionsException) + { + m_args.push_back("-maxconcurrency"); + m_args.push_back("10,20"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + // + + TEST_F(CommandLineOptionsTestFixture, TestTargetTimeoutHasEmptyOption_ExpectCommandLineOptionsException) + { + m_args.push_back("-ttimeout"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(CommandLineOptionsTestFixture, TestTargetTimeoutHasInRangeOptions_ExpectInRangeTestTargetTimeout) + { + m_args.push_back("-ttimeout"); + m_args.push_back("10"); + InitOptions(); + EXPECT_EQ(m_options->GetTestTargetTimeout(), AZStd::chrono::seconds(10)); + } + + TEST_F(CommandLineOptionsTestFixture, TestTargetTimeoutHasOutOfRangeOption_ExpectCommandLineOptionsException) + { + m_args.push_back("-ttimeout"); + m_args.push_back("-1"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(CommandLineOptionsTestFixture, TestTargetTimeoutInvalidOption_ExpectCommandLineOptionsException) + { + m_args.push_back("-ttimeout"); + m_args.push_back("foo"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(CommandLineOptionsTestFixture, TestTargetTimeoutHasMultipeValues_ExpectCommandLineOptionsException) + { + m_args.push_back("-ttimeout"); + m_args.push_back("10,20"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + // + + TEST_F(CommandLineOptionsTestFixture, GlobalTimeoutHasEmptyOption_ExpectCommandLineOptionsException) + { + m_args.push_back("-gtimeout"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(CommandLineOptionsTestFixture, GlobalTimeoutHasInRangeOptions_ExpectInRangeGlobalTimeout) + { + m_args.push_back("-gtimeout"); + m_args.push_back("10"); + InitOptions(); + EXPECT_EQ(m_options->GetGlobalTimeout(), AZStd::chrono::seconds(10)); + } + + TEST_F(CommandLineOptionsTestFixture, GlobalTimeoutHasOutOfRangeOption_ExpectCommandLineOptionsException) + { + m_args.push_back("-gtimeout"); + m_args.push_back("-1"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(CommandLineOptionsTestFixture, GlobalTimeoutInvalidOption_ExpectCommandLineOptionsException) + { + m_args.push_back("-gtimeout"); + m_args.push_back("foo"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(CommandLineOptionsTestFixture, GlobalTimeoutHasMultipeValues_ExpectCommandLineOptionsException) + { + m_args.push_back("-gtimeout"); + m_args.push_back("10,20"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + // + + TEST_F(CommandLineOptionsTestFixture, SafeModeHasEmptyOption_ExpectCommandLineOptionsException) + { + m_args.push_back("-safemode"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(CommandLineOptionsTestFixture, SafeModeHasOnOption_ExpectOnSafeMode) + { + m_args.push_back("-safemode"); + m_args.push_back("on"); + InitOptions(); + EXPECT_TRUE(m_options->HasSafeMode()); + } + + TEST_F(CommandLineOptionsTestFixture, SafeModeHasOffOption_ExpectOffSafeMode) + { + m_args.push_back("-safemode"); + m_args.push_back("off"); + InitOptions(); + EXPECT_FALSE(m_options->HasSafeMode()); + } + + TEST_F(CommandLineOptionsTestFixture, SafeModeInvalidOption_ExpectCommandLineOptionsException) + { + m_args.push_back("-safemode"); + m_args.push_back("foo"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(CommandLineOptionsTestFixture, SafeModeHasMultipeValues_ExpectCommandLineOptionsException) + { + m_args.push_back("-safemode"); + m_args.push_back("on,off"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(CommandLineOptionsTestFixture, SuiteFilterEmptyOption_ExpectCommandLineOptionsException) + { + m_args.push_back("-suite"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } + catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } + catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(CommandLineOptionsTestFixture, SuiteFilterMultipleOptions_ExpectCommandLineOptionsException) + { + m_args.push_back("-suite"); + m_args.push_back("periodic,smoke"); + + try + { + InitOptions(); + + // Do not expect the command line options construction to succeed + FAIL(); + } catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) + { + // Expect a command line options to be thrown + SUCCEED(); + } catch (...) + { + // Do not expect any other exceptions + FAIL(); + } + } + + TEST_F(CommandLineOptionsTestFixture, SuitesFilterMainOption_ExpectMainSuiteFilter) + { + m_args.push_back("-suites"); + m_args.push_back("main"); + InitOptions(); + EXPECT_EQ(m_options->GetSuiteFilter(), TestImpact::SuiteType::Main); + } + + TEST_F(CommandLineOptionsTestFixture, SuitesFilterPeriodicOption_ExpectPeriodicSuiteFilter) + { + m_args.push_back("-suites"); + m_args.push_back("periodic"); + InitOptions(); + EXPECT_EQ(m_options->GetSuiteFilter(), TestImpact::SuiteType::Periodic); + } + + TEST_F(CommandLineOptionsTestFixture, SuitesFilterSandboxOption_ExpectSandboxSuiteFilter) + { + m_args.push_back("-suites"); + m_args.push_back("sandbox"); + InitOptions(); + EXPECT_EQ(m_options->GetSuiteFilter(), TestImpact::SuiteType::Sandbox); + } +} diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Code/Tests/TestImpactConsoleMainTest.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Code/Tests/TestImpactConsoleMainTest.cpp new file mode 100644 index 0000000000..e69de29bb2 diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Code/Tests/TestImpactConsoleTestSequenceEventHandlerTest.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Code/Tests/TestImpactConsoleTestSequenceEventHandlerTest.cpp new file mode 100644 index 0000000000..52422daa31 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Code/Tests/TestImpactConsoleTestSequenceEventHandlerTest.cpp @@ -0,0 +1,42 @@ +///* +// * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +// * its licensors. +// * +// * For complete copyright and license terms please see the LICENSE at the root of this +// * distribution (the "License"). All use of this software is governed by the License, +// * or, if provided, by the license below or the license accompanying this file. Do not +// * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * +// */ +// +//#include +//#include +// +//#include +// +//#include +//#include +//#include +//#include +// +//namespace UnitTest +//{ +// class ConsoleTestSequenceTestFixture +// : public AllocatorsTestFixture +// { +// }; +// +// TEST_F(ConsoleTestSequenceTestFixture, CheckEmptyArgs_ExpectDefaultValues) +// { +// AZStd::unordered_set suites = { "PERIODIC","MAIN" }; +// TestImpact::Console::TestSequenceEventHandler seq(&suites); +// AZStd::vector selectedTests = { "Test1", "Test2", "Test3", "Test4", "Test5" }; +// seq.operator()(TestImpact::Client::TestRunSelection(selectedTests, {"Test6"})); +// +// for (auto i = 0; i < selectedTests.size(); i++) +// { +// seq.operator()(TestImpact::Client::TestRun(selectedTests[i], (TestImpact::Client::TestRunResult)i, AZStd::chrono::milliseconds(i))); +// } +// } +//} diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Code/Tests/TestImpactFrontendConsoleStaticTestMain.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Code/Tests/TestImpactFrontendConsoleStaticTestMain.cpp new file mode 100644 index 0000000000..9e128f90af --- /dev/null +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Code/Tests/TestImpactFrontendConsoleStaticTestMain.cpp @@ -0,0 +1,34 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include +#include + +class TestImpactTestEnvironment + : public AZ::Test::ITestEnvironment +{ +protected: + void SetupEnvironment() override + { + AZ::AllocatorInstance::Create(); + AZ::AllocatorInstance::Create(); + } + + void TeardownEnvironment() override + { + AZ::AllocatorInstance::Destroy(); + AZ::AllocatorInstance::Destroy(); + } +}; + +AZ_UNIT_TEST_HOOK(new TestImpactTestEnvironment); diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Code/Tests/TestImpactRuntimeConfigurationFactoryTest.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Code/Tests/TestImpactRuntimeConfigurationFactoryTest.cpp new file mode 100644 index 0000000000..4b1b3f162e --- /dev/null +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Code/Tests/TestImpactRuntimeConfigurationFactoryTest.cpp @@ -0,0 +1,27 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include + +#include + +#include + +namespace UnitTest +{ + //TEST(FOO, BAR) + //{ + // std::ifstream configFile("C:\\o3de_TIF_Feature\\windows_vs2019\\bin\\debug\\tiaf.debug.json"); + // AZStd::string configData((std::istreambuf_iterator(configFile)), std::istreambuf_iterator()); + // const auto configuration = TestImpact::ConfigurationFactory(configData); + //} +} diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Executable/Code/testimpactframework_frontend_console_files.cmake b/Code/Tools/TestImpactFramework/Frontend/Console/Code/testimpactframework_frontend_console_files.cmake similarity index 100% rename from Code/Tools/TestImpactFramework/Frontend/Console/Executable/Code/testimpactframework_frontend_console_files.cmake rename to Code/Tools/TestImpactFramework/Frontend/Console/Code/testimpactframework_frontend_console_files.cmake diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/testimpactframework_frontend_console_static_files.cmake b/Code/Tools/TestImpactFramework/Frontend/Console/Code/testimpactframework_frontend_console_static_files.cmake similarity index 100% rename from Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/testimpactframework_frontend_console_static_files.cmake rename to Code/Tools/TestImpactFramework/Frontend/Console/Code/testimpactframework_frontend_console_static_files.cmake diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/testimpactframework_frontend_console_static_tests_files.cmake b/Code/Tools/TestImpactFramework/Frontend/Console/Code/testimpactframework_frontend_console_static_tests_files.cmake similarity index 67% rename from Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/testimpactframework_frontend_console_static_tests_files.cmake rename to Code/Tools/TestImpactFramework/Frontend/Console/Code/testimpactframework_frontend_console_static_tests_files.cmake index 5714be5dfb..4d8e65b2cc 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/testimpactframework_frontend_console_static_tests_files.cmake +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Code/testimpactframework_frontend_console_static_tests_files.cmake @@ -10,4 +10,9 @@ # set(FILES + Tests/TestImpactCommandLineOptionsTest.cpp + Tests/TestImpactRuntimeConfigurationFactoryTest.cpp + Tests/TestImpactFrontendConsoleStaticTestMain.cpp + Tests/TestImpactConsoleMainTest.cpp + Tests/TestImpactConsoleTestSequenceEventHandlerTest.cpp ) diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Executable/CMakeLists.txt b/Code/Tools/TestImpactFramework/Frontend/Console/Executable/CMakeLists.txt deleted file mode 100644 index 8298bb7123..0000000000 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Executable/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -# -# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or -# its licensors. -# -# For complete copyright and license terms please see the LICENSE at the root of this -# distribution (the "License"). All use of this software is governed by the License, -# or, if provided, by the license below or the license accompanying this file. Do not -# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# - -add_subdirectory(Code) \ No newline at end of file diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Executable/Code/CMakeLists.txt b/Code/Tools/TestImpactFramework/Frontend/Console/Executable/Code/CMakeLists.txt deleted file mode 100644 index 88da472e1e..0000000000 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Executable/Code/CMakeLists.txt +++ /dev/null @@ -1,25 +0,0 @@ -# -# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or -# its licensors. -# -# For complete copyright and license terms please see the LICENSE at the root of this -# distribution (the "License"). All use of this software is governed by the License, -# or, if provided, by the license below or the license accompanying this file. Do not -# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# - -ly_add_target( - NAME TestImpact.Frontend.Console EXECUTABLE - OUTPUT_NAME tiaf - NAMESPACE AZ - FILES_CMAKE - testimpactframework_frontend_console_files.cmake - INCLUDE_DIRECTORIES - PRIVATE - Source - BUILD_DEPENDENCIES - PRIVATE - AZ::TestImpact.Frontend.Console.Static -) - diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/CMakeLists.txt b/Code/Tools/TestImpactFramework/Frontend/Console/Static/CMakeLists.txt deleted file mode 100644 index 8298bb7123..0000000000 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -# -# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or -# its licensors. -# -# For complete copyright and license terms please see the LICENSE at the root of this -# distribution (the "License"). All use of this software is governed by the License, -# or, if provided, by the license below or the license accompanying this file. Do not -# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# - -add_subdirectory(Code) \ No newline at end of file diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/CMakeLists.txt b/Code/Tools/TestImpactFramework/Runtime/Code/CMakeLists.txt index 4c021a12b4..bcfeeacb33 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/CMakeLists.txt +++ b/Code/Tools/TestImpactFramework/Runtime/Code/CMakeLists.txt @@ -40,48 +40,49 @@ ly_add_target( # Tests ################################################################################ -ly_add_target( - NAME TestImpact.Runtime.Tests ${PAL_TRAIT_TEST_TARGET_TYPE} - NAMESPACE AZ - FILES_CMAKE - testimpactframework_runtime_tests_files.cmake - INCLUDE_DIRECTORIES - PRIVATE - Include - Source - Tests - BUILD_DEPENDENCIES - PRIVATE - AZ::AzTestShared - AZ::AzTest - AZ::TestImpact.Runtime.Static - RUNTIME_DEPENDENCIES - AZ::AzTestRunner - AZ::TestImpact.TestProcess.Console - AZ::TestImpact.TestTargetA.Tests - AZ::TestImpact.TestTargetB.Tests - AZ::TestImpact.TestTargetC.Tests - AZ::TestImpact.TestTargetD.Tests - COMPILE_DEFINITIONS - PRIVATE - LY_TEST_IMPACT_AZ_TESTRUNNER_BIN="$" - LY_TEST_IMPACT_TEST_PROCESS_BIN="$" - LY_TEST_IMPACT_TEST_TARGET_A_BIN="$" - LY_TEST_IMPACT_TEST_TARGET_B_BIN="$" - LY_TEST_IMPACT_TEST_TARGET_C_BIN="$" - LY_TEST_IMPACT_TEST_TARGET_D_BIN="$" - LY_TEST_IMPACT_TEST_TARGET_A_BASE_NAME="$" - LY_TEST_IMPACT_TEST_TARGET_B_BASE_NAME="$" - LY_TEST_IMPACT_TEST_TARGET_C_BASE_NAME="$" - LY_TEST_IMPACT_TEST_TARGET_D_BASE_NAME="$" - LY_TEST_IMPACT_TEST_TARGET_ENUMERATION_DIR="${GTEST_XML_OUTPUT_DIR}/TestImpact/Temp/Exclusive/Enum" - LY_TEST_IMPACT_TEST_TARGET_RESULTS_DIR="${GTEST_XML_OUTPUT_DIR}/TestImpact/Temp/Exclusive/Result" - LY_TEST_IMPACT_TEST_TARGET_COVERAGE_DIR="${GTEST_XML_OUTPUT_DIR}/TestImpact/Temp/Exclusive/Coverage" - LY_TEST_IMPACT_INSTRUMENTATION_BIN="${LY_TEST_IMPACT_INSTRUMENTATION_BIN}" - LY_TEST_IMPACT_MODULES_DIR="${CMAKE_BINARY_DIR}" - LY_TEST_IMPACT_COVERAGE_SOURCES_DIR="${CMAKE_CURRENT_SOURCE_DIR}" -) - -ly_add_googletest( - NAME AZ::TestImpact.Runtime.Tests -) +# Disabled: SPEC-7246 +#ly_add_target( +# NAME TestImpact.Runtime.Tests ${PAL_TRAIT_TEST_TARGET_TYPE} +# NAMESPACE AZ +# FILES_CMAKE +# testimpactframework_runtime_tests_files.cmake +# INCLUDE_DIRECTORIES +# PRIVATE +# Include +# Source +# Tests +# BUILD_DEPENDENCIES +# PRIVATE +# AZ::AzTestShared +# AZ::AzTest +# AZ::TestImpact.Runtime.Static +# RUNTIME_DEPENDENCIES +# AZ::AzTestRunner +# AZ::TestImpact.TestProcess.Console +# AZ::TestImpact.TestTargetA.Tests +# AZ::TestImpact.TestTargetB.Tests +# AZ::TestImpact.TestTargetC.Tests +# AZ::TestImpact.TestTargetD.Tests +# COMPILE_DEFINITIONS +# PRIVATE +# LY_TEST_IMPACT_AZ_TESTRUNNER_BIN="$" +# LY_TEST_IMPACT_TEST_PROCESS_BIN="$" +# LY_TEST_IMPACT_TEST_TARGET_A_BIN="$" +# LY_TEST_IMPACT_TEST_TARGET_B_BIN="$" +# LY_TEST_IMPACT_TEST_TARGET_C_BIN="$" +# LY_TEST_IMPACT_TEST_TARGET_D_BIN="$" +# LY_TEST_IMPACT_TEST_TARGET_A_BASE_NAME="$" +# LY_TEST_IMPACT_TEST_TARGET_B_BASE_NAME="$" +# LY_TEST_IMPACT_TEST_TARGET_C_BASE_NAME="$" +# LY_TEST_IMPACT_TEST_TARGET_D_BASE_NAME="$" +# LY_TEST_IMPACT_TEST_TARGET_ENUMERATION_DIR="${GTEST_XML_OUTPUT_DIR}/TestImpact/Temp/Exclusive/Enum" +# LY_TEST_IMPACT_TEST_TARGET_RESULTS_DIR="${GTEST_XML_OUTPUT_DIR}/TestImpact/Temp/Exclusive/Result" +# LY_TEST_IMPACT_TEST_TARGET_COVERAGE_DIR="${GTEST_XML_OUTPUT_DIR}/TestImpact/Temp/Exclusive/Coverage" +# LY_TEST_IMPACT_INSTRUMENTATION_BIN="${LY_TEST_IMPACT_INSTRUMENTATION_BIN}" +# LY_TEST_IMPACT_MODULES_DIR="${CMAKE_BINARY_DIR}" +# LY_TEST_IMPACT_COVERAGE_SOURCES_DIR="${CMAKE_CURRENT_SOURCE_DIR}" +#) +# +#ly_add_googletest( +# NAME AZ::TestImpact.Runtime.Tests +#) diff --git a/cmake/LYTestWrappers.cmake b/cmake/LYTestWrappers.cmake index eebc281d06..1bacc4feb0 100644 --- a/cmake/LYTestWrappers.cmake +++ b/cmake/LYTestWrappers.cmake @@ -315,6 +315,7 @@ function(ly_add_pytest) ${ly_add_pytest_UNPARSED_ARGUMENTS} ) + set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_pytest_NAME}_SCRIPT_PATH ${ly_add_pytest_PATH}) set_tests_properties(${LY_ADDED_TEST_NAME} PROPERTIES RUN_SERIAL "${ly_add_pytest_TEST_SERIAL}") endfunction() diff --git a/scripts/build/Platform/Windows/build_config.json b/scripts/build/Platform/Windows/build_config.json index 9201e13a80..81100eead7 100644 --- a/scripts/build/Platform/Windows/build_config.json +++ b/scripts/build/Platform/Windows/build_config.json @@ -88,7 +88,7 @@ "COMMAND": "python_windows.cmd", "PARAMETERS": { "SCRIPT_PATH": "scripts/build/TestImpactAnalysis/tiaf_driver.py", - "SCRIPT_PARAMETERS": "--testFailurePolicy abort --safeMode --suite main --pipeline !PIPELINE_NAME! --destCommit !CHANGE_ID! --config \"build\\windows_vs2019\\bin\\TestImpactFramework\\persistent\\tiaf.profile.json\"" + "SCRIPT_PARAMETERS": "--testFailurePolicy abort --safeMode --suite main --pipeline !PIPELINE_NAME! --destCommit !CHANGE_ID! --config \"!OUTPUT_DIRECTORY!\\TestImpactFramework\\persistent\\tiaf.profile.json\"" } }, "debug_vs2019": { diff --git a/scripts/build/TestImpactAnalysis/tiaf.py b/scripts/build/TestImpactAnalysis/tiaf.py index c040ae1147..19c1b2754d 100644 --- a/scripts/build/TestImpactAnalysis/tiaf.py +++ b/scripts/build/TestImpactAnalysis/tiaf.py @@ -219,49 +219,4 @@ class TestImpact: else: print(f"The test impact analysis runtime returned with error: '{result.returncode}'.") return result.returncode - - - - #args = [] - #print("Please note: test impact analysis sequences will be run in read-only mode (seed sequences are unaffected).") - #if sequence_type == SequenceType.REGULAR: - # print("Sequence type: regular.") - # args.append("--sequence=regular") - # args.append("--fpolicy=abort") - #elif sequence_type == SequenceType.SEED: - # print("Sequence type: seed.") - # args.append("--sequence=seed") - # args.append("--fpolicy=continue") - #elif sequence_type == SequenceType.TEST_IMPACT_ANALYSIS: - # print("Sequence type: test impact analysis (no write).") - # args.append("--fpolicy=abort") - # if self.__has_change_list: - # args.append(f"-changelist={self.__change_list_path}") - # args.append("--sequence=tianowrite") - # else: - # print(f"No change list was generated, falling back to a regular sequence.") - # print("Sequence type: Regular.") - # args.append("--sequence=regular") - #else: - # raise ValueError(sequence_type) - # - #if test_timeout != None: - # args.append(f"--ttimeout={test_timeout}") - # print(f"Test target timeout is set to {test_timeout} seconds.") - #if global_timeout != None: - # args.append(f"--gtimeout={global_timeout}") - # print(f"Global sequence timeout is set to {test_timeout} seconds.") - # - #print("Args: ", end='') - #print(*args) - #result = subprocess.run([self.__tiaf_bin] + args) - #if result.returncode == 0: - # print("Test impact analysis runtime returned successfully.") - # if sequence_type == SequenceType.SEED: - # print("Writing historical meta-data...") - # self.__write_last_run_hash(self.__dst_commit) - # print("Complete!") - #else: - # print(f"The test impact analysis runtime returned with error: '{result.returncode}'.") - #return result.returncode - + \ No newline at end of file diff --git a/scripts/build/TestImpactAnalysis/tiaf_driver.py b/scripts/build/TestImpactAnalysis/tiaf_driver.py index 7ff73e505b..f452b9738f 100644 --- a/scripts/build/TestImpactAnalysis/tiaf_driver.py +++ b/scripts/build/TestImpactAnalysis/tiaf_driver.py @@ -57,4 +57,5 @@ if __name__ == "__main__": args = parse_args() tiaf = TestImpact(args.config, args.pipeline, args.dst_commit) return_code = tiaf.run(args.suite, args.test_failure_policy, args.safe_mode, args.test_timeout, args.global_timeout) - sys.exit(return_code) \ No newline at end of file + sys.exit(return_code) + \ No newline at end of file From af0bcb06365969d17f71b705862448b473a6556f Mon Sep 17 00:00:00 2001 From: John Date: Thu, 10 Jun 2021 14:58:34 +0100 Subject: [PATCH 092/102] Remove test files --- ...ImpactBuildTargetDescriptorFactoryTest.cpp | 370 ---- .../TestImpactModuleCoverageFactoryTest.cpp | 522 ----- ...TestImpactTargetDescriptorCompilerTest.cpp | 150 -- ...tImpactTestEnumerationSuiteFactoryTest.cpp | 589 ------ .../TestImpactTestRunSuiteFactoryTest.cpp | 592 ------ ...TestImpactTestTargetMetaMapFactoryTest.cpp | 239 --- .../TestImpactProcessSchedulerTest.cpp | 712 ------- .../Tests/Process/TestImpactProcessTest.cpp | 491 ----- .../Target/TestImpactBuildTargetTest.cpp | 352 ---- .../Code/Tests/TestImpactExceptionTest.cpp | 150 -- .../Tests/TestImpactTestJobRunnerCommon.h | 159 -- .../Code/Tests/TestImpactTestUtils.cpp | 1843 ----------------- .../Runtime/Code/Tests/TestImpactTestUtils.h | 220 -- .../Code/Tests/TestProcess/CMakeLists.txt | 12 - .../Tests/TestProcess/Code/CMakeLists.txt | 24 - .../Code/Source/TestImpactTestProcess.cpp | 93 - .../Code/Source/TestImpactTestProcess.h | 38 - .../Source/TestImpactTestProcessLargeText.cpp | 531 ----- .../Source/TestImpactTestProcessLargeText.h | 19 - .../Code/Source/TestImpactTestProcessMain.cpp | 19 - ...estimpactframework_testprocess_files.cmake | 18 - .../Code/Tests/TestTargetA/CMakeLists.txt | 12 - .../Tests/TestTargetA/Code/CMakeLists.txt | 28 - .../Code/Tests/TestImpactTestTargetA.cpp | 73 - ...actframework_testtargeta_tests_files.cmake | 14 - .../Code/Tests/TestTargetB/CMakeLists.txt | 12 - .../Tests/TestTargetB/Code/CMakeLists.txt | 29 - .../Code/Tests/TestImpactTestTargetB.cpp | 78 - ...actframework_testtargetb_tests_files.cmake | 14 - .../Code/Tests/TestTargetC/CMakeLists.txt | 12 - .../Tests/TestTargetC/Code/CMakeLists.txt | 29 - .../Code/Tests/TestImpactTestTargetC.cpp | 63 - ...actframework_testtargetc_tests_files.cmake | 14 - .../Code/Tests/TestTargetD/CMakeLists.txt | 12 - .../Tests/TestTargetD/Code/CMakeLists.txt | 29 - .../Code/Tests/TestImpactTestTargetD.cpp | 188 -- ...actframework_testtargetd_tests_files.cmake | 14 - 37 files changed, 7764 deletions(-) delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactBuildTargetDescriptorFactoryTest.cpp delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactModuleCoverageFactoryTest.cpp delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactTargetDescriptorCompilerTest.cpp delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactTestEnumerationSuiteFactoryTest.cpp delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactTestRunSuiteFactoryTest.cpp delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactTestTargetMetaMapFactoryTest.cpp delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/Process/TestImpactProcessSchedulerTest.cpp delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/Process/TestImpactProcessTest.cpp delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/Target/TestImpactBuildTargetTest.cpp delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactExceptionTest.cpp delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactTestJobRunnerCommon.h delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactTestUtils.cpp delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactTestUtils.h delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/CMakeLists.txt delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/CMakeLists.txt delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/Source/TestImpactTestProcess.cpp delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/Source/TestImpactTestProcess.h delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/Source/TestImpactTestProcessLargeText.cpp delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/Source/TestImpactTestProcessLargeText.h delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/Source/TestImpactTestProcessMain.cpp delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/testimpactframework_testprocess_files.cmake delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetA/CMakeLists.txt delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetA/Code/CMakeLists.txt delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetA/Code/Tests/TestImpactTestTargetA.cpp delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetA/Code/testimpactframework_testtargeta_tests_files.cmake delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetB/CMakeLists.txt delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetB/Code/CMakeLists.txt delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetB/Code/Tests/TestImpactTestTargetB.cpp delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetB/Code/testimpactframework_testtargetb_tests_files.cmake delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetC/CMakeLists.txt delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetC/Code/CMakeLists.txt delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetC/Code/Tests/TestImpactTestTargetC.cpp delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetC/Code/testimpactframework_testtargetc_tests_files.cmake delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetD/CMakeLists.txt delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetD/Code/CMakeLists.txt delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetD/Code/Tests/TestImpactTestTargetD.cpp delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetD/Code/testimpactframework_testtargetd_tests_files.cmake diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactBuildTargetDescriptorFactoryTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactBuildTargetDescriptorFactoryTest.cpp deleted file mode 100644 index a0a4a7309e..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactBuildTargetDescriptorFactoryTest.cpp +++ /dev/null @@ -1,370 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include - -#include -#include - -#include -#include - -namespace UnitTest -{ - class BuildTargetDescriptorFactoryTestFixture - : public AllocatorsTestFixture - { - protected: - const AZStd::vector m_staticExclude = {".cmake"}; - const AZStd::vector m_inputExclude = {".jinja"}; - const AZStd::string m_autogenMatcher = {"(.*)\\..*"}; - - const AZStd::vector m_autogenInputs = - { - "Gems/ScriptCanvasDiagnosticLibrary/Code/Source/Log.ScriptCanvasNode.xml", - "Gems/ScriptCanvasDiagnosticLibrary/Code/Source/DrawText.ScriptCanvasNode.xml", - "Gems/ScriptCanvas/Code/Include/ScriptCanvas/AutoGen/ScriptCanvasNode_Header.jinja", - "Gems/ScriptCanvas/Code/Include/ScriptCanvas/AutoGen/ScriptCanvasNode_Source.jinja" - }; - - const AZStd::vector m_autogenOutputs = - { - "windows_vs2019/Gems/ScriptCanvasDiagnosticLibrary/Code/Azcg/Generated/Source/Log.generated.h", - "windows_vs2019/Gems/ScriptCanvasDiagnosticLibrary/Code/Azcg/Generated/Source/DrawText.generated.h", - "windows_vs2019/Gems/ScriptCanvasDiagnosticLibrary/Code/Azcg/Generated/Source/Log.generated.cpp", - "windows_vs2019/Gems/ScriptCanvasDiagnosticLibrary/Code/Azcg/Generated/Source/DrawText.generated.cpp" - }; - - const AZStd::vector m_staticSources = - { - "Gems/ScriptCanvasDiagnosticLibrary/Code/Source/precompiled.cpp", - "Gems/ScriptCanvasDiagnosticLibrary/Code/Source/precompiled.h", - "Gems/ScriptCanvasDiagnosticLibrary/Code/scriptcanvasdiagnosticlibrary_autogen_files.cmake", - "windows_vs2019/Gems/ScriptCanvasDiagnosticLibrary/Code/Azcg/Generated/Source/Log.generated.h", - "windows_vs2019/Gems/ScriptCanvasDiagnosticLibrary/Code/Azcg/Generated/Source/DrawText.generated.h", - "windows_vs2019/Gems/ScriptCanvasDiagnosticLibrary/Code/Azcg/Generated/Source/Log.generated.cpp", - "windows_vs2019/Gems/ScriptCanvasDiagnosticLibrary/Code/Azcg/Generated/Source/DrawText.generated.cpp" - }; - - const AZStd::vector m_expectedStaticSources = - { - "Gems/ScriptCanvasDiagnosticLibrary/Code/Source/precompiled.cpp", - "Gems/ScriptCanvasDiagnosticLibrary/Code/Source/precompiled.h", - "windows_vs2019/Gems/ScriptCanvasDiagnosticLibrary/Code/Azcg/Generated/Source/Log.generated.h", - "windows_vs2019/Gems/ScriptCanvasDiagnosticLibrary/Code/Azcg/Generated/Source/DrawText.generated.h", - "windows_vs2019/Gems/ScriptCanvasDiagnosticLibrary/Code/Azcg/Generated/Source/Log.generated.cpp", - "windows_vs2019/Gems/ScriptCanvasDiagnosticLibrary/Code/Azcg/Generated/Source/DrawText.generated.cpp" - }; - - const AZStd::string m_name = "ScriptCanvasDiagnosticLibrary.Static"; - const AZStd::string m_outputName = "ScriptCanvasDiagnosticLibrary"; - const AZStd::string m_path = "Gems/ScriptCanvasDiagnosticLibrary/Code"; - - const TestImpact::AutogenSources m_expectedAutogenSources = - { - { - "Gems/ScriptCanvasDiagnosticLibrary/Code/Source/Log.ScriptCanvasNode.xml", - { - "windows_vs2019/Gems/ScriptCanvasDiagnosticLibrary/Code/Azcg/Generated/Source/Log.generated.h", - "windows_vs2019/Gems/ScriptCanvasDiagnosticLibrary/Code/Azcg/Generated/Source/Log.generated.cpp" - } - }, - { - "Gems/ScriptCanvasDiagnosticLibrary/Code/Source/DrawText.ScriptCanvasNode.xml", - { - "windows_vs2019/Gems/ScriptCanvasDiagnosticLibrary/Code/Azcg/Generated/Source/DrawText.generated.h", - "windows_vs2019/Gems/ScriptCanvasDiagnosticLibrary/Code/Azcg/Generated/Source/DrawText.generated.cpp" - } - } - }; - }; - - TEST_F(BuildTargetDescriptorFactoryTestFixture, NoRawData_ExpectArtifactException) - { - // Given an empty raw descriptor string - const AZStd::string rawTargetDescriptor; - - try - { - // When attempting to construct the build target descriptor - const TestImpact::BuildTargetDescriptor buildTarget = - TestImpact::BuildTargetDescriptorFactory(rawTargetDescriptor, m_staticExclude, m_inputExclude, m_autogenMatcher); - - // Do not expect this statement to be reachable - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::ArtifactException& e) - { - // Expect an artifact exception - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(BuildTargetDescriptorFactoryTestFixture, InvalidRawData_ExpectArtifactException) - { - // Given a raw descriptor string of invalid data - const AZStd::string rawTargetDescriptor = "abcde"; - - try - { - // When attempting to construct the build target descriptor - const TestImpact::BuildTargetDescriptor buildTarget = - TestImpact::BuildTargetDescriptorFactory(rawTargetDescriptor, m_staticExclude, m_inputExclude, m_autogenMatcher); - - // Do not expect this statement to be reachable - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::ArtifactException& e) - { - // Expect an artifact exception - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(BuildTargetDescriptorFactoryTestFixture, NoAutogenMatcher_ExpectArtifactException) - { - // Given a valid raw descriptor string - const AZStd::string rawTargetDescriptor = - GenerateBuildTargetDescriptorString(m_name, m_outputName, m_path, m_staticSources, m_autogenInputs, m_autogenOutputs); - - try - { - // When attempting to construct the build target descriptor with an empty autogen matcher - const TestImpact::BuildTargetDescriptor buildTarget = - TestImpact::BuildTargetDescriptorFactory(rawTargetDescriptor, m_staticExclude, m_inputExclude, ""); - - // Do not expect this statement to be reachable - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::ArtifactException& e) - { - // Expect an artifact exception - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(BuildTargetDescriptorFactoryTestFixture, EmptyName_ExpectArtifactException) - { - // Given a invalid raw descriptor string lacking build meta-data name - const AZStd::string rawTargetDescriptor = - GenerateBuildTargetDescriptorString("", m_outputName, m_path, m_staticSources, m_autogenInputs, m_autogenOutputs); - - try - { - // When attempting to construct the build target descriptor - const TestImpact::BuildTargetDescriptor buildTarget = - TestImpact::BuildTargetDescriptorFactory(rawTargetDescriptor, m_staticExclude, m_inputExclude, m_autogenMatcher); - - // Do not expect this statement to be reachable - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::ArtifactException& e) - { - // Expect an artifact exception - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(BuildTargetDescriptorFactoryTestFixture, EmptyOutputName_ExpectArtifactException) - { - // Given a invalid raw descriptor string lacking build meta-data output name - const AZStd::string rawTargetDescriptor = - GenerateBuildTargetDescriptorString(m_name, "", m_path, m_staticSources, m_autogenInputs, m_autogenOutputs); - - try - { - // When attempting to construct the build target descriptor - const TestImpact::BuildTargetDescriptor buildTarget = - TestImpact::BuildTargetDescriptorFactory(rawTargetDescriptor, m_staticExclude, m_inputExclude, m_autogenMatcher); - - // Do not expect this statement to be reachable - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::ArtifactException& e) - { - // Expect an artifact exception - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(BuildTargetDescriptorFactoryTestFixture, EmptyPath_ExpectArtifactException) - { - // Given a invalid raw descriptor string lacking build meta-data path - const AZStd::string rawTargetDescriptor = - GenerateBuildTargetDescriptorString(m_name, m_outputName, "", m_staticSources, m_autogenInputs, m_autogenOutputs); - - try - { - // When attempting to construct the build target descriptor - const TestImpact::BuildTargetDescriptor buildTarget = - TestImpact::BuildTargetDescriptorFactory(rawTargetDescriptor, m_staticExclude, m_inputExclude, m_autogenMatcher); - - // Do not expect this statement to be reachable - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::ArtifactException& e) - { - // Expect an artifact exception - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(BuildTargetDescriptorFactoryTestFixture, NoStaticSources_ExpectValidDescriptor) - { - const TestImpact::BuildTargetDescriptor expectedBuiltTarget = - GenerateBuildTargetDescriptor(m_name, m_outputName, m_path, {}, m_expectedAutogenSources); - - // Given a valid raw descriptor string with no static sources - const AZStd::string rawTargetDescriptor = - GenerateBuildTargetDescriptorString(m_name, m_outputName, m_path, {}, m_autogenInputs, m_autogenOutputs); - - // When attempting to construct the build target descriptor - const TestImpact::BuildTargetDescriptor buildTarget = - TestImpact::BuildTargetDescriptorFactory(rawTargetDescriptor, {}, m_inputExclude, m_autogenMatcher); - - // Expect the constructed build target descriptor to match the specified descriptor - EXPECT_TRUE(buildTarget == expectedBuiltTarget); - } - - TEST_F(BuildTargetDescriptorFactoryTestFixture, NoAutogenSources_ExpectValidDescriptor) - { - const TestImpact::BuildTargetDescriptor expectedBuiltTarget = - GenerateBuildTargetDescriptor(m_name, m_outputName, m_path, m_expectedStaticSources, {}); - - // Given a valid raw descriptor string with no autogen sources - const AZStd::string rawTargetDescriptor = - GenerateBuildTargetDescriptorString(m_name, m_outputName, m_path, m_staticSources, {}, {}); - - // When attempting to construct the build target descriptor - const TestImpact::BuildTargetDescriptor buildTarget = - TestImpact::BuildTargetDescriptorFactory(rawTargetDescriptor, m_staticExclude, m_inputExclude, m_autogenMatcher); - - // Expect the constructed build target descriptor to match the specified descriptor - EXPECT_TRUE(buildTarget == expectedBuiltTarget); - } - - TEST_F(BuildTargetDescriptorFactoryTestFixture, NoStaticOrAutogenSources_ExpectValidDescriptor) - { - const TestImpact::BuildTargetDescriptor expectedBuiltTarget = GenerateBuildTargetDescriptor(m_name, m_outputName, m_path, {}, {}); - - // Given a valid raw descriptor string with no static or autogen sources - const AZStd::string rawTargetDescriptor = GenerateBuildTargetDescriptorString(m_name, m_outputName, m_path, {}, {}, {}); - - // When attempting to construct the build target descriptor - const TestImpact::BuildTargetDescriptor buildTarget = - TestImpact::BuildTargetDescriptorFactory(rawTargetDescriptor, m_staticExclude, m_inputExclude, m_autogenMatcher); - - // Expect the constructed build target descriptor to match the specified descriptor - EXPECT_TRUE(buildTarget == expectedBuiltTarget); - } - - TEST_F(BuildTargetDescriptorFactoryTestFixture, AutogenOutputSourcesButNoAutogenInputSources_ExpectArtifactException) - { - // Given a valid raw descriptor string with autogen output sources but no autogen input sources - const AZStd::string rawTargetDescriptor = - GenerateBuildTargetDescriptorString(m_name, m_outputName, m_path, m_staticSources, {}, m_autogenOutputs); - - try - { - // When attempting to construct the build target descriptor - const TestImpact::BuildTargetDescriptor buildTarget = - TestImpact::BuildTargetDescriptorFactory(rawTargetDescriptor, m_staticExclude, m_inputExclude, m_autogenMatcher); - - // Do not expect this statement to be reachable - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::ArtifactException& e) - { - // Expect an artifact exception - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(BuildTargetDescriptorFactoryTestFixture, AutogenInputSourcesButNoAutogenOutputSources_ExpectArtifactException) - { - // Given a valid raw descriptor string with autogen inout sources but no autogen output sources - const AZStd::string rawTargetDescriptor = - GenerateBuildTargetDescriptorString(m_name, m_outputName, m_path, m_staticSources, m_autogenInputs, {}); - - try - { - // When attempting to construct the build target descriptor - const TestImpact::BuildTargetDescriptor buildTarget = - TestImpact::BuildTargetDescriptorFactory(rawTargetDescriptor, m_staticExclude, m_inputExclude, m_autogenMatcher); - - // Do not expect this statement to be reachable - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::ArtifactException& e) - { - // Expect an artifact exception - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(BuildTargetDescriptorFactoryTestFixture, StaticAndAutogenSources_ExpectValidDescriptor) - { - const TestImpact::BuildTargetDescriptor expectedBuiltTarget = - GenerateBuildTargetDescriptor(m_name, m_outputName, m_path, m_expectedStaticSources, m_expectedAutogenSources); - - // Given a valid raw descriptor string with static and autogen sources - const AZStd::string rawTargetDescriptor = - GenerateBuildTargetDescriptorString(m_name, m_outputName, m_path, m_staticSources, m_autogenInputs, m_autogenOutputs); - - // When attempting to construct the build target descriptor - const TestImpact::BuildTargetDescriptor buildTarget = - TestImpact::BuildTargetDescriptorFactory(rawTargetDescriptor, m_staticExclude, m_inputExclude, m_autogenMatcher); - - // Expect the constructed build target descriptor to match the specified descriptor - EXPECT_TRUE(buildTarget == expectedBuiltTarget); - } -} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactModuleCoverageFactoryTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactModuleCoverageFactoryTest.cpp deleted file mode 100644 index 137addc360..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactModuleCoverageFactoryTest.cpp +++ /dev/null @@ -1,522 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include - -#include -#include - -#include -#include - -namespace UnitTest -{ - TEST(CoberturaModuleCoveragesFactory, ParseEmptyString_ThrowsArtifactException) - { - // Given an empty string - const AZStd::string rawCoverage; - - try - { - // When attempting to parse the empty coverage - const AZStd::vector coverage = TestImpact::Cobertura::ModuleCoveragesFactory(rawCoverage); - - // Do not expect the parsing to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::ArtifactException& e) - { - // Expect an artifact exception - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST(CoberturaModuleCoveragesFactory, ParseInvalidString_ThrowsArtifactException) - { - // Given an invalid string - const AZStd::string rawCoverage = "!@?"; - - try - { - // When attempting to parse the invalid coverage - const AZStd::vector coverage = TestImpact::Cobertura::ModuleCoveragesFactory(rawCoverage); - - // Do not expect the parsing to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::ArtifactException& e) - { - // Expect an artifact exception - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST(CoberturaModuleCoveragesFactory, ParseEmptyCoverage_ExpectEmptyModuleCoverages) - { - // Given an empty coverage string - const AZStd::string rawCoverage = - "" - "" - " " - " " - "" - ""; - - // When attempting to parse the empty coverage - const AZStd::vector coverage = TestImpact::Cobertura::ModuleCoveragesFactory(rawCoverage); - - // Expect an empty module coverages - EXPECT_TRUE(coverage.empty()); - } - - TEST(CoberturaModuleCoveragesFactory, ParseTestTargetLineCoverageA_ReturnsValidLineCoverage) - { - const AZStd::vector expectedCoverage = GetTestTargetALineModuleCoverages(); - - // Given the raw line coverage output of TestTargetA - const AZStd::string rawCoverage = - "" - "" - " " - " C:" - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - "" - ""; - - // When the raw line coverage text is parsed - const AZStd::vector coverage = TestImpact::Cobertura::ModuleCoveragesFactory(rawCoverage); - - // Expect the generated suite data to match that of the raw coverage text - EXPECT_TRUE(coverage == expectedCoverage); - } - - TEST(CoberturaModuleCoveragesFactory, ParseTestTargetSourceCoverageA_ReturnsValidSourceCoverage) - { - const AZStd::vector expectedCoverage = GetTestTargetASourceModuleCoverages(); - - // Given the raw source coverage output of TestTargetA - const AZStd::string rawCoverage = - "" - "" - " " - " C:" - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - "" - ""; - - // When the raw source coverage text is parsed - const AZStd::vector coverage = TestImpact::Cobertura::ModuleCoveragesFactory(rawCoverage); - - // Expect the generated suite data to match that of the raw coverage text - EXPECT_TRUE(coverage == expectedCoverage); - } - - TEST(CoberturaModuleCoveragesFactory, ParseTestTargetLineCoverageB_ReturnsValidLineCoverage) - { - const AZStd::vector expectedCoverage = GetTestTargetBLineModuleCoverages(); - - // Given the raw line coverage output of TestTargetB - const AZStd::string rawCoverage = - "" - "" - " " - " C:" - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - "" - ""; - - // When the raw line coverage text is parsed - const AZStd::vector coverage = TestImpact::Cobertura::ModuleCoveragesFactory(rawCoverage); - - // Expect the generated suite data to match that of the raw coverage text - EXPECT_TRUE(coverage == expectedCoverage); - } - - TEST(CoberturaModuleCoveragesFactory, ParseTestTargetSourceCoverageB_ReturnsValidSourceCoverage) - { - const AZStd::vector expectedCoverage = GetTestTargetBSourceModuleCoverages(); - - // Given the raw source coverage output of TestTargetB - const AZStd::string rawCoverage = - "" - "" - " " - " C:" - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - "" - ""; - - // When the raw source coverage text is parsed - const AZStd::vector coverage = TestImpact::Cobertura::ModuleCoveragesFactory(rawCoverage); - - // Expect the generated suite data to match that of the raw coverage text - EXPECT_TRUE(coverage == expectedCoverage); - } - - TEST(CoberturaModuleCoveragesFactory, ParseTestTargetLineCoverageC_ReturnsValidLineCoverage) - { - const AZStd::vector expectedCoverage = GetTestTargetCLineModuleCoverages(); - - // Given the raw line coverage output of TestTargetC - const AZStd::string rawCoverage = - "" - "" - " " - " C:" - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - "" - ""; - - // When the raw line coverage text is parsed - const AZStd::vector coverage = TestImpact::Cobertura::ModuleCoveragesFactory(rawCoverage); - - // Expect the generated suite data to match that of the raw coverage text - EXPECT_TRUE(coverage == expectedCoverage); - } - - TEST(CoberturaModuleCoveragesFactory, ParseTestTargetSourceCoverageC_ReturnsValidSourceCoverage) - { - const AZStd::vector expectedCoverage = GetTestTargetCSourceModuleCoverages(); - - // Given the raw source coverage output of TestTargetC - const AZStd::string rawCoverage = - "" - "" - " " - " C:" - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - "" - ""; - - // When the raw source coverage text is parsed - const AZStd::vector coverage = TestImpact::Cobertura::ModuleCoveragesFactory(rawCoverage); - - // Expect the generated suite data to match that of the raw coverage text - EXPECT_TRUE(coverage == expectedCoverage); - } - - TEST(CoberturaModuleCoveragesFactory, ParseTestTargetLineCoverageD_ReturnsValidLineCoverage) - { - const AZStd::vector expectedCoverage = GetTestTargetDLineModuleCoverages(); - - // Given the raw line coverage output of TestTargetD - const AZStd::string rawCoverage = - "" - "" - " " - " C:" - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - "" - ""; - - // When the raw line coverage text is parsed - const AZStd::vector coverage = TestImpact::Cobertura::ModuleCoveragesFactory(rawCoverage); - - // Expect the generated suite data to match that of the raw coverage text - EXPECT_TRUE(coverage == expectedCoverage); - } - - TEST(CoberturaModuleCoveragesFactory, ParseTestTargetSourceCoverageD_ReturnsValidSourceCoverage) - { - const AZStd::vector expectedCoverage = GetTestTargetDSourceModuleCoverages(); - - // Given the raw source coverage output of TestTargetD - const AZStd::string rawCoverage = - "" - "" - " " - " C:" - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - "" - ""; - - // When the raw source coverage text is parsed - const AZStd::vector coverage = TestImpact::Cobertura::ModuleCoveragesFactory(rawCoverage); - - // Expect the generated suite data to match that of the raw coverage text - EXPECT_TRUE(coverage == expectedCoverage); - } -} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactTargetDescriptorCompilerTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactTargetDescriptorCompilerTest.cpp deleted file mode 100644 index 1e72902b4e..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactTargetDescriptorCompilerTest.cpp +++ /dev/null @@ -1,150 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include - -#include -#include - -#include -#include - -namespace UnitTest -{ - class TargetDescriptorCompilerTestFixture - : public AllocatorsTestFixture - { - public: - void SetUp() override - { - AllocatorsTestFixture::SetUp(); - - m_buildTargetDescriptors.emplace_back(TestImpact::BuildTargetDescriptor{{"TestTargetA", "", ""}, {}}); - m_buildTargetDescriptors.emplace_back(TestImpact::BuildTargetDescriptor{{"TestTargetB", "", ""}, {}}); - m_buildTargetDescriptors.emplace_back(TestImpact::BuildTargetDescriptor{{"ProductionTargetA", "", ""}, {}}); - m_buildTargetDescriptors.emplace_back(TestImpact::BuildTargetDescriptor{{"ProductionTargetB", "", ""}, {}}); - m_buildTargetDescriptors.emplace_back(TestImpact::BuildTargetDescriptor{{"ProductionTargetC", "", ""}, {}}); - - m_testTargetMetaMap.emplace("TestTargetA", TestImpact::TestTargetMeta{"", TestImpact::LaunchMethod::TestRunner}); - m_testTargetMetaMap.emplace("TestTargetB", TestImpact::TestTargetMeta{"", TestImpact::LaunchMethod::StandAlone}); - } - - protected: - AZStd::vector m_buildTargetDescriptors; - TestImpact::TestTargetMetaMap m_testTargetMetaMap; - }; - - TestImpact::ProductionTargetDescriptor ConstructProductionTargetDescriptor(const AZStd::string& name) - { - return TestImpact::ProductionTargetDescriptor{TestImpact::BuildTargetDescriptor{{name, "", ""}, {{}, {}}}}; - } - - TestImpact::TestTargetDescriptor ConstructTestTargetDescriptor(const AZStd::string& name, TestImpact::LaunchMethod launchMethod) - { - return TestImpact::TestTargetDescriptor{ - TestImpact::BuildTargetDescriptor{{name, "", ""}, {{}, {}}}, TestImpact::TestTargetMeta{"", launchMethod}}; - } - - TEST_F(TargetDescriptorCompilerTestFixture, EmptyBuildTargetDescriptorList_ExpectArtifactException) - { - try - { - // Given an empty build target descriptor list but valid test target meta map - // When attempting to construct the test target - const auto& [productionTargetDescriptors, testTargetDescriptors] = - TestImpact::CompileTargetDescriptors({}, AZStd::move(m_testTargetMetaMap)); - - // Do not expect this statement to be reachable - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::ArtifactException& e) - { - // Expect an artifact exception - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(TargetDescriptorCompilerTestFixture, EmptyTestTargetMetaMap_ExpectArtifactException) - { - try - { - // Given a valid build target descriptor list but empty test target meta map - // When attempting to construct the test target - const auto& [productionTargetDescriptors, testTargetDescriptors] = - TestImpact::CompileTargetDescriptors(AZStd::move(m_buildTargetDescriptors), {}); - - // Do not expect this statement to be reachable - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::ArtifactException& e) - { - // Expect an artifact exception - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(TargetDescriptorCompilerTestFixture, TestTargetWithNoMatchingMeta_ExpectArtifactException) - { - try - { - // Given a valid build target descriptor list but a test target meta map with an orphan entry - m_testTargetMetaMap.emplace("Orphan", TestImpact::TestTargetMeta{"", TestImpact::LaunchMethod::TestRunner}); - - // When attempting to construct the test target - const auto& [productionTargetDescriptors, testTargetDescriptors] = - TestImpact::CompileTargetDescriptors(AZStd::move(m_buildTargetDescriptors), {}); - - // Do not expect this statement to be reachable - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::ArtifactException& e) - { - // Expect an artifact exception - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(TargetDescriptorCompilerTestFixture, ValidProductionTargetsAndTestTargetMetas_ExpectValidProductionAndTestTargets) - { - // Given a valid build target descriptor list and a valid test target meta map - // When attempting to construct the test target - const auto& [productionTargetDescriptors, testTargetDescriptors] = - TestImpact::CompileTargetDescriptors(AZStd::move(m_buildTargetDescriptors), AZStd::move(m_testTargetMetaMap)); - - // Expect the production targets to match the expected targets - EXPECT_TRUE(productionTargetDescriptors.size() == 3); - EXPECT_TRUE(productionTargetDescriptors[0] == ConstructProductionTargetDescriptor("ProductionTargetA")); - EXPECT_TRUE(productionTargetDescriptors[1] == ConstructProductionTargetDescriptor("ProductionTargetB")); - EXPECT_TRUE(productionTargetDescriptors[2] == ConstructProductionTargetDescriptor("ProductionTargetC")); - - // Expect the test targets to match the expected targets - EXPECT_TRUE(testTargetDescriptors.size() == 2); - EXPECT_TRUE(testTargetDescriptors[0] == ConstructTestTargetDescriptor("TestTargetA", TestImpact::LaunchMethod::TestRunner)); - EXPECT_TRUE(testTargetDescriptors[1] == ConstructTestTargetDescriptor("TestTargetB", TestImpact::LaunchMethod::StandAlone)); - } - -} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactTestEnumerationSuiteFactoryTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactTestEnumerationSuiteFactoryTest.cpp deleted file mode 100644 index d6e1dc5920..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactTestEnumerationSuiteFactoryTest.cpp +++ /dev/null @@ -1,589 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include - -#include -#include - -#include -#include -#include -#include -#include -#include - -namespace UnitTest -{ - TEST(GTestEnumerationSuiteFactory, ParseEmptyString_ThrowsArtifactException) - { - // Given an empty string - const AZStd::string rawEnum; - - try - { - // When attempting to parse the empty suite - const AZStd::vector suites = TestImpact::GTest::TestEnumerationSuitesFactory(rawEnum); - - // Do not expect the parsing to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::ArtifactException& e) - { - // Expect an artifact exception - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST(GTestEnumerationSuiteFactory, ParseInvalidString_ThrowsArtifactException) - { - // Given an invalid string - const AZStd::string rawEnum = "!@?"; - - try - { - // When attempting to parse the empty suite - const AZStd::vector suites = TestImpact::GTest::TestEnumerationSuitesFactory(rawEnum); - - // Do not expect the parsing to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::ArtifactException& e) - { - // Expect an artifact exception - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST(GTestEnumerationSuiteFactory, ParseTestTargetA_ReturnsValidSuitesAndTests) - { - const AZStd::vector expectedSuites = GetTestTargetATestEnumerationSuites(); - - // Given the raw enum output of TestTargetA - const AZStd::string rawEnum = - "\n" - "\n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - "\n" - "\n"; - - // When the raw enumeration text is parsed - const AZStd::vector suites = TestImpact::GTest::TestEnumerationSuitesFactory(rawEnum); - - // Expect the generated suite data to match that of the raw enum text - EXPECT_TRUE(suites == expectedSuites); - } - - TEST(GTestEnumerationSuiteFactory, ParseTestTargetB_ReturnsValidSuitesAndTests) - { - const AZStd::vector expectedSuites = GetTestTargetBTestEnumerationSuites(); - - // Given the raw enum output of TestTargetB - const AZStd::string rawEnum = - "\n" - "\n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - "\n" - "\n"; - - // When the raw enumeration text is parsed - const AZStd::vector suites = TestImpact::GTest::TestEnumerationSuitesFactory(rawEnum); - - // Expect the generated suite data to match that of the raw enum text - EXPECT_TRUE(suites == expectedSuites); - } - - TEST(GTestEnumerationSuiteFactory, ParseTestTargetC_ReturnsValidSuitesAndTests) - { - const AZStd::vector expectedSuites = GetTestTargetCTestEnumerationSuites(); - - // Given the raw enum output of TestTargetC - const AZStd::string rawEnum = - "\n" - "\n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - "\n" - "\n"; - - // When the raw enumeration text is parsed - const AZStd::vector suites = TestImpact::GTest::TestEnumerationSuitesFactory(rawEnum); - - // Expect the generated suite data to match that of the raw enum text - EXPECT_TRUE(suites == expectedSuites); - } - - TEST(GTestEnumerationSuiteFactory, ParseTestTargetD_ReturnsValidSuitesAndTests) - { - const AZStd::vector expectedSuites = GetTestTargetDTestEnumerationSuites(); - - // Given the raw enum output of TestTargetD - const AZStd::string rawEnum = - "\n" - "\n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - "\n" - "\n"; - - // When the raw enumeration text is parsed - const AZStd::vector suites = TestImpact::GTest::TestEnumerationSuitesFactory(rawEnum); - - // Expect the generated suite data to match that of the raw enum text - EXPECT_TRUE(suites == expectedSuites); - } -} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactTestRunSuiteFactoryTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactTestRunSuiteFactoryTest.cpp deleted file mode 100644 index 0f0a7dfdd8..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactTestRunSuiteFactoryTest.cpp +++ /dev/null @@ -1,592 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include - -#include -#include - -#include -#include -#include -#include -#include -#include - -namespace UnitTest -{ - TEST(GTestRunSuiteFactory, ParseEmptyString_ThrowsArtifactException) - { - // Given an empty string - const AZStd::string rawRun; - - try - { - // When attempting to parse the empty suite - const AZStd::vector suites = TestImpact::GTest::TestRunSuitesFactory(rawRun); - - // Do not expect the parsing to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::ArtifactException& e) - { - // Expect a artifact exception - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST(GTestRunSuiteFactory, ParseInvalidString_ThrowsArtifactException) - { - // Given an invalid string - const AZStd::string rawRun = "!@?"; - - try - { - // When attempting to parse the invalid suite - const AZStd::vector suites = TestImpact::GTest::TestRunSuitesFactory(rawRun); - - // Do not expect the parsing to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::ArtifactException& e) - { - // Expect a artifact exception - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST(GTestRunSuiteFactory, ParseTestTargetA_ReturnsValidSuitesAndTests) - { - const AZStd::vector expectedSuites = GetTestTargetATestRunSuites(); - - // Given the raw run output of TestTargetA - const AZStd::string rawRun = - "" - "" - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - "" - ""; - - // When the raw run text is parsed - const AZStd::vector suites = TestImpact::GTest::TestRunSuitesFactory(rawRun); - - // Expect the generated suite data to match that of the raw run text - EXPECT_TRUE(suites == expectedSuites); - } - - TEST(GTestRunSuiteFactory, ParseTestTargetB_ReturnsValidSuitesAndTests) - { - const AZStd::vector expectedSuites = GetTestTargetBTestRunSuites(); - - // Given the raw run output of TestTargetB - const AZStd::string rawRun = - "" - "" - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - "" - ""; - - // When the raw run text is parsed - const AZStd::vector suites = TestImpact::GTest::TestRunSuitesFactory(rawRun); - - // Expect the generated suite data to match that of the raw run text - EXPECT_TRUE(suites == expectedSuites); - } - - TEST(GTestRunSuiteFactory, ParseTestTargetC_ReturnsValidSuitesAndTests) - { - const AZStd::vector expectedSuites = GetTestTargetCTestRunSuites(); - - // Given the raw run output of TestTargetC - const AZStd::string rawRun = - "" - "" - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - "" - ""; - - // When the raw run text is parsed - const AZStd::vector suites = TestImpact::GTest::TestRunSuitesFactory(rawRun); - - // Expect the generated suite data to match that of the raw run text - EXPECT_TRUE(suites == expectedSuites); - } - - TEST(GTestRunSuiteFactory, ParseTestTargetD_ReturnsValidSuitesAndTests) - { - const AZStd::vector expectedSuites = GetTestTargetDTestRunSuites(); - - // Given the raw run output of TestTargetD - const AZStd::string rawRun = - "" - "" - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - " " - "" - ""; - - // When the raw run text is parsed - const AZStd::vector suites = TestImpact::GTest::TestRunSuitesFactory(rawRun); - - // Expect the generated suite data to match that of the raw run text - EXPECT_TRUE(suites == expectedSuites); - } -} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactTestTargetMetaMapFactoryTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactTestTargetMetaMapFactoryTest.cpp deleted file mode 100644 index 588c46d1ce..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactTestTargetMetaMapFactoryTest.cpp +++ /dev/null @@ -1,239 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include - -#include -#include - -#include -#include - -namespace UnitTest -{ - TEST(TestTargetMetaMapFactoryTest, NoRawData_ExpectArtifactException) - { - // Given an empty meta data string - const AZStd::string rawTestTargetMetaData; - - try - { - // When attempting to construct the test target - const TestImpact::TestTargetMetaMap testTargetMetaData = TestImpact::TestTargetMetaMapFactory(rawTestTargetMetaData); - - // Do not expect this statement to be reachable - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::ArtifactException& e) - { - // Expect an artifact exception - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST(TestTargetMetaMapFactoryTest, InvalidRawData_ExpectArtifactException) - { - // Given a raw meta data string of invalid data - const AZStd::string rawTestTargetMetaData = "abcde"; - - try - { - // When attempting to construct the test target - const TestImpact::TestTargetMetaMap testTargetMetaData = TestImpact::TestTargetMetaMapFactory(rawTestTargetMetaData); - - // Do not expect this statement to be reachable - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::ArtifactException& e) - { - // Expect an artifact exception - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST(TestTargetMetaMapFactoryTest, EmptyTestMetaArray_ExpectArtifactException) - { - // Given a raw meta data string of valid JSON that contains no tests - const AZStd::string rawTestTargetMetaData = - "{" - " \"google\": {" - " \"test\": {" - " \"tests\": [" - " ]" - " }" - " }" - "}"; - - try - { - // When attempting to construct the test target - const TestImpact::TestTargetMetaMap testTargetMetaData = TestImpact::TestTargetMetaMapFactory(rawTestTargetMetaData); - - // Do not expect this statement to be reachable - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::ArtifactException& e) - { - // Expect an artifact exception - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST(TestTargetMetaMapFactoryTest, EmptyName_ExpectArtifactException) - { - // Given a raw meta data string with a test that has no name value - const AZStd::string rawTestTargetMetaData = - "{" - " \"google\": {" - " \"test\": {" - " \"tests\": [" - " { \"name\": \"\", \"namespace\": \"Legacy\", \"suite\": \"main\", \"launch_method\": \"test_runner\" }" - " ]" - " }" - " }" - "}"; - - try - { - // When attempting to construct the test target - const TestImpact::TestTargetMetaMap testTargetMetaData = TestImpact::TestTargetMetaMapFactory(rawTestTargetMetaData); - - // Do not expect this statement to be reachable - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::ArtifactException& e) - { - // Expect an artifact exception - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST(TestTargetMetaMapFactoryTest, EmptyBuildType_ExpectArtifactException) - { - // Given a raw meta data string with a test that has no build type - const AZStd::string rawTestTargetMetaData = - "{" - " \"google\": {" - " \"test\": {" - " \"tests\": [" - " { \"name\": \"TestName\", \"namespace\": \"Legacy\", \"suite\": \"main\", \"launch_method\": \"\" }" - " ]" - " }" - " }" - "}"; - - try - { - // When attempting to construct the test target - const TestImpact::TestTargetMetaMap testTargetMetaData = TestImpact::TestTargetMetaMapFactory(rawTestTargetMetaData); - - // Do not expect this statement to be reachable - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::ArtifactException& e) - { - // Expect an artifact exception - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST(TestTargetMetaMapFactoryTest, InvalidBuildType_ExpectArtifactException) - { - // Given a raw meta data string with a test that has an invalid build type - const AZStd::string rawTestTargetMetaData = - "{" - " \"google\": {" - " \"test\": {" - " \"tests\": [" - " { \"name\": \"TestName\", \"namespace\": \"Legacy\", \"suite\": \"main\", \"launch_method\": \"Unknown\" }" - " ]" - " }" - " }" - "}"; - - try - { - // When attempting to construct the test target - const TestImpact::TestTargetMetaMap testTargetMetaData = TestImpact::TestTargetMetaMapFactory(rawTestTargetMetaData); - - // Do not expect this statement to be reachable - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::ArtifactException& e) - { - // Expect an artifact exception - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST(TestTargetMetaMapFactoryTest, ValidMetaData_ExpectValidTestMetaData) - { - // Given a raw meta data string valid test meta-data - const AZStd::string rawTestTargetMetaData = - "{" - " \"google\": {" - " \"test\": {" - " \"tests\": [" - " { \"name\": \"TestA\", \"namespace\": \"Legacy\", \"suite\": \"main\", \"launch_method\": \"test_runner\" }," - " { \"name\": \"TestB\", \"namespace\": \"\", \"suite\": \"main\", \"launch_method\": \"test_runner\" }," - " { \"name\": \"TestC\", \"namespace\": \"\", \"suite\": \"\", \"launch_method\": \"test_runner\" }," - " { \"name\": \"TestD\", \"namespace\": \"Legacy\", \"suite\": \"main\", \"launch_method\": \"stand_alone\" }" - " ]" - " }" - " }" - "}"; - - // When attempting to construct the test target - const auto testTargetMetaData = TestImpact::TestTargetMetaMapFactory(rawTestTargetMetaData); - - // Expect the constructed test meta-data to match that of the supplied raw data - EXPECT_EQ(testTargetMetaData.size(), 4); - EXPECT_TRUE(testTargetMetaData.find("TestA") != testTargetMetaData.end()); - EXPECT_TRUE((testTargetMetaData.at("TestA") == TestImpact::TestTargetMeta{"main", TestImpact::LaunchMethod::TestRunner})); - EXPECT_TRUE(testTargetMetaData.find("TestB") != testTargetMetaData.end()); - EXPECT_TRUE((testTargetMetaData.at("TestB") == TestImpact::TestTargetMeta{"main", TestImpact::LaunchMethod::TestRunner})); - EXPECT_TRUE(testTargetMetaData.find("TestC") != testTargetMetaData.end()); - EXPECT_TRUE((testTargetMetaData.at("TestC") == TestImpact::TestTargetMeta{"", TestImpact::LaunchMethod::TestRunner})); - EXPECT_TRUE(testTargetMetaData.find("TestD") != testTargetMetaData.end()); - EXPECT_TRUE((testTargetMetaData.at("TestD") == TestImpact::TestTargetMeta{"main", TestImpact::LaunchMethod::StandAlone})); - } -} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Process/TestImpactProcessSchedulerTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Process/TestImpactProcessSchedulerTest.cpp deleted file mode 100644 index 2b32067688..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Process/TestImpactProcessSchedulerTest.cpp +++ /dev/null @@ -1,712 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include - -#include - -#include -#include -#include -#include -#include -#include -#include - -#include - -namespace UnitTest -{ - using SchedulerPermutation = AZStd::tuple - < - size_t, // Number of concurrent processes - size_t // Number of processes to launch - >; - - class ProcessSchedulerTestFixtureWithParams - : public AllocatorsTestFixture - , private AZ::Debug::TraceMessageBus::Handler - , public ::testing::WithParamInterface - { - private: - bool OnOutput(const char*, const char*) override; - void SetUp() override; - void TearDown() override; - - protected: - void ConfigurePermutation(); - void ExpectSuccessfulLaunch(TestImpact::ProcessId pid); - void ExpectGracefulExit(TestImpact::ProcessId pid); - void ExpectUnsuccessfulLaunch(TestImpact::ProcessId pid); - void ExpectExitCondition( - TestImpact::ProcessId pid, AZStd::optional expectedExitCondition = AZStd::nullopt); - void DoNotExpectExitCondition(TestImpact::ProcessId pid); - void ExpectTerminatedProcess(TestImpact::ProcessId pid); - void ExpectTimeoutProcess(TestImpact::ProcessId pid); - void ExpectUnlaunchedProcess(TestImpact::ProcessId pid); - void ExpectLargeStdOutput(TestImpact::ProcessId pid); - void ExpectLargeStdError(TestImpact::ProcessId pid); - void ExpectSmallStdOutput(TestImpact::ProcessId pid); - void ExpectSmallStdError(TestImpact::ProcessId pid); - void DoNotExpectStdOutput(TestImpact::ProcessId pid); - void DoNotExpectStdError(TestImpact::ProcessId pid); - - struct ProcessResult - { - AZStd::optional m_launchResult; - AZStd::optional m_exitStatus; - AZStd::optional m_createTime; - AZStd::optional m_exitTime; - AZStd::chrono::milliseconds m_duration = AZStd::chrono::milliseconds(0); - AZStd::optional m_returnCode; - TestImpact::StdContent m_std; - }; - - size_t m_numMaxConcurrentProcesses = 0; - size_t m_numProcessesToLaunch = 0; - - AZStd::unique_ptr m_processScheduler; - TestImpact::ProcessLaunchCallback m_processLaunchCallback; - TestImpact::ProcessExitCallback m_processExitCallback; - AZStd::vector m_processInfos; - AZStd::vector m_processResults; - }; - - // Permutation values for small process batches - const std::vector SmallNumMaxConcurrentProcesses = {1, 4, 8}; - const std::vector SmallNumProcessesToLaunch = {1, 8, 32}; - - // Permutation values for large process batches - const std::vector LargeNumMaxConcurrentProcesses = {16, 32, 64}; - const std::vector LargeNumProcessesToLaunch = {128, 256, 512}; - - void ProcessSchedulerTestFixtureWithParams::ConfigurePermutation() - { - const auto& [numMaxConcurrentProcesses, numProcessesToLaunch] = GetParam(); - m_numMaxConcurrentProcesses = numMaxConcurrentProcesses; - m_numProcessesToLaunch = numProcessesToLaunch; - } - - // Consume AZ log spam - bool ProcessSchedulerTestFixtureWithParams::OnOutput([[maybe_unused]] const char*, [[maybe_unused]] const char*) - { - return true; - } - - void ProcessSchedulerTestFixtureWithParams::SetUp() - { - UnitTest::AllocatorsTestFixture::SetUp(); - AZ::Debug::TraceMessageBus::Handler::BusConnect(); - ConfigurePermutation(); - - m_processLaunchCallback = [this]( - TestImpact::ProcessId pid, - TestImpact::LaunchResult launchResult, - AZStd::chrono::high_resolution_clock::time_point createTime) - { - m_processResults[pid].m_launchResult = launchResult; - m_processResults[pid].m_createTime.emplace(createTime); - return TestImpact::CallbackResult::Continue; - }; - - m_processExitCallback = [this]( - TestImpact::ProcessId pid, - TestImpact::ExitCondition exitStatus, - TestImpact::ReturnCode returnCode, - TestImpact::StdContent&& std, - AZStd::chrono::high_resolution_clock::time_point exitTime) - { - m_processResults[pid].m_std = std::move(std); - m_processResults[pid].m_returnCode.emplace(returnCode); - m_processResults[pid].m_exitStatus = exitStatus; - m_processResults[pid].m_exitTime = exitTime; - m_processResults[pid].m_duration = - AZStd::chrono::duration_cast(exitTime - *m_processResults[pid].m_createTime); - return TestImpact::CallbackResult::Continue; - }; - - m_processResults.resize(m_numProcessesToLaunch); - } - - void ProcessSchedulerTestFixtureWithParams::TearDown() - { - AZ::Debug::TraceMessageBus::Handler::BusDisconnect(); - UnitTest::AllocatorsTestFixture::TearDown(); - } - - // Expects the process to have exited under the specified circumstances - void ProcessSchedulerTestFixtureWithParams::ExpectExitCondition( - TestImpact::ProcessId pid, AZStd::optional expectedExitCondition) - { - const auto& [launchResult, exitStatus, createTime, exitTime, duration, returnCode, std] = m_processResults[pid]; - - // Expect the processes to have exited - EXPECT_TRUE(exitStatus.has_value()); - - // Expect the return code to be valid - EXPECT_TRUE(returnCode.has_value()); - - if (expectedExitCondition.has_value()) - { - // Expect the process to have exited under the specified conditions - EXPECT_EQ(exitStatus, expectedExitCondition); - - // Expect the return code to be that of the process's is (for gracefull exists) or that of the error code - // associated with the abnormal exit condition - EXPECT_EQ(returnCode, - expectedExitCondition == TestImpact::ExitCondition::Gracefull - ? pid - : static_cast(*exitStatus)); - } - - // Expect the duration to be non zero and the create time and exit time to have values as the processes has been in-flight - EXPECT_GT(duration.count(), 0); - EXPECT_TRUE(createTime.has_value()); - EXPECT_TRUE(exitTime.has_value()); - - // Expect the exit time to be later than the start time - EXPECT_GT(exitTime, createTime); - } - - // Expects the process to not have exited for to lack of being in-flight - void ProcessSchedulerTestFixtureWithParams::DoNotExpectExitCondition(TestImpact::ProcessId pid) - { - const auto& [launchResult, exitStatus, createTime, exitTime, duration, returnCode, std] = m_processResults[pid]; - - // Expect the processes to not have exited as the process has not been launched successfully - EXPECT_FALSE(exitStatus.has_value()); - - // Expect the return code to be invalid - EXPECT_FALSE(returnCode.has_value()); - - // Expect the duration to be zero as the process has not been in-flight - EXPECT_EQ(duration.count(), 0); - - // Do not expect the exit time to have a value - EXPECT_FALSE(exitTime.has_value()); - } - - // Expects the process to have been launched successfully, makes no assumptions about how it exited - void ProcessSchedulerTestFixtureWithParams::ExpectSuccessfulLaunch(TestImpact::ProcessId pid) - { - const auto& [launchResult, exitStatus, createTime, exitTime, duration, returnCode, std] = m_processResults[pid]; - - // Expect a launch to have been attempted by this process (not still in queue) - EXPECT_TRUE(launchResult.has_value()); - - // Expect the process to have launched successfully - EXPECT_EQ(launchResult, TestImpact::LaunchResult::Success); - } - - // Expects the process to have failed to launch - void ProcessSchedulerTestFixtureWithParams::ExpectUnsuccessfulLaunch(TestImpact::ProcessId pid) - { - const auto& [launchResult, exitStatus, createTime, exitTime, duration, returnCode, std] = m_processResults[pid]; - - // Expect a launch to have been attempted by this process (not still in queue) - EXPECT_TRUE(launchResult.has_value()); - - // Expect the process to have launched unsuccessfully - EXPECT_EQ(launchResult, TestImpact::LaunchResult::Failure); - - // Expect the create time to have a value as the process was technically created - EXPECT_TRUE(createTime.has_value()); - - DoNotExpectExitCondition(pid); - } - - // Expects the process to have exited gracefully of its own accord (i.e. not terminated for any reason by the scheduler) - void ProcessSchedulerTestFixtureWithParams::ExpectGracefulExit(TestImpact::ProcessId pid) - { - ExpectSuccessfulLaunch(pid); - ExpectExitCondition(pid, TestImpact::ExitCondition::Gracefull); - } - - // Expects the process to have been terminated by the client or scheduler - void ProcessSchedulerTestFixtureWithParams::ExpectTerminatedProcess(TestImpact::ProcessId pid) - { - ExpectSuccessfulLaunch(pid); - ExpectExitCondition(pid, TestImpact::ExitCondition::Terminated); - } - - // Expects the process to have been terminated by the scheduler due to the process or scheduler timing out - void ProcessSchedulerTestFixtureWithParams::ExpectTimeoutProcess(TestImpact::ProcessId pid) - { - ExpectSuccessfulLaunch(pid); - ExpectExitCondition(pid, TestImpact::ExitCondition::Timeout); - } - - // Expects the process to have not been attempted to launch (was still in the queue) - void ProcessSchedulerTestFixtureWithParams::ExpectUnlaunchedProcess(TestImpact::ProcessId pid) - { - const auto& [launchResult, exitStatus, createTime, exitTime, duration, returnCode, std] = m_processResults[pid]; - - // Expect a launch to not have been attempted by this process (still in queue) - EXPECT_FALSE(launchResult.has_value()); - - // Expect the create time to have a value as the process was technically created - EXPECT_FALSE(createTime.has_value()); - - DoNotExpectExitCondition(pid); - } - - // Expects the std output to have been captured from the process with a large volume of text - void ProcessSchedulerTestFixtureWithParams::ExpectLargeStdOutput(TestImpact::ProcessId pid) - { - const auto& stdOut = m_processResults[pid].m_std.m_out; - - // Expect standard output to have a value - EXPECT_TRUE(stdOut.has_value()); - - // Expect the output length to be that of the large text output from the child process - EXPECT_EQ(stdOut.value().length(), LargeTextSize); - } - - // Expects the std error to have been captured from the process with a large volume of text - void ProcessSchedulerTestFixtureWithParams::ExpectLargeStdError(TestImpact::ProcessId pid) - { - const auto& stdErr = m_processResults[pid].m_std.m_err; - - // Expect standard error to have a value - EXPECT_TRUE(stdErr.has_value()); - - // Expect the error length to be that of the large text output from the child process - EXPECT_EQ(stdErr.value().length(), LargeTextSize); - } - - // Expects the std output to have been captured from the process with a small known text string - void ProcessSchedulerTestFixtureWithParams::ExpectSmallStdOutput(TestImpact::ProcessId pid) - { - const auto& stdOut = m_processResults[pid].m_std.m_out; - - // Expect standard output to have a value - EXPECT_TRUE(stdOut.has_value()); - - // Expect the output to match the known stdout of the child - EXPECT_EQ(stdOut.value(), KnownTestProcessOutputString(pid)); - } - - // Expects the std error to have been captured from the process with a small known text string - void ProcessSchedulerTestFixtureWithParams::ExpectSmallStdError(TestImpact::ProcessId pid) - { - const auto& stdErr = m_processResults[pid].m_std.m_err; - - // Expect standard error to have a value - EXPECT_TRUE(stdErr.has_value()); - - // Expect the output to match the known stderr of the child - EXPECT_EQ(stdErr.value(), KnownTestProcessErrorString(pid)); - } - - // Expects no standard output from the child process - void ProcessSchedulerTestFixtureWithParams::DoNotExpectStdOutput(TestImpact::ProcessId pid) - { - const auto& stdOut = m_processResults[pid].m_std.m_out; - - // Do not expect standard output to have a value - EXPECT_FALSE(stdOut.has_value()); - } - - // Expects no standard error from the child process - void ProcessSchedulerTestFixtureWithParams::DoNotExpectStdError(TestImpact::ProcessId pid) - { - const auto& stdErr = m_processResults[pid].m_std.m_err; - - // Do not expect standard error to have a value - EXPECT_FALSE(stdErr.has_value()); - } - - TEST_P(ProcessSchedulerTestFixtureWithParams, ValidProcesses_SuccessfulLaunches) - { - // Given a set of processes to launch with valid arguments - for (size_t pid = 0; pid < m_numProcessesToLaunch; pid++) - { - m_processInfos.emplace_back( - pid, - ValidProcessPath, - ConstructTestProcessArgs(pid, NoSleep)); - } - - // When the process scheduler launches the processes - m_processScheduler = AZStd::make_unique( - m_processInfos, - m_processLaunchCallback, - m_processExitCallback, - m_numMaxConcurrentProcesses, - AZStd::nullopt, - AZStd::nullopt - ); - - for (size_t pid = 0; pid < m_numProcessesToLaunch; pid++) - { - ExpectGracefulExit(pid); - } - } - - TEST_P(ProcessSchedulerTestFixtureWithParams, ValidAndInvalidProcesses_LaunchSuccessesAndFailures) - { - const size_t invalidProcessGroup = 4; - - // Given a mixture of processes to launch with valid and invalid arguments - for (size_t pid = 0; pid < m_numProcessesToLaunch; pid++) - { - if (pid % invalidProcessGroup == 0) - { - m_processInfos.emplace_back( - pid, - InvalidProcessPath, - ConstructTestProcessArgs(pid, NoSleep)); - } - else - { - m_processInfos.emplace_back( - pid, - ValidProcessPath, - ConstructTestProcessArgs(pid, NoSleep)); - } - } - - // When the process scheduler launches the processes - m_processScheduler = AZStd::make_unique( - m_processInfos, - m_processLaunchCallback, - m_processExitCallback, - m_numMaxConcurrentProcesses, - AZStd::nullopt, - AZStd::nullopt - ); - - for (size_t pid = 0; pid < m_numProcessesToLaunch; pid++) - { - if (pid % invalidProcessGroup == 0) - { - ExpectUnsuccessfulLaunch(pid); - } - else - { - ExpectGracefulExit(pid); - } - - DoNotExpectStdOutput(pid); - DoNotExpectStdError(pid); - } - } - - TEST_P(ProcessSchedulerTestFixtureWithParams, ProcessTimeouts_InFlightProcessesTimeout) - { - const size_t timeoutProcessGroup = 4; - - // Given a mixture of processes to launch with some processes sleeping indefinately - for (size_t pid = 0; pid < m_numProcessesToLaunch; pid++) - { - if (pid % timeoutProcessGroup == 0) - { - m_processInfos.emplace_back(pid, ValidProcessPath, ConstructTestProcessArgs(pid, LongSleep)); - } - else - { - m_processInfos.emplace_back(pid, ValidProcessPath, ConstructTestProcessArgs(pid, NoSleep)); - } - } - - // When the process scheduler launches the processes with a process timeout value - m_processScheduler = AZStd::make_unique( - m_processInfos, - m_processLaunchCallback, - m_processExitCallback, - m_numMaxConcurrentProcesses, - AZStd::chrono::milliseconds(100), - AZStd::nullopt - ); - - for (size_t pid = 0; pid < m_numProcessesToLaunch; pid++) - { - if (pid % timeoutProcessGroup == 0) - { - ExpectTimeoutProcess(pid); - } - else - { - ExpectExitCondition(pid); - } - - DoNotExpectStdOutput(pid); - DoNotExpectStdError(pid); - } - } - - TEST_P( - ProcessSchedulerTestFixtureWithParams, ProcessLaunchCallbackAbort_InFlightProcessesTerminatedAndQueuedProcessesUnlaunched) - { - const size_t processToAbort = 8; - - // Given a set of processes to launch - for (size_t pid = 0; pid < m_numProcessesToLaunch; pid++) - { - m_processInfos.emplace_back( - pid, - ValidProcessPath, - ConstructTestProcessArgs(pid, NoSleep)); - } - - // Given a launch callback that will return the abort value for the process to abort - const auto abortingLaunchCallback = [this, processToAbort]( - TestImpact::ProcessId pid, - TestImpact::LaunchResult launchResult, - AZStd::chrono::high_resolution_clock::time_point createTime) - { - m_processLaunchCallback(pid, launchResult, createTime); - - if (pid == processToAbort) - { - return TestImpact::CallbackResult::Abort; - } - else - { - return TestImpact::CallbackResult::Continue; - } - }; - - // When the process scheduler launches the processes - m_processScheduler = AZStd::make_unique( - m_processInfos, - abortingLaunchCallback, - m_processExitCallback, - m_numMaxConcurrentProcesses, - AZStd::nullopt, - AZStd::nullopt - ); - - for (size_t pid = 0; pid < m_numProcessesToLaunch; pid++) - { - if (pid < processToAbort) - { - ExpectSuccessfulLaunch(pid); - } - else if (pid == processToAbort) - { - ExpectTerminatedProcess(pid); - } - else - { - ExpectUnlaunchedProcess(pid); - } - - DoNotExpectStdOutput(pid); - DoNotExpectStdError(pid); - } - } - - TEST_P(ProcessSchedulerTestFixtureWithParams, ProcessExitCallbackAbort_InFlightProcessesTerminated) - { - const size_t processToAbort = 4; - - // Given a set of processes to launch - for (size_t pid = 0; pid < m_numProcessesToLaunch; pid++) - { - m_processInfos.emplace_back( - pid, - ValidProcessPath, - ConstructTestProcessArgs(pid, NoSleep)); - } - - // Given an exit callback that will return the abort value for the process to abort - const auto abortingExitCallback = [this, processToAbort]( - TestImpact::ProcessId pid, - TestImpact::ExitCondition exitStatus, - TestImpact::ReturnCode returnCode, - TestImpact::StdContent&& std, - AZStd::chrono::high_resolution_clock::time_point exitTime) - { - m_processExitCallback(pid, exitStatus, returnCode, std::move(std), exitTime); - - if (pid == processToAbort) - { - return TestImpact::CallbackResult::Abort; - } - else - { - return TestImpact::CallbackResult::Continue; - } - }; - - // When the process scheduler launches the processes - m_processScheduler = AZStd::make_unique( - m_processInfos, - m_processLaunchCallback, - abortingExitCallback, - m_numMaxConcurrentProcesses, - AZStd::nullopt, - AZStd::nullopt - ); - - for (size_t pid = 0; pid < m_numProcessesToLaunch; pid++) - { - if (pid < processToAbort) - { - ExpectSuccessfulLaunch(pid); - } - else if (pid == processToAbort) - { - ExpectGracefulExit(pid); - } - else - { - if (m_processResults[pid].m_launchResult.has_value()) - { - ExpectSuccessfulLaunch(pid); - } - else - { - ExpectUnlaunchedProcess(pid); - } - } - - DoNotExpectStdOutput(pid); - DoNotExpectStdError(pid); - } - } - - TEST_P(ProcessSchedulerTestFixtureWithParams, SchedulerTimeout_QueuedAndInFlightProcessesTerminated) - { - const size_t processToTimeout = 0; - - // Given a set of processes to launch where one process will sleep indefinately - for (size_t pid = 0; pid < m_numProcessesToLaunch; pid++) - { - if (pid == processToTimeout) - { - m_processInfos.emplace_back( - pid, - ValidProcessPath, - ConstructTestProcessArgs(pid, LongSleep)); - } - else - { - m_processInfos.emplace_back( - pid, - ValidProcessPath, - ConstructTestProcessArgs(pid, NoSleep)); - } - } - - // When the process scheduler launches the processes with a scheduler timeout value - m_processScheduler = AZStd::make_unique( - m_processInfos, - m_processLaunchCallback, - m_processExitCallback, - m_numMaxConcurrentProcesses, - AZStd::nullopt, - AZStd::chrono::milliseconds(100) - ); - - for (size_t pid = 0; pid < m_numProcessesToLaunch; pid++) - { - if (pid == processToTimeout) - { - ExpectTimeoutProcess(pid); - } - else - { - if (m_processResults[pid].m_launchResult.has_value()) - { - ExpectSuccessfulLaunch(pid); - } - else - { - ExpectUnlaunchedProcess(pid); - } - } - - DoNotExpectStdOutput(pid); - DoNotExpectStdError(pid); - } - } - - TEST_P(ProcessSchedulerTestFixtureWithParams, RedirectStdOut_StdOutputIsLargeTextString) - { - // Given a set of processes to launch - for (size_t pid = 0; pid < m_numProcessesToLaunch; pid++) - { - m_processInfos.emplace_back( - pid, - TestImpact::StdOutputRouting::ToParent, - TestImpact::StdErrorRouting::None, - ValidProcessPath, - ConstructTestProcessArgsLargeText(pid, NoSleep)); - } - - // When the process scheduler launches the processes - m_processScheduler = AZStd::make_unique( - m_processInfos, - m_processLaunchCallback, - m_processExitCallback, - m_numMaxConcurrentProcesses, - AZStd::nullopt, - AZStd::nullopt - ); - - for (size_t pid = 0; pid < m_numProcessesToLaunch; pid++) - { - ExpectGracefulExit(pid); - ExpectLargeStdOutput(pid); - DoNotExpectStdError(pid); - } - } - - TEST_P(ProcessSchedulerTestFixtureWithParams, RedirectStdError_StdErrorIsLargeTextString) - { - // Given a set of processes to launch - for (size_t pid = 0; pid < m_numProcessesToLaunch; pid++) - { - m_processInfos.emplace_back( - pid, - TestImpact::StdOutputRouting::None, - TestImpact::StdErrorRouting::ToParent, - ValidProcessPath, - ConstructTestProcessArgsLargeText(pid, NoSleep)); - } - - // When the process scheduler launches the processes - m_processScheduler = AZStd::make_unique( - m_processInfos, - m_processLaunchCallback, - m_processExitCallback, - m_numMaxConcurrentProcesses, - AZStd::nullopt, - AZStd::nullopt - ); - - for (size_t pid = 0; pid < m_numProcessesToLaunch; pid++) - { - ExpectGracefulExit(pid); - DoNotExpectStdOutput(pid); - ExpectLargeStdError(pid); - } - } - - INSTANTIATE_TEST_CASE_P( - SmallConcurrentProcesses, - ProcessSchedulerTestFixtureWithParams, - ::testing::Combine( - ::testing::ValuesIn(SmallNumMaxConcurrentProcesses), - ::testing::ValuesIn(SmallNumProcessesToLaunch) - )); - - INSTANTIATE_TEST_CASE_P( - LargeConcurrentProcesses, - ProcessSchedulerTestFixtureWithParams, - ::testing::Combine( - ::testing::ValuesIn(LargeNumMaxConcurrentProcesses), - ::testing::ValuesIn(LargeNumProcessesToLaunch) - )); -} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Process/TestImpactProcessTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Process/TestImpactProcessTest.cpp deleted file mode 100644 index df4ea5a967..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Process/TestImpactProcessTest.cpp +++ /dev/null @@ -1,491 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include - -#include -#include - -#include -#include -#include - -namespace UnitTest -{ - class ProcessTestFixture - : public AllocatorsTestFixture - { - protected: - void SetUp() override - { - UnitTest::AllocatorsTestFixture::SetUp(); - } - - void TearDown() override - { - if (m_process && m_process->IsRunning()) - { - m_process->Terminate(0); - } - - UnitTest::AllocatorsTestFixture::TearDown(); - } - - AZStd::unique_ptr m_process; - static constexpr const TestImpact::ProcessId m_id = 1; - static constexpr const TestImpact::ReturnCode m_terminateErrorCode = 666; - }; - - TEST_F(ProcessTestFixture, LaunchValidProcess_ProcessReturnsSuccessfully) - { - // Given a process launched with a valid binary - TestImpact::ProcessInfo processInfo(m_id, ValidProcessPath); - m_process = TestImpact::LaunchProcess(processInfo); - - // When the process has exited - m_process->BlockUntilExit(); - - // Expect the process to no longer be running - EXPECT_FALSE(m_process->IsRunning()); - } - - TEST_F(ProcessTestFixture, LaunchInvalidProcessInfo_ThrowsProcessException) - { - try - { - // Given a process launched with an invalid binary - TestImpact::ProcessInfo processInfo(m_id, ""); - - // Do not expect this code to be reachable - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::ProcessException& e) - { - // Except a process exception to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(ProcessTestFixture, LaunchInvalidBinary_ThrowsProcessException) - { - try - { - // Given a process launched with an ivalid binary - TestImpact::ProcessInfo processInfo(m_id, "#!#zz:/z/z/z.exe.z@"); - - // When the process is attempted launch - m_process = TestImpact::LaunchProcess(processInfo); - - // Do not expect this code to be reachable - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::ProcessException& e) - { - // Except a process exception to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(ProcessTestFixture, GetProcessInfo_ReturnsProcessInfo) - { - // Given a process launched with a valid binary and args - const AZStd::string args = ConstructTestProcessArgs(m_id, NoSleep); - TestImpact::ProcessInfo processInfo(m_id, ValidProcessPath, args); - m_process = TestImpact::LaunchProcess(processInfo); - - // When the process has exited - m_process->BlockUntilExit(); - - // Expect the retrieved process info to match to process info used to launch the process - EXPECT_EQ(m_process->GetProcessInfo().GetId(), m_id); - EXPECT_EQ(m_process->GetProcessInfo().GetProcessPath(), ValidProcessPath); - EXPECT_EQ(m_process->GetProcessInfo().GetStartupArgs(), args); - } - - TEST_F(ProcessTestFixture, GetReturnCodeAfterExit_ReturnsArg) - { - // Given a process launched with a valid binary and args - const AZStd::string args = ConstructTestProcessArgs(m_id, NoSleep); - TestImpact::ProcessInfo processInfo(m_id, ValidProcessPath, args); - m_process = TestImpact::LaunchProcess(processInfo); - - // When the process has exited - m_process->BlockUntilExit(); - - // Expect the process to no longer be running - EXPECT_FALSE(m_process->IsRunning()); - - // Expect the return value to be the process id - EXPECT_EQ(m_process->GetReturnCode(), m_id); - } - - TEST_F(ProcessTestFixture, GetReturnCodeInFlight_ReturnsNullOpt) - { - // Given a process launched with a valid binary and args to sleep for 100 seconds - const AZStd::string args = ConstructTestProcessArgs(m_id, LongSleep); - TestImpact::ProcessInfo processInfo(m_id, ValidProcessPath, args); - - // When the process is running - m_process = TestImpact::LaunchProcess(processInfo); - - // Expect the return value to be empty - EXPECT_EQ(m_process->GetReturnCode(), AZStd::nullopt); - } - - TEST_F(ProcessTestFixture, TerminateWithErrorCode_ReturnsErrorCode) - { - // Given a process launched with a valid binary and args to sleep for 100 seconds - const AZStd::string args = ConstructTestProcessArgs(m_id, LongSleep); - TestImpact::ProcessInfo processInfo(m_id, ValidProcessPath, args); - m_process = TestImpact::LaunchProcess(processInfo); - - // When the process is terminated by the client with the specified error code - m_process->Terminate(m_terminateErrorCode); - - // Expect the return value to be the error code specified in the Terminate call - EXPECT_EQ(m_process->GetReturnCode(), m_terminateErrorCode); - - // Expect the process to no longer be running - EXPECT_FALSE(m_process->IsRunning()); - } - - TEST_F(ProcessTestFixture, CheckIsRunningWhilstRunning_ProcessIsRunning) - { - // Given a process launched with a valid binary and args to sleep for 100 seconds - const AZStd::string args = ConstructTestProcessArgs(m_id, LongSleep); - TestImpact::ProcessInfo processInfo(m_id, ValidProcessPath, args); - - // When the process is running - m_process = TestImpact::LaunchProcess(processInfo); - - // Expect the process to be running - EXPECT_TRUE(m_process->IsRunning()); - } - - TEST_F(ProcessTestFixture, CheckIsRunningWhilstNotRunning_ReturnsFalse) - { - // Given a process launched with a valid binary - TestImpact::ProcessInfo processInfo(m_id, ValidProcessPath); - m_process = TestImpact::LaunchProcess(processInfo); - - // When the process is terminated by the client - m_process->Terminate(0); - - // Expect the process to be running - EXPECT_FALSE(m_process->IsRunning()); - } - - TEST_F(ProcessTestFixture, RedirectStdOut_OutputIsKnownTestProcessOutputString) - { - // Given a process launched with a valid binary and standard output redirected to parent - const AZStd::string args = ConstructTestProcessArgs(m_id, NoSleep); - TestImpact::ProcessInfo processInfo( - m_id, - TestImpact::StdOutputRouting::ToParent, - TestImpact::StdErrorRouting::None, - ValidProcessPath, - args); - m_process = TestImpact::LaunchProcess(processInfo); - - // When the process exits - m_process->BlockUntilExit(); - - // Expect stdoutput to be rerouted to the parent - EXPECT_TRUE(m_process->GetProcessInfo().ParentHasStdOutput()); - - // Do not expect stdouterr to be rerouted to the parent - EXPECT_FALSE(m_process->GetProcessInfo().ParentHasStdError()); - - // Expect the output to match the known stdout of the child - EXPECT_EQ(m_process->ConsumeStdOut(), KnownTestProcessOutputString(m_id)); - } - - TEST_F(ProcessTestFixture, RedirectStdErr_OutputIsKnownTestProcessOutputString) - { - // Given a process launched with a valid binary and standard error redirected to parent - const AZStd::string args = ConstructTestProcessArgs(m_id, NoSleep); - TestImpact::ProcessInfo processInfo( - m_id, - TestImpact::StdOutputRouting::None, - TestImpact::StdErrorRouting::ToParent, - ValidProcessPath, - args); - m_process = TestImpact::LaunchProcess(processInfo); - - // When the process exits - m_process->BlockUntilExit(); - - // Do not expect stdoutput to be rerouted to the parent - EXPECT_FALSE(m_process->GetProcessInfo().ParentHasStdOutput()); - - // Expect stderror to be rerouted to the parent - EXPECT_TRUE(m_process->GetProcessInfo().ParentHasStdError()); - - // Expect the output to match the known stdout of the child - EXPECT_EQ(m_process->ConsumeStdErr(), KnownTestProcessErrorString(m_id)); - } - - TEST_F(ProcessTestFixture, RedirectStdOutAndTerminate_OutputIsEmpty) - { - // Given a process launched with a valid binary and standard output redirected to parent - const AZStd::string args = ConstructTestProcessArgs(m_id, NoSleep); - TestImpact::ProcessInfo processInfo( - m_id, - TestImpact::StdOutputRouting::ToParent, - TestImpact::StdErrorRouting::None, - ValidProcessPath, - args); - m_process = TestImpact::LaunchProcess(processInfo); - - // When the process is terminated - m_process->Terminate(0); - - // Expect stdoutput to be rerouted to the parent - EXPECT_TRUE(m_process->GetProcessInfo().ParentHasStdOutput()); - - // Do not expect stdouterr to be rerouted to the parent - EXPECT_FALSE(m_process->GetProcessInfo().ParentHasStdError()); - - // Expect the output to be empty - EXPECT_FALSE(m_process->ConsumeStdOut().has_value()); - } - - TEST_F(ProcessTestFixture, RedirectStdErrAndTerminate_OutputIsEmpty) - { - // Given a process launched with a valid binary and standard error redirected to parent - const AZStd::string args = ConstructTestProcessArgs(m_id, NoSleep); - TestImpact::ProcessInfo processInfo( - m_id, - TestImpact::StdOutputRouting::None, - TestImpact::StdErrorRouting::ToParent, - ValidProcessPath, - args); - m_process = TestImpact::LaunchProcess(processInfo); - - // When the process is terminated - m_process->Terminate(0); - - // Do not expect stdoutput to be rerouted to the parent - EXPECT_FALSE(m_process->GetProcessInfo().ParentHasStdOutput()); - - // Expect stderror to be rerouted to the parent - EXPECT_TRUE(m_process->GetProcessInfo().ParentHasStdError()); - - // Expect the output to be empty - EXPECT_FALSE(m_process->ConsumeStdErr().has_value()); - } - - TEST_F(ProcessTestFixture, RedirectStdOutAndStdErrorRouting_OutputsAreKnownTestProcessOutputStrings) - { - // Given a process launched with a valid binary and both standard output and standard error redirected to parent - const AZStd::string args = ConstructTestProcessArgs(m_id, NoSleep); - TestImpact::ProcessInfo processInfo( - m_id, - TestImpact::StdOutputRouting::ToParent, - TestImpact::StdErrorRouting::ToParent, - ValidProcessPath, - args); - m_process = TestImpact::LaunchProcess(processInfo); - - // When the process exits - m_process->BlockUntilExit(); - - // Expect stdoutput to be rerouted to the parent - EXPECT_TRUE(m_process->GetProcessInfo().ParentHasStdOutput()); - - // Expect stderror to be rerouted to the parent - EXPECT_TRUE(m_process->GetProcessInfo().ParentHasStdError()); - - // Expect the output to match the known stdout and stderr of the child - EXPECT_EQ(m_process->ConsumeStdOut(), KnownTestProcessOutputString(m_id)); - EXPECT_EQ(m_process->ConsumeStdErr(), KnownTestProcessErrorString(m_id)); - } - - TEST_F(ProcessTestFixture, NoStdOutOrStdErrRedirect_OutputIsEmpty) - { - // Given a process launched with a valid binary and both standard output and standard error redirected to parent - const AZStd::string args = ConstructTestProcessArgs(m_id, NoSleep); - TestImpact::ProcessInfo processInfo( - m_id, - TestImpact::StdOutputRouting::None, - TestImpact::StdErrorRouting::None, - ValidProcessPath, - args); - m_process = TestImpact::LaunchProcess(processInfo); - - // When the process exits - m_process->BlockUntilExit(); - - // Do not expect stdoutput to be rerouted to the parent - EXPECT_FALSE(m_process->GetProcessInfo().ParentHasStdOutput()); - - // Do not expect stdouterr to be rerouted to the parent - EXPECT_FALSE(m_process->GetProcessInfo().ParentHasStdError()); - - // Expect the output to to be empty - EXPECT_FALSE(m_process->ConsumeStdOut().has_value()); - EXPECT_FALSE(m_process->ConsumeStdErr().has_value()); - } - - TEST_F(ProcessTestFixture, LargePipeNoRedirects_OutputsAreEmpty) - { - // Given a process outputting large text with no standard output and no standard error redirected to parent - const AZStd::string args = ConstructTestProcessArgsLargeText(m_id, NoSleep); - TestImpact::ProcessInfo processInfo( - m_id, - TestImpact::StdOutputRouting::None, - TestImpact::StdErrorRouting::None, - ValidProcessPath, - args); - m_process = TestImpact::LaunchProcess(processInfo); - - // When the process exits - m_process->BlockUntilExit(); - - // Expect the output to to be empty - EXPECT_FALSE(m_process->ConsumeStdOut().has_value()); - EXPECT_FALSE(m_process->ConsumeStdErr().has_value()); - } - - TEST_F(ProcessTestFixture, LargePipeNoRedirectsAndTerminated_OutputsAreEmpty) - { - // Given a process outputting large text with no standard output and standard error redirected to parent - const AZStd::string args = ConstructTestProcessArgsLargeText(m_id, LongSleep); - TestImpact::ProcessInfo processInfo( - m_id, - TestImpact::StdOutputRouting::None, - TestImpact::StdErrorRouting::None, - ValidProcessPath, - args); - - // When the process is running - m_process = TestImpact::LaunchProcess(processInfo); - - // Expect the output to to be empty - EXPECT_FALSE(m_process->ConsumeStdOut().has_value()); - EXPECT_FALSE(m_process->ConsumeStdErr().has_value()); - - // When the process is terminated - m_process->Terminate(0); - - // Expect the output to to be empty - EXPECT_FALSE(m_process->ConsumeStdOut().has_value()); - EXPECT_FALSE(m_process->ConsumeStdErr().has_value()); - } - - TEST_F(ProcessTestFixture, LargePipeRedirectsAndReadWhilstRunning_OutputsAreEmpty) - { - // Given a process outputting large text with no standard output and standard error redirected to parent - const AZStd::string args = ConstructTestProcessArgsLargeText(m_id, ShortSleep); - TestImpact::ProcessInfo processInfo( - m_id, - TestImpact::StdOutputRouting::None, - TestImpact::StdErrorRouting::None, - ValidProcessPath, - args); - m_process = TestImpact::LaunchProcess(processInfo); - - // When the outputs are read as the process is running - while (m_process->IsRunning()) - { - // Expect the outputs to be empty - EXPECT_FALSE(m_process->ConsumeStdOut().has_value()); - EXPECT_FALSE(m_process->ConsumeStdErr().has_value()); - } - } - - TEST_F(ProcessTestFixture, LargePipeRedirectsAndTerminated_OutputsAreEmpty) - { - // Given a process outputting large text with both standard output and standard error redirected to parent - const AZStd::string args = ConstructTestProcessArgsLargeText(m_id, LongSleep); - TestImpact::ProcessInfo processInfo( - m_id, - TestImpact::StdOutputRouting::ToParent, - TestImpact::StdErrorRouting::ToParent, - ValidProcessPath, - args); - m_process = TestImpact::LaunchProcess(processInfo); - - // When the process is terminated - m_process->Terminate(0); - - // Expect the outputs to be empty - EXPECT_FALSE(m_process->ConsumeStdOut().has_value()); - EXPECT_FALSE(m_process->ConsumeStdErr().has_value()); - } - - TEST_F(ProcessTestFixture, LargePipeRedirectsAndBlockedUntilExit_OutputsAreLargeTextFileSize) - { - // Given a process outputting large text with both standard output and standard error redirected to parent - const AZStd::string args = ConstructTestProcessArgsLargeText(m_id, NoSleep); - TestImpact::ProcessInfo processInfo( - m_id, - TestImpact::StdOutputRouting::ToParent, - TestImpact::StdErrorRouting::ToParent, - ValidProcessPath, - args); - m_process = TestImpact::LaunchProcess(processInfo); - - // When the process exits - m_process->BlockUntilExit(); - - // Expect the output lengths to be that of the large text output from the child process - EXPECT_EQ(m_process->ConsumeStdOut().value().length(), LargeTextSize); - EXPECT_EQ(m_process->ConsumeStdErr().value().length(), LargeTextSize); - } - - TEST_F(ProcessTestFixture, LargePipeRedirectsAndReadWhilstRunning_TotalOutputsAreLargeTextFileSize) - { - // Given a process outputting large text with both standard output and standard error redirected to parent - const AZStd::string args = ConstructTestProcessArgsLargeText(m_id, NoSleep); - TestImpact::ProcessInfo processInfo( - m_id, - TestImpact::StdOutputRouting::ToParent, - TestImpact::StdErrorRouting::ToParent, - ValidProcessPath, args); - m_process = TestImpact::LaunchProcess(processInfo); - - size_t stdOutBytes = 0; - size_t stdErrBytes = 0; - while (m_process->IsRunning()) - { - // When the outputs are read as the process is running - if (auto output = m_process->ConsumeStdOut(); output.has_value()) - { - stdOutBytes += output.value().length(); - } - - if (auto output = m_process->ConsumeStdErr(); output.has_value()) - { - stdErrBytes += output.value().length(); - } - } - - // Expect the output lengths to be that of the large text output from the child process - EXPECT_EQ(stdOutBytes, LargeTextSize); - EXPECT_EQ(stdErrBytes, LargeTextSize); - - // Expect the outputs to be empty - EXPECT_FALSE(m_process->ConsumeStdOut().has_value()); - EXPECT_FALSE(m_process->ConsumeStdErr().has_value()); - } -} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Target/TestImpactBuildTargetTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Target/TestImpactBuildTargetTest.cpp deleted file mode 100644 index e7ebec17d1..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Target/TestImpactBuildTargetTest.cpp +++ /dev/null @@ -1,352 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include - -#include -#include - -#include -#include - -namespace UnitTest -{ - namespace - { - AZStd::string GenerateBuildTargetName(size_t index) - { - return AZStd::string::format("Target%u", index); - } - - AZStd::string GenerateBuildTargetOutputName(size_t index) - { - return AZStd::string::format("Output%u", index); - } - - AZ::IO::Path GenerateBuildTargetPath(size_t index) - { - return AZStd::string::format("C:\\Repo\\Dir%u", index); - } - - AZStd::string GenerateTestTargetSuite(size_t index) - { - return AZStd::string::format("Suite%u", index); - } - - TestImpact::LaunchMethod GenerateLaunchMethod(size_t index) - { - return index % 2 == 0 ? TestImpact::LaunchMethod::StandAlone : TestImpact::LaunchMethod::TestRunner; - } - - AZStd::string GenerateStaticSourceFile(size_t index) - { - return AZStd::string::format("StaticSource%u", index); - } - - TestImpact::AutogenPairs GenerateAutogenSourceFiles(size_t index) - { - TestImpact::AutogenPairs autogen; - autogen.m_input = AZStd::string::format("InputSource%u", index); - autogen.m_outputs.push_back(AZStd::string::format("OutputSource%u", index)); - autogen.m_outputs.push_back(AZStd::string::format("OutputHeader%u", index)); - return autogen; - } - - TestImpact::TargetSources GenerateTargetSources(size_t index) - { - TestImpact::TargetSources sources; - sources.m_staticSources.resize(index + 1); - for (size_t i = 0; i < sources.m_staticSources.size(); i++) - { - sources.m_staticSources[i] = GenerateStaticSourceFile(i); - } - - // Only generate autogen sources for even-numbered indexes - if (index % 2 == 0) - { - sources.m_autogenSources.resize(index + 1); - for (size_t i = 0; i < sources.m_autogenSources.size(); i++) - { - sources.m_autogenSources[i] = GenerateAutogenSourceFiles(i); - } - } - - return sources; - } - - TestImpact::ProductionTargetDescriptor GenerateProductionTargetDescriptor(size_t index = 0) - { - return TestImpact::ProductionTargetDescriptor( - { - TestImpact::BuildMetaData - { - GenerateBuildTargetName(index), GenerateBuildTargetOutputName(index), GenerateBuildTargetPath(index) - }, - GenerateTargetSources(index) - }); - } - - TestImpact::TestTargetDescriptor GenerateTestTargetDescriptor(size_t index = 0) - { - return TestImpact::TestTargetDescriptor( - { - TestImpact::BuildMetaData - { - GenerateBuildTargetName(index), GenerateBuildTargetOutputName(index), GenerateBuildTargetPath(index) - }, - GenerateTargetSources(index) - }, - TestImpact::TestTargetMeta - { - GenerateTestTargetSuite(index), GenerateLaunchMethod(index) - }); - } - } // namespace - - template - typename Target::Descriptor GenerateTargetDescriptor(size_t index = 0) - { - static_assert( - AZStd::is_same_v || AZStd::is_same_v, - "Unexpected target type, wants TestImpact::ProductionTarget or TestImpact::TestTarget"); - - if constexpr (AZStd::is_same_v) - { - return GenerateProductionTargetDescriptor(index); - } - else - { - return GenerateTestTargetDescriptor(index); - } - } - - void ValidateSources(const TestImpact::TargetSources& sources, size_t index = 0) - { - EXPECT_EQ(sources.m_staticSources.size(), index + 1); - for (size_t i = 0; i < sources.m_staticSources.size(); i++) - { - EXPECT_EQ(sources.m_staticSources[i], GenerateStaticSourceFile(i)); - } - - // Even numbered indexes have autogen sources - if (index % 2 == 0) - { - EXPECT_EQ(sources.m_autogenSources.size(), index + 1); - for (size_t i = 0; i < sources.m_autogenSources.size(); i++) - { - enum - { - OutputSource = 0, - OutputHeader = 1 - }; - - const TestImpact::AutogenPairs expectedAutogenSources = GenerateAutogenSourceFiles(i); - EXPECT_EQ(sources.m_autogenSources[i].m_input, expectedAutogenSources.m_input); - EXPECT_EQ(sources.m_autogenSources[i].m_outputs[OutputSource], expectedAutogenSources.m_outputs[OutputSource]); - EXPECT_EQ(sources.m_autogenSources[i].m_outputs[OutputHeader], expectedAutogenSources.m_outputs[OutputHeader]); - } - } - else - { - EXPECT_TRUE(sources.m_autogenSources.empty()); - } - } - - void ValidateTarget(const TestImpact::ProductionTarget& target, size_t index = 0) - { - EXPECT_EQ(target.GetName(), GenerateBuildTargetName(index)); - EXPECT_EQ(target.GetOutputName(), GenerateBuildTargetOutputName(index)); - EXPECT_EQ(target.GetPath(), GenerateBuildTargetPath(index)); - EXPECT_EQ(target.GetType(), TestImpact::TargetType::Production); - ValidateSources(target.GetSources(), index); - } - - void ValidateTarget(const TestImpact::TestTarget& target, size_t index = 0) - { - EXPECT_EQ(target.GetName(), GenerateBuildTargetName(index)); - EXPECT_EQ(target.GetOutputName(), GenerateBuildTargetOutputName(index)); - EXPECT_EQ(target.GetPath(), GenerateBuildTargetPath(index)); - EXPECT_EQ(target.GetType(), TestImpact::TargetType::Test); - EXPECT_EQ(target.GetSuite(), GenerateTestTargetSuite(index)); - EXPECT_EQ(target.GetLaunchMethod(), GenerateLaunchMethod(index)); - ValidateSources(target.GetSources(), index); - } - - template - class TargetListFixture - : public AllocatorsTestFixture - { - public: - using TargetListType = TargetList; - using TargetType = typename TargetList::TargetType; - - static constexpr size_t m_numTargets = 10; - }; - - using TargetTypes = testing::Types; - TYPED_TEST_CASE(TargetListFixture, TargetTypes); - - TYPED_TEST(TargetListFixture, CreateTarget_ExpectValidTarget) - { - // Given a target of the specified type - TargetType target(GenerateTargetDescriptor()); - - // Expect the target to match the procedurally generated target descriptor - ValidateTarget(target); - } - - TYPED_TEST(TargetListFixture, CreateEmptyTargetList_ExpectTargetException) - { - try - { - // Given an empty target list - TargetListType targetList({}); - - // Do not expect the target list construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::TargetException& e) - { - // Expect a target exception to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TYPED_TEST(TargetListFixture, CreateTargetListWithDuplicateDescriptor_ExpectTargetException) - { - try - { - // Given a set of target descriptors containing a single duplicate - AZStd::vector descriptors; - descriptors.reserve(m_numTargets); - for (size_t i = 0; i < m_numTargets; i++) - { - // Wrap the last index round to repeat the first index - descriptors.push_back(GenerateTargetDescriptor(i % (m_numTargets - 1))); - } - - // When constructing the target list containing the duplicate target descriptor - TargetListType targetList(AZStd::move(descriptors)); - - // Do not expect the target list construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::TargetException& e) - { - // Expect a target exception to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TYPED_TEST(TargetListFixture, CreateTargetListWithValidDescriptors_ExpectValidTargetList) - { - // Given a valid set of target descriptor - AZStd::vector descriptors; - descriptors.reserve(m_numTargets); - for (size_t i = 0; i < m_numTargets; i++) - { - descriptors.push_back(GenerateTargetDescriptor(i)); - } - - // When constructing the target list containing the valid target descriptors - TargetListType targetList(AZStd::move(descriptors)); - - // Expect the number of targets in the list to match the number of target descriptors used to construct the list - EXPECT_EQ(targetList.GetNumTargets(), m_numTargets); - - for (size_t i = 0; i < targetList.GetNumTargets(); i++) - { - // Expect the target to match the procedurally generated target descriptor - ValidateTarget(targetList.GetTargets()[i], i); - - // Expect the target obtained by name to match the procedurally generated target descriptor - auto target = targetList.GetTarget(GenerateBuildTargetName(i)); - EXPECT_TRUE(target); - EXPECT_EQ(target->GetName(), GenerateBuildTargetName(i)); - } - } - - TYPED_TEST(TargetListFixture, FindNonExistantTargets_ExpectEmptyResults) - { - // Given a valid set of target descriptor - AZStd::vector descriptors; - descriptors.reserve(m_numTargets); - for (size_t i = 0; i < m_numTargets; i++) - { - descriptors.push_back(GenerateTargetDescriptor(i)); - } - - // When constructing the target list containing the valid target descriptors - TargetListType targetList(AZStd::move(descriptors)); - - // Expect the number of targets in the list to match the number of target descriptors used to construct the list - EXPECT_EQ(targetList.GetNumTargets(), m_numTargets); - - for (size_t i = 0; i < targetList.GetNumTargets(); i++) - { - // When attempting to find a target that does not exist - auto target = targetList.GetTarget(GenerateBuildTargetName(i + targetList.GetNumTargets())); - - // Expect an empty result - EXPECT_FALSE(target); - } - } - - TYPED_TEST(TargetListFixture, FindNonExistantTargetsAndThrow_ExpectTargetExceptionss) - { - // Given a valid set of target descriptor - AZStd::vector descriptors; - descriptors.reserve(m_numTargets); - for (size_t i = 0; i < m_numTargets; i++) - { - descriptors.push_back(GenerateTargetDescriptor(i)); - } - - // When constructing the target list containing the valid target descriptors - TargetListType targetList(AZStd::move(descriptors)); - - // Expect the number of targets in the list to match the number of target descriptors used to construct the list - EXPECT_EQ(targetList.GetNumTargets(), m_numTargets); - - for (size_t i = 0; i < targetList.GetNumTargets(); i++) - { - try - { - // When attempting to find a target that does not exist - targetList.GetTargetOrThrow(GenerateBuildTargetName(i + targetList.GetNumTargets())); - - // Do not expect the target list construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::TargetException& e) - { - // Expect a target exception to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - } -} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactExceptionTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactExceptionTest.cpp deleted file mode 100644 index bf2ccb6bc7..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactExceptionTest.cpp +++ /dev/null @@ -1,150 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include -#include -#include - -namespace UnitTest -{ - constexpr const char* Message = "String Constructor"; - - TEST(ExceptionTest, DefaultConstructor_HasEmptyMessageString) - { - // Given an exception instantiated with the default constructor - const TestImpact::Exception e; - - // Expect the message to be an empty string - EXPECT_STREQ(e.what(), "\0"); - } - - TEST(ExceptionTest, StringConstructor_HasSpecifiedMessageString) - { - // Given an exception instantiated with a string - const AZStd::string msg(Message); - const TestImpact::Exception e(msg); - - // Expect the message to be the specified string - EXPECT_STREQ(e.what(), Message); - } - - TEST(ExceptionTest, StringLiteralConstructor_HasSpecifiedMessageString) - { - // Given an exception instantiated with a string literal - const TestImpact::Exception e(Message); - - // Expect the message to be the specified string - EXPECT_STREQ(e.what(), Message); - } - - TEST(ExceptionTest, InitializedWithLocalString_HasCopyOfLocalStringString) - { - try - { - // Given an exception instantiated with a string in local scope - throw(TestImpact::Exception(AZStd::string::format("%s", Message))); - - // Do not expect this code to be reachable - FAIL(); - } - catch (const TestImpact::Exception& e) - { - // Expect the message to be the specified out-of-scope string - EXPECT_STREQ(e.what(), Message); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST(ExceptionTest, InitializedWithLocalStringLiteral_HasCopyOfLocalStringString) - { - try - { - // Given an exception instantiated with a copy of the message string in local scope - throw(TestImpact::Exception(AZStd::string::format("%s", Message).c_str())); - - // Do not expect this code to be reachable - FAIL(); - } - catch (const TestImpact::Exception& e) - { - // Expect the message to be the specified out-of-scope string literal - EXPECT_STREQ(e.what(), Message); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST(ExceptionTest, EvalMacroFails_ExceptionTestThrown) - { - try - { - // Given an exception instantiated with a string literal in local scope - AZ_TestImpact_Eval(false, TestImpact::Exception, Message); - - // Do not expect this code to be reachable - FAIL(); - } - catch (const TestImpact::Exception& e) - { - // Expect the message contain the specified string - EXPECT_STREQ(e.what(), Message); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST(ExceptionTest, EvalMacroSucceeds_ExceptionTestNotThrown) - { - try - { - // Given an exception instantiated with a string literal in local scope - AZ_TestImpact_Eval(true, TestImpact::Exception, Message); - - // Expect this code to be reachable - SUCCEED(); - } - catch (...) - { - // Do not expect any exceptions - FAIL(); - } - } - - TEST(ExceptionTest, Throw_ExceptionTestThrown) - { - try - { - // Given an exception instantiated with a string literal in local scope - throw(TestImpact::Exception(Message)); - } - catch (const TestImpact::Exception& e) - { - // Expect the message contain the specified string - EXPECT_STREQ(e.what(), Message); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } -} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactTestJobRunnerCommon.h b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactTestJobRunnerCommon.h deleted file mode 100644 index 9d5ddd6c8c..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactTestJobRunnerCommon.h +++ /dev/null @@ -1,159 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#pragma once - -#include - -#include -#include - -#include -#include -#include - -template -constexpr bool IsFlagSet(Flags flags, Flags flag) -{ - return TestImpact::Bitwise::IsFlagSet(flags, flag); -} - -namespace UnitTest -{ - // Named constants for array of targets lookup - enum - { - TestTargetA, - TestTargetB, - TestTargetC, - TestTargetD - }; - - // Named constants for max concurrency values - enum - { - OneConcurrentProcess = 1, - FourConcurrentProcesses = 4 - }; - - // Validates that the specified job was executed and returned successfully - template - void ValidateJobExecutedSuccessfully(const Job& job) - { - EXPECT_EQ(job.GetResult(), TestImpact::JobResult::ExecutedWithSuccess); - EXPECT_TRUE(job.GetDuration() > AZStd::chrono::milliseconds(0)); - EXPECT_TRUE(job.GetReturnCode().has_value()); - EXPECT_EQ(job.GetReturnCode(), 0); - EXPECT_TRUE(job.GetPayload().has_value()); - } - - // Validates that the specified job has not been executed - template - void ValidateJobNotExecuted(const Job& job) - { - EXPECT_EQ(job.GetResult(), TestImpact::JobResult::NotExecuted); - EXPECT_EQ(job.GetStartTime(), AZStd::chrono::high_resolution_clock::time_point()); - EXPECT_EQ(job.GetEndTime(), AZStd::chrono::high_resolution_clock::time_point()); - EXPECT_EQ(job.GetDuration(), AZStd::chrono::milliseconds(0)); - EXPECT_FALSE(job.GetReturnCode().has_value()); - EXPECT_FALSE(job.GetPayload().has_value()); - } - - // Validates that the specified job failed to execute - template - void ValidateJobFailedToExecute(const Job& job) - { - EXPECT_EQ(job.GetResult(), TestImpact::JobResult::FailedToExecute); - EXPECT_EQ(job.GetStartTime(), AZStd::chrono::high_resolution_clock::time_point()); - EXPECT_EQ(job.GetEndTime(), AZStd::chrono::high_resolution_clock::time_point()); - EXPECT_EQ(job.GetDuration(), AZStd::chrono::milliseconds(0)); - EXPECT_FALSE(job.GetReturnCode().has_value()); - EXPECT_FALSE(job.GetPayload().has_value()); - } - - // Validates that the specified job executed but returned with error - template - void ValidateJobExecutedWithFailure(const Job& job) - { - EXPECT_EQ(job.GetResult(), TestImpact::JobResult::ExecutedWithFailure); - EXPECT_TRUE(job.GetDuration() > AZStd::chrono::milliseconds(0)); - EXPECT_TRUE(job.GetReturnCode().has_value()); - EXPECT_GT(job.GetReturnCode().value(), 0); - EXPECT_FALSE(job.GetPayload().has_value()); - } - - // Validates that the specified job was executed but was terminated by the job runner - template - void ValidateJobTimeout(const Job& job) - { - EXPECT_EQ(job.GetResult(), TestImpact::JobResult::Terminated); - EXPECT_TRUE(job.GetDuration() > AZStd::chrono::milliseconds(0)); - EXPECT_TRUE(job.GetReturnCode().has_value()); - EXPECT_EQ(job.GetReturnCode().value(), TestImpact::ProcessTimeoutErrorCode); - EXPECT_FALSE(job.GetPayload().has_value()); - } - - // Validates that the specified job executed but returned with error - template - void ValidateJobExecutedWithFailedTests(const Job& job) - { - EXPECT_EQ(job.GetResult(), TestImpact::JobResult::ExecutedWithFailure); - EXPECT_TRUE(job.GetDuration() > AZStd::chrono::milliseconds(0)); - EXPECT_TRUE(job.GetReturnCode().has_value()); - EXPECT_GT(job.GetReturnCode().value(), 0); - EXPECT_TRUE(job.GetPayload().has_value()); - } - - // Validates whether a test run completed (passed/failed) - template - void ValidateTestRunCompleted(const Job& job, TestImpact::TestRunResult result) - { - if (result == TestImpact::TestRunResult::Passed) - { - ValidateJobExecutedSuccessfully(job); - } - else - { - ValidateJobExecutedWithFailedTests(job); - } - } - - // Validates that the specified test run matches the expected output - inline void ValidateTestTargetRun(const TestImpact::TestRun& actualResult, const TestImpact::TestRun& expectedResult) - { - EXPECT_TRUE(CheckTestRunsAreEqualIgnoreDurations(actualResult, expectedResult)); - EXPECT_EQ(actualResult.GetNumTestSuites(), CalculateNumTestSuites(expectedResult.GetTestSuites())); - EXPECT_EQ(actualResult.GetNumTests(), CalculateNumTests(expectedResult.GetTestSuites())); - EXPECT_EQ(actualResult.GetNumEnabledTests(), CalculateNumEnabledTests(expectedResult.GetTestSuites())); - EXPECT_EQ(actualResult.GetNumDisabledTests(), CalculateNumDisabledTests(expectedResult.GetTestSuites())); - EXPECT_GT(actualResult.GetDuration(), AZStd::chrono::milliseconds{0}); - EXPECT_EQ(actualResult.GetNumPasses(), CalculateNumPassedTests(expectedResult.GetTestSuites())); - EXPECT_EQ(actualResult.GetNumFailures(), CalculateNumFailedTests(expectedResult.GetTestSuites())); - EXPECT_EQ(actualResult.GetNumRuns(), CalculateNumRunTests(expectedResult.GetTestSuites())); - EXPECT_EQ(actualResult.GetNumNotRuns(), CalculateNumNotRunTests(expectedResult.GetTestSuites())); - } - - // Delete any existing data in the test run folder as not to pollute tests with data from previous test runs - // Note: the file IO operations of this fixture means it cannot be sharded by the test sharder due to file race conditions - inline void DeleteFiles(const AZStd::string& path, const AZStd::string& pattern) - { - AZ::IO::SystemFile::FindFiles(AZStd::string::format("%s/%s", path.c_str(), pattern.c_str()).c_str(), [&path](const char* file, bool isFile) - { - if (isFile) - { - AZ::IO::SystemFile::Delete(AZStd::string::format("%s/%s", path.c_str(), file).c_str()); - } - - return true; - }); - }; -} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactTestUtils.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactTestUtils.cpp deleted file mode 100644 index 6b2fa72310..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactTestUtils.cpp +++ /dev/null @@ -1,1843 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include - -#include - -#include -#include - -namespace UnitTest -{ - void WriteTextToFile(const AZStd::string& text, const AZ::IO::Path& path) - { - const AZStd::vector textBytes(text.begin(), text.end()); - AZ::IO::SystemFile textFile; - AZ_TestImpact_Eval( - textFile.Open( - path.c_str(), - AZ::IO::SystemFile::SF_OPEN_CREATE | AZ::IO::SystemFile::SF_OPEN_CREATE_PATH | AZ::IO::SystemFile::SF_OPEN_WRITE_ONLY), - TestImpact::Exception, "Couldn't open cache file for writing"); - const auto bytesWritten = textFile.Write(textBytes.data(), textBytes.size()); - AZ_TestImpact_Eval(bytesWritten == textBytes.size(), TestImpact::Exception, "Couldn't write text file data"); - } - - AZStd::string ConstructTestProcessArgs(TestImpact::ProcessId pid, AZStd::chrono::milliseconds sleepTime) - { - return AZStd::string::format("--id %i --sleep %u", pid, sleepTime.count()); - } - - AZStd::string ConstructTestProcessArgsLargeText(TestImpact::ProcessId pid, AZStd::chrono::milliseconds sleepTime) - { - return ConstructTestProcessArgs(pid, sleepTime) + " large"; - } - - AZStd::string KnownTestProcessOutputString(TestImpact::ProcessId pid) - { - return AZStd::string("TestProcessMainStdOut" + AZStd::to_string(pid)); - } - - AZStd::string KnownTestProcessErrorString(TestImpact::ProcessId pid) - { - return AZStd::string("TestProcessMainStdErr" + AZStd::to_string(pid)); - } - - AZStd::string GenerateSuiteOrFixtureName(const AZStd::string& name) - { - return name; - } - - AZStd::string GenerateTypedFixtureName(const AZStd::string& name, size_t typeNum) - { - return AZStd::string::format("%s/%u", name.c_str(), typeNum); - } - - AZStd::string GenerateParameterizedFixtureName(const AZStd::string& name, const AZStd::optional& prefix) - { - if (prefix.has_value()) - { - return AZStd::string::format("%s/%s", prefix->c_str(), name.c_str()); - } - - return AZStd::string::format("%s", name.c_str()); - } - - AZStd::string GenerateParameterizedTestName(const AZStd::string& name, size_t testNum) - { - return AZStd::string::format("%s/%u", name.c_str(), testNum); - } - - AZStd::string StringVectorToJSONElements(const AZStd::vector strings) - { - AZStd::string output; - - for (size_t i = 0, last = strings.size() - 1; i < strings.size(); i++) - { - output += AZStd::string::format("\"%s\"", strings[i].c_str()); - - if (i != last) - { - output += ","; - } - - output += "\n"; - } - - return output; - } - - AZStd::string GenerateBuildTargetDescriptorString( - const AZStd::string& name, - const AZStd::string& outputName, - const AZStd::string& path, - const AZStd::vector& staticSources, - const AZStd::vector& autogenInputs, - const AZStd::vector& autogenOutputs) - { - constexpr const char* const targetTemplate = - "{\n" - " \"sources\": {\n" - " \"input\": [\n" - "%s\n" - " ],\n" - " \"output\": [\n" - "%s\n" - " ],\n" - " \"static\": [\n" - "%s\n" - " ]\n" - " },\n" - " \"target\": {\n" - " \"name\": \"%s\",\n" - " \"output_name\": \"%s\",\n" - " \"path\": \"%s\"\n" - " }\n" - "}\n" - "\n"; - - AZStd::string output = AZStd::string::format(targetTemplate, - StringVectorToJSONElements(autogenInputs).c_str(), - StringVectorToJSONElements(autogenOutputs).c_str(), - StringVectorToJSONElements(staticSources).c_str(), - name.c_str(), - outputName.c_str(), - path.c_str()); - - return output; - } - - TestImpact::BuildTargetDescriptor GenerateBuildTargetDescriptor( - const AZStd::string& name, - const AZStd::string& outputName, - const AZStd::string& path, - const AZStd::vector& staticSources, - const TestImpact::AutogenSources& autogenSources) - { - TestImpact::BuildTargetDescriptor targetDescriptor; - targetDescriptor.m_buildMetaData = {name, outputName, path}; - targetDescriptor.m_sources = {staticSources, autogenSources}; - return targetDescriptor; - } - - TestImpact::TestEnumerationSuite GenerateParamterizedSuite( - const AZStd::pair& fixture, - const AZStd::optional& permutation, - const AZStd::vector> tests, - size_t permutationCount) - { - TestImpact::TestEnumerationSuite suite = - TestImpact::TestEnumerationSuite{GenerateParameterizedFixtureName(fixture.first, permutation), fixture.second, {}}; - - for (const auto& test : tests) - { - for (auto i = 0; i < permutationCount; i++) - { - suite.m_tests.push_back(TestImpact::TestEnumerationCase{GenerateParameterizedTestName(test.first, i), test.second}); - } - } - - return suite; - } - - void GenerateTypedSuite( - const AZStd::pair& fixture, - const AZStd::vector> tests, - size_t permutationCount, - AZStd::vector& parentSuiteList) - { - for (auto i = 0; i < permutationCount; i++) - { - parentSuiteList.push_back(TestImpact::TestEnumerationSuite{GenerateTypedFixtureName(fixture.first, i), fixture.second, {}}); - - for (const auto& test : tests) - { - parentSuiteList.back().m_tests.push_back(TestImpact::TestEnumerationCase{test.first, test.second}); - } - } - } - - size_t CalculateNumPassedTests(const AZStd::vector& suites) - { - size_t numPassedTests = 0; - for (const auto& suite : suites) - { - numPassedTests += AZStd::count_if(suite.m_tests.begin(), suite.m_tests.end(), [](const auto& test) { - return test.m_result.has_value() && test.m_result.value() == TestImpact::TestRunResult::Passed; - }); - } - - return numPassedTests; - } - - size_t CalculateNumFailedTests(const AZStd::vector& suites) - { - size_t numFailedTests = 0; - for (const auto& suite : suites) - { - for (const auto& test : suite.m_tests) - { - if (test.m_result.has_value() && test.m_result.value() == TestImpact::TestRunResult::Failed) - { - numFailedTests++; - } - } - } - - return numFailedTests; - } - - size_t CalculateNumRunTests(const AZStd::vector& suites) - { - size_t numRunTests = 0; - for (const auto& suite : suites) - { - for (const auto& test : suite.m_tests) - { - if (test.m_status == TestImpact::TestRunStatus::Run) - { - numRunTests++; - } - } - } - - return numRunTests; - } - - size_t CalculateNumNotRunTests(const AZStd::vector& suites) - { - size_t numNotRunTests = 0; - for (const auto& suite : suites) - { - for (const auto& test : suite.m_tests) - { - if (test.m_status == TestImpact::TestRunStatus::NotRun) - { - numNotRunTests++; - } - } - } - - return numNotRunTests; - } - - AZStd::vector GetTestTargetATestEnumerationSuites() - { - AZStd::vector suites; - - suites.push_back(TestImpact::TestEnumerationSuite{GenerateSuiteOrFixtureName("TestCase"), true, {}}); - suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test1_WillPass", true}); - suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test2_WillPass", true}); - suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test3_WillPass", true}); - suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test4_WillPass", true}); - suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test5_WillPass", true}); - suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test6_WillPass", true}); - suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test7_WillFail", true}); - - suites.push_back(TestImpact::TestEnumerationSuite{GenerateSuiteOrFixtureName("TestFixture"), true, {}}); - suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test1_WillPass", true}); - suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test2_WillPass", true}); - suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test3_WillPass", true}); - - return suites; - } - - AZStd::vector GetTestTargetBTestEnumerationSuites() - { - AZStd::vector suites; - - suites.push_back(TestImpact::TestEnumerationSuite{GenerateSuiteOrFixtureName("TestCase"), true, {}}); - suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test1_WillPass", true}); - suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test2_WillPass", true}); - suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test3_WillPass", true}); - - suites.push_back(TestImpact::TestEnumerationSuite{GenerateSuiteOrFixtureName("TestFixture"), true, {}}); - suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test1_WillPass", true}); - - const size_t numParams = 27; - suites.push_back(GenerateParamterizedSuite( - {"TestFixtureWithParams", true}, "PermutationA", {{"Test1_WillPass", true}, {"Test2_WillPass", true}}, numParams)); - suites.push_back(GenerateParamterizedSuite( - {"TestFixtureWithParams", true}, AZStd::nullopt, {{"Test1_WillPass", true}, {"Test2_WillPass", true}}, numParams)); - - return suites; - } - - AZStd::vector GetTestTargetCTestEnumerationSuites() - { - AZStd::vector suites; - - suites.push_back(TestImpact::TestEnumerationSuite{GenerateSuiteOrFixtureName("TestFixture"), true, {}}); - suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test1_WillPass", true}); - suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test2_WillPass", true}); - - const size_t numTypes = 4; - for (auto i = 0; i < numTypes; i++) - { - suites.push_back(TestImpact::TestEnumerationSuite{GenerateTypedFixtureName("TestFixtureWithTypes", i), true, {}}); - for (auto j = 1; j < numTypes + 1; j++) - { - suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{AZStd::string::format("Test%u_WillPass", j), true}); - } - } - - return suites; - } - - AZStd::vector GetTestTargetDTestEnumerationSuites() - { - AZStd::vector suites; - - suites.push_back(TestImpact::TestEnumerationSuite{GenerateSuiteOrFixtureName("TestCase"), true, {}}); - suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test1_WillPass", true}); - suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"DISABLED_Test2_WillPass", false}); - suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test3_WillPass", true}); - suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test4_WillPass", true}); - suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test5_WillPass", true}); - - suites.push_back(TestImpact::TestEnumerationSuite{GenerateSuiteOrFixtureName("TestFixture1"), true, {}}); - suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test1_WillPass", true}); - suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test2_WillPass", true}); - - suites.push_back(TestImpact::TestEnumerationSuite{GenerateSuiteOrFixtureName("DISABLED_TestFixture2"), false, {}}); - suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test1_WillPass", true}); - suites.back().m_tests.push_back(TestImpact::TestEnumerationCase{"Test2_WillPass", true}); - - const size_t numTypes = 4; - GenerateTypedSuite( - {"TestFixtureWithTypes1", true}, {{"Test1_WillPass", true}, {"DISABLED_Test2_WillPass", false}, {"Test3_WillPass", true}}, - numTypes, suites); - - GenerateTypedSuite( - {"DISABLED_TestFixtureWithTypes2", false}, - {{"Test1_WillPass", true}, {"DISABLED_Test2_WillPass", false}, {"Test3_WillPass", true}}, numTypes, suites); - - const size_t numParams = 27; - suites.push_back(GenerateParamterizedSuite( - {"TestFixtureWithParams1", true}, "PermutationA", {{"Test1_WillPass", true}, {"DISABLED_Test2_WillPass", false}}, numParams)); - suites.push_back(GenerateParamterizedSuite( - {"TestFixtureWithParams1", true}, AZStd::nullopt, {{"Test1_WillPass", true}, {"DISABLED_Test2_WillPass", false}}, numParams)); - suites.push_back(GenerateParamterizedSuite( - {"DISABLED_TestFixtureWithParams2", false}, "PermutationA", {{"Test1_WillPass", true}, {"DISABLED_Test2_WillPass", false}}, - numParams)); - suites.push_back(GenerateParamterizedSuite( - {"DISABLED_TestFixtureWithParams2", false}, AZStd::nullopt, {{"Test1_WillPass", true}, {"DISABLED_Test2_WillPass", false}}, - numParams)); - - return suites; - } - - TestImpact::TestRunCase TestRunCaseFromTestEnumerationCase(const TestImpact::TestEnumerationCase& enumCase) - { - TestImpact::TestRunCase runCase; - runCase.m_name = enumCase.m_name; - runCase.m_enabled = enumCase.m_enabled; - return runCase; - } - - TestImpact::TestRunSuite TestRunSuiteFromTestEnumerationSuite(const TestImpact::TestEnumerationSuite& enumSuite) - { - TestImpact::TestRunSuite runSuite; - runSuite.m_name = enumSuite.m_name; - runSuite.m_enabled = enumSuite.m_enabled; - - for (const auto& enumCase : enumSuite.m_tests) - { - runSuite.m_tests.push_back(TestRunCaseFromTestEnumerationCase(enumCase)); - } - - return runSuite; - } - - AZStd::vector TestRunSuitesFromTestEnumerationSuites( - const AZStd::vector& enumSuites) - { - AZStd::vector runSuites; - runSuites.reserve(enumSuites.size()); - - for (const auto& enumSuite : enumSuites) - { - runSuites.push_back(TestRunSuiteFromTestEnumerationSuite(enumSuite)); - } - - return runSuites; - } - - void SetTestRunSuiteData(TestImpact::TestRunSuite& testSuite, AZStd::chrono::milliseconds duration) - { - testSuite.m_duration = duration; - } - - void SetTestRunCaseData( - TestImpact::TestRunCase& testCase, AZStd::chrono::milliseconds duration, TestImpact::TestRunStatus status, - AZStd::optional result = AZStd::nullopt) - { - testCase.m_duration = duration; - testCase.m_status = status; - testCase.m_result = result; - } - - AZStd::vector GetTestTargetATestRunSuites() - { - AZStd::vector suites(TestRunSuitesFromTestEnumerationSuites(GetTestTargetATestEnumerationSuites())); - - enum - { - TestCaseIndex = 0, - TestFixtureIndex - }; - - { - auto& suite = suites[TestCaseIndex]; - SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{3}); - SetTestRunCaseData( - suite.m_tests[0], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[1], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[2], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[3], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[4], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[5], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[6], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Failed); - } - { - auto& suite = suites[TestFixtureIndex]; - SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{38}); - SetTestRunCaseData( - suite.m_tests[0], AZStd::chrono::milliseconds{4}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[1], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[2], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - } - - return suites; - } - - AZStd::vector GetTestTargetBTestRunSuites() - { - AZStd::vector suites(TestRunSuitesFromTestEnumerationSuites(GetTestTargetBTestEnumerationSuites())); - - enum - { - TestCaseIndex = 0, - TestFixtureIndex, - PermutationATestFixtureWithParamsIndex, - TestFixtureWithParamsIndex - }; - - { - auto& suite = suites[TestCaseIndex]; - SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{202}); - SetTestRunCaseData( - suite.m_tests[0], AZStd::chrono::milliseconds{3}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[1], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[2], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - } - { - auto& suite = suites[TestFixtureIndex]; - SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{62}); - SetTestRunCaseData( - suite.m_tests[0], AZStd::chrono::milliseconds{5}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - } - { - auto& suite = suites[PermutationATestFixtureWithParamsIndex]; - SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{3203}); - SetTestRunCaseData( - suite.m_tests[0], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[1], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[2], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[3], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[4], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[5], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[6], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[7], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[8], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[9], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[10], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[11], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[12], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[13], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[14], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[15], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[16], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[17], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[18], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[19], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[20], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[21], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[22], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[23], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[24], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[25], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[26], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[27], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[28], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[29], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[30], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[31], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[32], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[33], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[34], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[35], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[36], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[37], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[38], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[39], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[40], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[41], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[42], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[43], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[44], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[45], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[46], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[47], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[48], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[49], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[50], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[51], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[52], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[53], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - } - { - auto& suite = suites[TestFixtureWithParamsIndex]; - SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{3360}); - SetTestRunCaseData( - suite.m_tests[0], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[1], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[2], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[3], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[4], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[5], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[6], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[7], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[8], AZStd::chrono::milliseconds{2}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[9], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[10], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[11], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[12], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[13], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[14], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[15], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[16], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[17], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[18], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[19], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[20], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[21], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[22], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[23], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[24], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[25], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[26], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[27], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[28], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[29], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[30], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[31], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[32], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[33], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[34], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[35], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[36], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[37], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[38], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[39], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[40], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[41], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[42], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[43], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[44], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[45], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[46], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[47], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[48], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[49], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[50], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[51], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[52], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[53], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - } - - return suites; - } - - AZStd::vector GetTestTargetCTestRunSuites() - { - AZStd::vector suites(TestRunSuitesFromTestEnumerationSuites(GetTestTargetCTestEnumerationSuites())); - - enum - { - TestCaseIndex = 0, - TestFixtureWithTypes0Index, - TestFixtureWithTypes1Index, - TestFixtureWithTypes2Index, - TestFixtureWithTypes3Index - }; - - { - auto& suite = suites[TestCaseIndex]; - SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{125}); - SetTestRunCaseData( - suite.m_tests[0], AZStd::chrono::milliseconds{4}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[1], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - } - { - auto& suite = suites[TestFixtureWithTypes0Index]; - SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{210}); - SetTestRunCaseData( - suite.m_tests[0], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[1], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[2], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[3], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - } - { - auto& suite = suites[TestFixtureWithTypes1Index]; - SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{208}); - SetTestRunCaseData( - suite.m_tests[0], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[1], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[2], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[3], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - } - { - auto& suite = suites[TestFixtureWithTypes2Index]; - SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{199}); - SetTestRunCaseData( - suite.m_tests[0], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[1], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[2], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[3], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - } - { - auto& suite = suites[TestFixtureWithTypes3Index]; - SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{49}); - SetTestRunCaseData( - suite.m_tests[0], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[1], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[2], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[3], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - } - - return suites; - } - - AZStd::vector GetTestTargetDTestRunSuites() - { - AZStd::vector suites(TestRunSuitesFromTestEnumerationSuites(GetTestTargetDTestEnumerationSuites())); - - enum - { - TestCaseIndex = 0, - TestFixture1Index, - DISABLEDTestFixture2Index, - TestFixtureWithTypes10Index, - TestFixtureWithTypes11Index, - TestFixtureWithTypes12Index, - TestFixtureWithTypes13Index, - DISABLEDTestFixtureWithTypes20Index, - DISABLEDTestFixtureWithTypes21Index, - DISABLEDTestFixtureWithTypes22Index, - DISABLEDTestFixtureWithTypes23Index, - PermutationATestFixtureWithParams1Index, - TestFixtureWithParams1Index, - PermutationADISABLED_TestFixtureWithParams2Index, - DISABLED_TestFixtureWithParams2Index, - }; - - { - auto& suite = suites[TestCaseIndex]; - SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{3}); - SetTestRunCaseData( - suite.m_tests[0], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData(suite.m_tests[1], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData( - suite.m_tests[2], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[3], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[4], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - } - { - auto& suite = suites[TestFixture1Index]; - SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{4}); - SetTestRunCaseData( - suite.m_tests[0], AZStd::chrono::milliseconds{2}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[1], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - } - { - auto& suite = suites[DISABLEDTestFixture2Index]; - SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{0}); - SetTestRunCaseData(suite.m_tests[0], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[1], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - } - { - auto& suite = suites[TestFixtureWithTypes10Index]; - SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{1}); - SetTestRunCaseData( - suite.m_tests[0], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData(suite.m_tests[1], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData( - suite.m_tests[2], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - } - { - auto& suite = suites[TestFixtureWithTypes11Index]; - SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{3}); - SetTestRunCaseData( - suite.m_tests[0], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData(suite.m_tests[1], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData( - suite.m_tests[2], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - } - { - auto& suite = suites[TestFixtureWithTypes12Index]; - SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{0}); - SetTestRunCaseData( - suite.m_tests[0], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData(suite.m_tests[1], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData( - suite.m_tests[2], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - } - { - auto& suite = suites[TestFixtureWithTypes13Index]; - SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{1}); - SetTestRunCaseData( - suite.m_tests[0], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData(suite.m_tests[1], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData( - suite.m_tests[2], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - } - { - auto& suite = suites[DISABLEDTestFixtureWithTypes20Index]; - SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{0}); - SetTestRunCaseData(suite.m_tests[0], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[1], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[2], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - } - { - auto& suite = suites[DISABLEDTestFixtureWithTypes21Index]; - SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{0}); - SetTestRunCaseData(suite.m_tests[0], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[1], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[2], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - } - { - auto& suite = suites[DISABLEDTestFixtureWithTypes22Index]; - SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{0}); - SetTestRunCaseData(suite.m_tests[0], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[1], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[2], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - } - { - auto& suite = suites[DISABLEDTestFixtureWithTypes23Index]; - SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{0}); - SetTestRunCaseData(suite.m_tests[0], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[1], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[2], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - } - { - auto& suite = suites[PermutationATestFixtureWithParams1Index]; - SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{173}); - SetTestRunCaseData( - suite.m_tests[0], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[1], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[2], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[3], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[4], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[5], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[6], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[7], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[8], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[9], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[10], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[11], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[12], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[13], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[14], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[15], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[16], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[17], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[18], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[19], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[20], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[21], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[22], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[23], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[24], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[25], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[26], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData(suite.m_tests[27], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[28], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[29], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[30], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[31], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[32], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[33], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[34], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[35], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[36], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[37], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[38], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[39], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[40], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[41], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[42], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[43], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[44], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[45], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[46], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[47], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[48], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[49], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[50], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[51], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[52], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[53], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - } - { - auto& suite = suites[TestFixtureWithParams1Index]; - SetTestRunSuiteData(suite, AZStd::chrono::milliseconds{102}); - SetTestRunCaseData( - suite.m_tests[0], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[1], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[2], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[3], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[4], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[5], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[6], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[7], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[8], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[9], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[10], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[11], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[12], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[13], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[14], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[15], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[16], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[17], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[18], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[19], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[20], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[21], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[22], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[23], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[24], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[25], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData( - suite.m_tests[26], AZStd::chrono::milliseconds{1}, TestImpact::TestRunStatus::Run, TestImpact::TestRunResult::Passed); - SetTestRunCaseData(suite.m_tests[27], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[28], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[29], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[30], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[31], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[32], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[33], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[34], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[35], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[36], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[37], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[38], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[39], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[40], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[41], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[42], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[43], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[44], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[45], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[46], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[47], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[48], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[49], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[50], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[51], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[52], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - SetTestRunCaseData(suite.m_tests[53], AZStd::chrono::milliseconds{0}, TestImpact::TestRunStatus::NotRun); - } - // The other two fixtures are all disabled (not run) - - return suites; - } - - AZStd::vector GetTestTargetASourceModuleCoverages() - { - AZStd::vector moduleCoverages; - - TestImpact::ModuleCoverage moduleCoverage; - moduleCoverage.m_path = "C:\\Lumberyard\\windows_vs2019\\bin\\debug\\TestImpact.TestTargetA.Tests.dll"; - - TestImpact::SourceCoverage sourceCoverage; - sourceCoverage.m_path = - "C:\\Lumberyard\\Code\\Tools\\TestImpactFramework\\Runtime\\Code\\Tests\\TestTargetA\\Code\\Tests\\TestImpactTestTargetA.cpp"; - - moduleCoverage.m_sources.push_back(AZStd::move(sourceCoverage)); - moduleCoverages.push_back(AZStd::move(moduleCoverage)); - return moduleCoverages; - } - - AZStd::vector GetTestTargetBSourceModuleCoverages() - { - AZStd::vector moduleCoverages; - - TestImpact::ModuleCoverage moduleCoverage; - moduleCoverage.m_path = "C:\\Lumberyard\\windows_vs2019\\bin\\debug\\TestImpact.TestTargetB.Tests.dll"; - - TestImpact::SourceCoverage sourceCoverage; - sourceCoverage.m_path = - "C:\\Lumberyard\\Code\\Tools\\TestImpactFramework\\Runtime\\Code\\Tests\\TestTargetB\\Code\\Tests\\TestImpactTestTargetB.cpp"; - - moduleCoverage.m_sources.push_back(AZStd::move(sourceCoverage)); - moduleCoverages.push_back(AZStd::move(moduleCoverage)); - return moduleCoverages; - } - - AZStd::vector GetTestTargetCSourceModuleCoverages() - { - AZStd::vector moduleCoverages; - - TestImpact::ModuleCoverage moduleCoverage; - moduleCoverage.m_path = "C:\\Lumberyard\\windows_vs2019\\bin\\debug\\TestImpact.TestTargetC.Tests.dll"; - - TestImpact::SourceCoverage sourceCoverage; - sourceCoverage.m_path = - "C:\\Lumberyard\\Code\\Tools\\TestImpactFramework\\Runtime\\Code\\Tests\\TestTargetC\\Code\\Tests\\TestImpactTestTargetC.cpp"; - - moduleCoverage.m_sources.push_back(AZStd::move(sourceCoverage)); - moduleCoverages.push_back(AZStd::move(moduleCoverage)); - return moduleCoverages; - } - - AZStd::vector GetTestTargetDSourceModuleCoverages() - { - AZStd::vector moduleCoverages; - - TestImpact::ModuleCoverage moduleCoverage; - moduleCoverage.m_path = "C:\\Lumberyard\\windows_vs2019\\bin\\debug\\TestImpact.TestTargetD.Tests.dll"; - - TestImpact::SourceCoverage sourceCoverage; - sourceCoverage.m_path = - "C:\\Lumberyard\\Code\\Tools\\TestImpactFramework\\Runtime\\Code\\Tests\\TestTargetD\\Code\\Tests\\TestImpactTestTargetD.cpp"; - - moduleCoverage.m_sources.push_back(AZStd::move(sourceCoverage)); - moduleCoverages.push_back(AZStd::move(moduleCoverage)); - return moduleCoverages; - } - - AZStd::vector GetTestTargetALineModuleCoverages() - { - AZStd::vector moduleCoverages; - - TestImpact::ModuleCoverage moduleCoverage; - moduleCoverage.m_path = "C:\\Lumberyard\\windows_vs2019\\bin\\debug\\TestImpact.TestTargetA.Tests.dll"; - - TestImpact::SourceCoverage sourceCoverage; - sourceCoverage.m_path = - "C:\\Lumberyard\\Code\\Tools\\TestImpactFramework\\Runtime\\Code\\Tests\\TestTargetA\\Code\\Tests\\TestImpactTestTargetA.cpp"; - sourceCoverage.m_coverage = AZStd::vector(); - - auto& lines = sourceCoverage.m_coverage; - lines.push_back(TestImpact::LineCoverage{22, 1}); - lines.push_back(TestImpact::LineCoverage{23, 1}); - lines.push_back(TestImpact::LineCoverage{24, 1}); - lines.push_back(TestImpact::LineCoverage{25, 1}); - lines.push_back(TestImpact::LineCoverage{27, 1}); - lines.push_back(TestImpact::LineCoverage{28, 1}); - lines.push_back(TestImpact::LineCoverage{29, 1}); - lines.push_back(TestImpact::LineCoverage{30, 1}); - lines.push_back(TestImpact::LineCoverage{32, 1}); - lines.push_back(TestImpact::LineCoverage{33, 1}); - lines.push_back(TestImpact::LineCoverage{34, 1}); - lines.push_back(TestImpact::LineCoverage{35, 1}); - lines.push_back(TestImpact::LineCoverage{37, 1}); - lines.push_back(TestImpact::LineCoverage{38, 1}); - lines.push_back(TestImpact::LineCoverage{39, 1}); - lines.push_back(TestImpact::LineCoverage{40, 1}); - lines.push_back(TestImpact::LineCoverage{42, 1}); - lines.push_back(TestImpact::LineCoverage{43, 1}); - lines.push_back(TestImpact::LineCoverage{44, 1}); - lines.push_back(TestImpact::LineCoverage{45, 1}); - lines.push_back(TestImpact::LineCoverage{47, 1}); - lines.push_back(TestImpact::LineCoverage{48, 1}); - lines.push_back(TestImpact::LineCoverage{49, 1}); - lines.push_back(TestImpact::LineCoverage{50, 1}); - lines.push_back(TestImpact::LineCoverage{52, 1}); - lines.push_back(TestImpact::LineCoverage{53, 1}); - lines.push_back(TestImpact::LineCoverage{54, 1}); - lines.push_back(TestImpact::LineCoverage{55, 1}); - lines.push_back(TestImpact::LineCoverage{57, 1}); - lines.push_back(TestImpact::LineCoverage{58, 1}); - lines.push_back(TestImpact::LineCoverage{59, 1}); - lines.push_back(TestImpact::LineCoverage{60, 1}); - lines.push_back(TestImpact::LineCoverage{62, 1}); - lines.push_back(TestImpact::LineCoverage{63, 1}); - lines.push_back(TestImpact::LineCoverage{64, 1}); - lines.push_back(TestImpact::LineCoverage{65, 1}); - lines.push_back(TestImpact::LineCoverage{67, 1}); - lines.push_back(TestImpact::LineCoverage{68, 1}); - lines.push_back(TestImpact::LineCoverage{69, 1}); - lines.push_back(TestImpact::LineCoverage{70, 1}); - lines.push_back(TestImpact::LineCoverage{73, 1}); - - moduleCoverage.m_sources.push_back(AZStd::move(sourceCoverage)); - moduleCoverages.push_back(AZStd::move(moduleCoverage)); - return moduleCoverages; - } - - AZStd::vector GetTestTargetBLineModuleCoverages() - { - AZStd::vector moduleCoverages; - - TestImpact::ModuleCoverage moduleCoverage; - moduleCoverage.m_path = "C:\\Lumberyard\\windows_vs2019\\bin\\debug\\TestImpact.TestTargetB.Tests.dll"; - - TestImpact::SourceCoverage sourceCoverage; - sourceCoverage.m_path = - "C:\\Lumberyard\\Code\\Tools\\TestImpactFramework\\Runtime\\Code\\Tests\\TestTargetB\\Code\\Tests\\TestImpactTestTargetB.cpp"; - sourceCoverage.m_coverage = AZStd::vector(); - - auto& lines = sourceCoverage.m_coverage; - lines.push_back(TestImpact::LineCoverage{29, 1}); - lines.push_back(TestImpact::LineCoverage{30, 1}); - lines.push_back(TestImpact::LineCoverage{31, 1}); - lines.push_back(TestImpact::LineCoverage{32, 1}); - lines.push_back(TestImpact::LineCoverage{34, 1}); - lines.push_back(TestImpact::LineCoverage{35, 1}); - lines.push_back(TestImpact::LineCoverage{36, 1}); - lines.push_back(TestImpact::LineCoverage{37, 1}); - lines.push_back(TestImpact::LineCoverage{39, 1}); - lines.push_back(TestImpact::LineCoverage{40, 1}); - lines.push_back(TestImpact::LineCoverage{41, 1}); - lines.push_back(TestImpact::LineCoverage{42, 1}); - lines.push_back(TestImpact::LineCoverage{44, 1}); - lines.push_back(TestImpact::LineCoverage{45, 1}); - lines.push_back(TestImpact::LineCoverage{46, 1}); - lines.push_back(TestImpact::LineCoverage{47, 1}); - lines.push_back(TestImpact::LineCoverage{49, 1}); - lines.push_back(TestImpact::LineCoverage{50, 1}); - lines.push_back(TestImpact::LineCoverage{51, 1}); - lines.push_back(TestImpact::LineCoverage{52, 1}); - lines.push_back(TestImpact::LineCoverage{54, 1}); - lines.push_back(TestImpact::LineCoverage{55, 1}); - lines.push_back(TestImpact::LineCoverage{56, 1}); - lines.push_back(TestImpact::LineCoverage{57, 1}); - lines.push_back(TestImpact::LineCoverage{59, 1}); - lines.push_back(TestImpact::LineCoverage{66, 1}); - lines.push_back(TestImpact::LineCoverage{68, 1}); - lines.push_back(TestImpact::LineCoverage{75, 1}); - lines.push_back(TestImpact::LineCoverage{78, 1}); - - moduleCoverage.m_sources.push_back(AZStd::move(sourceCoverage)); - moduleCoverages.push_back(AZStd::move(moduleCoverage)); - return moduleCoverages; - } - - AZStd::vector GetTestTargetCLineModuleCoverages() - { - AZStd::vector moduleCoverages; - - TestImpact::ModuleCoverage moduleCoverage; - moduleCoverage.m_path = "C:\\Lumberyard\\windows_vs2019\\bin\\debug\\TestImpact.TestTargetC.Tests.dll"; - - TestImpact::SourceCoverage sourceCoverage; - sourceCoverage.m_path = - "C:\\Lumberyard\\Code\\Tools\\TestImpactFramework\\Runtime\\Code\\Tests\\TestTargetC\\Code\\Tests\\TestImpactTestTargetC.cpp"; - sourceCoverage.m_coverage = AZStd::vector(); - - auto& lines = sourceCoverage.m_coverage; - lines.push_back(TestImpact::LineCoverage{32, 1}); - lines.push_back(TestImpact::LineCoverage{33, 1}); - lines.push_back(TestImpact::LineCoverage{34, 1}); - lines.push_back(TestImpact::LineCoverage{35, 1}); - lines.push_back(TestImpact::LineCoverage{37, 1}); - lines.push_back(TestImpact::LineCoverage{38, 1}); - lines.push_back(TestImpact::LineCoverage{39, 1}); - lines.push_back(TestImpact::LineCoverage{40, 1}); - lines.push_back(TestImpact::LineCoverage{42, 1}); - lines.push_back(TestImpact::LineCoverage{43, 1}); - lines.push_back(TestImpact::LineCoverage{44, 1}); - lines.push_back(TestImpact::LineCoverage{45, 1}); - lines.push_back(TestImpact::LineCoverage{47, 1}); - lines.push_back(TestImpact::LineCoverage{48, 1}); - lines.push_back(TestImpact::LineCoverage{49, 1}); - lines.push_back(TestImpact::LineCoverage{50, 1}); - lines.push_back(TestImpact::LineCoverage{52, 1}); - lines.push_back(TestImpact::LineCoverage{53, 1}); - lines.push_back(TestImpact::LineCoverage{54, 1}); - lines.push_back(TestImpact::LineCoverage{55, 1}); - lines.push_back(TestImpact::LineCoverage{57, 1}); - lines.push_back(TestImpact::LineCoverage{58, 1}); - lines.push_back(TestImpact::LineCoverage{59, 1}); - lines.push_back(TestImpact::LineCoverage{60, 1}); - lines.push_back(TestImpact::LineCoverage{63, 1}); - - moduleCoverage.m_sources.push_back(AZStd::move(sourceCoverage)); - moduleCoverages.push_back(AZStd::move(moduleCoverage)); - return moduleCoverages; - } - - AZStd::vector GetTestTargetDLineModuleCoverages() - { - AZStd::vector moduleCoverages; - - TestImpact::ModuleCoverage moduleCoverage; - moduleCoverage.m_path = "C:\\Lumberyard\\windows_vs2019\\bin\\debug\\TestImpact.TestTargetD.Tests.dll"; - - TestImpact::SourceCoverage sourceCoverage; - sourceCoverage.m_path = - "C:\\Lumberyard\\Code\\Tools\\TestImpactFramework\\Runtime\\Code\\Tests\\TestTargetD\\Code\\Tests\\TestImpactTestTargetD.cpp"; - sourceCoverage.m_coverage = AZStd::vector(); - - auto& lines = sourceCoverage.m_coverage; - lines.push_back(TestImpact::LineCoverage{56, 1}); - lines.push_back(TestImpact::LineCoverage{57, 1}); - lines.push_back(TestImpact::LineCoverage{58, 1}); - lines.push_back(TestImpact::LineCoverage{59, 1}); - lines.push_back(TestImpact::LineCoverage{61, 1}); - lines.push_back(TestImpact::LineCoverage{62, 0}); - lines.push_back(TestImpact::LineCoverage{63, 0}); - lines.push_back(TestImpact::LineCoverage{64, 0}); - lines.push_back(TestImpact::LineCoverage{66, 1}); - lines.push_back(TestImpact::LineCoverage{67, 1}); - lines.push_back(TestImpact::LineCoverage{68, 1}); - lines.push_back(TestImpact::LineCoverage{69, 1}); - lines.push_back(TestImpact::LineCoverage{71, 1}); - lines.push_back(TestImpact::LineCoverage{72, 1}); - lines.push_back(TestImpact::LineCoverage{73, 1}); - lines.push_back(TestImpact::LineCoverage{74, 1}); - lines.push_back(TestImpact::LineCoverage{76, 1}); - lines.push_back(TestImpact::LineCoverage{77, 1}); - lines.push_back(TestImpact::LineCoverage{78, 1}); - lines.push_back(TestImpact::LineCoverage{79, 1}); - lines.push_back(TestImpact::LineCoverage{81, 1}); - lines.push_back(TestImpact::LineCoverage{82, 1}); - lines.push_back(TestImpact::LineCoverage{83, 1}); - lines.push_back(TestImpact::LineCoverage{84, 1}); - lines.push_back(TestImpact::LineCoverage{86, 1}); - lines.push_back(TestImpact::LineCoverage{87, 1}); - lines.push_back(TestImpact::LineCoverage{88, 1}); - lines.push_back(TestImpact::LineCoverage{89, 1}); - lines.push_back(TestImpact::LineCoverage{91, 1}); - lines.push_back(TestImpact::LineCoverage{92, 0}); - lines.push_back(TestImpact::LineCoverage{93, 0}); - lines.push_back(TestImpact::LineCoverage{94, 0}); - lines.push_back(TestImpact::LineCoverage{96, 1}); - lines.push_back(TestImpact::LineCoverage{97, 0}); - lines.push_back(TestImpact::LineCoverage{98, 0}); - lines.push_back(TestImpact::LineCoverage{99, 0}); - lines.push_back(TestImpact::LineCoverage{101, 1}); - lines.push_back(TestImpact::LineCoverage{102, 1}); - lines.push_back(TestImpact::LineCoverage{103, 1}); - lines.push_back(TestImpact::LineCoverage{104, 1}); - lines.push_back(TestImpact::LineCoverage{106, 1}); - lines.push_back(TestImpact::LineCoverage{107, 0}); - lines.push_back(TestImpact::LineCoverage{108, 0}); - lines.push_back(TestImpact::LineCoverage{109, 0}); - lines.push_back(TestImpact::LineCoverage{111, 1}); - lines.push_back(TestImpact::LineCoverage{112, 0}); - lines.push_back(TestImpact::LineCoverage{113, 0}); - lines.push_back(TestImpact::LineCoverage{114, 0}); - lines.push_back(TestImpact::LineCoverage{116, 1}); - lines.push_back(TestImpact::LineCoverage{117, 0}); - lines.push_back(TestImpact::LineCoverage{118, 0}); - lines.push_back(TestImpact::LineCoverage{119, 0}); - lines.push_back(TestImpact::LineCoverage{121, 1}); - lines.push_back(TestImpact::LineCoverage{128, 1}); - lines.push_back(TestImpact::LineCoverage{130, 1}); - lines.push_back(TestImpact::LineCoverage{137, 1}); - lines.push_back(TestImpact::LineCoverage{139, 1}); - lines.push_back(TestImpact::LineCoverage{146, 1}); - lines.push_back(TestImpact::LineCoverage{148, 1}); - lines.push_back(TestImpact::LineCoverage{155, 1}); - lines.push_back(TestImpact::LineCoverage{157, 1}); - lines.push_back(TestImpact::LineCoverage{158, 1}); - lines.push_back(TestImpact::LineCoverage{159, 1}); - lines.push_back(TestImpact::LineCoverage{160, 1}); - lines.push_back(TestImpact::LineCoverage{162, 1}); - lines.push_back(TestImpact::LineCoverage{163, 0}); - lines.push_back(TestImpact::LineCoverage{164, 0}); - lines.push_back(TestImpact::LineCoverage{165, 0}); - lines.push_back(TestImpact::LineCoverage{167, 1}); - lines.push_back(TestImpact::LineCoverage{168, 1}); - lines.push_back(TestImpact::LineCoverage{169, 1}); - lines.push_back(TestImpact::LineCoverage{170, 1}); - lines.push_back(TestImpact::LineCoverage{172, 1}); - lines.push_back(TestImpact::LineCoverage{173, 0}); - lines.push_back(TestImpact::LineCoverage{174, 0}); - lines.push_back(TestImpact::LineCoverage{175, 0}); - lines.push_back(TestImpact::LineCoverage{177, 1}); - lines.push_back(TestImpact::LineCoverage{178, 0}); - lines.push_back(TestImpact::LineCoverage{179, 0}); - lines.push_back(TestImpact::LineCoverage{180, 0}); - lines.push_back(TestImpact::LineCoverage{182, 1}); - lines.push_back(TestImpact::LineCoverage{183, 0}); - lines.push_back(TestImpact::LineCoverage{184, 0}); - lines.push_back(TestImpact::LineCoverage{185, 0}); - lines.push_back(TestImpact::LineCoverage{188, 1}); - - moduleCoverage.m_sources.push_back(AZStd::move(sourceCoverage)); - moduleCoverages.push_back(AZStd::move(moduleCoverage)); - return moduleCoverages; - } - - template - bool CheckTestCasesAreEqual(const TestCase& lhs, const TestCase& rhs) - { - if (lhs.m_enabled != rhs.m_enabled) - { - AZ_Error("CheckTestCasesAreEqual", false, "lhs.m_enabled: %u, rhs.m_enabled: %u", lhs.m_enabled, rhs.m_enabled); - return false; - } - - if (lhs.m_name != rhs.m_name) - { - AZ_Error("CheckTestCasesAreEqual", false, "lhs.m_name: %s, rhs.m_name: %s", lhs.m_name.c_str(), rhs.m_name.c_str()); - return false; - } - - return true; - } - - template - bool CheckTestSuitesAreEqual(const TestSuite& lhs, const TestSuite& rhs) - { - if (lhs.m_enabled != rhs.m_enabled) - { - AZ_Error("CheckTestSuitesAreEqual", false, "lhs.m_enabled: %u, rhs.m_enabled: %u", lhs.m_enabled, rhs.m_enabled); - return false; - } - - if (lhs.m_name != rhs.m_name) - { - AZ_Error("CheckTestSuitesAreEqual", false, "lhs.m_name: %s, rhs.m_name: %s", lhs.m_name.c_str(), rhs.m_name.c_str()); - return false; - } - - return AZStd::equal(lhs.m_tests.begin(), lhs.m_tests.end(), rhs.m_tests.begin(), [](const auto& left, const auto& right) { - return left == right; - }); - } - - template - bool CheckTestSuiteVectorsAreEqual(const AZStd::vector& lhs, const AZStd::vector& rhs) - { - return AZStd::equal(lhs.begin(), lhs.end(), rhs.begin(), [](const TestSuite& left, const TestSuite& right) { - return left == right; - }); - } - - template - bool CheckTestContainersAreEqual(const TestContainer& lhs, const TestContainer& rhs) - { - if (lhs.GetTestSuites().size() != rhs.GetTestSuites().size()) - { - return false; - } - - return AZStd::equal( - lhs.GetTestSuites().begin(), lhs.GetTestSuites().end(), rhs.GetTestSuites().begin(), [](const auto& left, const auto& right) { - return left == right; - }); - } - - bool operator==(const TestImpact::TestEnumerationCase& lhs, const TestImpact::TestEnumerationCase& rhs) - { - return CheckTestCasesAreEqual(lhs, rhs); - } - - bool operator==(const TestImpact::TestRunCase& lhs, const TestImpact::TestRunCase& rhs) - { - if (!CheckTestCasesAreEqual(lhs, rhs)) - { - return false; - } - - if (lhs.m_status != rhs.m_status) - { - AZ_Error( - "TestRunCase ==", false, "lhs.m_status: %u, rhs.m_status: %u", static_cast(lhs.m_status), - static_cast(rhs.m_status)); - return false; - } - - if (lhs.m_duration != rhs.m_duration) - { - AZ_Error("TestRunCase ==", false, "lhs.m_duration: %u, rhs.m_duration: %u", lhs.m_duration.count(), rhs.m_duration.count()); - return false; - } - - if (lhs.m_result != rhs.m_result) - { - if (!lhs.m_result.has_value() && rhs.m_result.has_value()) - { - AZ_Error("TestRunCase ==", false, "lhs.m_result: null, rhs.m_result: %u", static_cast(rhs.m_result.value())); - } - else if (lhs.m_result.has_value() && !rhs.m_result.has_value()) - { - AZ_Error("TestRunCase ==", false, "lhs.m_result: %u, rhs.m_result: null", static_cast(lhs.m_result.value())); - } - else - { - AZ_Error( - "TestRunCase ==", false, "lhs.m_result: %u, rhs.m_result: %u", static_cast(lhs.m_result.value()), - static_cast(rhs.m_result.value())); - } - - return false; - } - - return true; - } - - bool operator==(const TestImpact::TestEnumerationSuite& lhs, const TestImpact::TestEnumerationSuite& rhs) - { - return CheckTestSuitesAreEqual(lhs, rhs); - } - - bool operator==(const TestImpact::TestRunSuite& lhs, const TestImpact::TestRunSuite& rhs) - { - if (!CheckTestSuitesAreEqual(lhs, rhs)) - { - return false; - } - - if (lhs.m_duration != rhs.m_duration) - { - AZ_Error( - "TestEnumerationSuite ==", false, "lhs.m_duration: %u, rhs.m_duration: %u", lhs.m_duration.count(), rhs.m_duration.count()); - return false; - } - - return true; - } - - bool operator==(const AZStd::vector& lhs, const AZStd::vector& rhs) - { - return CheckTestSuiteVectorsAreEqual(lhs, rhs); - } - - bool operator==(const AZStd::vector& lhs, const AZStd::vector& rhs) - { - return CheckTestSuiteVectorsAreEqual(lhs, rhs); - } - - bool operator==(const TestImpact::TestEnumeration& lhs, const TestImpact::TestEnumeration& rhs) - { - return CheckTestContainersAreEqual(lhs, rhs); - } - - bool operator==(const TestImpact::TestRun& lhs, const TestImpact::TestRun& rhs) - { - if (lhs.GetDuration() != rhs.GetDuration()) - { - AZ_Error("TestRun ==", false, "lhs.GetDuration(): %u, rhs.GetDuration(): %u", lhs.GetDuration(), rhs.GetDuration()); - return false; - } - - if (lhs.GetNumDisabledTests() != rhs.GetNumDisabledTests()) - { - AZ_Error( - "TestRun ==", false, "lhs.GetNumDisabledTests(): %u, rhs.GetNumDisabledTests(): %u", lhs.GetNumDisabledTests(), - rhs.GetNumDisabledTests()); - return false; - } - - if (lhs.GetNumEnabledTests() != rhs.GetNumEnabledTests()) - { - AZ_Error( - "TestRun ==", false, "lhs.GetNumEnabledTests(): %u, rhs.GetNumEnabledTests(): %u", lhs.GetNumEnabledTests(), - rhs.GetNumEnabledTests()); - return false; - } - - if (lhs.GetNumFailures() != rhs.GetNumFailures()) - { - AZ_Error("TestRun ==", false, "lhs.GetNumFailures(): %u, rhs.GetNumFailures(): %u", lhs.GetNumFailures(), rhs.GetNumFailures()); - return false; - } - - if (lhs.GetNumNotRuns() != rhs.GetNumNotRuns()) - { - AZ_Error("TestRun ==", false, "lhs.GetNumNotRuns(): %u, rhs.GetNumNotRuns(): %u", lhs.GetNumNotRuns(), rhs.GetNumNotRuns()); - return false; - } - - if (lhs.GetNumPasses() != rhs.GetNumPasses()) - { - AZ_Error("TestRun ==", false, "lhs.GetNumPasses(): %u, rhs.GetNumPasses(): %u", lhs.GetNumPasses(), rhs.GetNumPasses()); - return false; - } - - if (lhs.GetNumRuns() != rhs.GetNumRuns()) - { - AZ_Error("TestRun ==", false, "lhs.GetNumRuns(): %u, rhs.GetNumRuns(): %u", lhs.GetNumRuns(), rhs.GetNumRuns()); - return false; - } - - return CheckTestContainersAreEqual(lhs, rhs); - } - - bool CheckTestRunCaseVectorsAreEqual( - const AZStd::vector& lhs, const AZStd::vector& rhs) - { - return AZStd::equal( - lhs.begin(), lhs.end(), rhs.begin(), [](const TestImpact::TestRunCase& leftCase, const TestImpact::TestRunCase& rightCase) { - if (!CheckTestCasesAreEqual(leftCase, rightCase)) - { - return false; - } - - if (leftCase.m_status != rightCase.m_status) - { - AZ_Error( - "CheckTestRunsAreEqualIgnoreDurations", false, "leftCase.m_status: %u, rightCase.m_status: %u", - static_cast(leftCase.m_status), static_cast(rightCase.m_status)); - return false; - } - - if (leftCase.m_result != rightCase.m_result) - { - if (!leftCase.m_result.has_value() && rightCase.m_result.has_value()) - { - AZ_Error( - "CheckTestRunsAreEqualIgnoreDurations", false, "leftCase.m_result: null, rightCase.m_result: %u", - static_cast(rightCase.m_result.value())); - } - else if (leftCase.m_result.has_value() && !rightCase.m_result.has_value()) - { - AZ_Error( - "CheckTestRunsAreEqualIgnoreDurations", false, "leftCase.m_result: %u, rightCase.m_result: null", - static_cast(leftCase.m_result.value())); - } - else - { - AZ_Error( - "CheckTestRunsAreEqualIgnoreDurations", false, "leftCase.m_result: %u, rightCase.m_result: %u", - static_cast(leftCase.m_result.value()), static_cast(rightCase.m_result.value())); - } - - return false; - } - - return true; - }); - } - - bool CheckTestRunsAreEqualIgnoreDurations(const TestImpact::TestRun& lhs, const TestImpact::TestRun& rhs) - { - return AZStd::equal( - lhs.GetTestSuites().begin(), lhs.GetTestSuites().end(), rhs.GetTestSuites().begin(), - [](const TestImpact::TestRunSuite& leftSuite, const TestImpact::TestRunSuite& rightSuite) { - if (leftSuite.m_enabled != rightSuite.m_enabled) - { - AZ_Error( - "CheckTestRunsAreEqualIgnoreDurations", false, "leftSuite.m_enabled: %u, rightSuite.m_enabled: %u", - leftSuite.m_enabled, rightSuite.m_enabled); - return false; - } - - if (leftSuite.m_name != rightSuite.m_name) - { - AZ_Error( - "CheckTestRunsAreEqualIgnoreDurations", false, "leftSuite.m_name: %s, rightSuite.m_name: %s", - leftSuite.m_name.c_str(), rightSuite.m_name.c_str()); - return false; - } - - return CheckTestRunCaseVectorsAreEqual(leftSuite.m_tests, rightSuite.m_tests); - }); - } - - bool operator==(const TestImpact::BuildMetaData& lhs, const TestImpact::BuildMetaData& rhs) - { - if (lhs.m_name != rhs.m_name) - { - return false; - } - else if (lhs.m_outputName != rhs.m_outputName) - { - return false; - } - else if (lhs.m_path != rhs.m_path) - { - return false; - } - - return true; - } - - bool operator==(const TestImpact::TargetSources& lhs, const TestImpact::TargetSources& rhs) - { - if (lhs.m_staticSources != rhs.m_staticSources) - { - return false; - } - else if (lhs.m_autogenSources.size() != rhs.m_autogenSources.size()) - { - return false; - } - else - { - for (size_t i = 0; i < lhs.m_autogenSources.size(); i++) - { - if (lhs.m_autogenSources[i].m_input != rhs.m_autogenSources[i].m_input) - { - return false; - } - else if (lhs.m_autogenSources[i].m_outputs.size() != rhs.m_autogenSources[i].m_outputs.size()) - { - return false; - } - - for (size_t j = 0; j < lhs.m_autogenSources[i].m_outputs.size(); j++) - { - if (lhs.m_autogenSources[i].m_outputs[j] != rhs.m_autogenSources[i].m_outputs[j]) - { - return false; - } - } - } - } - - return true; - } - - bool operator==(const TestImpact::BuildTargetDescriptor& lhs, const TestImpact::BuildTargetDescriptor& rhs) - { - return lhs.m_buildMetaData == rhs.m_buildMetaData && lhs.m_sources == rhs.m_sources; - } - - bool operator==(const TestImpact::TestTargetMeta& lhs, const TestImpact::TestTargetMeta& rhs) - { - if (lhs.m_suite != rhs.m_suite) - { - return false; - } - else if (lhs.m_launchMethod != rhs.m_launchMethod) - { - return false; - } - - return true; - } - - bool operator==(const TestImpact::ProductionTargetDescriptor& lhs, const TestImpact::ProductionTargetDescriptor& rhs) - { - return lhs.m_buildMetaData == rhs.m_buildMetaData; - } - - bool operator==(const TestImpact::TestTargetDescriptor& lhs, const TestImpact::TestTargetDescriptor& rhs) - { - return lhs.m_buildMetaData == rhs.m_buildMetaData && lhs.m_sources == rhs.m_sources && lhs.m_testMetaData == rhs.m_testMetaData; - } - - bool operator==(const TestImpact::LineCoverage& lhs, const TestImpact::LineCoverage& rhs) - { - if (lhs.m_hitCount != rhs.m_hitCount) - { - AZ_Error("LineCoverage ==", false, "lhs.m_hitCount: %u, rhs.m_hitCount: %u", lhs.m_hitCount, rhs.m_hitCount); - return false; - } - - if (lhs.m_lineNumber != rhs.m_lineNumber) - { - AZ_Error("LineCoverage ==", false, "lhs.m_lineNumber: %u, rhs.m_lineNumber: %u", lhs.m_lineNumber, rhs.m_lineNumber); - return false; - } - - return true; - } - - bool operator==(const TestImpact::SourceCoverage& lhs, const TestImpact::SourceCoverage& rhs) - { - if (lhs.m_path != rhs.m_path) - { - AZ_Error("LineCoverage ==", false, "lhs.m_path: %s, rhs.m_path: %s", lhs.m_path.c_str(), rhs.m_path.c_str()); - return false; - } - - if (lhs.m_coverage.empty() != rhs.m_coverage.empty()) - { - AZ_Error( - "LineCoverage ==", false, "lhs.m_coverage.empty(): %u, rhs.m_coverage.empty(): %u", lhs.m_coverage.empty(), - rhs.m_coverage.empty()); - return false; - } - - if (!lhs.m_coverage.empty()) - { - return AZStd::equal( - lhs.m_coverage.begin(), lhs.m_coverage.end(), rhs.m_coverage.begin(), - [](const TestImpact::LineCoverage& left, const TestImpact::LineCoverage& right) { - return left == right; - }); - } - - return true; - } - - bool operator==(const TestImpact::ModuleCoverage& lhs, const TestImpact::ModuleCoverage& rhs) - { - if (lhs.m_path != rhs.m_path) - { - AZ_Error("ModuleCoverage ==", false, "lhs.m_path: %s, rhs.m_path: %s", lhs.m_path.c_str(), rhs.m_path.c_str()); - return false; - } - - return AZStd::equal( - lhs.m_sources.begin(), lhs.m_sources.end(), rhs.m_sources.begin(), - [](const TestImpact::SourceCoverage& left, const TestImpact::SourceCoverage& right) { - return left == right; - }); - } - - bool operator==(const AZStd::vector& lhs, const AZStd::vector& rhs) - { - if (lhs.size() != rhs.size()) - { - AZ_Error("ModuleCoverage ==", false, "lhs.size(): %u, rhs.size(): %u", lhs.size(), rhs.size()); - return false; - } - - return AZStd::equal( - lhs.begin(), lhs.end(), rhs.begin(), [](const TestImpact::ModuleCoverage& left, const TestImpact::ModuleCoverage& right) { - return left == right; - }); - } - - bool operator!=(const AZStd::vector& lhs, const AZStd::vector& rhs) - { - return !(lhs == rhs); - } - - bool operator==(const TestImpact::TestCoverage& lhs, const TestImpact::TestCoverage& rhs) - { - if (lhs.GetNumModulesCovered() != rhs.GetNumModulesCovered()) - { - return false; - } - - if (lhs.GetNumSourcesCovered() != rhs.GetNumSourcesCovered()) - { - return false; - } - - if (lhs.GetModuleCoverages() != rhs.GetModuleCoverages()) - { - return false; - } - - if (lhs.GetSourcesCovered().size() != rhs.GetSourcesCovered().size()) - { - return false; - } - - return true; - } -} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactTestUtils.h b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactTestUtils.h deleted file mode 100644 index cc8d4190ec..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactTestUtils.h +++ /dev/null @@ -1,220 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -namespace UnitTest -{ - // Common parameters for process related tests - inline constexpr const char* ValidProcessPath = LY_TEST_IMPACT_TEST_PROCESS_BIN; - inline constexpr const char* InvalidProcessPath = "!!!@@@---???"; - inline constexpr const AZStd::chrono::milliseconds LongSleep = AZStd::chrono::minutes(60); - inline constexpr const size_t LargeTextSize = 0xFFFF - 1; // 65,535 chars less the null terminator - inline constexpr const AZStd::chrono::milliseconds ShortSleep = AZStd::chrono::milliseconds(500); - inline constexpr const AZStd::chrono::milliseconds NoSleep = AZStd::chrono::milliseconds(0); - - // Writes the specified text string to the specified file - void WriteTextToFile(const AZStd::string& text, const AZ::IO::Path& path); - - // Construct the arguments for launcing the test process - AZStd::string ConstructTestProcessArgs(TestImpact::ProcessId pid, AZStd::chrono::milliseconds sleepTime); - - // Construct the arguments for launcing the test process with large text dump - AZStd::string ConstructTestProcessArgsLargeText(TestImpact::ProcessId pid, AZStd::chrono::milliseconds sleepTime); - - // Known standard output string of the test process - AZStd::string KnownTestProcessOutputString(TestImpact::ProcessId pid); - - // Known standard error string of the test process - AZStd::string KnownTestProcessErrorString(TestImpact::ProcessId pid); - - // Generate a gtest typed test fixture name string based on the specified name and type - AZStd::string GenerateTypedFixtureName(const AZStd::string& name, size_t typeNum); - - // Generate a gtest parameterized test fixture name string based on the specified name and permutation number - AZStd::string GenerateParameterizedFixtureName( - const AZStd::string& name, - const AZStd::optional& prefix = AZStd::nullopt); - - // Generate a gtest parameterized test name string based on the specified name and permutation number - AZStd::string GenerateParameterizedTestName(const AZStd::string& name, size_t testNum); - - // Generate a JSON string of array elements from the specified vector - AZStd::string StringVectorToJSONElements(const AZStd::vector strings); - - // Generate a build target descriptor string in JSON format from the specified build target description - AZStd::string GenerateBuildTargetDescriptorString( - const AZStd::string& name, - const AZStd::string& outputName, - const AZStd::string& path, - const AZStd::vector& staticSources, - const AZStd::vector& autogenInputs, - const AZStd::vector& autogenOutputs); - - // Generate a build target descriptor from the specified build target description - // Note: no check for correctness of arguments is peformed - TestImpact::BuildTargetDescriptor GenerateBuildTargetDescriptor( - const AZStd::string& name, - const AZStd::string& outputName, - const AZStd::string& path, - const AZStd::vector& staticSources, - const TestImpact::AutogenSources& autogenSources); - - // Procedurally generate a parameterized test suite based on the supplied parameters - TestImpact::TestEnumerationSuite GenerateParamterizedSuite( - const AZStd::pair& fixture, - const AZStd::optional& permutation, - const AZStd::vector> tests, - size_t permutationCount); - - // Procedurally generate a typed test suite based on the supplied parameters - void GenerateTypedSuite( - const AZStd::pair& fixture, - const AZStd::vector> tests, - size_t permutationCount, - AZStd::vector& parentSuiteList); - - // Helper functions for calculating test suite meta-data - size_t CalculateNumPassedTests(const AZStd::vector& suites); - size_t CalculateNumFailedTests(const AZStd::vector& suites); - size_t CalculateNumRunTests(const AZStd::vector& suites); - size_t CalculateNumNotRunTests(const AZStd::vector& suites); - - template - size_t CalculateNumTestSuites(const AZStd::vector& suites) - { - return suites.size(); - } - - template - size_t CalculateNumTests(const AZStd::vector& suites) - { - size_t numTests = 0; - for (const auto& suite : suites) - { - numTests += suite.m_tests.size(); - } - - return numTests; - } - - template - size_t CalculateNumEnabledTests(const AZStd::vector& suites) - { - size_t numEnabledTests = 0; - for (const auto& suite : suites) - { - if (!suite.m_enabled) - { - continue; - } - - for (const auto& test : suite.m_tests) - { - if (test.m_enabled) - { - numEnabledTests++; - } - } - } - - return numEnabledTests; - } - - template - size_t CalculateNumDisabledTests(const AZStd::vector& suites) - { - size_t numDisabledTests = 0; - for (const auto& suite : suites) - { - if (!suite.m_enabled) - { - numDisabledTests += suite.m_tests.size(); - continue; - } - - for (const auto& test : suite.m_tests) - { - if (!test.m_enabled) - { - numDisabledTests++; - } - } - } - - return numDisabledTests; - } - - // Test enumeration suite representation of the test targets used for testing - AZStd::vector GetTestTargetATestEnumerationSuites(); - AZStd::vector GetTestTargetBTestEnumerationSuites(); - AZStd::vector GetTestTargetCTestEnumerationSuites(); - AZStd::vector GetTestTargetDTestEnumerationSuites(); - - // Test run suite representation of the test targets used for testing - AZStd::vector GetTestTargetATestRunSuites(); - AZStd::vector GetTestTargetBTestRunSuites(); - AZStd::vector GetTestTargetCTestRunSuites(); - AZStd::vector GetTestTargetDTestRunSuites(); - - // Line coverage representation of the test targets used for testing - AZStd::vector GetTestTargetALineModuleCoverages(); - AZStd::vector GetTestTargetBLineModuleCoverages(); - AZStd::vector GetTestTargetCLineModuleCoverages(); - AZStd::vector GetTestTargetDLineModuleCoverages(); - - // Source coverage representation of the test targets used for testing - AZStd::vector GetTestTargetASourceModuleCoverages(); - AZStd::vector GetTestTargetBSourceModuleCoverages(); - AZStd::vector GetTestTargetCSourceModuleCoverages(); - AZStd::vector GetTestTargetDSourceModuleCoverages(); - - // Helper comparisons for test validation (could potentially be moved to production source in the future) - bool operator==(const TestImpact::TestEnumerationCase& lhs, const TestImpact::TestEnumerationCase& rhs); - bool operator==(const TestImpact::TestEnumerationSuite& lhs, const TestImpact::TestEnumerationSuite& rhs); - bool operator==(const AZStd::vector& lhs, const AZStd::vector& rhs); - bool operator==(const TestImpact::TestEnumeration& lhs, const TestImpact::TestEnumeration& rhs); - - bool operator==(const TestImpact::TestRunCase& lhs, const TestImpact::TestRunCase& rhs); - bool operator==(const TestImpact::TestRunSuite& lhs, const TestImpact::TestRunSuite& rhs); - bool operator==(const AZStd::vector& lhs, const AZStd::vector& rhs); - bool operator==(const TestImpact::TestRun& lhs, const TestImpact::TestRun& rhs); - bool CheckTestRunsAreEqualIgnoreDurations(const TestImpact::TestRun& lhs, const TestImpact::TestRun& rhs); - - bool operator==(const TestImpact::BuildMetaData& lhs, const TestImpact::BuildMetaData& rhs); - bool operator==(const TestImpact::TargetSources& lhs, const TestImpact::TargetSources& rhs); - bool operator==(const TestImpact::BuildTargetDescriptor& lhs, const TestImpact::BuildTargetDescriptor& rhs); - bool operator==(const TestImpact::TestTargetMeta& lhs, const TestImpact::TestTargetMeta& rhs); - bool operator==(const TestImpact::ProductionTargetDescriptor& lhs, const TestImpact::ProductionTargetDescriptor& rhs); - bool operator==(const TestImpact::TestTargetDescriptor& lhs, const TestImpact::TestTargetDescriptor& rhs); - - bool operator==(const TestImpact::LineCoverage& lhs, const TestImpact::LineCoverage& rhs); - bool operator==(const TestImpact::SourceCoverage& lhs, const TestImpact::SourceCoverage& rhs); - bool operator==(const TestImpact::ModuleCoverage& lhs, const TestImpact::ModuleCoverage& rhs); - bool operator==(const AZStd::vector& lhs, const AZStd::vector& rhs); - bool operator==(const TestImpact::TestCoverage& lhs, const TestImpact::TestCoverage& rhs); -} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/CMakeLists.txt b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/CMakeLists.txt deleted file mode 100644 index 8298bb7123..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -# -# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or -# its licensors. -# -# For complete copyright and license terms please see the LICENSE at the root of this -# distribution (the "License"). All use of this software is governed by the License, -# or, if provided, by the license below or the license accompanying this file. Do not -# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# - -add_subdirectory(Code) \ No newline at end of file diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/CMakeLists.txt b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/CMakeLists.txt deleted file mode 100644 index 4b8e9d2809..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/CMakeLists.txt +++ /dev/null @@ -1,24 +0,0 @@ -# -# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or -# its licensors. -# -# For complete copyright and license terms please see the LICENSE at the root of this -# distribution (the "License"). All use of this software is governed by the License, -# or, if provided, by the license below or the license accompanying this file. Do not -# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# - -ly_add_target( - NAME TestImpact.TestProcess.Console EXECUTABLE - NAMESPACE AZ - FILES_CMAKE - testimpactframework_testprocess_files.cmake - INCLUDE_DIRECTORIES - PRIVATE - Source - BUILD_DEPENDENCIES - PRIVATE - AZ::AzCore - AZ::AzFramework -) \ No newline at end of file diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/Source/TestImpactTestProcess.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/Source/TestImpactTestProcess.cpp deleted file mode 100644 index cdfad1404f..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/Source/TestImpactTestProcess.cpp +++ /dev/null @@ -1,93 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include "TestImpactTestProcess.h" -#include "TestImpactTestProcessLargeText.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -namespace TestImpact -{ - TestProcess::TestProcess(int argc, char* argv[]) - { - StartupEnvironment(); - ParseArgs(argc, argv); - } - - TestProcess::~TestProcess() - { - TeardownEnvironment(); - } - - void TestProcess::StartupEnvironment() - { - AZ::AllocatorInstance::Create(); - } - - void TestProcess::TeardownEnvironment() - { - AZ::AllocatorInstance::Destroy(); - } - - void TestProcess::ParseArgs(int argc, char* argv[]) - { - const AZStd::string idArg = "id"; - const AZStd::string sleepArg = "sleep"; - const AZStd::string largeArg = "large"; - - AZ::CommandLine commandLine; - commandLine.Parse(argc, argv); - - m_id = AZStd::stoi(commandLine.GetSwitchValue(idArg, 0)); - m_sleep = AZStd::stoi(commandLine.GetSwitchValue(sleepArg, 0)); - - m_dumpLargeText = false; - for (auto i = 0; i < commandLine.GetNumMiscValues(); i++) - { - const auto& miscValue = commandLine.GetMiscValue(i); - if (miscValue == largeArg) - { - m_dumpLargeText = true; - break; - } - } - } - - int TestProcess::MainFunc() - { - if (m_dumpLargeText) - { - // Dump the large text blob to stdout and stderr - std::cout << LongText; - std::cerr << LongText; - } - else - { - // Dump the short known output string with id appended to stdout and stderr - std::cout << "TestProcessMainStdOut" << m_id; - std::cerr << "TestProcessMainStdErr" << m_id; - } - - AZStd::this_thread::sleep_for(AZStd::chrono::milliseconds(m_sleep)); - - return m_id; - } -} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/Source/TestImpactTestProcess.h b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/Source/TestImpactTestProcess.h deleted file mode 100644 index cfe18aa3df..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/Source/TestImpactTestProcess.h +++ /dev/null @@ -1,38 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#pragma once - -#include -#include - -namespace TestImpact -{ - class TestProcess - { - public: - TestProcess(int argc, char* argv[]); - ~TestProcess(); - - int MainFunc(); - - private: - void StartupEnvironment(); - void TeardownEnvironment(); - void ParseArgs(int argc, char* argv[]); - - int m_id; - int m_sleep; - bool m_dumpLargeText = false; - }; - -} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/Source/TestImpactTestProcessLargeText.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/Source/TestImpactTestProcessLargeText.cpp deleted file mode 100644 index a4b899dd65..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/Source/TestImpactTestProcessLargeText.cpp +++ /dev/null @@ -1,531 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include "TestImpactTestProcessLargeText.h" - -namespace TestImpact -{ - // Long text blob measuring 65,535 in length (inc. null terminator) - const char* const LongText = - "m65NIFtFmZV9CiZj1go6fPV6xo2NcBeEHoJ1JktsoT9mHUclADmJhjTMT5qaKd0FD8GiredOA6wSUiEGFkc62QwNxuEFUQIuOeQAH8NE4MuhvjSWnw0MCXnf5TppuCfU" - "5Xx7fimsd8wYoxDiPSatXSaXWcCdn7QNttzmGbwuldr3Fmbntiz5aliAcVrlB82RCVQ4v4aT41wCivgPJ0CNbfQyiPDBzZd3cGwLmvt6Nu9pP4kbTVHRsdif0tvs8j2V" - "93fotwV7Q50Xl3jniX2AUnwO6OgZdl97lPHn9wZbO7V5vRGQe1n9aweXK4u3qlBfplvsZAVOoC1RuY1JmCz2V2iOfucmIbpxL5jR6qobgFZI0Z2ioi34ssqqyH9xFgwG" - "muF69vqJjYUzJGgneGQUqWTH1yr4AIFyqfBFvBe04UAsYfo7jPQWxCpCRXNQfTuN2609HPafOQ4IJh8IdL77CtRd8B1y1Ah8oqWzQZ54HZFOWQ1FZ1yCIxcnaBNAhauS" - "RJnYB4gcTmdtrZlFysea28CjyjroTKTzeyzqfNhfteIs3urS3IyspinKEFIAWQ8z1NYS0ks6o47r777gzQcQZ51T6fAdHO346PBHyeZBdXZxCXK615SKctxlG1p2TvhQ" - "2TMOtb9xTr24JrlijnR9WFGzG1CuoLeTN7cgm6rk86KVbFnG74XwKNvIZ5xbHaiD0qaSrZzoGtBxrDdldw8BjcXLZS9FFOToOxdUASqX8TkWEP9oA4CDImFRr3AlUtsS" - "g6m4nzc7Lp4wdM8Jd9pPl2rmF7HgZ5yjO3dew9G83ewcAyN3XhLjVH68ExbdN4RTKbHsczWEPKDWQAoEzGLn0YILmrKh1f2GaobAcnqbyPkN7pMax7CqYfUJao1DvedD" - "gfqcRzqEb9F90Xn4Il3SaW0Qmsxy6OGTJDSSOJr7hBsthALqpKdFpMbFYeAvh2PYhS6G8KvzcNeKLcZfsWdONwVWnenuyQDRxQjOzp87pHMEtWGDJ2UxTsIKGvvxjrqC" - "bBalQhDZk0mIwUJdb5hf5qkJLD5tPdozFDe53FkFYOKDlsWdRpSvTnYeBk1WMxGD31yywHFUlOfUf2a17A76dBYu6Jb2KMfHcNfxJhIJOXWfT8nnIAJwvir4SHqlzPbr" - "IEh35MHMM5XpbreQZjD7sB3ovEmZfXx4dCbztrIXs17zpskuW36cVQDxR4Mz1ZEbdUtdZg7r8AxUNHZJhe0v96QmodqTp8qh7WP8EUmmaTr8F3PMyyMXgJfuAczpmMfr" - "rXvcFE48Cr0ZRcMDuWpPthkVPFxXAcaOAn6eqaJdHRQlioODCbjjw1kEcV2VcONDXtS7pV3YguletmZOUxYIcqVqWYAyztnam06QtmuCTTjg0l4a6R3obuugJZAt1gBS" - "IN6gaVPKQwJQLvLM7ChKDB0jmYE3nIRliYjIZ02blecFbvqCxJMakyc818tULWFQNV3msn43rNLCO3VAUKaQrXGn3zWjuqeur9WFoUnANTZhndyiuspIx0mxrCuRXqpO" - "h4zNhN0d7Rh0v22PrkzgLvB23UcgxbX6xcw7UQ5g3Cq77sq1ko191ZpKuxrBXhDLye9GXv7BKh4Wcfc8qHAP3qjKmHZInIVlpholPrxgrxfPW9MZSGDFPxJfhkyb6VSG" - "SNcyMPMhdtoCtUUAfDXoqLCbQmN7On7eJ0NJZdZA9BWtzjbFIxYYmnxl39QLFmpEkgZaq0AtAsYh8L7sbm8v0yGB0wTCWLFYqAx8NBsooS2YlgT9SX4XFgG29HibgsX3" - "VkCoJ3m4nzukHgnpzI6WIoiNjj4Z8sHR5NLWYWfN3ZOCj9dw23dhcTnRVBbUxkpyOMybV2S7ytwd0vb3ZJ6hlOh6dYZNdw2M3iN0NaKYs55a5zxNAMyKBHfemBFLN18E" - "zVEQgKz9qJv8EEBmgZ2p8iKoeeQrSRLz8blVW1OPilcpKGvIh9kW4lgssMZ7qRju2oJ0DyJ3joEnFOvtGwljLgoy7AzHLpN0dF5lrD94IL3Kn7b5g2h8yAhwRAV3Pmtf" - "GHLnb0gKzLbyJVEoBhvH41B5MOHtLfiAtcyJ2zMV5R4ldJV1vJm0t79xK0X99YykIlUuguwxnzWwd7tQO6F00gaBGvpVcz1zEwJEenBbjncVzXqAQwnO9zKtzUOcC5nR" - "7CfvLz0l4dMfAAMHZ7wGlCnWdBVKO6L6OyUSLGvrpz6jhdlhS45QJKkzTB50ZgOmK7owJwPrTnvDvH3Dxq8SDbmynSfvESmx4utROmawwY9R3fySBFKsLSB3lYlmg8Q8" - "xjabKm0PCDMjeslYXgohxklk7XMCOzkutm2o72sOGgqvFNXFQz9BJAnid2f9KpGc29oIUobf1RyIcTXpnPMedrM8brEd4v1aGdcb9rJv0eE3v9xt3eznh85sf2AN7PwS" - "tROP09CXfnq0qj6SOr2Wop7uCypqKx3gRSTggIZJL2Q7bFvwhYmIIpxuGMuKOBf2xxzo4pqME0HfIuXNkElZpj729pyxas04iOheSqurWhM2EKKmaBrkvFVK0mEa2T9I" - "9Rjh1ZNYvJU59S1lrD0DP4jkOdQ9bHYA9jgpXmFFN8jgWRRu14eKdrTYsZ6w3HNAiN0nqpmQUl30d10UAeMJHkv4xD889zQRMmmPCgLLJfvhZYfiwlqzBjbh7OQhydxm" - "dHPaOcFZpEQwbvR3KhHSVcbhKewJGHLSbHNWsRRsJrfOBnwSm0TfU3FD3xEqkPEl8mmsgQvakUAxI49RV9B3ueJJ9K8mvsZBBIRhnSZoqwipEskXqIc2VIW1CnW9qi0y" - "OObnoJ7qBpu4aueF3lGn0rgPXiXpcw0KaFxoq3kUTSEJXsj7mwHVYxKJfoPdaTuWDfJZ5Hk4Xgy16pUiD11SxMl7GZcXuAGl319LqX3nIAiahaoHTAhYimesTaf8x5h6" - "kgPxITKZXkqdD21Buykh4jKvcPLVivIr7yM8NOKvDf6zgQmm6VHihI4XFLgvEUe6EoYoE0RJS7Luc8dVFdNsVcGICayaaznjL3sFV6J7d4TVd35PEGmXtvLKKSvQ6Be9" - "gb5KZEk8pcHvYqpqzyrX4ZFve3cpv0IyLAlsWSawYV5L7eluYJNsC0vBImelV5oCecW7J6WmYNxTBpQt4M1MWcFzw920KaE9mXf4gkNCQb14FrbT8sNzt5JVFgz9Fjy7" - "aoRbG9xGCqyZkUa7RwmiTgFuk5CRSCgMu2lUNydfnAHbapUpkFyYRAShQr5Hjnu9H0OPhWUSTM6tz5Zzj8OzAwTTTb3Zp5DBywqQXZZPVcUutswpPVHgRRI6t65rd9nB" - "7XGXuJNfSQHotH5V3smIAk5gJezFb31PsfLkJ898DYGkmCwWtOvJqCglscSDSLv0EUoQGIRpJu9W9PQR9pU2IKpJhYks388vSwtAnYrUNOxEJAmUfnKlUZsHbJtFezAM" - "YtcS1I1kBK4eaP61zPWBWQ0uTR2UMXE6oVNuHXKLlQZFVADVTNBNRFvRO8s7LTmE4iMEeuRiOHbOkzexztJuxi4NDnyG964iJ580zHsehJckVyRh9w8AMaRs7HE7BwKx" - "gGwKY9DmZ8V2CyvqDLjxtdt9yIaPxg4ZtsAHTzYY4gXCQGoZRp9AsWaoNSJKt2JRmZE8CwJ6kPUyN9DCqPH33c8mu1zU2JHLBrsjNpgAaOlcr95W9YRYi6eBHoE9sT4i" - "a1zNr1veiFCh281PLhH52Bh3L4eXj9Z9vitWE9nJ6U58aDGj7zP3ITxRBni1UVlvVOYkInOQysSB6Qc3kjKq2dpYjCUQBZRaJO4fGyK8GwsXkBJAVJ5cFFCY2DARgsB4" - "9vksxVhIC8iNCUHeuzaIqqoDVD4EABtN5sgbjqsTmA76iojm99w0sfOGPdDmqKfxybKi8Lyg7xOQUu0HIIF4HYOcnsG6xuHfW1CNqvkcZR9tnOeyax19PgZaAX0Ulo2L" - "0HNi98GXoKbmuD9l1eFWPRQVVFfnDDH84KuPZcciXWWAIMDhGwtfUd72GcgeUju0NqI6DsEf9WPDDAw8VS5U6rHeXmhp9hR7vbOnyAstoarOqPlx7L3q9P2jGJu7Lhzx" - "fmUY2iGty6xoG7MM4jnrywA15ErsQXjQbe1JOwVx7YycaIH3pLjd72qLp0Oy1lINFHQnbqDqdnWWYHHNFnnYrndj9fXueBzctmlxoUM2duxpZJQ6BB2CXQ7y9guWWx9N" - "PrrYO7QpMAzi5MOLLIqkbkQbta7SlxHh7eouqkBHOvWaAJR6iI52rEJkMUudh5VevD5Sh4QMXUR3OsfPTLyENMhT50zwLzBzSHKQ3EaT3MuyERp33j3KXvME05lZpUfy" - "g7UkcSCAm0duYh7RxWBYykoXmLxBtlGLs7XtqjGvzvuQohfJGMnUnYCxNTpxjIlmhORhc8lUBPZ8Flq51MhnYp6sYztV3hvyDKY1wSgCvauNNNxjdguRzMOyBl8PcNcV" - "a02kDeE2wqVZvhv48vyzMITPZvMKOzhA2U1giReSYereQOehKtyeXXqhTGooUxUfOfE5dKCbfLyBW5wuptRtY0LZ1VdxWQluqErsSNJvX6JVXic4n3vwc7ockTlU8iFk" - "UCG2tfegjf8EpSlydI4vTjphHnIkiTTG4g26ILlGEDnoxJL3U8jZhWgdf5aGNaJ2wiOLy7FZlJkD8StKWRFjyGyNM3LEDOFAmkyzWwwWOV0hBjAKwkne0Wdbn1vV88b3" - "w2u4Cfi1ZK3n7NeZwI96qpzWV4oA0aj7yl4mkjFtouALOD09aKqspPNBMkY80pMwNrwkp9EIZNDvAslGAdCs4LmmsAsXMNQqfDo9ubMeDGqgjV4FhhrA26UljfKbCKCe" - "05jYo8VhtjfRUAutYjCE3H4VSVGEs8OIkQyNdvPcI8o1kqDoG0OdadaId1v4z9Qos5KCvPh6wx1mh8OW6x8MOCUwWKy7kDbIk81zjfMPADNTHe4u5xRsir5VSiaBGb3G" - "TADf5NpPnxAOpUAJlbiaLRA4TRFAuBmITqBeL8htAJIggtZsuxUZkTIs5acgpRBYgdV3gAN8mmkBb0m78x2buNHIqdbV8ggNCuJICD14AK8PKSuxNvvnzqhEEkUZEr8h" - "dF4bDnZ5kL3djQDEl0C1eDGOOL0LqyspoADQFmxkZV2fJAJQnYU0ewqvRqZTCBNtRtg1vneRzDUDxlSkMZP6PddDOkJ03VxNDnttG250htR0M8elz9sHCzMxBh6T4k16" - "lwOGctboSL0GhW7ZFjeE7sfXgCfdH80r0K28pw2sm8l1cdY14ZWuqpRP25xI6DYlWAx78T6yQ9IdVIrF5eVYbAkVs7pUCgNgOWdJmhet21WJgc25sRRcMpy9Gxq7rugS" - "EhhVAtfM4HiA3ofo3WwDuWcsUCjqVcQjh209w0O4vRDhHediAccuO62DwZRG4Gh4xAG0yFEOsrrk9znlSV4XMc3HK2ARPEguqVCVgpjJgfr5jCZnwfwmTz4L6DAAVb9f" - "ctBNSpYhr22klJgG96CHGnnI5ybpv1BGTYlJwRS1DUDeoWh537pcRmD2l5LgyFu2aWuwPxEypG9hpPmf5Gj3uwdyL0ABlKTUleyBeEVCu5B1RgnzKe6BTnInP3XtQKJu" - "l49rhKfDo8JOtcyyQ4LXEVaWnOrx4hs6Atu1L0mWEZ2M1UGwfYrv4Cd1ORt3WU30ZWwr5EsVBj4i9WtVogL3DL66CtYNM4HTUdU9p8VUjEMiKoxt7XkligvrvgcDxqe4" - "gGRAsMd1tDSUXCxYOv9aMO64DEYOAEkODj6neQg2zZwANgRY4wAagV4vzL4YomGQmSA5xE5mE0InfMJJdSVLPdamy2wNshqwe5ow7aj5c0itCMyL0EZek5NmMBqBAn5F" - "9pULD0wljEo8TgcvPkVSmIWrA4iTMraEHeHS7KgzDYC0FtM8KHnE5j7SvGxvykldyoXfjHn8HMe5XdHqNxNREZarZcZ6nkP9CJYCaWpuYoCDza4RBp80HTTDSWZXjKpY" - "H2PTPsztToaX8GjgAfjjICrUBiuKdOUUmznqa4SUeKvKoKbLK1GIRnOOk38aS69xB8t89iuhkymMZrczCWKdVTdEaDMFKI0ILTy3wyfkbkiXjEVQ62byURuHDr3tF5K5" - "aqLl11VLvZzNsTbJ6DhWaKD0HzSD0C2r4LtRzSNtsUkWIqETP1tkqCXAzD1KeP90KAVsj9TLOa8AagEau3tF041igctfjTa0vrrXm47zncAKn6MpklJjoBrtE8c8r3p6" - "rOPEtDdM2dZxaWwukrtLao3F5aezSu1462z2PgnYMmv1FlMNe3eETZ7thoyChiQrxyxrkzTKjwj6ra1NNYok5dZE6bgUZTxhC6Y44eAOIqWLn0tTHb8mryBSwiBkrPAx" - "RzVkYak80HfVgxFyL9Rb1nsq39QZLxrWuniLssgEFrutQA89B3nvfCV13NacROJdL309VUPMbDAZXxc193UYtIEZngCj7H8DGdezs2OQ8j8eIvSizweHcbFSPanqtlKN" - "BJ4rQYsdh3Zfq5OYMdNytxp2CNoUHg36oI4zxSMwyYKPC4KicGSQnwLMEwmyXKu57CazUHdc7PZkK3Lea7kNWA5hUdnBmrxd0xIOVjhFi4e3wSgsHnGphH4UF2rDPvC2" - "GRvuPSbEwEMyXJL159oeR0O5z7DG683VtOuLQPCyLm1tgqAGUaVg2iZULOfOLZuHlC8ByzwKQnQGvTNL1g6lHxh1ygaDdO3weStxgpMe9O2mV9mfj8CsOZSbx12NtDvR" - "46yqInqtHCT5nIbzTYQloULXEuXWL5AQ67g7rUsInEqzO2iwh1LwxqLMvoB42DPBEvpb7fmVOFddxzaEE0kd4p6i7t9cUi0JR1e2ZMrMNK7zFyLmx9gvAzarDSjGOY3n" - "EvKdBn63lLSANMwoInAt4Aept7tWydGScGtaj1yusS5iwO0THRP177dKlAfCjkFGx6mR942AnCxo6oMmJ91gATIKWm2fgBaW5BUKy1fQ5MqQDnQzEzRYEjMi9k6vQEur" - "VkVM5059sI5UJxlVr9yaKGHt3KTeu1F7PGNWQCwPIMqpVMlx7CbQ3bhQWSJRNISWUWFrWCOY1n8jbtPdwojNG23ugMCGwZYedB29EHuBfHIDX2RPdKt2dJAHg4aXjx6i" - "1TRgq5nm4K0Avaq86WlMfK3JkK7VM7Tt4XzfLLx6Uutkv7XU7GGG0bbBbUrtIsug5NFvhvrwcRR9KL5kl9z9nS1iJu7tJdljEcGggHiqjRBe6hoSPxvAa7AF6kDV5iOV" - "Ca9LkWZjg9C2P6Ri4KcmiK3OwZhFzn6P6jFAF8Y0raLbZSZBLJwUUzT5fYLYzpiMMsmgT2HXl3tOzEEZKhdxRjBPR5br51LanQypKBNgFuR7FlRuSEJaNiEPZmXK3GeU" - "Pk6PSpR0AdFaEKbowx4wxsJvzKT96pSzR7LaEEqOa9phydtTh85HoIuPnKkVcFIHGpbynCGL4TUAbqv14rfQfmNrVTqx9hqpJApofNu9tyhBYJUrQ7PrOCPG2ur34lOw" - "sUg5MkHEyzVfq0WGqlC7TTGoO3bA3TWsO5SHGOhHsq5wuo4JXjPrOz5vEI2tQQlG2Pn275cZiCAJ2XZmxG7k3LRUJOQ6NpFg1Rv509bnfB1MBet2KCA05UYQMG5MMaWN" - "WlauS0eN6pka7p1l2D39aDiiIeOjwJvWfc6Tf4FM7muhWABXwDymT41VX5140sKrFZ6h84jdBDKBoTaqANexdXrs9D3Zr2BlgiP0hDX9bqmwvvxP4JACCBFFcA2ZSM0E" - "K9J9P15AjyZg6iWei3FjJ9t3B81dN7NwG2x8bGgw8RZt5m8A9D8F3D6LeVgGVLGuAa55DFnhCwtkCviBSTJn0btERDELpFhQU5H3A19P26dEFpIZIqM5kj9xQwqMUVbC" - "riaOOL3BVc9mGt6jSJn1TI7YyibFoT06lSybiMGB3CO6T2tov9puMTiVoWEQJDGNlUUhqLyokzdbo3d8lBe4aWUFq19Cs0HZt50o9xdbIQcyEtHBMXNZ3gwECJ6QbYC5" - "igpTeTLc2aXk1003QJsZ0qgr6OkeOqfdBFr37zB9EEYw29uP7en85bKbqk3Xdnvo8HbIeCIDyb4A5u3RH9BJbre6CCsWbqP9P6bCkJrtT7JuDcpb2q7sWKuP6vR3fILk" - "eR30Ptmb6XX6C6LL69TRXPDE5jkuWiO4Sb3SJi1zSqztVvDgyfrar1P0LXnE6vkJ4Anqws8raHKz9EYuo9ZLZoYqxkqieyzRiTkqyrz7e3pAk7VLpHItA1HjV5NNBWyB" - "2Z7xnMziRxWtgv6APf4huLiLE7WcTTkkdyv5vm7uPjtZHq5ziUx30IjXW4fexcBcp4kmFPwYDBTyI7yOLHPrXxJIFgcdalZTEbwxSVLdlJpucFkXfG6uIVWSTWR27O5F" - "ClBzl09FLL5w0EON184alTAtonqJou8HJjxPQC9zaqqgshTvdgLw9IVZLFGTQwfs3bgSiFH2foWClTcdns2SnY1BZf2EdPUETwJKaFlAcz9DO1shcPN7Jo1gjm0xz2um" - "vznuMIOu48BSYrkvev0C5P3fTNoaYsAmmA2c0oWfgeJNN0SgEJ1FuNWc00yLWOblaOEakAkIysiRemkijteLN2p7OoW5hzOqwADExvpRsy4gdHbCGfHGQPKf0wvhqXvm" - "OqdkPLqp2gynWmiUuKcWPulZlyhgQXAePzS0oVkYTrbv6yy5mLSvgro9L6Vf7Z0OsEa5fWOM0mC718tmE4vbtBXPgXiiTDQVqqOaUlAMndzA18rzlNLRlZtv7XW4bN4k" - "eOcuSPbNe9tTWo9XABv5tqe94TRBPyakEpX9hQ7Fc3yy0wQLXrFSzzZDPlKTajNnOOb1nBksaoD6N6x8vXBTh5rhsG2DL9jKnfrEveVJt6H5o2HrjYaNFPQuddKJb5pW" - "ns98Xb8puf5QSVs4VN8wyzW5veX5PfTUOHjDRrBAzMtmb0C0a0qCuZq8dRkqigJMRv3LCmjtoYYYs1uIuZMWyaqj1BD8g2oOxNc411hkQpwwxIqzm7cevxMFKm1SBkqb" - "JoZHNX7P91ufuEeStHxRXlwUXep029JUyu6DkxuGCxbQIAEvNTdI8zpThTbOxI0jlVKOpCNEBo9LetsePnoA0Hj7qha4owWWfElKHzdxPB5RBxxkjeVAXOQLL5waKW6V" - "nufDcpSYJkpySbB6cR1n0qeX5lIqMBTxyY2rhCq3lK8mauRd3KxTOoDRuWh06FeR6kUindQ3GmaSDvKDF3aWunZtjcpqFmg7thwPhdsw5QyUlS4JFRbpBlc7YU2PJ8zJ" - "UYY2aecQmncf9ixMiYMNQbFX9k9SPM1T0DdzF8DP7tmgKLDsTYqhOutxBRIYCxAS27jBZPsoxBYAw8oxO1NXeCCJ4SETjJw1sLGz9HcBPzONy5slxvbYpdNZumHg3bp6" - "FRXdNDFYKiHkjnWIHP23HEAc3m3gqO4ZR4t3uMORHPFWkMOPbBwoKf6RMlLJsGCEVi0f3hZ9bAMaLVdGgvNRKMvpHNELMw197TbFZiznwIfwwt8tX07AtcDMQmqatYss" - "WKReaIWsECI1KHLfAehJtSqQYy1XPnhbbE7aSWlrkHl7WIbxt9jY4huOuxxhabKaZ1mbIaeyOOefMXSwzbKY49MrblqAJG2mS5Kuj5njMpSjJXhrNndZwYHIEFaquWqk" - "PEo9mbmcDwQI432dz9hY4wmeL3GxJO2318EzCPTITS0mZ8gkNg2DzFmU7IRSSW1hb9SxjdiBZXEuwlMWLzawO3MhiWQQiQfeWp3eISuZZ84IUuC49kKrmDiWkGRxLjnW" - "6KxvADT097jBN8blD56XtitDKkCgnDVtnmQhiNPOReweIesKxwTYzCnXH7jNkZKQtqXjunfjORabiXUAe6iiWLet7Qls4frgiSSjS8oH27zifNCmgfkvTEjxQXXTTFeI" - "TdrjTU5HYmTQJ348OFFYNNuQkZhdhGKaloaMkfhKiB9zWHY6s2hSO9nAfwBNmPD80WhNQPP9WIwaW0hMTFCAGoZNMRcHrlSgdn9pW10yoErcm2yrSx3NKu8paYUwmtPs" - "1qtEla6PfuAJzyJ0VsLWoegOH3Y53UUhggjqIkfqoqIokAsw798RLLlsuMWUGl0UgGKsViEXmBTjCXGZEQNX6BdDhr32ysNyybnPvWOoHF9pNCYssG6dtdMzyxydLYQj" - "l7vnIwwhi92QKHr4qluQCmFqOHvWEc2Wdo5zVeZVjmSycRaFR08b6SZZS0Q09WyGhXO6jy6Zw45nUCY0KwmtB7vIcp0mq9mXSrvnMZt7rKivuHh8H5jOnDTXHrZa3hxW" - "QbTyE1SBTKavdQWknVhUFxvas6tOZk4H7W7JIHT7DYUoRYHQkDPhD2CbOH3ReIL2QvESfjbUBVej22j79E36JoaPGjukIXPpbiA2jsefOicv4T35jEj0bdSbQ3yEv9vb" - "z0YOZPutnxwLfffZmLnnzgU9ydVXnn5ih27Vzj7Lv9fB3spJcf2j02GbNc0rp9gOpu5PhgW5fbCe6IUyfCL3oNnjSm7BSMuumMVGFFJjIOX2idgXevLQTGbkeMwv0KHl" - "tEcMaVj6HrbKYUKvmAIhQ2lDGBF3l9ox1k2lFCRKzs9VvMZohSTLei3ueO57hBOnVCxmMKMLVEOJRTgv7ZX3yQ8CvlZQT7nTJ18O5k4HingPWGjENFctUQ2VfDQK8Out" - "ZKOoLEzCE5HzlyUEJGIjtEhvbKq8EWtBAx4p8v4O4ufAGcIlTzoOveKb7UyXhjqpp4o9BBMxMLh0mH7mNrNh7cmLPDpd1QaR2PdOF5BGs5YtCdHlw8Sb47B9Q0vWAsBz" - "P39MHbITqKLClcUKgJ8a7AEaKnXKVXJGXul6tjwuiuEqfQtpnmY1I0nj75OhV1H2w7CtoXqEMDjrk9yT3mxFPwwVLgCVBNvdAaPdKSTa9U2zw2sNatYgMHN9ors4Vb9y" - "H1elBvlBbl7HoXyIbV64P19xipFrCtZYV7otEtSFtKDkMygolFbc07bnnJrelGMxOpISP77dRfVMEFqLG3hYr8vW6AojA7p2uZ0sVMEwO45ItjLo68e9nl7jhK9DDou7" - "Aj0v1VV5fxZXB0I09r1fGCM65wR75rE6bGC3vUfclYIspmziq04GApz7XCnUyBv8llsK7nHzIglGUynfnPlHYO6tliqLHEE682nPTH8wbUggvPRcrfve09ALkHsSDUIU" - "5HgRPXwQS34C9xFUTNkoOsgzAW9nsnJz3kwadwRqFSHJR8F7IkRQvGsg4G0FKPUiIs01mJ82Lkm53AJ2a8JBYICsWPEFN7e50BqMioWt9b6OQ0vwvPhZ0NmjBDKSwf3O" - "8QxFuPdsQ8nALXz3MiM5UcEykdOElKHtah3DUwJKjX6I3Ie1rSB8qKPEqjrid2AAwJo89TlwrNjfgvLnjz0hjTUhcB71YhM3BJbl0hW0JWg5wLgNQuFp21gzIl96wEVR" - "yD93LuDNmqsEELurUUllJ2rF929e6gwA8aidNYAFmej1OWw2ULGgCXt5Ib6BjUVeexWxQqMsgOb3I7YKsmX7CNgBq5odN7FEbW4MY4VDpgx26bcWYFoV0FY58al7Y2Um" - "abSKv1jW4zEuCOS7NDaceJtPMXUqi4h9pFw3jdJdKP8JbWhHGCJwFotSc3xe57KPlpzZ08pafdZefJq1t2eMK3Z9tRvLnWNSmcuicTMJvdRbgDSONRRvpwxwyJpwuGxk" - "gqSr6hY5QV9xFO0u3JcY2zuYvaJ8wcB4CxqtNBVVghf5HvvPKOYS188mVw9FmrlWuE5dHCquIVj90gp5vbAYJgUCUrRxNzVaL7sTS6jSA1Li83Q1I1NsaJyZa1SKgyen" - "zjddCeUK8Tn3xHj2HVlK85C0s6drwfyxIrbnCbQc5KGEDBM4W9H3Pe4E7zB58UZCOYDq59JD5horMpFT1Ny4CIZz1heh3n5mb4jH2f5AdXnKLqnAPBCBWB2GW90xYEPO" - "ZvUzpGHELmD6Maf6OOlC86wjj9CjrQqI4ER3m0Fb4StARxGXkV0yvRQvttaCZkrMOXogiRvtlQjimLNfFfcsNBEK8UHwqcH5tB5oQ7ix4dNa3sdCrtSSZIdcfKhefcjF" - "xO0YLbtWKnWttWOUDkZx7RTPMYtdZXDF2jJ4mR71pP1vYcHhSGVjBymb7pN4ZqbiAhi8AqCBCch7pGV87bDnLEkxAXdZRs8ItHIT6ztTM1HEI7pzzKPYzhFjVGzQ41a5" - "xnpWXGGuj0p6cVSV23s8fbNIfv7Gh2M6lS7uOjSlltlD3Gicv1HHRV2MEYMwHbJMFiFSPykXge8BIjhE4cCawxau4rKpJFp84kHu7HPlE6G6fUadIXJmyaDyIZ5TTAPP" - "YiShh1HOghRKpNZLEOkmJpfRUwbV4NnCJalLFQKSR3eTRSBqw932wLS1CUcj4bXmHbKDtNOEIkdUp9lT0lOyVP9IUOOFH709dDTlf3Dd6IZNJpXzi3GScIxsFaZVq7NC" - "6cdcKwipRUsD1aghJqSpAJZFHFIU8nPsPJlu9CpWCUCwKIwV0eSWonw2tSGvJToFqhyHmBeBTXsbymRCXJEcA8WVqjkmiR2nHmh6AnGBhdiTZfBxu5lGotYDfR5b46zx" - "fVGo7Ys1LPj1g5U81hl27NqbgutFQ3maGBu5xZkpeka3JCmdbpaPRIO076emwiJYi03Bb6Ncogv3Eg8Cf9xrNmIdvFRVi3iPws6HynQaq48lDYPTRcVB2l29UiA0nJPv" - "bomNxXa2RTMYDs9PzoI9CLRbxvz51XRRrAP9XzASOu5Y7SqzRFYmoCyS6drTrxMts1lywiY2Mzl3022xsQOcHz1Z6g7urt1UrSCtvQYJzIzgtWuHkVN6m9c7qfor7bbG" - "xctZRuHSrQaZUqeTmFduVLQlJuOZkO6VVwyDRUz48pzPCAfQKtDKUj9uuDSNpBlRnreaNsrIL8U28FDk4RO9TYrn2JgWLbo1bYewmyJd3hPCJg6HlLYyv7YH29SoAVyl" - "haTjaIfnjijq8cOHS7Tm6DANE5iuP2MjomrIwI9XjwDtLViGzWEoM7JC7hspvkTqaGnaaDKVoHnLLfAcEeF6dEeJtzSkGmOmt9LrN7Kg5sWBXRfbyUKBsnPIkCwlrbeK" - "fTHdvlzFgBrgdrkadA6GB6h0HMqWcWvzyZoV3aSJqSQXMg3hAYBwPP1iGxygyAY33kKRQJcpW4738TOBPuEd1GXn6WaIlB9DDBKJomssrefT8TuOeTKUEy3HlWPkyixQ" - "1tvX3AGy13TmbTva6Bg48QncMhRQlpht7xuGJ8TTRSeyvcTWFB68tvA6Is15sVdvnvyiwTy75Q2XHM4K06BKGH4PzOm8WRM26Jp2FMcsyZka7134eZVDjQ7Vb0MG9W8M" - "qioMWg5BM3h3Gz5zYKU2iqLZ9qTsLxESdwetbCSb3kn9pIL39qdEcQ5DPkp49nVHmS1NlPquBDBlD3ZCdDvPHVQ57VKMGZ8JdMU1xOyurJBq53FUpxt0UJStd4aeJwRm" - "i3WD7OYA0TYzarDn9webrHrg5XLJL1Da49wQVFuDcoZK1RL5E2Cqd1xA6Bw3jUZ6kd3hEaPTff6J7yBdkMwSnV7hLLthPCexiOZ0PQuyITyzs7S9qaWXTu8IAm6Z9IsU" - "hoNFxetpiEQkVNkLOhDxmTDcks3Ot1vXYaIVJs7mmEaqHYdDDjvccMi7RJfyh8hRADyEWlMkqoY8io3NcZt4p6FtLv9btEwVkJ7cb6wjrqki8V2jas6vpjXwE26PWxh3" - "yBqfBZMG5YItqZBonduR9cM7xyCVQyocuHvTjOFLThhTgvBnNKgjv3twZL9mxlWIG6AEip0xLt0feP4GZDS7c0ojbOYomc01mZkq5NY9nC8SlUPuk4llPL7ToaA6VSGb" - "rgTx3JUx1LQj3Yq9oSJ1A4FqKxVN5MxuPQ4D7CvClVz09Mw8tjnvnCAv5fJdxGEvPT9Ma4vnivT2wrdA7Wl2HtH3KbakBXd4kfwfHgqdQo30sAr8jxZcRr8AavLgx43g" - "PKe307AulqTgKyRy3iiCrbTUUJKkq2BbptM4lzDi3GljCA9W2t1Qs4W4PWWptuYpMy4r2r3ikAlqLgKooKWwhZ5uhgzGQjb1YYpXlEYHMNDLBgXTDGGoqOO0z4Xlcg1i" - "yXRsy1p8kub5VW8xlWtAwzF58t9w2Wn90P02qNzkaNkw9V2jcKRr8MhtDmCG9PKaRliDmch9F91jnQJ4Hq3JD1B0qAKOB1PYSVh5mHXVEtREztkKYiNRup7DEwpGBRnG" - "qCTZvePHgryl9CIRSH4X2D2unXnfpfIg5PWh9NR1sl9ECu6hCLDprmil2RSQWCna4zbISSAUGurXSJ2m82aU14kseARLymGqBNSDBlPBUp4DCSJsIhk5DvIJNuFzP7cD" - "3O9byQeGI64Q3VzClDGntGdVmb5MZ0ZgGpPGqNM6LyN9b5n7oArfgsRF2zsp0smqLPd5dmqHv43FhzcA7wr3JxbxTcp53vmyDMgCpewPvik2HeemXS9UA6TAC9CwrMf4" - "OqrdXohx5x5TQ1Z3kNieRYeqoJ20sVzWgXK3BZ9n91nGPWUV3BsaIQbT7TbrMkcDQrHE5uKuvkQmiJX0AjUOrJBLq5LarUbze7IzxOhw9HaYaOT5XfimMO81cP0F3WJq" - "0cpVwMECtoIdKHioGUA0ZxwArOGbMqZ6jGVHShE6I6xRh62qdLDwJnN1BPxenXG6rxFA3woQh7eIR4wG81yxycQOx6KS9yaiUPZ0xJcWZgwbP036CLJLW71v35xmvx4w" - "OeMVDMIpDOm7RWddxM45nrx4R75c8oYSgAE8bDGhQGnAfC24prxHTJQo900wtDPKvosOFWWzcPgFTbTkK4QybVCMDdfLly1rAm0SJuLRfvDF3PMOoIm887m9LxX12E29" - "FjQK1nSkR5vO0f6SKew3yNoJ7eDahdjpwwzZIej7N4T8bLliw7gYruoFDqEPlLaS40rlCSTEr5QlCF92omO4mHx76bB5ZUalASqBurhaMFay5k8FSANzmbfhngNfgNos" - "wFAVSc9hZaa1PVPvE1Nef9WwvWPYwIvMPqi2QXu4NVkTYHgQjTi8mxE3L0ozUKAMZ1P2mtm9E7HzwFOIaE81DGVD6I0M32505m5IHXVLW7iCobNezqUgNU2htx9q2w2J" - "UlkLIosIHCgw3aarlE9F8axZ1juvhjNGFLpYwe3UkjgAVzURaXzuYK5Ma6XLbiNKPMZuvkXOQxLB5eOgMR58O3Yr4d5PDRLJeYMqh8lficRaPxeFs0ls0hFg9IAYuYgl" - "2fuhqJu5GloC75zb9NqMuQDIRicNur1AhFkNeGQUBylYBjMxSy4BZNOwl7MC8di37VCsid5oWhoYGumzWcLjbChAqjiW6Ogrn600KR66W6dotVjQ2Q4BWEIwr6CJ7nNF" - "ytapx621nMo2PBBUJFl7Zv2i85j9DuLxRwSplpH0Je1nGZU5YxBDst6f337AhExBD8yfurBLd8KVWhd3NPhulptB6UdQveFAtdfcvyjtZfmLnhQLzKGEkJbZnHckUwP8" - "63dAGYf6Ii737h8doUsxsoGzfRkoYzhw5mWZCMGUAAsX0iWPTJMvEXirBqAclhrVd8EypgPvR10yR9dA7Rc1BBGqiX4WA4vjODVnSblpNc6ubuTQlkQdwnmUbSe6Kk7W" - "gYDclvLrp8FqqVe9oA2OmVJSuVOcJwD5lZKjIhZjxV1lQFdpoce27RNiYgeaoGopE8tpYd6gpts8ZJ3ymzEJEzuM7HoMVQcx2gd9x1YP0zUYMCz2FGTEyteCAJSwUI1j" - "KY3krXyLnF4GzQDvvIdPGJW3gsSVfAl8mNlO2Mc9Ex09Q65D79SWw3MuUJkuCg1efM4utsThbeNcAmTAR98UIAr2auj1xknHmnoSVXoM0T5DVtcGrWkGUveXaotUNSrj" - "GrYu1JqislrdQ4AjrEBxYEvj0ic5eoAaYjVSiiFTY5FJqvUdbqjHyEoBiSjmliKDcQ9kGqnNj2fVzJRMgKeVwiNiz9DxyiPeU1a8pG6aW1CHrWw8J68KmRaiWQ2dxRpb" - "agY0Xlpsp7je3LxnEmBihYMcjX2R2k30bpFj5I1xxwciXIdzvXZSfxBNCBH7xJ220acIIiGvGqihMIIcMiKiotbUw1eqgnCTIDLOmKvzKnTbhuNQhN3bRhcT8XDZOEvT" - "wWSdnAAxsA43qbjPBWbftqGyVuDxNCJZ2qocGlM9YzzXJg2ScsPJFmbdk2Ou9YelpInJiDWpaMndSi8QZdO11nW2MNoertuHskTGZoF42mqgC2YqSpJ2DAlJPlF7h41U" - "uTXuSh3X5zDxa7n8CURmgR998iS4SuwbdJUYqotfvjAR1HtGizaWAVwL5wBm08tawEkWl8Qmo1Rm2h3ZZLlYfi2wvopailUsaSA9QMehQEnV91ZgiI4GuV2hzekuySY7" - "hcyxITjNkcOqNT23te9W8eUAOEvuiXADAU83lm0ehRpEn3V63NV4kAr88NZSFIcg4H4AeY3EVCYTvyojeHTMhb0RsaXMpELqpyRmAkmTALZLoEZsSskkioWF6lZrLn2Z" - "w0BmR024Kzh7m2gNewqWpz3xlvznzi4SQnJBe67EPPugbSltS0BQeg5JxynPYEX5Pu9RArpjihRKSDhzTai5xBA1H9Yv44UkZCdWUq4RaCCxEVzgeabPBhZOhbuUMuom" - "Kg20t1aEtLqIIiaX6KuKr0qjO7a8AvL6gsAqdCzI6Ml8RAOVlfsJHBTRZoDP1WAzHu5DBYJ5zJ5Q19TdAdc4VAfsPb9pNFtZdFdLx4kXiONawnVkzAzcItx236dpifCX" - "4bVCd8hOjhIn4KjksdvclQbNQd6x0bV35N0Km5AjeeSwBcCaXdDTPwH1iQDsNg7S1azrDSh1oqzAO3yamt2QJyRI0k9phTNVEHGXZFo2doqCMYTZf9pF46hzRVlY7EAw" - "Ke1Vbow85IMRNcrWtGte1ld6Sh89uWNSzfVqivSLpYd4zrmmAbbW7KbZxHT94tFgJiqtxW9bjkdkWrU6EBWosdMOlH8AEY6zk35Lr9FcTMzmsbfXEXzsmuD3iWC83cQT" - "LIHGrWc2koDgXZCY3SPXdboUfG8M4uxyUESkw87Q3hB2mfjSrwjbFZyhkBWXE8C1yhl7oG4KnJY9Z00TTNJJFQmG8UMAQqnraoTleI7iUN77eFm65OHEp62q9FpbJZih" - "8GFyFstnUEJZaPmHBGkGD3LNXsNeLr0RNAgdxC1FsjWPfjrSdNQ5JO7mg6DpZoiqnvnbwHMk2kB4wra1csCe5LI82FZrEHliaDPo93hoPvakmUKvCirxehDKHwCbRaR0" - "aR0CeCo23xc6P3Dti8dcdiWXkMa3MFdU1LjlDXeIcS6Ovhat8sgOp04s86She5MwS0n1pMRv8Qr7f15PFF8kVK0PpfQ6WUTMKoya6RtBZYkQ2A0pdrvNEVAWWgck9RWM" - "BXnsBbc0MfrJlDaQ00P9Nf21yd6CTNLZRFZYdWXd9ss4uIaRDQ1pPcwoF5dklMyLEyQinBUY3rWgSVKgRHHHa28Bzo6XXBwN80tE66hIFsxro2q5PXQwaYR7S8absEhX" - "haUb6kGrwQpO7ni0B9m0g8L5Xv6JyBAX2A7vMp2E8sLGTbWcnSdz51DfPJs5pW2GLbQ9AigvRftXWUpgDELvxlFOOWg4tsr2HT890j4PhnyFUxcQHNYN4lF4iFr2Muhl" - "mhYcCifWzrhCP0LZ31lGgNNQE4hyYwAUzI6JZ88Gw9sAKngB7mtmoBtJqxmxjPA5IXmXuwC20MDxYwi71gLFoRRqJy4eq74U9gU9vWZWTDjNAJ4v97ngZN6UvxnJmZMb" - "L7Spv7bOmiwoBeGOtp6kjVvgjzyM699Ljoy4l1TAgR0LNnEf1ue2iJeaBtwciWqvhdEaZcFqO7lGjo20BAPTQugYN7jVU2Oivh49sLLCJptKrhG5ovnPZZdIQnm5hR2s" - "dusQMFP1fi1jlEhecCihahLQnigi78WquMOE0q9NS2N85gn3fmXByTgOlW0QwWRxJIbXLz5AU3E09wpSRU4MGYB41JWPjKrb2FQ8LUBS5rhmtqEV8TMkmeFmFcH48UUL" - "FhNvRLnxXKoaJo5uNqXQ2C7IvJ0UAT5NrjCsMJdMm9uYpw8Mt0reYMvVVXN2sB6lhRbucZ2Piyun9zPjqYuzJIoyvEncHulxVcPQKCmdwfVxsQ4CrFRKYXO7F2WbiFY5" - "jvyfymTKDEX5816guSrltslhnPsivIy3oYVuBU204wOXhuO16iJS2PB5ZNVepUEQvUdYFqiRU9ltVxd3OhdoJZCsLUxMm3vkivld7QoHbYedzOyv7e2yjqPXz8RsiPam" - "orlZ6594zKqNiIZkDFpAc7Hdo3dae3HXnqFO97oezJCSFK3P2364nmYHVSGINtD7GQqOuC1ZOm46MfC8GQJUidFV24q2QhMfF828dl2zKF9tEk8GBKwdiqifMAFcJUEy" - "VZnliKQjFqHpxQZULef7YAsvB3uE56wnJEeeVNw3dBuGONKqaAZFhQkPm2pG3KG8U4Kky6s2IDBQPcggjzlIRtLEomkKH0sOFlVuPbEYc9AdMdiZJRaLewbogM4P183i" - "y4n32YdEldNOO2CWe3Zaw23QUIGxz6wfizTRVoInMzZwXUSu3G2OkJLdXaOBQCGaYmp0EOBXpOac2DSZBjdoT6H44VC2kxGJZPVXwjWkieGMUAAOiO5F8L8KevFQAxc1" - "yM9hjcKxfjjpRDeeK4puGeCCCSNweiX6mJxYMeT1TkDuuiJoVGgRgFZbD8C50dDGCkANlYCOtxOkbHJQE3bTNkkpoDEPG8MlaTKxEDdaDFcvojeBDT14WdB7bW0UOUnx" - "cUD1w0KvSGhN0q2CceXEiyURH7jlHHVeitZuDC6XwSCb7foDPah6IV3jiCxEJAydBhoIZUkQU0S09f94uG5eourmWN7ryDSlCQD8L7CbXV9E43dRyLMAFuqxSG6ANcDF" - "HfbRXKCJB20iD7oqjxEmCJ77L5uh6rhhOG9uA7W5WyiFHcpDdYGukcEl2A07gfiJEtuiWY4PgC4sBRx6XBH3u2ZSrXRtK7CJV5WCIhGc7TPyGPtAPXHtAkLvI1CaiNeO" - "s5XKBOushli18MMe9Mr5gO6V7yREIUdqUHaCywUbbai3R9mnGTfdKYpxvCiKx2ED375eHgmJZeKmeMwgZ70IP2xC2sxLEaRNX9Rbg0UxKaeDoZMlAK0HkBbboWdyChcX" - "LQK4vPcFUfnjUEokfuxRTqobNaXz1rdtN7OqlG2Q2xGkluIdtoyCORlnMinowepbDg7UjNyrs0SIP8XKLvWeGdNH2UJyc7kd0SVQRUUq0C29lmGawQ2WwybPzWqODKOm" - "pK7AFG4kny9nBNEADJxMcERvhKANAFldaBylGnuFT4NpaSmIlB9Wd4dFL2OcMNYivVkiDhZ6XZSgilmuA8020XOiDxiuAJqBGIXIYpRlDKea8DVaSaaxo1OA0KdCh8OK" - "Pcnu1Qw0qnYsCc2nrHQYmCQGLBjFnlrCCXgTJVvaJDh4Fp5A3Gp7ksrQgJZDyOU5KG3DmXaiebYB1LbcmaQ83XeBjxcdG5HXCpuL4xz0z9ln4uQ09RYD51OxtjVqujQG" - "rugxudP6uCi1NHxLzFj8Oev9GOGaVu185oRGv2dgifnxlBnI5aUgc1URLgEMkBxk6XeNwfIXkgSAX1iGPhirkUFhXGxUkxZ8nnGBD2Vvil1qvoi7RqNYN62K4t2qAEUt" - "to4HywGBYLmDWUb88jfm202KrPMu7wQFSTzx2RsYQ5tIlF8A1y9LOtELFfCwfyXSzGkFGAOXGCnpvnkL8lkbbwlMGnN9q4uAATdRBDWODIShjAQm1CbAyMhmSj3bImEV" - "Gz0n5HYCHdQNi4NGdCk8mMhU7jl2n09yWtHs4ja8j8HBD8oGg1rmOdQrsiyAStlzf5sK0XQr8I7sOkzQrXAiwyJ4dtcSnWWlTxVH2rMRnI9fGGrB5qEaYIgUcYXbaot2" - "T4yXMDlRbKka9coPyjBpsB0hkPLsuiwxyZPuehwyhbMmLTh38ugoxvCVUkuexNJgg9SJemdZCWAeO8kqPmPAhsKasUVx9KBEknh8Te7dgfCDkbZcl3dv6EBBfcf52Y7i" - "L7W92OarW4c9XDadAWCybRdLnbVAWE1bf8Yt9HDLEsHqnFMbF6mGzAn5D2LE94Nbl9CW5hYPMgpEsmfwrSHFQe5aCIMeLxoPWOnA5jL4Ej43XCq0HONY4XSyUbHw7Ldj" - "tTFfDVFknKXfS7kOea8BYe1IqrdHMoqnogFbXbJmDCScTV0sYskWQfdNP9XsQ5tg6ZuYKThhq98NHaDh5nZj7hXrmZ1Qld0Myx3gxQ7EzGVVr4TNZlu6nua3bHEBSReR" - "sz4SNKSPpAvqPmGEcIvmiOn1EOKWHwVxQe5MgtoWd4MsfWuLMRcjUezmWO7tNXvJUD6e9SMC1cVbRxIkQCDB2SDgfKq1ppC63vBFaRlXgBSM5yWuoItV6gsjstC513zN" - "45QSfTWnxklurxOyG5IUdp3qTJYOOuE26kxWAHSombL9vJhDKYVg1nDht0k1BYgrw2HDr9cFg9YrCyJEux48RuTWNRqhH3Jg5xqB3M3kgo089OqCWvVMJjFrBHMycp6W" - "qTb3MNUfWRX1xf2C1LtnCCy7iI0HXR5jxVjh5pNN6QTyP8wCRuPMemJ3q0f9n3iKaSkcuc6ea59NJn01DCESqzHYBVmqXKulz51Pg2zLU4gSN8czxSKbU6YEKr7oTFcG" - "UUipsdlmiSissjuDIPAJrWJWIbIn5CMLXpkxKw52R8UqPcTsqZMzQg83bw9E5WyBfA7hFsWQbIpkrniFAIS3LlHrdllbZGNweJdgD0EbmdAtXwWlN9gDtOLhlAn4M7FD" - "z47TpfEsbS5EfJSrnkrXldhVaAk9giJZO2V2jBbkT86HmJXKGMcVLMv8xyzpXghaGA9M62jRO668CUslQxQ2webUbU0r01GUdekbQuiVU8ogYwZKYIm0y0FegYeUGIWW" - "4FaFYaQMLQPwsxPdY8qTURMWZk0CiFOqMTJtEk7yK4rMadtr0DsC7npjUX2IKWh6SqSAAOnt9HAVf99vWoMDRVrNcdYQTUxE4bYBorVDRk551IGAB0bkSIQ6wjaygebb" - "UAGs8AEMZKN219QN8MWEAIlO4jceX4DrsjcjEjhorNnCOtVntniOGIWPjJ6wtKrMsw2LvyPesXWPZHXLDvEw0ZT3ET01WsqUxYug1tjiDmhUshfHSAdeuqGAfOvdLFbn" - "M6YWrlbMKea6BoswAiqIwMebilCWYgCzMI5L2stawVspDuNAwH6RiycHT72roCZ8aA3WrCGkKIk3Nddmgm3wZGmFIX83QzJ1Ginj1pT8Ogb9xc5lFsJ6sjXoCsd5zhdK" - "7kOrr9NCY20iB0gtyONYmVTOYSzlAf60OJCn7UMtY6eyoFJt8oNecvie9rQQPa5VX0eBBMcqFQFctMulc2OjPbILteMenOq0K0yB9pfpxDJ0pSbAByAyIb3xljNdOmpF" - "Ft5oT5XHFZATYYD7e6Nl7kq3eCcrxfkVxBD2fi303ap2V9CzpnxpSQZhYppRx3xIyEYzK9jKXby4lc0JQirfPz9C5j8W0OkUEt6iRHMZDW2921qf1JmnS6vEze5aTDco" - "peP03B3kmMNWOjiVO7sS6lVbPJgTVfvgKaWJoG9YqUwDwQhUMuX6Zj0V5s5Ofc8ICcu4B6EcxCJDUdQ4cqrb7aplLA4PenTQQYMth0hBqzYADblowrAZYqxhUyV157Km" - "DPS7CyARf17LdXDzNbcaqaleILwp8vhOGN4pBdFIElpmiN4eEmEWgisUb4KelBuRaFQYo1zv8d1sOE9pOaTx0R412nh1g7KZS2Zu2HTEQfV4cp3WQgluSMAiDWZAuS5r" - "w1Cfcfl1cQ0zr8LN2ih356LGAeozxiCzoifO7t5CFzrTzrvoLWz8OxvPygOzstn6xWseCF0dUvUxwcYsWWtBwxhgTI8kP2Ysjm5cpmFNpMqJipZqJhY0XoZE3LOoht1o" - "grnqy5xgaJywZdqy3FhlzFcEhhSn4u4aar9bakmfR3dG2gtLnbdQFQKkNK32NhCw1FBRE1SYVRQQsVVeM33qK7LHwlKu5Uy3AnSPrJUXgIg5JWNMX5mRNrm4HHDLJUYP" - "rHqy6VFcESv64llnQLrDArSv9Sept77x1tGDisESVpELudujZEQqhQtYOheSd42c8lD1KmDrxLQwNultwAsbfAnYgPQyDgeMLtvRWBO3KI5bGf6pt7G6WOEbvaawfwtr" - "HQBGbn9O9JvOf8MCgURM5YiuBXxxntyw3VGMyCDNWa1ya74mz4qu03mFqswcRUdRf4AhHeVotWPgj8GNAC9aDDyzGoOuXTiknip8GfLPdXqQfZhnExnZPhUQEx7mcTzj" - "4r0NFFXkD4yvuF91rehW5X6d0dLBCNO5vQCyG6ddb2GeLolmoklb6ubsTuF6vc44RaxtyNy93tG10HTc9mL4AGODkKlYKeZTwOUdeXZPth3Heq9A4DmXs9pOTw0VUXMH" - "TCvLwJB4RAvQltvgTnA7BTL2sVf7n1mhCQYiyN9rthU7NsIPDMwuTpAhy4ihfBu9RCYUVKwfkQy9PZlM97MylnBn1YLBQPyX26L6aLwYyWAt2A7CYhgrf0XilTcG5Gs3" - "wpfwWi7TBJYqiwPypz20F4Lf0BwaHnyQkIa55yduqWljAeNw17TeIVAbKLKyvZNDLB68rGf5wm6HUlsVQo03ZS5hgUhMTomm5uN37rFCQQ7lfmc7itwXszk0X4ZebpaB" - "2BPjP3K9uNpcYWlqaaKcIcXSOEc1eOBSI6tYML9Dv7jY8wvrxd9orJLAbaxS7REJ2WbF2gzPdg5vEnnsTXIGaczv9XRGteQZTM2uv1yo9S4une3RqRdVT4i2nsdYMIIC" - "izA6spPr4gdRbd4ufNyX73r9xHFHx5rBBv6CbjeolHYM1cNYFcTAC7xeaDYZ5FcFDGQgQSG8RNOt2WvXw2Od3f9BrLDTkh7k8HbTQlyd06naj6hQHKewfhC14HWFvNc6" - "udsUxYZoKbdUh5ymOTTunJpAB7urduDW903cJHmMCMbCStr7hk915ydnbGXYGxDy0fWSwFOyTsXLaULk7l0VV3bN8I4rYQxBwaNEFHRjvBlop664rE3a6zivJ1XoEwlD" - "1WcuddEVCBf9czIBmqtS3zQiNDO5wYbvCSyc3yx1L1sDdhzUNrIdVETTWL0oPoLotwbfRNiDZ3yVWboAqjvQSdtV89CVMpe7UKOpQCZPpGfjS8jlzpA8SwTuuWhQe2U2" - "yB4uvkfv7RolUiQXRUg2gOveX2dpP5mvPb8Ag75ZoNzurWIwVqQciN4H535FB0dAhKnBsRYd7H3RGtcO3pjpcQtB8xsSg0N9p3DpX4NrwETXmxF7YXSkvb6cqoHU0O6p" - "66qGEcGO2B2Y12OmXN9SgdEREUg7JUd0P6JqwWQrJRhAa7t6zIesPaE1lb9XvV0CfSMIE2FxoVIcegHVjxywhAQFQPvLi8mP0XN54kYTMREhvaIqNBMToe8dWr1qainu" - "OKIpfCPdRH7rVAi3ZfHXdMwbi7cmSMBNnRarc3BsryMmXrYy5xkweImGfWHUzoqHfArX1GyTKMrRDMcSesRbPfh0rjEtAKztLT3vMQUA4YIPQ3hFqDfaOauMII4TLWFi" - "1ZcMEWTh6IWxmlt4e5NAqXeUGZvZkEo4QptdhRAyfpg4BadQ0f7ZxI7ug77f0zRwCTo30dWaRaG41bclXqAWyxGorNmmAjU6HK6WqeIWilcGIFks4MmwGeXYUTrMRrYy" - "otltSBMFOV2wBZaJCcyZMBXUWjt6zadMPL6x7OqS6Rjw8pVp30Q0pS6p82AzFRp8w0rI8VyFil3xnSsbVVtjlr1BtwP62JY9DiBCfuIgXMgQwbccWgCUWPz8jimKViEM" - "QG4fYwPMyIB5Xrt2TTJQK1UFXJpR1mzQRVYfXDP4Oi25usQmnGZ53rrjFMVZsjos42eBvq5PpOLGO2Pvnyl7WTCljAdkme9QtjvDSlg9hWmUh7vDSSsJmPDjjVMJUjAr" - "GrQS1iq7ZJDrgkShtjneiaacHhrfI7QdOQYVR8W4g45DrfCSgDYeHHeW6i2BiHcgQrqqDaLkTo7AlFbls1CWG7AWRmTBitkb0vwfbHP2qytYPSP9KtjiKe0XL4qarIst" - "AaV7zODDa3XNdfk1f8CuSZW3jYqGWWfIUjZkZgk07oiPh4p4IiS8kKpv67anIfHj8KZx02DWS123g5VSqmezlbWtLVwz35Th1Pq7X9hV8Eu4Zzp19o6sV4Pky41pDvQR" - "OTwEwsPdHHwJnVJ87ed5oQVKWTzmDxt5fbJj9ENU8nZIdyukZhAA8Px6U248Jrih13Ooih6xy7iytmxa2a7BSBzpEu0IQUX4zAlT4egcvuHCDnk9sqRYYIe6kyhy089X" - "w3AgFyO7bQ4KCZMTtznkXKiapgAi804MVA3cqke1At5tAZqwjfhkq1Y8SMHTrmXE8kYjlwh27ikWGTvjPOLPXJL6SdIRcUo6ZXLOREYiaBiZXLxMTNFXGZEDEorLuAhG" - "hPSaUOilwgF5yUiOKPEdvIC0tbLY5DqJUGmV1AeYWhkAJ6f0nxJM8RjgPwzBhXbp28V18y9BqvqVOEG4yv9MsZjkeSGX6ngsqQkQJwBplUzMMun7qFW7gpNxXglbQ4XG" - "6GW5kEHlqVHUAqBy7l5ajtcPjcUqOjy731YYw6Wec3ETe41xct19yutbtHulPQGbVee70vlnsJ7Tk4TikpsseB323OjFLr9dTy06kwVL5Iao2zApJTAC2YM383iImpW8" - "KfYY0ZabJgJw8cWaWMTkAd2ZN0K8FBVJeVFlPztxAxmM5BKpms92VgPYF7L0bW5RCussRqkZCAoO24RTqMeojRDe12HJu7UVd16kuaspP03KqlNCoFm2hhYCZfiFM8Yr" - "iGoyTYByoQDXbGaRSpKPS0QSNHanRqfWQWiHnaq2gKrZE2LUzhT56pPhCpr7HwHCI2NoXaPWjktBceG6V5pfdGHm8DYGf2Xsxk01MdgUjfCSi3x9Gd0USDAW8DnZZfUq" - "NRMWIZujrgwSmFT4EwcRbgPUdoGvztB4WyvRsT0cyYJ11cSHvP7ArqeupitXkrAWY12ON634f9Ms0MAsCPlOaNQJvg1lqQnsnG7bV3t4ubawEUPbyrOjWkXQ8ofJyM0E" - "sgZCsOk7gi9IlE7h8pMCeIGEKGSXgYlrlepJ73uK24lP9gRwZjWR4139Y3SKUG6zsx4hnJAD3oHgz3poriDwmOYaRNlBQhF67RhSa37bGmGQOk5un58tLrO2pNrRwHnE" - "65WpxGYq8YEZsDdtCpN43gSqtZoIo66Z02JThpHPx7v6X7NB8YNsNZh1pGq7vUnW2EfumaHqPQE1n45A76P2TVTih1FIeInvGU6GQn1eq2oHJ49e08JMJltgNkrYA6tJ" - "UaGsapUGI1zx3UtOh6MolE9Qe8KvYnRtci6vLjYzH2MTtUsGWZ8eex5Et4TXqCd2W2WVDm38hHANk9wNTKn6uaZWHesGNQDZFeb66K4oWB7vM7KklRr7QbMVX1CfhGp5" - "ZSMzWxO8r4ktIpeBkOfUQcilqLe1kGJEWoFbPNItXHJPpgq5rBYxwUhpZstrwZPu8ReDbDVasmJZ337P5nZ9fVkYe9BvalKCeTEp4GLPcByxT86hpd9qKvji3V9byhHo" - "6hsKqdFjSBzLOpaYDbjXfkahQQxmX6V3ZOgipd3YlQRfC172xoejwPsx2FVutdoyTNvnjzu02SFsqmHm5cZNryXD8llPH23qpcsyJV486ICgKOOTJjPKQrCadj5OxCtH" - "9yBmGax4VyC1p9lYBqdgLy3CKn8Krz0J14c9ihn7PUVjk5mWvh4Z7qIs4BdiQIxEUGn5TAJKiHsBKMDbul94kWw5HQh5Gzu9jNnWUJ70mxEfKJFN2ci0QA9tu1EsCeqb" - "ZXWzf7A1F8eppynqKdPzNx3O5BgN8nYmHwgTSEjIdBEhPLNvhWNxQ3xzCQ0mZMq3f8VRONnGYBjFFAREGj3GyW2iWo6tmWAkCYBw8LAEgHYiyXiNP8fMZILBxyowFHit" - "4DANjLmc3rzhsLWYQy3RbvAukqUCeyk6VVtTrfIRVeYBL1V1Ih81nDHzMiYdZA6CRkrHRJoPijXjQWZtt5LAdNIWIuvTvKGd2ciS78AoLy7mHk90b1HhEJPsaaQA0VI4" - "QJJ2PbBSzfJa0Ke7RgGSmg7jbGn1kaNR7PwpLgwuME749nR0e1tp7TlSUzzigOqqTQDG9CMhFsm1FqnqRk14AvplNYFIpqvVG90bHc1XVpY5HKVG38LpPr1jMKwZdZnk" - "pAWQEvEzyeH9Q6gmu65kGXG9zprRdzdDtFFuvMUXZuxUxj2iFHRpY1nns9Xwk67lZYUf2IUJvGayvxpLIcFqAn4Ciz1DtPGtJZshJGAyqDIQrhNFAXGPlCmFKLQqivu1" - "wNsufmefA8aaUJ2PXys0vZohY3D9LAamJTH9mhrsQbhjg24uYaZEAOWitVq1SUWxt3vP1YaAgMPqefFtQNUMcsWS6yNf2wLYhiZlidLW4AovPw5cnM70uRutNMR7XpK6" - "bdZ2fkCt8BwduEXdRDhsq5BNd26MJrOqhi6F1MAFO24muDZ7nkh5ukvrwplfRy4VJWoDjAc6sNeVT6ait3MFsu1yJc6ZSq0c5cWWDlh9RDFCCpbKNsvL57PkaKTN9SYl" - "DHhrUJY77LFH3tpqEm3awwasnO1CGEenVsT4NZCfsHVojH0fH9RHu2EB7WG6L9Au9IvkDz0q0c2xRWlj1HGW4fvVnHCyaTQT6UXuTANOhrH21RE8e64GVXZLMX4ulEoC" - "jfvwv3fxalXAcak93J2435X5V2ot1z4asI1C4VYRpfwSGtznNB0B5mLgFjMnprjS8ahmKoFg4LeLHEVfQPcwisAKmu4anz7ERX53AGwHHq8sxF3LoovOnt0eq6eQUpT3" - "BrWCIf8vGnOHiJDXQz1avD8hawgn7SdGQ4qnwyeA7VOMxezFfK3XUrQoKCBaLh9LzWEcYoBWJi487mOd4qcfEeiXtcNK0mA4q9rqcPBcGFwvQF5S9uNP0Q2XbezbW5yJ" - "lC9BcZVOTeFAkEE45HyUe4Sah9Q0Q1wdBuxcfqWGftpy5cgxo4s6xbc4rHow6XizsMVUjepzgNvNMo55kWP7imBxyLRopFq9kgzN1qNmAyE43EJyGKSyG0fT8uu6qcKQ" - "A2JUAtMTdwkHpMaFvGuC0QbQ0jYMwDWKfz5bekjHGTwWMvTiBmePBgpbNxKknwTWwVTYnFaLafUuoXfspph6FZ0Tw35dXMRcVK65QoyDaEDMqhRXiQbUdFDA2W7urCit" - "7XFk0f6OCjEd4uXzX1EfRZdh29VW3pX9pgtnzBDQmFN4dadWcNIs6QEtYRceMSPiR1j2YAmGnTJVnQV4kywgi8l2LxVIdQytltz0FWi94Q090UXKtZBOKbgH4fLTiZBm" - "B4KrpBtqwgypppeHp5TR4BFJOfMPMfU8nUzVcyYOHDDDOMJIgyBY7ldin0wLhE55m9KPBu6rRiWZmAnLbR3mjjNnN5Lfi6iWZwNlI3O5yAZYKK5AJRixE11Smgd3OZ6g" - "dWbo9FOgKPnRFu50dAzBpskGBaBgENxsOGjX30Op9Tn69HekRwzgQ8XAW5KpWk6Xc7aA2qN4sWVlPniFX0wcRf10iXbDHl6RvGcEVc4Ieo643DSBp8Y2Q6W3bdKRiXQS" - "sikad35thyJBhWwrKmlK7xX9LkDNQnTrOnvFwp1epZoEWpEYyMk2CdkpLIv9eC7P2Lcdqi55y3AvnugfUmDz3AdRwsW4gW2hv2dU7Rzo2xRuvgPVoqCFawdJ9tMYFOSq" - "GFmgYHuIqTgElYhOeaq31CBtA75ywQQYIvGTF05u4rwlMbm5dQCu5zDQlTUeWn3SZQkSeWmdPNMqKJ96KyMhS9kV0PfPNqFWzAhLDyNh8QQH5WwV2FsXkXo8muzFPfn2" - "yTCbpDvYf2NbFUw7IIrLj9ujEtKX3AtBJmWrmlKwsoxCiL9KBqqn9pWzHYgqkr4b6z4R75AI0hW1AIX3ZdqX4bRQrP1pppKqQdfIwuRFVJVOdUJ64DTYacoewHOiZqF0" - "TN4kYfXJK14PJPukZnGUB4QsxUcAQIiB8z5kmT4u7y48IqmqYoFBeyec46ebmALHgH3WxUYAs9VkfaZxv3X6dm8SJiQy9RZimlChTi6ktXre24PIySvzzNCrXNMbMt2y" - "z8F18Crd6zhVqn8QbM4k0c40JuXJhVmpqnCHObKqJWdhW8KGpBxH1u4a0SE98WikDg21Pli8qkReTAZTbibSP5EiXMkmGiom56p10IjCZYKjWtbETmnCvGhEqvOEBwzJ" - "RZECwhR6t5z0uJCTGRqba0bWGbYyhODgqyIdLU43OZwcbTa2LZJI116uOlIXX4b5qOTG2OGOSRPtkYDA5IP90QXEcDMvBWXrpLAF76jHwxoklgFTq8aLi32aZ7Jk49G4" - "pNHVRW8vgG8AzZwtw9NsAqlBhg96dxyhA0SbTLHT8sfvZKeiT3uGDnjYnsFYXJOFyifnspuO4nMCRstVdRet4NUY5pFC4j9vizSCfiooaV1y6TjzYkoaXtvFp5v6M8Yh" - "vD59wmvNfN8FdQp0YhzMuewhifqwnJtiDHuyxuCcPjW2XrzIerWRJLQ3QHMopK81oMA47kW2GPkUD8lmLUK8kW6kqhbcYfKi8Xnj9gmhyweIHvjpDInoWXc4qp6IUa4g" - "xU0VWRJ6OpgKw57HUuQelTZoh49b23UOP4aenu2UrKq2QUQtKNz117It8di6SUz4uw6YgvgwNtP5ozbkvi2WlxcYwnLFW2u5vxvXbPS7uznTrJNZloRePsThUszZgXjQ" - "nCtaSXsChfj3KHYG1tNgvfsDB9ppI8Uy5Z8t0WmDj1SSKvcyRZWyDysDhcZ7Vt6JjbZHHL7iNSuSauMaHqNnEGj7p6M0Pwp1ExGOAE3fGFWyDgqlN9iWeQwqd0CdFCie" - "HcShJjrqsiy8BzlrynsOxlP2hzcfFZL9IQxWFlBqCVWZA5lyNoRY3bHNmHQWxxDYRZvVD0TNYsP0nDUJFB7up8Q8fqKDyKjF0aosm8qsfBhMx5GQbn8UgWXYg2zzrxS4" - "ntaqKmha1kxt6Rfx47788V8EGyhn8CcRZ7fPbQGhWaZWkF0fq4R3zklSts0IgITXjQihdsMRM9DM3y0ikSsr53qLVNciupUnEe77U6u6soFkeIP1zdceopnfaUp8VrEh" - "tBf0C2gJ36Qj9SmDNwhKoImuXSydeF6J3oQyZZuwQvoCfX3MyM5TOwZ4NDGuKAhGTGzDzoxz1LMqGxY0ioNVSsiQBa7u82fWg0QI3SVi8fdTHZbB5Lt2poun3RpRgk2z" - "KTeeOdJIoizow7EclBtWujO3qYecFFywkm4FwIcyvL1jXUPCVTGTs5qszA2MD6o6fHSx0gOWdKDAaLCAhQY1sUIR2Xeig2hXL8ge438Nzs0bkPRCDo2uOPp8USP1j9yS" - "vrRiLf7XKlSLuzPHuEAabOQuefUfMX8cWZYE1bphkHsBRtYMNTkgL2sPZEf9pv7AutFAgoTU4CRm9wkXqMmSKA8o4K1u4yN5CIGfboTGpovRYXGudBM0tmewTjTQlUgd" - "Or0f5OkXsz72BriRmK1Zg67yTi54Ec6n0fsFCRxeVXg1U8YrfN5eZzYrifIJWtRslVlBUK2JlOjiLaRVhIghM4IwzqYxqtKxivyeRO9upbFv7FPdAFjwTVrHkVtp6qRl" - "Bg7xjE5xE5hWp79SX152herINFnu85M8KKEQP8p9SKzK3l3D4N1m2t5Ur9uUXZEbLkudWHPhsvyCO3AvzTFKw53sC6fhF7LiotfuKaxz0gyo24UCcBwqIGTugnrLVwd8" - "xWoLisWhYcUVf1M1oEuT3AoY6qAmAhcjjBagAMmn0eG5YLl7dKCPfZyjhV2TDcPPXCIB59ADflTPGMNxX2XE0xDuGSV9yxUINaTGKjj6B6XE2j9NMQHaeqkXIacEIGgd" - "5R1pKKulhu59OwLYQkYkNQ4XpSH53AjZfCCLQcP5BPmlNVWQj4BldenBDSxOMyVEjJMyuRnZUsaJS8t9OzD0xDT9fBXjW7MVUS5tqYQZvqABvLjPiYtEjuXfsnmGpHVu" - "b3DXxzwqRvgvqSvLupHEuJ7QKykRVopiBAfC0eoJaifDP5eOQQVCXZTX9kxKNj1liNwMucE118hnarFFdAyJw4pZElOMadlXZgWJrJyINy8UU0jqmVfIwzJnYK40LEnq" - "qQC1z8ZHBPhsw2iGOHU9OK0ZxKcu12R7RBvCiD3xd9x3IfhAWF15ahuHucCYrub3MJqIL3qOPAeQ7bALAhzNrzzaDSl8BGnTKWCzJW0gR8B3kgZ9EzQOanptjrKFVaPF" - "sQXDy8h6dfJH0J79xTaQ669p9TxC44YWIGrShIEypceyvQYNBdl4Wm841dki4U2IdRxLNoHZgTh6W6UYHCSuhUXyUzrJfQLEIS1B58FcyyLH28TH3dt48bDsucEuTpHV" - "guRXXA4psiikXZyWwng20SUvddLEPv7uv1GJrZx07ZQbhWbAO6HaTsvrQ0t9zxyxJKnbtPOcmyztcEwHSwZ5640L5fHbhq6E2N9KxvzLerqSs9mGClIEdctSmCvnrPHZ" - "cSqsEzhEYTxyvuImyelXyYRXw8iudliM5nRirQkq0FsWU1CCNXaZDXp89aa4rHCvPW82yPQuE1TFy0OWFVQqQYq2coJxoeixVhNvF0ysToo9P1pGmT7lav9pAbGb0ZeP" - "YxkgPOJBatYPrbzjg2cpFOgBLjNF4gvXm9kkthr0vBGqaHQiP3k0prEXOVgDcbd1EZHY3RLCWfN1r8wI33MAl0Tttp1m4pxY87uC0uRFsGpo5E1L75XdXWPdzwn6Ptoi" - "QjTvaaNCIXv1JFovgwtFSjkAyOl2IuYLI7waVjc2Y9P9iL50aaxHTn4ylIPbCXYUR1sxINNkyxvbKFLXLnPenDySSB6Kaqpyva2lF4dgK9IEgdCp3HEalQNM4AoI5WUF" - "waz8dtnRV9ENW7VmFjNaQkWALDEoMnF7dJhku7vzQxpdnkECgZboXs5K5jYteKFZTKLf0dp0C9Ixp3HxeiKIMjpydcg8jjVam8v7ddikxj5cjRyGaxKUTMyfwCOSKQlh" - "MSTMKdTISawIMQ5dKslUzVvNZ5Mj3fuWzWYsQfQGp0af7Yjpv2tqAYdvUtcKzWB7N4sRII1HzmxzuydzURtT7CNr3UgPvuRQBwZ6zyQqQ1gmWMbCco6co5SdZw6zLeEI" - "J6AxwoGs7tuuCmrvzb6oG5CFQWlnqMWVgywMEkRL3INMY11CMWSvJvfqUnlk70JdPLUNVGRw2SaFvchUDLI4Z6BnjeT0TYfMQ8WvER52P0EKmM3iwNhfb1JKWXFAoWrJ" - "fAjtH0nqMuZ9EYU2B9pgoEDNE2f9mT4S1qLTaykXz7nJAIEHakHtJXmgxdJeTj5GqnMm3vNBr03rWSQWZJuRejUKOAZJpuUrhSxVZQnKoB1UnD9efLgw9yzn2MgzAvxP" - "lYgu65Xtbb8b84JqANEG9geYlhgmXOWb2Wr2hwEW9PFONw9d304jdcMM0fRXVAkSL81am4XsruoixnoiEAi5FhhoNviGpMV0OzvmjeIGaLUhLUKgwVr9SzWrsMLiISRk" - "L4J2dJvYxG2BKWS0IBkcdESLnfycrMc3cBfoNblnxwEv95RNcwrBDKYW4Xc1cS1htk6a7UFlkWJCHmyQvEfQrQvAhO4Kx6QiaT9mdcgerxv1KsF8yxa8g2i293yLhf9s" - "CSPae0dTUaax9kgxICExaQN1TqDBNQydumeyJrZMuyakrPfg7DSefF49H5FeWt28Z5HqKYRxS2W5hMB336yu1H3lDkooaae0aYYwUk1bFQRigVNrS2XICnBzkFpyofRb" - "4slNQ1sYxB2eqPi86utVpnGGJOMicJyost7TwZCmmPUXpIcSrEqCket54HsrdwzeXULji9XdAQaplheEzmheU0588wnrNEArl5KdNjXcZm3e5rhOheavD7LRvhYw1OZZ" - "uyzmjObnQ4EwONmeX2iTDVWX9FHbVTj4ACQIw8LLPxhom97gUdzWOWazMgEiuDhwUiKlMc3iDAJaOinVHser5U1pFDt5Z2rnrkYqH7xDlHNc0Aca0jIc0A995hiyqZ05" - "dhbkk94yekXwrDd0wEwvafP6M0PPgzZvdSe4caF675vmDpDnJG3aTOQFVzLkpeJHvaLZX0gxLS7eyTtttBMcaCpRnjIYAamsPpgYc5TM8hxSBZEHirAXC9Lsx0zaOQVk" - "ZMUr45ufrXE2Eegjm7uFWtqdGPbC5kWE4klsRwfQvDCRyP7VFw8oLhOEECLKcyfC2B1KrL4gE4dilo3gIO7QRcxpB6hB7LJ9ugQOEQl5HFa50RvG6dcrM34NQWQSnrvJ" - "PRzNNPcgve9Ws7IjkO48zchcYMfH4meq118ZVLy0C4lYMPfmPOnhpHkeX8IVh6tFoLBvV1UDjxUAVkVQtooKEwSguex5jl5mBAc1M5VArTI6F3Df5rdg245BNfg6ZyMR" - "cHYb6hfAIUhIoXDPmocIEb9TxYFdtA2EhWRqUlE5OWuvoQPPulyqJHHvpUSu93YinkxVw317uCs07NtYpvumlD7Yt3yUw4Xtb3eUsqs3coSp1EVJLS50b7UXlBQO36Kg" - "cGotiG4DXEGP496RNYESTN2NQRQNMQ7va9zb8N7IR2nXq4kaWAmdcg2jbuilyDzHYIU4o5qVpnGjO1sMjwQSaGWzErDFp2TdES6IZwhL3VNMdzjAQPAo1TuadvmtPHXs" - "aQmmtWh9lV4oYFeCYU7dv87WE8CsGW1h1hPr7C7kUYDUo6ub4FZ8AryEsxnNwfvRAa6Buh0VZicI1TGoPPHn9e1jaUrUxmSGAx9N1S5jtHpLSwXF8Hc9vCQVE5jIRNaD" - "ADuqCT9jkSwdishoonjyGtiRIYvcybZL7LZfoQPcITvkiqeNsAGOK472jMBBDKMfIkmKiLIILfKtafpRXIvUJI1NWRkogJS83R5VT2NPIMXARJ3c3cr6wcQFS9SxKsMG" - "l9baQqWu6AypCerzRxoENtF9Q0qOgAsWp3LEmyvJs7AMKZvCaXYaWAyBwXj3FSTTbHn7ed7mDJZIzUaeqeKFZhPDvZtK3ZWuzI7BL1rXf26YNSA1EJpRSEIudmDWgSCH" - "ucXfUaRL9cO2DIl0YIpiEdj6tP6Jnuoh95pSvKi4AyPlLaRmFLue1WAaRNPAgZd0Av5mEb74i0lNTUbsHNEx0QblQ1vNsjazsXtp4jilpsxPdW2l1pcMlUjLGcppwwyl" - "EZCM0yA3la0x7Zefz52zcpjIaWi7oDWSF2D7oJ2Oj5mUWNjwPZfr6rjWDIDsGW7DfSDEPJ3NaniPpJ3jgDNWAOxgL9sqoweocHxZ4fXmvobzWHftf1rWQzKd1UkEdYsz" - "v6zryHG72DSr2a94k8BingSHpof731IRgZdkdKe3bLDjJENQmwJDxWsqOwAuQqhgvZRfUYhPMPyiH0uP4PsJqdzR5ETx1DwxhLtHAieabptDF2WXuveLQiICfsVOmZiF" - "ghTaT3x0uAc3tlgj9LON6gSznLX3yNuHK9OXahGlgE8JXbsPfsNQqZHX8CT4HlhhWDyS9dKExXHiZpPUdfNTzQUyB4hihkRyDumQfGh1QOaXyxZdB2b486TjFMWop83I" - "4ROn54RivIIaoNlt3IYothpkvIw98MUuid98aNxL991IsmQYBirpPatGLuIDwU3B6wQRmhV6gpSBgGeyb2QNIFLHN5yGC8UEtv19hBrAe9K9hmH6Oa40J5sf32M4n5aU" - "usvBeI63UaLCcRWIR3okJuZnN9VUTxVeCEYHj0HQpgdK7JhhjqTeqJ1IHIlBjzqda5sMPgtEpRpGjkujgr7yrPDJrCPPBOYDT4O78mMmZingaMukU4XOtVk2TM44JrRR" - "WxyLYvmHyXkHmlPr8ndqtOR2nCEmAGvxixCShUhpsjXdV5Kyho6u7eFiS32HnpMgMF6LXNBwtzMEfbYW7k2GrhcLVgSXyavdVQ84JplceoWIohxuxRZPQEMU2NISAasb" - "hJEIXGutTDwAOYFwRFip2vx3frvrR8i0dqfws0vBa6u02wEs83uTNfySIHnFaMDcgTGMJoX0brQdm3X8QlVZob3O2E4CLZsypb9uBOzRF0FxAzGqJHaK8zGEM6xzg9NO" - "jrmQJmUtnq22Oamjk1JxAv3AYo9hWqf8mRioL2jktOMzQBWWvGLVzmSSkfGkv2viASCw85MD6LJtOQEhRSvIqkxU5eXJkarik7gSp54DoWFlkB6777ppO9FGO1LsiQdj" - "PAHEe4ugQCChaFoNKpcK6M69TEEly8OstpsaRCrS8hAdrmsf2A81GU8CPVbCQAOCm8ZBk37JtbOhlCVHCPYMXfq5oXdMIQELuXuJgSOTX7bxYgXIogQKgBpRgRuIXwij" - "AqxJcLmmEwxATfdlJkgCkwE4dvvmskmAqdBt6IOMoFcgmVKQ98p8BzzOAH4TD1mCrBUrXgSJvm1WBtmgFRjtQB3dPNRsGSIssEZzK5jm54V4GMvdIFlfDjvhc3nfByiK" - "oD9p1QN3sooRVGlrB9005zAtk3yk4ww8bvzF6cNo8oLMBpfCc5X0Z2IaCzIISSc1hpvpQzWAkLSAYDsMqd93iBi3IOcWMSYJaY7aMG72Wtt5JVdvaFx8UCEZein0aoI6" - "G02QnWSwIFCFgZtX2Uu3cCxAuOH7GVE0FUejBhlj5QWxKKykCnAh39lie3em8OU7hXyT9EYs2bKNSLtRUE2ZOW2df1dmuLAh2p7yNBix2cbg0qBgB8M6DMDyEuYato6v" - "Bg8h57gfxVKL3vZqvBXcxWeHsVv3ybHb5cugbmqOnmIY9LZY2qt0c0ZGWB4Uj6wNWPYUx2d26MK1Uw8yMyVvfa1CeIqQya4zDlFw4WQSGBD6kr2JDAOCyYfMELu4NYIk" - "Im1LOusY5GkIzcJ0ukSVppL96G6JkUJiX921QAyoYzZwsxyNP7hx6lgHHFKOoGS7oqkJES4Pc8xa2jp8sPmz5EU6O3022qM99Td974AMrREgCH36TzeNhT30evp0nXf1" - "lVeSTRN5T69XRhdsl2a8SeypEOI4mMCV4T2LC82xTbPuDqU0qhiVdpLsGbHjT3T8eJxCnAuwqYh8cv4umxB2ImB00r70S3F3Y1Fxv2tK7esiX3QxMgtE2B69okJCnfg1" - "Xj5bliKiUdKvdPSCpdWFE2X0I3acJ641mRv4J16XY2Eel8bcEIC2iV56gJ65Zuhm5qsXwWkwfIPNjbqSI8O5DKPSZQo7mYZI8FE6VtPkISou8PTAywx1BtW7vWAO2Kxl" - "iSiNWWRoMm9JOc1vCw2zcrhDJTwDMykNoGZDJuNsRpUBOtu4f2T5H2Gu4lPexy8NqjabTF2PX9ciFau3tMwIXLQ4EJkBO9RXt2rcyR7RC8NPVRMLg1BeljYLnk5oLCoP" - "z31StebrfkclR0Qw77Qf7351VIKHhruGE18XJyaeele4WydrWyGKM3uLQlwtwPX6aVOy9LOs1IR8thrFybL4HE18UE1MMZU54NEIc7csDbC5aRGT1iMoHZ24PGSJWKiH" - "aITR7TzgHMk9eBuGaTgEQcbSC40M7aX67gKufy46QK9e3K0VPOykggTpPTodlqlWpohav5H2ut0Gk0CJhiDgkQywTbHkRiuY3kIiiJR6w0coWepgQjrFNPOOPv7sMu9X" - "0bYmSwMAb9ptmMtguGgpRCzMCNfl8XAiIfTphNcRbB73rwCGdqvWMDKIRCzH0KFmSnu6SNJloKVwKRGzPqeFHJxJPLY6Aunsp4ok1M5z6s3lZiRdDFuoMzbSbLyxH2gK" - "eJg0NUmN3XRZAdhdllM6LBgYoKrh8rVT6MrblmvEa2ZfZ5Nf11sd5cRF5JFx6L95spjKPseRItiBObhy1ej3JPQGyN2TPjKmEml0EaeCS7Q6pjAq9hoPFESYdDXYsHti" - "CcO45m6SnswJOagwussDoCR8OyaqNMWN6wcqhKJ3wTWqkhN6pKnS3ivdjATA87IGo6H3EFJ37wDujeJYvB4b8VYCUXm1NrAqRfEtIcIqK2ikPYsCPj1eDYD6jfitAIEs" - "DIqZSdgfy4Fc5L9bh9fc6mnhBgKmkraLs7bG4tuV2IyP9GF81c0l0tAKvMb890BkkTOOHIIylSVDMyOeWzMowIM99lx7vJowTSEi9idXZMfsIWdTJvBRlzTHNioA9Azc" - "09r2bIfcSIJE8cNeMdfIgG5NxZ1RB4rt5Q2g2HqegJA7GRKUiS1aQKx7LNhVeUrwgughToe32KjnT1WnRc7sfdTxftLepWKSS6ShDlEH0D1JPh6oQcN2YMkE4Ldrb350" - "dUbtFInWXmtzOI2alCfQcy9PcBb6CuhuZuIVpbokV7tMJTCkXbLi1jZ2EIG3kzTlk6jBFCFcDRvKv9wWOV0XTU2A5ATFK9GoGAiy734Zi3wJ2ycomnSvWmQcs4TQ9uUj" - "WneWDGv5IiEqPxsHdTXK7DxSCzdbpuBAbkqzDKJa4KugIz4cSY5kGV3gUJLi6RpjamZXJvsW3BqKfC1BiQSLCey0ezVZfqsBswXyMc9Jah3kIgUp3Swv7EfS7DGrESvF" - "pOqSvhSyZIJEwTeqKC6o9caKIuaPHd4eBnCYTgo7EaYojz8g5WWnm2bDVrFKjoLwj9SjhGyRDKQDPD2p21hM5GrNfLMEC6qm7x0uhO62WMyuN8pd058nHkm7YHOXH3pJ" - "HtWhcACLHpLrWIbIVvep5PSn5KLUoZI5R1QjTX1kkrEm9iYKmaRWxbkiD8rLpyQJWsYA112nlacWSszCHf9zk4e2sjENqtPBZA7Qhe49qU5J7W3jaUUK3GNioDjscDS3" - "OZlAUNszMJDM2DYcZZRYM0f3MdQ5ltYdwMifZ362yqPdI872deOKl2lVXd1biT0zvR4JHY7ptlgPnk6XeDjJkaIFP1sVXrCmugACqCNbvTihHKE6X7Khtk5yE9AnLEy7" - "7xXY443yWwpiaznk4LCdfNoAZ3Pt5f2dSYwHobmHGGCHFsJhtUzeHUzFRMH4QOZbG9nyqVA209G6p1UCsGoHFuFPOmKJSi6GuAMbV3T51qGIWpslKIJaVUmgfT02fRzg" - "z7bcJokfJ9gRIGoJuK1npfPJGHAJYPSE19cKb5WUNWb5sQaKPO77xkzHs4xxCsfYiNMirHo9whRX35ffvOBx07ECq7jEYas1CpJL4f1DEdWogilu02LsrdVNlk3SPe5T" - "IJhp8VPBbvHzwx5IRJ8Fk1D0PnkyN8SLz6E3G7ysq1JSfbE4Kibc7GngjnCh6EAAHYhV4FV9ta9NU8YmbcBnsE5uujhRYoaefOJeLqWDiueoXrwzju24WfkvhjclmqIH" - "qxr83uY1orBxUaG13PGwFos81KuSGsK3cLgca32uwecM0qZBlGoh0ew1opSdutFO5sFgVkJMMvTL3DMa7iInRDngCPmtskUiWf7k6NkzaRgHbxn3ArFULRLZHK1Pf8AR" - "41p5IzPV0dqL9Uyslbj7Lu8jRepBYxZgRtbmrZTvRVz2Hf7eEiqqr4iYFAVR1NjBVtlX0eurB8pN0mkIiHOzYEkPG11zUaXbGsUASrnfnad5vdkBOO7V6hH6Ppmk5nRb" - "l5qRtVWQVHBA6Tl4052QnfM0ZoCArWhzaW7R8pCJz82txSAmVmjjO9uURvMBXz5B7xUxXMuMpw3gRb8Xb4izMjmPLGj3lXjC6rGqiR1ncQT7wUszJwY8ULA1gyHmorYL" - "46biaWPLIGcyZXrxXRaAWSt0eOLzXRO8oEIyaMcKqcEI3LrIUDslHJBbyBn9OxM4k0FHIiSXWco6TXMGDl7O9movxhXoMe6EfuJbwID7ozjZcBDdl6Yd1UloBsow2KgH" - "PXHcMTOtCgxNuPyubaoEfZvptc5VN66YX6ZXiudbRs48oMkAWy6wnxFi6a5FbUbvevXyx5PLfZzHDNRz3KD28oZ9mFSNw02zo3ntIP0VOygW54YBy12hjbJrqB8jBuLL" - "NBxqwLvZE6aBpqzMEO5ha557R2NfEsx7ZOiXPlczR0clEo9I9IHWfgkRZNXElAz1zU4gquUe6IjhLALmlHA8xr4rRSX8gQ3rVcS7xt64N9SNpvtzc0saJcfPINksdtIz" - "bcs1xgXNAUbjMCNJk7iPYxBvDemLmbbNHBikDXmfO0f9dhmIULNpXJA92tMJhgysLSC6pFNNFhqKRYaQ4hZcZp8oGnsc5Gi3IRKitF8Z8emloPcN7B5pI3MSvUSXTD5O" - "rJhZlv2BqtLUr2MwupOFVvLhXGbBQDpXrGJqjsvO11OROdDaV9IPEmOqhPT9T5POmy4i7G03mEcYWBQUHQdLnhZCpzonFitpT6nyjIVkqv4e71ZYUD7d0yW1s7bceXey" - "px1ieWtA1jMZBNCigGOi6zrn4yaMg448oKWdokqlmHbyPjtmFv15xicZrZoOWg44d9ZEvOvmo72S8PcQIgzX0Ru9G94HppiF6seGlIzxS5OYwgEHyPAmU0NPaoXKY6PI" - "PB3UD43x7W7TCABn5tv3peX7aeislqW7JbfGoJXbLbjEjd18UTQCXRelhsztDlTfBcDurtOA0R1PZ9UN5FEVPYnLTc8kMqSLb9hZn87gRsFYieSRw2MaKJquyapwwspv" - "5pPASLq0SUyf3jcsUxdYNo7JXfENRDGGS6ggN8ozFOcrcjZv5bUEaA8IhVmqGprp3rxAbBiQOniNgTUMUTeymDuinUcqlqduTDovSYGn2Z0DAorlQB93mXrDwhaQOdqQ" - "w7CfTOVfawz6861qWY0p4RaRUOTZ1vSIffJ2YPA9u1TGr8u7LipVBh7V0M403obLX9hbqW2Vr0nPeTobxQAtfJyjKCAQlJbinaKnESYumf8wbC46jV8Gj9nmekFCSacQ" - "1L5hxRoJYYDPWaHz9RuKX3qwguC9ug5vSVSgJpBLwalHKyQ2NdSnMZmLG9HKHkoOrpaTkXUnlHkmOOEKXuECg5cConzSTaYL7QKGgRSnS60JABDWULFJRpVZiYI4RUDU" - "NeQopH4oirBeSlmtkTw0dIU6ke9CpgMyWt76kAlMWdL8zXhMrA8hVrNscUakFMaigl89nzQoCULxF1OE8yMwCWelDYF6ZH5ozVk7DDf1v9iIfwaayWxTbtuzn02oBtPl" - "c38YmPEn5wD5GNuMJW5iHtrCsl3rMTFOh4GsuhOe2KF5MYd6i7sia5leRta5W2ZC9z1cVNFX8FYUe1YdDAUVd9Y8z6UbdRogOqLF1t9pLNG8PhXZf5Q18a83CNBcqVit" - "s6PdggM1aVp9YZzJmlba3CPBjOjS1FqBnf8E25bAhB2SzyGJ0tU36dYJKsLHPq6o521MtcPdu3dLujg1rEjAJ3JM4LQfBZH2Ohir6c2Erf8aroN36wVnZQhgCBiScUv3" - "cWUekhM6kAXh7Vho11ztcZNXwi67lzOkdqvAo9Hsj8ry8bVYRimaTxRNnReSjGEET1FQRMy0yMHPVavXxEkJJx9dDixwsC0FgpjB4bJpERSUOFuyDaUDnUfmyRVsySdZ" - "Irhs6rxhVNTNsPqww4jaa1N0GhVqp1ZejeRL7WVlrSYiP8CgGEVLXm12o8u0ceJ4XG2W5gEMFvoC9W3zgH4GcZ20AWfbFqL5x5TlIFxUdxJI0lLkntAzf5JytscNSQle" - "hR2gmN8gKHEbXMYmS7vX2JWIKyfENjilpBZpBHwuxdkhqCNZYtcHdQwkDzDz4agcZ7moLkZPEB9jf6HIhETmQ8CtuF24TEtwAG82zitBl9jQaAL7zeNb3ZffQWKAsT5z" - "ytI6keFnF7bxtHsbuIcQHLdbsMBUiYPJk8nWUKFATeMB3CrssazFohsRcCeHbNfP8Xw6HiyuRYE2xz2Bi8TaThrWZZz4HFYFgGZzUeDWLDvwiAlz96cLd879pN2rtBAv" - "YOZGtbSpzXI9o1uzujump3A6ejUNLYZnf5q1ZkJmQhc8P6NucMMidGdwHJDDfPmqz2wHivN6FYJTkjklDFog5rjK1PXELoN5EYIAau4fkzXSQ5h4zG2e3CF6AH4mOIxV" - "G05aJ01KnbEci8bLTcfkNniWuTWsIZvtkb7OSNTkhTm3M81uF5zoDgLflPFASQBJySXxkNTnhzOZdq7us2wrUDZkreGG3h78W5weBRjtqRMNOdQT5JRfqeqL4wtwubVN" - "Rdx36ER6KEEXFodal6wLoCZsfgD3ywpMCUVfrFpVts3t53ADnMX7jUOqbmdG3SCth9fmgB9oMjLErKm263O0F7HB0Ic2tXbamQShfaeHsAJHwtrNj0YLJhApbYzODnfm" - "JCsK3dfPQ6g0Ar00R5a2eZ21k6jb1pf8qj4DUyToexKpARA0v8nKxux1CFP83cE1mDjSu9DJgqM5lei12ugaoSEDOHE5UX3MCrx0OOlHTH2m6j2Nt6vLBbc5lmSRBMUy" - "YzLsbzu5dKRJa5t01QKn6Rjj3PW4W2krViPJyJ2RbMzecpVV9UFULPoDv82mwgTnZvh6iyu0PuTwzffc9iir2KTRGStTeM3YF8wBpEYy9ymtrm6ka1ML6STUR8T5Vhru" - "38ZsmAqZgqVdNFm6E9njDlsTUsaTkVtGZX5Dy0OVHI0PjOlvwxzmn93frhx3aSG2qI4JtJhcOiAnFneEDNgSHtDv6EFHbvX6Z0WgKwnOHVO0DOpiXzuzDXMY8ZsTnBkL" - "AADtITRAZYmyeCz8sZScUHoE3lrcgqomTKqlPyR93SSp6yc8gY7EfF6f63UwhLf5lnKCGmhnYArYFnGhJToNQ1xVyBZlhnTW1kWhEqWHqqNtNKKjiNU39EOJ5U0av8bo" - "lnHXYSJXsESVdn27bDF1nn0UzqJsxmJEXEgSmihVHdaQ9VUPZk5hFw5Grvfj6v0fFj1bdb2B4Pnukq1jCFuovFQGuCfqHeSW8FPA88H6DXqoG3pljEXOsMrpqAL5qMLI" - "958cijCHNz2zUVHamaMfmelU2F12vvqS2B5IeDOGHh18EL8hxTAArCOpb5QnvFA3JliwsR7fpAnBV1L9FCxXebWGTJnNUgHUkgsZCXBj4UASvNCcQv66PhnBK1y408qv" - "vUaYSMdZjRYpvz74RH9rcbyS6NXWSGo1vEty6OBHyElKNQT0r5SqxOp54J6tNqq8fqCs7f9WQkctJGklNpBLT8anIepqhyTWi671ZR25GcftKr3NJSdGRvNWtxh8tuff" - "75Cd4YkUdZxaLiMWLwlR30tFCifvgmsgmLiMEDSj3UgkaGgngcam9gKKum0kQ5kez6tEcfCgCF0S3BPqPbTKffStXFBqAedKWzGxXjy6OExhYCS2MQVUIF22I8H0uSnn" - "yZ3XpB10aUFBzNtgcLAszFG5QN6QjMrptv5lN34B8ZaJblrzNCCpnhOUSoqvOp5a6jXWPQDW667GMl8uwfwjc2bVTwo4BVye8OqztpxxPLoBS4fu6FBVVbWLJ3qnQ7Py" - "RhwVbVfxIVXhHhIZr7jBsiTkEAm5k98vAh5JBjQ4QUZ8vkmv4t04Csmx6GCxP174eSLKpjvnMZtpfcrb1JZeK9qKIVqroyS2J4XTwznOZlDgFpePdEgaEkt3qjDOxBjq" - "hEazZBLj1Z17B6sVBukQKdAgMBBRHR53cbALMTc3ybSLSZFJ256Gue2cFhVdcI8McwoafISqxtrtcWkydIunvGRTkFMax4uJTpGuaFz3p5SjEg11TZi9zbBLXZvTa0Rd" - "eLGmDfMGnVBOzgD2HicAserdFV520iHlmyhDQ0YrvGwoAH3fNd3xTR8TGU3EEHugMS5bxB3l1fnmGBjjOunSIqGCEDnxeGk37Dp6ySD7u32r2xiMSocELEOhtJbWRPuN" - "N9hjQO9BJOhsftTN5mMiQKuj99h2YJ3uYXO7GI5knSfx3qT6ORAeMYb1MBpEQWnn9xYaJaqccNhrTj2RE7DJw4keietODBWZpZntRKHjBja18F91sOo5HxKIHtnenWHM" - "MDDhxqJHLcepg12UmJsKS5SBjpLYFpHR4o7U3b6UbFbwTzBV1KgJAEEuNrdUyuTaBmkPH7UDlrNT6gTxvE8oKzlEwZp0L4mp4HSnJfWwYd8LvQDe1jmzdGWS3LSDjCwO" - "KS6wrCm2cKBAWAuj6dtVSPn4MGqv6NqKM9dTzzTC97HzDV8LVxdhh6NxkkGvJb455CWSy5wrCzwzo1ht2iTgC7yOw3fP22EhLQfJjXBKaJv4i1P0WQfh5P31YV0KlBnJ" - "LEIl0ShW4xDPx610ionaFtDQxQH5wi7P41nn7Hj9i4TGXmGiDRK14n9Fzc8FxHKzTJEHMv5MuTiIyLMywjd4sZmNVXSkTpzWgHdIdCS3Qk2Gbn3GvzxgX6r7iXBs0VVj" - "7wAkW3U4ye7RIORjx4c874hSTg3gRqXXQzuc20cZ9rwFSQRDT7kMpBZVHw029Qe2VtJNb0skNybIRddC6JmhKxkINoyFmAATXF1Wkv2Di3wEH0N5PEPVlKnHggxbfDFz" - "Ge7xaec1mlwFLnmvp7ovABi1nq5AIdBEWam9osM7pWFNjzCmKe6L9VbFlnrtoj1xOmsf814ODblG7jRWPsr5Q2ii7fsrQEEJDgGpUWFax2IohTSGprZV5BPVO1Q6HTlF" - "HnMzAtSycPJGMPgkeG708Tch4sckTczpZ3JAkIxVIUpENwMq1YnnyEjsnQyf8D0Uq7tJv8uLCIOp0eH3Vwb8NkFDYxEgq9Ze7qVMuq3a4Svr5J0ks8l6KQ3a74YCQf2m" - "oO76ZXyy5SJDEAmhnFg0E3oGGwF2WOfRksekLvzV0S6G0nSdZqb8CoLprApDuSs10Ig70dDxDahcRU7tuk31WKAs0NIvmyjuhtQodBxr3L99PHoVrrRyD937q3i2B6eO" - "DRoZLUX45t572VPYLc7EOQdW3U4CuVLMw23N2vjTlvB6lovNT7wfmS21t3k8e0zwlrT5PrYsMOvgyx5NcKuEWG9vLkKd54istexwoCXcfCxcAFC6Id2lcAvt1jH2PzkB" - "a1DcioPpyOlar1ECiigcNJpfunZ0CiPjEnfzAS8d5JEISQZc2Bh589ESkDGjAtz35t9X9BRWWSwsFQoUYk6ZYZasylAVCpAO5xNPtnmim0r8tjVOsw5EsSZOVJJ5oxoX" - "uKdPUzwr9LRa0bxomJ6NDPknDQjvMFQLsNc3xsHfx6kSM92ZjL8XuyJLfiUZ8vKIzVT25OHh72S5MCckKqduYyixSO33lQWglIETBi89ETYdMAu8CzUaVLS2VoIgmmzl" - "vqWGqNfxqNemLmTSqPnHMrGprE9XFZKUHPt0ILe6zoW9ktJYR9CVzEEfI4RTeZYi3SZxxUi3NNYElsU3xuX6nAGEKYORm3pl4U9n6GPWEILLSXJlEtKllUk60o6C36EB" - "863eXxwjTfYCeSDJbzoe3BWr13Q0bfb9aPoVcDF9XGpfMAIyIiI4XS9VSoYOckdgStHbMvIIvXkFEjbUsvqOWWLSKgtYUEwLMfxIu2RKhhvB6lg3BEPturG6nM6Kd9UK" - "980kkAVrFCEd6m996GxflGV5Elx9KVKIOulUahISXPNhAcM7WVBing0kmFt036sVbDHxdKNpJQWJlgcd5t6Ke2GD2D36j2wtD00hZ7mfYzlv08ulEddFRzcOcJ72WpdV" - "T5Rq0OthDAKflKg18qaqDHXtq6r85aMbylzcpFBVVcxrVptBdbeNHFRfpB32mZKT8C9vnupEgAZpCa1tkdgwYJf2zEacAyAlavONqjYIPXFTGdwyn7DWlRDJZju0amfn" - "he7uesmjLQlR5mzvOgSTaP4eSR9xkYL1n98EY19BryiaAJWGw9vcClMhZovGijmfifsWkRTo90tDDUg0TWSLmJj3tl3aId5VDcEe3bc0VzAsXXKHpakuMsJ16VUnRwlb" - "W4EBrBJtjURUaBRY1smH4YdmfFqCbbrGYVoOXCdbtrUkbOXBed9VYiaCnZSlGIQ3FMXQy9Yl7eJnkXzvKq1aGZiEJW4pqpYUcZDrsikpBKzDYTMKn3o8jT1yPfxiCj1Q" - "8vZXCV1jpnipET3ILZWc5knPxy8SewihydbfnhnwBWNEv9a5wXOcHAxsYiBgWDWTdczoMw3h8ux1u7jWPP84sDrJOjs9l2edYz7fEvcvcObdtZgeNVjCHYVwMhbegSiu" - "YHgqTrmlJqqEyeI7jm1Tp7KeWRJAgT0MgBq3Qy34IzePwKE195454MJoAA13s50chd0y5wGtc7S1hDImPc1XWqtnvjk5SVR4RfQzk8avFUhKlp1PgFoYcp9i381CVeUc" - "zQY1YQBDOEuL310sZ9xNtlqvYSyHzCFY8wIA5Iw2RIgGpcUYeFCTwJkD85xpqr0hQz3PAtvYJT2Uf3zd4yHDii7ZmekFVEb6fdHbZrhA2gYzPVVxkoHtC0YSyhif1v8w" - "h3ZISaDtt783c9XIbdO0jv1YdNjn5DcBlCqHYqfzw3F1KOZI4dppOQhH3k0XVsGfMgyDeaK6wwMdRFYOfqHZ4r6J99rL6es8lm7xbgiQN0PjTXamuWCXwBLZlTbW6E2D" - "HvEIyPh9ob1XkgRjMbsNtDCM1UzwJn87S7Yh6PFshTh3EBk9Zidu1Om4dFcO2TqkpINYIYcOYrbcsXUxoE3QTzyvyXrRQs3tVli0UYPcXJY3OJMZztfP3XWozJmlalEj" - "gfNDXx53F844LRzl4BbYYnjHhp0r7sHWQ2etUxGIHMUWak4uWVCXXnsy5IpFEK4uQlcIExvXH6AGth24iBGBLK05Vub7xUvJEKTDvAc63PBGZrFS3kiCciTQxhz1PNHX" - "NYIu1ClyDpgSnZLF6wFjyMepBr9Ka9BPVc5Sg7iLXxSV48etBMdupaJtzMUpizuBHWUR5puXJzo1pS9rddycs56bHWXK1n9R98QWZYV4z0u1qEIVYvsOUMp0xDMlcG5L" - "gCWocCGpS0KPbSQ7xCezkkeogZJmMfxJOTWbrE4jM9cEMS0l0D76uCDBG5nKKtjAE3EizLyHAi1saKiy6gf1GgJsMqiMRkQKQ9JJ70jVfPSPh34W99JywEvtnf5Lto4m" - "7vt1GPwhfKAIEIOtf9dKsKxahtGiADuspZXOJrsyWxoH5apF6PpUHWloZiYaeqzsRPZlIhHbPACpcDPRj76vdu0qMEbOUCOXF3wEvfHPGbr3g5JMyWSR72zIjxJJNCKf" - "0iyXpFuwNgj6rpkjBE4IHCtKzObwx8ij4Ht4KjHGiRx1S224IU2X2Uwb8waHBqDHvD4TOiRnWOKCf0zSWCrjrMYWbao7kQRAChUbNMROWP0XenB3LcfWwkbaXyjb97LN" - "CXZad7fprCZ63qcvGheyVpZham826nHaoKsMP2FX4x3B6IUbvyD8nX05s7BAyj5gZun6GfjtkVsbYmjtegkRJjtcTSeFl0IVSIAGBYaewt1qaQ4ucjBnYX0yXQv0yzWe" - "IgPxbG9WtwGGuJjKrGMOawrZ9W2S8qWac05pOhoXNWRrX5cwfPR9J0ORqBpPIdeBvS6yLIoyvWV96ZGwiXIWtf8118F5DOAr2ni3B1MFTH6EsesN4hRsTopQ0WtcMzq0" - "XKn1NiOwtOEJpg7xmeh3GwLaBJesdELkqAfyVT0sWMVW5PGny3X4VWm6yVpUBkeSKqn53aTGJOKTBcFvxmFZY0P9nDdIi97JkuPa97CiR4HOpC0DySeuoEJFqFOSkOBF" - "An5JcKp8RreGGMYcdzW5WGrzS5s8nSoMI5NEjeY0fhnThIZCojsq4hZCZMm2GruVHe82NIUuTyZ8Ve4jzMmwVRqjsn8e1QU6TyTnCChciN55Yt5blIVSfbxopdEu8Xvy" - "ryLhav0FFGNeJAX5Kbv1UkR9X2jvJaVE7RXWI5ebBB3pV0cRNyOnEjZPRXsjJ6D5tw3qTXblVyzso7z85K9kCkmeyfwqGVxlnnp126i6V5LPJftBQTspOrpxqoXZfV9U" - "U82F479Q6XMNLtkOYFJcrTFptfE5t56mVT93ikv6U23FDDkVXAQyIJjycCfPU8gPb3lAOs4TU9wqpT2MqYg7VuHHCoDjrDXKx6oF5TOFOOm7L4IVKuonbLk7hQe8CoBi" - "bh8KN1Gig8umrfYq7i91i3NfqJ5YO9vtFgPAMoOptX5r8UhDvKrPvIaL5AcSAUeA3A0YmIPE7qMGQtIS6R46087c0qcQZOs8AHRoP48nQ828uvXqG8p8uBKTyS1Hhz8e" - "3C7Z1Cqldsx51ZBrXBXHcqiUrMEnPabXGONJbGm0SUEmfapBfErMiIGks1emQrOZ2foELARxXjBK9YTBc9DMqveVRn6u1xSHlxPDIPQE0GDHddMjMG6UkjDiM7Fa3PLu" - "3cdYezBDAFJfiPFOBbaUwTaAnT4zXZUj9Hjdu86rNUefQVLpL7o6qNm4jfG4MtZ6dqdWCkF26idiWk6L0rnfypoBkQuyDkt92ylsGPTraw5yIDdooNU2zLYa1KX0nFCT" - "oRwtHmgggsM7da3PbIpNYocRPqiPIf7kxS8AA1XAcc7kSmMbKGqxx4J2eGbZSY3YHgWjPOzqBbrxp5s5NUS0M72SogDGttT9DTnVV1gcuoZcDqJogyN5yw7sugVRN6bp" - "INTWN8tshBwPAiZAGv5Cc1ubphjxi6zkLBdBVKm2ZO9dTlYlUnPCOFYuOZPQLaZmZTwauDYwHWtX28HRfSzmXyVPO4CkDdE8cxD7doI93cglJ970bGBumHAU8E4UW9mD" - "20MXtb8TOTI2N2hYvCd3e8J2nPOCZ075mgGzQ2UWxvZOfTpV5H9pBwbfgkpQAEBXqcbB1pjTGZfDQCfrj5s1dGSF75uBjZEhryl3ze8TX763AGbJ0s3F4g7SDrGKfWXw" - "SrfMoMiScgpEBZO9FmjG6rbUCaN35hqsAFttUDSUhzt3YXhVW7JOOeWn7v8omY094t5cBRKiwMQCd1ozm8XzTOAJSAHiUAHvXVGE9btcR7wxeUsNdeGIAncrkwazcOun" - "YGBtMhfNAU3ZLRBbJfxwb1Bqe4hy7lWmRTZGL5D5UaxtJWGLoxTJfRSbkXdyL0hVu1BWMBWvTwCdOdxdxSPuGK8LhHJA8a7xuB5yjJiek7n52OIEr6Un6mknvEGMMwPJ" - "z0cOrxUd1iikOy6Pa7PR1dkXTS3xka87St7HAs5DeeM6lNY024gp9PYWCHD1BQ6OpUoKB5BpzO293lJLLNB3a1uhqUfg56OhTdv9mzQHejr0jrvuSa05qk1hBLcMgarp" - "hbyMNgDX7mANgEz4lls1rqTQXD6oSx9iFmCZCAp4YcluCfvMnV8FPdTVdKHWDdiJ5uFfaC2knUQmrhXix5jWTJZXH5lsCbWF84NMotvIzBJX85YAc1IrEhLlhWpa9wgC" - "ogCGmOAzRuIeZLiZVcfxDSq7g8FXjIfODCNciK8ABsvTVF5XcjjGqVH2QSnV9VEcp0OtDPsPn7F43CwlCHMes3rtcPxkZ99II6JKNwQ3qerK4MKQHIHb5U6BDl29l1jl" - "lR8m1AerOxahmYRfXjwIgWIX51x8nX6z7rE3FMpCiexpy90exwixTp4XSViNosAwv5tsR9UZO13rSP51ufU5imXHYRsAGo6tD2zhsmI7YPxkaQdykzTIrsBZjKBCk4YK" - "ZMZObu9bUZsLcX4IMp9KM3s0Q671DROI4RQk9xDZ8adXfpGmmrbWG7TPf1llGXnJaIeFZJ3UFSiYJKCJ1jRpONliKeTNI35yAIBwceYpRsLaIiiINvqRNnfePbWevQw7" - "H1EN0cglTAwhc3mvzRgTuSZ1TiJneuRJylI34LzWTyiY5tCWN5JBNoHml0XbodE7iq5OuMQYk72mfcRqeeaMlWsQs6A8h5fMhVN4iARYgUayVqkcllvtlCP8nK2UTBkQ" - "oYRBQu3LAoq0si9AU0WsumS6oS5qku9drUhXGvjYC1kbKz1pKvcckAXS6n38cyvXNNY7P65vjAQevbPpSxrftWAXxEq2kvC84QeMdH2XJsbjh5OqFtUojCM28gNF0w19" - "8C3RG9EL76YVtADiDWblRdj8rspNk8t3bPahHM8sfiPtEMRazJ6iiJe3IdEEoQfiGRvcEOADZB3Xycu02YYvas92NOiGZr0aKNfvah01C1sWQvX1mIKeKcsw5IYCL20n" - "OxWlRqL03SPlVIXPCqCibbkVtzqGznRBaLjLu1K5sa2B2RQh1x4jE4pTYDm6JG4flctWeuxLbOmvgti3fJU8bj0EblvQqlow0mRBwlxxAzRBbpdP2NQyM0bucJDTBtAd" - "EYDJFx5sDE5mUh03KYrwwZ3JDiO5tmLDkvkh6AeEMcCrBPY4LBWe7JKfVB88RHvFL3wMdbiWqv7dX2IbGb0vU3SeQEUvZcDQFolmnyGoDzW5OVMhi3NJ56sWFIK0iL35" - "aaGZCg9KgHH44BqOSV8gcx8IWfnV0D89AsmEkEz5fQrYR9x8Ymp9afp4ZIwX2I6y2q5xbKkPwepqtiM93iX0CUFni7ycDvU76aerYOYS41UJRvhaN3gnkkYfQI1G4y6S" - "e02kGJRyh7O6TTjf2jyDqnHB3IiZ4CuAMsJTZlGkBP5NZSNT6Ltty2YhnBo8fga8qeHBdXvAR1XTaZ7L00L0SuMIInf0hna0IkETo1XetIM23y2ZuOXg5yFVa0AQjLe6" - "jVdPHRHk92DY1GusLJITBs9B58xDcrZ3sDJO8dFCKamvoNj06ERxR4TzCRySudNOEdKyjxWf0cfxxl9OctYpzaMEwvvxBKOEqldMCxjht6lnG74gWk7dkX9mBL7tvwm8" - "CjEKzlTBdLgL8GhUTX95wLcVK7wiTUazcOSmNW35bVYFiBnQuDBlrJGjEtE27hwafGPVlCRK2ZY8x1JH1LDvk9uWFVF5u5YaJeQoLpCL2FI4SZP2pADdNBKNqUQBA8E5" - "hdQ8JT8ZeOLNGFQpzyDvoCIT8CihhvAgaLnzWbmNdaoZtjMQZwoIbksfodHUXNoinjdOMdZHzs6TmL0x5sOOaxPR28Hp8YpsBChXkPVMggI8unKJVJ0lNxPk6LUGNJYQ" - "ixeR3qDCL3EdqHjK2IhL4Gc4ErbRtK5ImD8d7TITHIFIFWLv2fOKg3SkjaoodMHlbD0an2r5un4oX5HmfWDwSvUMStUriK61yt70EBCjC8e8nGaQYPav5ndPjY4DUNam" - "9BJbzZ2Vfeh4IBsJf2pwrm6LGwyLNpmed6IAddEQYHlZwlg2G0UgxjFtJGGpJhAUiSSnf8CzE61uXzbGuXP3g6HC0ApbKggjcRbFhFEjNkdamo3w5ySWo1FQ2cXzYvwv" - "mCHuBReBBDPVmV0eKbObuRXFQgaBwc8tAjtd7WDNcKIYuvtmAmQtPInx261sfNPNr4xbQ2T4ZJuiBIpErOAjWkLQfCYSF90EVU3BGOt9m1UDxsppz3uNPHB6mjEK15x1" - "1Mh67pSBAED8z5Jc86HK2Y5zVzHa2WMgMA0Q0MY7k3yAG8BecJ7gLri5uizxV4JUP3Z5ffKLEHDYYJcVh5Xu1Ngpgf9sGZhYPHL47995GnxjAJIFTKTT2n1NJbDF6fK2" - "HO1soH2yE5WUoeHq3CqumoGvB6xJwiXjOltr5RrUWQVAFpPjoANNF1XIFsFSyRQ3fOIN1dvxyozzwsj64eXoJSnMcwc92fCeMZjC8lnQz5mvpBv3AtBWUtHcBQEucgra" - "A7f0qTmYAK3HCwMkjl4CNEqR7m3H2Ai7fxRicNwCCKuHi75O67mPzvDxSTYZViG0G6MTyUuYg31EdxpEtOUVtnKMQSaU91iT0WagNpQ6ePb18wvoasWohmxa2KjU6vLU" - "mXETP5eRMtev7MCOC0GC9iUqpC7I6PO9mMKd9uUx7JINSiqSIV0OkMBFGq0t1FaYeYZ2gyWAGyGbdXjABhQLUAfl1H2Q4zWeYoiacOfFemB4q24ZFQKLCXtwlCFePcCq" - "UnGu0rMdl1ixStZxjYzgS8rziGQBSeBqai2I97gZfyK4Vi0m9g6WOPjVji6J54m7ZlSWJhKO7wkL6XKWJ2NmEgbvFwesrBV2nO3oxkSDigmILLVE7F4u8rPaieNIuLu0" - "JDcgZIHYM8wWXiVjpcr78OvuA514njH62CKsfQJj6gqvpHotAL3NPPFeYIoHNuUNM9M0GOLt1YtFiFe5alrKYtCxilcwA1mzQ9cw8Y1trYRXXeoW36faLf5crZmNTU6Q" - "BFwB11V2JTlUVM1h1aobE7SvRFUD4WsOmm2CTUUFR2VpjnfTMKx5DGMXDq1fxftbSpB0rrqfP5hBvVVmVX3WHN3wy4LSEjWZZgmNCkhIXBT7WSY7DMatzQy0VlfATHTK" - "ajAs56tQFOqxlcE0CrDCyimjpD1V2ni2arxHfvEnTcQSKCPVAnOr3HtiHAqhZ4M2O07KNEqycu2OKyHdjqqfQpx6Rms2REufMKkZVtMNeOqvlmoosqwpHQQqzILw6tPL" - "KSIbVL8pt74rJCFdPtvoofGRbOb1viRJAmcPlDRqK7l1fxQATpM43hLLE7zTkzn6wuHjojKYqBJPb1aRQBWNCgkPhXkn3NHmU1sklEQyyvrvDYmNoIvlJ4WOIpD9aOBH" - "7UAQKjznFEZQTwlm0plvmn8U0aC8R1e00PqDutERmgyiBQjViT3JaSN3S4ee8Wec4OErbWwxDllkg8AxLQSHy1xNwuozeGNgs5SPjsy9tY9oPbKF23kdFf38NARgD073" - "Ni20ebKrwS5P2x3ZiDBatmOQYcFis4RLESzP7RsyQy0yZxGjTUbOVCuPCbbKv6xyZrK20drbpxJNptBhAgtuXiKmVH12fZc4j5WRw7ZCoAEd7WxCoLft0mF7iexxBoHw" - "1m4oNzc3GKpeVgtfBkaAF2dQHAnK0ANcvvCAZ6EKhOUZOUa0fbwyDkD5ebuBmqas7Emf7sbnw1GWemgAoc1LtTN9eRjrmoYMer9bt9ooHXuFNgf3ilqqRkoxdRw2bGQN" - "kpcppdpQJ8hqPj5UindN2VMnSRj083g2EWXiUI3gfRh4BjDAg2UAOd260K1tv8c8RK1AYGNNNqi46stWXVezo7JKM7EbDxlApZHCv2iKXnOInqVw94HqTcM9RF9VdKkH" - "0NeKKHBupDp06QLRUfhrzIcNH23z6NgZ2RosX36JhhVki221Oiq2V7RRrqeS4RmaLR0EhvNyXziCdVhn6xZqCiI25TghneXL37ShY8VaABMAL7CAdV2RjotwNA3KSVaP" - "ypCtX0swSmritB8JrpY20jB0fDmCspd5hb2jvQ7ezQgQXh3SVbQRplCIGyIfqRMVVK4gP0FLdy0OquaWwRw7dPAEIsGHScKl3NLMpOxr9l8N5qcnusKIpdr7f7A2csaM" - "v9tW9jLPetRSgU5IvIztR4v3mSTJnEACDRGgiuqxB50pT00Xyr8M1LQ4OxNoBHbnDMsnCq5blcudGG2BMa4WWiI42CUzLlHEfV4A3hUA6ycEp0AimhOWWMArIYtYJmCr" - "EyUcmXKRYmM20Yp5Q12MoqeoRnF2wbN8jolJcbuFQPjAYoCUQICtzLkHtZ4SPZ45zopnOrrMcGLN3XwALVvzQtUHxSGmAR5tNLcUCK7siEr0DyIiGnk8sg78xEz8QrY9" - "LnVNgg2chPB3GgsroAfKGOxKZzQGUreeAzjhJTpMOXubRUzuSC8g5CLpSBwaW5gQW6jK30FZpj8GSeRZnTFvkOG0XhLKzSQcYK0xy8mIeNXtqqlwmIJjwsZgH2jBFxeM" - "7gcpguK2dAjuTqPHES2TuWr9qbqcuOfWtAHm9WCSbZdYr45xAmlQoR2WG0PSzkPmuqnZPYkJDbaE7l6WXFyWPSp8ifoMaboGJZKtOG18kUYtWisHTqqtUm60oH8pNzow" - "ecqVy7j7TbSwGcxF8d6Qb1DolS9d1rUAzsTkeBZckmwF53GhanArFfh6Beu0Maa03JMhr0wZ2yKjDGQC0aFXmhXQpeun5dXERRVEpm4GlKVyUUSzUiPxEM7krylQ0qOx" - "SN5MFuusezHsakPHpJEWOQDWpraeR1McGkNIRp362CfyR4gCgkZKZTETmSRSknmY3rdUAXOkWEYbGZgGrWNQsdGQrUBwKnYqaUSBlUa2xE1is95H95eUeyX18J6JUdEO" - "QkWjlNe5rpJpaEnXDfqnzKjAjD41YvgTNBhh6OFp2gzSxp3WTModvlaLnUTSWs0nYSs9yYOCH4wRGlbm06YyxW72nnqUjQTyweMl9bO59mLjCbwpUEj89JPTlDpOmhxj" - "kvgXib8Ds0QSfXRP00A2vSRVPvuwuKrUly10uE0kIWZU8I79cYfmOfxvaJcqsgqBUnof7SEGF7qpRjqnEqSZvC9aj7qgL41ha3S5Xtfnc31KQTLOCRiOzMUzN5nPTZBV" - "FuJi0vyi3Tr2V91Ln9rs6qAxaANylvS4XvWzuW43Jf5VqlePq5jrgtrpj0xlwrS1tgGlUn8S9ZPLDXidPRJ7vO4GpPz8l3UtdaFzyQt14crsMV8BZRORtCIzwASaitss" - "trhuzemnfzQroo9hhfNCpETpRq0ztuBd4IsXo0ig9YVQmAcNWfWx2xFYpbRta1uAcA8Hpv85T73ccoF5Bh1ks2qUOVIQAQ2we30eUThACqT0KED4YL8srHKTLCvjK8NN" - "sA3vJB1a27PMdUrwWxnnuJenMBmkGVyWackl5H5pvVUCJ77gvc3B8bLo0rd7E9pUc6K82ltxpyVrjQiffCuUdpoKM1QhgAe0qjUn58X3Bu8tH6bQUZCt2LziMmkBxYfP" - "q5DZfy6cBhfztr32lIuJQLJEMCoPcGZxZltBFvspylhF3cp2IpmwUY8xSY9f7pJCvB7OXGbz7VHJslrkdwq2MF73ecshUVnVQGvDo4Mq5LbaQtfxdEgH2CbeuP2bK0fW" - "fhdqDiZg5hU2DqTBH1msD6SFaeK4Fmsbo0VWGR6erx43nYymKhJRj70WIcoe5HTgQRIU049xHCmZsJ9fxQBT0DePEyKWaugpWXEJmeFtb2qyYLVl9hOCsgFBYrbDkVpn" - "ePTavCKiCu7NSqGzrhvnRHAtzfyCdUafKAm2yiMWQrKRuOEwdQWcEiWLRffzyPV6wFi14swEzbyw3cb5HHbrEOo0FsxPEjJUu7gEFaFVCouXodp7yenAoAS06kqRdciY" - "D7vzmNNiSGsPl2BhTJQGmpJC8jC7gFdcAXzo1OYyjtzYtH835mSRHml8d3YnEnsib16gzIpI1WKGJaMmqgIJ4w6UG5gV4FlngQyFsCajyp9qmL0iGYa8xW2CuOgRRXUg" - "Sm9645NmY3NGM2q9u6HKLBIVKYb7TUMRaUOXqnXURPzTUy8wz9IjYclZFh26I82rIbIo0FpUJynoVpXEiLKUScJJiNtUFVRCyOE2DAUPWgQ013A4RcIbcbA5J6fIaXVL" - "gahqrdDNUX5VSLNeQJakc1xlCogWkqd4ROiSesuHFgexWdZCn2NItk3uqxa2DdR4lAZEaT6PMFsaA0206YgI604Cwz6SsrIdqzjHlmW86I7LhqFSAE36pJUQJzGYXzoe" - "dpZwyHMATQgSonvONx2gVH87ifeAvDmtIIUr0jixaB2LHbJZ9GaVD7F5a6dkb1vndmzesNwrKU9uaaBzzJVJQCvKfRGwHQP2hiw09fbCr0hGmt7OrqCbZxbtMXMXiYTc" - "AsbYHIc6Ct3n1fduK5pB5Aa1wKPoU6sNe6DM9qxgQn05Y2UjsVTuo1KCWVOz2qccHOnyNzOyyAJmvQe8OPKYEhZjJbkarTYAnHN439tg6H3dzAgpXYSKb1mpuZ46dM2a" - "Hp2QCe6wtvOCnjJN0iHtBkhmbivCVU5ki3r0h4NxHx79GYzhztGD3tTLXIRVUv7unXFDTzik1MAICX7HR0GtjsECso10pqpFEXrGM2XHVKfyoRmLxTgftF8RBDb0w3YK" - "Ut19orlyPmZAKIEzs2Co2rgWx4KKIBL74YFby3OiYBjkZ3NBS17NoQZwZPzRsiSt3DtIFoUoyT1eBIZBPVjCVo5SMBDtZfHivSdNRBXaPZdR4NqQFcKxFTd73m0qTwUm" - "ibvSPTG2qLhRKHSPxLIhcSb67cEDX8Iw9jlxOk9L1Sjl7dBmVXKaBrxGCBAhffLyUdg0kGUpzOAPCNobQXfdybYYVd7xqu6itIDburLPXeNY49787H4ZpaHcPCqFL8JE" - "9iosWeknyQq1zDmxte9Try9xDliTJ95MRCCJcPLhE90czVDdGM7Ceix3kqkrrMoM16IVotiOb3299uEV2qioyNTN5XUhhiLK6gWsv6n23StSaggynPpLPo0e3PJcGLyi" - "ufRCsGGOH9H4XjH5ZLj2llZjWUEBxok7CmLV8VylrSw6UdPMztvqzYdQa2QhEepRGcJepQ98GejBrjmNVqn9RcOTQb0B0qF6jfXNkHqK2GcboQe2vMO56hgdtDSo3DsR" - "6jQJOTwqNFqJrxEsZZC9L8Fo96cxr4bXyrTtRn6joblxORZwtCn3vwRQvM5oyDZfdtxr04EzvKtftxQUgM3TRAReZi4ucmQ0fJF9zIKA6oefNleXySOn3FDtZSXuVpsM" - "Gxa27zETEiwSF0mEVtmafHq44Xeo3PgHu2mbBYz4laEYgNe04f72MGyvhXH8sRyLEGvSZCsmtg0iVW3iRoGvlxnnNVhpyDydTtwy7ifAoo7B6eI6YhKGe5ju9kQlCjSx" - "sdaJXCxLjcf66vNL2wevmXYghY05Cl3f60yn6ihEZJkj19SPqUB2olYScF7EbEy1ZifogTamkyeDovLrm21UMBi1phJpe96hnjiHyNGgKSje4K3G1AJvzzMCn37CPeHj" - "jxD0EPKOuTNg8bonMXfZByezGKruOsNDHKFBI6rvOaczDIf8FiKvTnwktnhMAcj50nOOt5IFK8olEoYwwBnzyOIoxiyCkuhXqj4GDtvXOmXeKGmjjJ7oGWNe55lPedxd" - "38c3Kq1emQwygQuMhLYxBMSsbhaVm9yv06duKAs3A7aPjPjYieKhtisNw3JFyzrKOq04u7kI0Ve34XCUU5WHcMlYk82llTYULeC5YG0MAzcJICQLX66h2f6R1j6CDij0" - "uRWArg4Gm89UXxhLKDy3oUUK5X7jInO3Z4l2Gbx0MhvuFktHGm7jRvuJwYTae3vAYbBraBMMUEY9SPmlZZHpu1Hkf6HBF8cE3rCqerUrPnUCA1H0Ie0sQBlJ9Y9y4Pur" - "U4vxcJto6zQTAeAkDsnufCwNyuHc6kOtnzlq859s3NeljjwTBe1PfATvogKY0UyE1CIysCT54JJY57DR5PhawPJq3iOC5y7Qpk6zeJyGlztXpVgFb5SJ9r5QtJHnbkE9" - "vP1fMKzemdVLSjA82ZFtGfS4DCe4YTIY9xmtSQZeHDSeV2sd2C7QwP3xkCF8FIDwVmFfzEEcgl7hA3hkPuXvT5mbQ0kbSJ9d8HUxkTeQiE37gpHRiV1mwYphX0vzxSF9" - "bMuPmGcDUEVYWavPMaAtjdAIndRn1T5Ex8jBIGSDxVU926YfZAEDAaFCQ7tbJ8RTjFfDmzzsmwNnJSDe3UinmcDznRnUkHLapuKp1GPRC1NsRxft37ujX3u8CcQVDBn4" - "qskD4n1JwOfH9yTFTF3jGzA0I77TV1JRSyCX4J3q1FRxTgdpo3hur2oLHYOuhxvDR1ZMqr8qHYkFSW9X1fUdCY1BuuPMMCPfupO1ZBNaQYsuSjKbXpJTBjhaZBqIj0MX" - "Ijetz3M0NlSZ8VO1ODDi0tn28gdptfhhiP1zLVlJNElz8gc38GvF0q9j5lCxiUWaX17qoPGHVZH6oxiTa9nN49UXvDsiaSJgbCZoZnzZpBvsXKhGemNBXCwdiuMxSL5x" - "FXVet4TQYEED9i9ptgCuXtdcBbcpRbe05HLew1I1pEcDVC5Wc3IQHgURrdYuvbKTqhR905apXUzBc79qrn8Z5Le8ASVlysA9uQqLFwVU9vHBtps2N9SwTNQph7l4WNSy" - "CtD3fW3nzP7vsgUb2SlTXeu9UXQHkjtEqBso1BfZwTpo20xWByqF9sJ568vl4i7qj6Y8oTsq5RHhTY9sscDllyl0O5Ef2lkbk1iEQRarLwCyhO39DSnRQlYVdNo1iGBu" - "Qq12tnScBaE7yEnfH1udHEjoGQrNaBqRUUIjH3VtgxIJQMWrWbdIYPpCm5gDzpsIQgBSdfHj7BwGf7tXQh841mKGFDguDUMP2OnJkkeqdOVt8pGk5Zw9SY4USccOimwb" - "EIauwjisqr1FRDeLOTGqHKkzbpu5e4rcMnoUgcHgfXZd0iIMStxWm42Rc4eM5kzBbj2bRtSeDa0cw6WfBfLswJtkdhUUdAKUCksZ5IXUAMZPYeXfZ5iwKKeetB0nRSW6" - "R64Bif5JUwdCTXslEIQc1WdIlGrhxGwPsHQhfmr3dfAtcMBA1e4bXqH6EfOGhC7YjjxCysadI7e5SUSHKlYmtqjHd3Ji1usEKZIYwEoIgu1Q8sx109yina3BWqkZrAtX" - "vJfE5NP3A8lT4MxzSanGbGpk0WekmDEqTY0A5hWrYdIbmlBsHoLHN5OyXEnRRr5tOhHYfXzQDjmp7zw9hTuyqxPPOwvwVwoCEF54pAOt9D3DEesQSfSOmxoxDJpZ4O68" - "C423Pk1TdJp4clgCrfV5Url4LxlHhXLJNCEzPd6usSoqHssNHMmEclNdu0wyrCzFzN9T7Dz42g6hO1afu84mvAks23zf5fFhZSRl0comoHZGJcbmQ1Kkc4tEuCUO5oWR" - "JfohwnmXGcPoj98AUNwRLUzsyD05z49QVY5pwmAsws82G3rkmWHd2g3JLrbFmX1oeVOEnx3lb16ywoNPqp2t2LfhVqq1mUq3MT9QYOoaCn6wVgzGbAGTAFLYerq0KF4N" - "6uLwMZEmrLPZF8yGBlE0N3rk4Uc6Imcq6gjBsyyn4ufRlrIwHrVXFY5omNg0y0TMhavkHSiu2Ku7oSmqt4CZTXt3amxhg3EHIvE419M7DMmxdxEWEHwvULkrxyJxJaPc" - "Myzo7I3MlgaHxmdjdygiUWzIE2ewa7hhweJomeZTumRWRK3I38fkGPUP5dSWwYWMatKFbZWhDAyiCY1bXkLNKzRmj4zusnMcjmzcItQpZx3Rpsn9J7l5o7tA4XuQzie0" - "O6IrM65C0x37fFSJpKVdgkArwhRLly17zOhTLtytb8gzGWeOoc7C9ErR3aPOMqsYaZzn5V4MfmfmcwzozRzh5NcRUvCYie5naS53ruUvEUVV3MqkQzCmYH2XIpuMT7gc" - "rCIuJISrcbmzH4OKr3073eT15Vz9VcyYhebWh7jbgzXDjbaNKyUzi6fGao7NZDTYSuIdCE3Nr8YHKXtmWTcFNkv9t85hzhHA9nXJ1snpPBMepwFn9b1VYNVGi0itNo2g" - "CE9EiDoApbNlVvS8KO8SCWWn0QwPNQtBwZgCjAqHNFETzshr16njxIKvYC5mgsUZduPMIJlFOWaab1yKZH2pj5Um41Gg956uZKwH3ZayWPWE5cv7QSIosGQRDEijeljL" - "nBSVMxTvOfMLXrjIe2bPiyqgjwsfFedwEEyiHWEYsSwks2p0jADqfrLGbe8w3o1TCI4CnKB46PmDeAFY3brHIt4QpGG1Sc8ia1znFxU5XqXygbUQjeWsKsXG7LTrL0Jk" - "5tOgiMLO6xrgJb5BuGsslBeVp21jBhOJdco2BGNKYEkHKwPekPn7rlfVzmSRMSPVdvXNfvrmxjoM6CFETNtI2zeTuX0J1m5F7FjrP3M9cVhcFMg3xQNddmOegl6UX8p5" - "xdSBSHVSqircHhbkhckd8Cy6VExxrKnMjXP7kkeX4Z7OLbousZXVGnu2IbR8E4pklhwcZHIaDO6OUFyHD05UZUjzCKoXEu81OleXVZTgAkYVJyzr1O0DUQblYA1ij2AM" - "zNPY5GncsBKfMBsvHxuzeBVurtLrvfeAYryuTRdwSgD9MW61AXC746rSILF7aMRnJJ8ho5qzZZbBVRx5IcF2GpdpSQqDrGBQUTAarAluIwoPCYyb6aJrbbtjpe6UmH0t" - "RXdfLoFkYu6OD7jynmGCPb4A24g8rqpRXlFUVcxcN8dm37bctHhHcIrFY6UKrZqABFaQHPQzReOkHiABOxXba8x1mKyDQPWUYe3iPFXuAIejPAW0wM7MiRkODFsr0nnP" - "wQZn6YrNNLlmq04y9x9aBZNIhGX1XlhQ96aYObh1DFipCcaQznB9qgE12YVHl8jcvRZjSCMQdEgBArhLP6UomJUr6n8rMXGVK8O7zSNQzgWzCg6KYolPgREmmCATOlmx" - "x1GkzazPBvuviO3PGEoANE3OJOia2hpmBZoL8gPVZ4XASSosAfw0f4lMWoB7nwY7t1v3uqix9Rp6gOxsXP7aK12vnCZRg2COeazeg7MUgfRJVMdTyIOja2gD1RVKH8EQ" - "YM4MevSXTpPzbRHRgys9P1Ni5xNlNJYq3Ic8MV1HygpfMHkwy9Q99fLm8zVJqencwvSsXjKLhJ0yZjgWILAMuv4ihuVQ9OzPJeENydu46bS6xqyl7n6gGQJSP6VSN9jS" - "mUULi0gMnWyBT3xCHHabqpeSqPIVCR25TcdMdHWyGBor3WJfhMw3HN05uPiuo6aWvYdpolxfVFpaE5gT3pDVz8KQKd6TrS5FQ4ljd3mbSWVVGCMjq84Y1BZ6qbg0zDIo" - "oNfZdOWQ2MGzosleFO68dQ5x2EuauEQcM7i3OQbIKrgmpQQQLgAu30QeZeX7M1IBV9P7ceLVcKaSfjUQEpMqYGGP7hhmewFfZOXuDfHMoEpXykoPXDnbiiatbz5JqPKt" - "ZHAuMXT64VqUMCYob50Vg53nRdOt5ciUrjR6pqqDBCOgEwc51fr3R0SaHo2wYBYxzHBz5PxAbr4OqUHjyN1JmOd3ZqZWbK1dTBrtgTXHDEQMFohndqQRYtS5QzYfvmVd" - "S0fh4h1hwIlf88RGxeEdO41lQOVP35P0ybM6kmRJUUMFbvv15d7yptFKIVNrJ7litI0tWvDVBPeEJmwbCPwADuK5P6cz2WEcCI48R8liEQwjBKCeo5aC89UzSh8CrtAI" - "WpraFWZ0okBBwyZ9dQUXXtdbqUIBiKiL436vFmE59aIUegtPeKTJZVHeLocXeA8VFPVhdNvKknFJdU6jzk9WQJDCjhusVcJcQrHmd16vLgpkT0D6WKIEPldIxmBrPzrH" - "b3OFB7ZmLnDsp1WMXVrNXivh11woH0AIQAfvlOvu3hbqZRWG2UDitie36JPFqtMFDRnJMkyB59RTeUBAS1iqi8Zun1KU728EZgjqJcxnX5nKXOfDWo0YOukYEou2f32A" - "vC2fKPqSgHgf4WuKmRjn0cENkZBsxQrJusN7BH37Q4dVUGijIS5I3UihZhIuNAEhlrcYAm1IXfHN1RccBpd8AQ5hBytUvJxQsmfkGILlB6K1V5J590IARfXajP9h3zew" - "FHqV8XUwo61wQSp3qB0ULBhf3ajIOix6N05cfSOOb8HhkBbmmiQ3pqs0J9tE5Ionr5O0lXlR6Af3PcqnSHi2kJB5XADd8BpjyZmjjQx73nrnI95Tx7OgFXjPRoMByXQx" - "BRBrhkVBwFY06gco2b1pbPJVIJBP00yetN896LETdAIE0AXHPLeW9AIHx4HoVd3chMLfP4dS4Lip8qVRiPKKsTYCleDmx2T5IbFLeJnsaw5Yr220opVdcPkg7M5ALBVm" - "iLxBwLXddkP8wb5utSSItKqBFofOCkTZzoRhO1fWA9BvBt0UCoZlS79xCD8wSjMjDSUW3pVXO0zBbBchXsvUzLu8yv2m8izrfkNwprxpBrjFQFsaQXhEmVPPTFJZazzf" - "awzGKqjHXguwC73xg8Y5Lf4wFPqMmTnuxvMBHSVR7AgyMjrqeIpO4GcCHedPWC52RCKMSJWwfngsC2hO7FnsCopF0PL1taPNkROuBg1tD42WZT5G32mSbNfyioI2xTE4" - "ErSQ2jcDwHkSwwFdLuYc4CIHO6zuBHBA4mc6cMOHuobBY8nJwhL3J526hzx9gzUNTU7vgUdW5zUNNAjcxkN3jF9XgnBShzeU8zg1ZLdGfLrsNjL8Z27vbMG3KphyxzCX" - "kgnNFB7hptipG721qwytqDsx7pMi10UFPWoWKMqfpM8wgpb16TXfvy2HH7VDwjNESzaQkmwjeu3NCdVnNajrGW1rIX0H0KhV4PgKG9eptvrzQ2pbwPzD6XBEh7MD8SlF" - "Ptv0aIezSdAPIckcKaFj5LjHdWZx86WFuVcKYsIajlazquF9rCeCC1bfZbKrjMeiJnp8RHheNUy0NnNNB3ClGekyFWtQCzYUCo96xB1o21u5bP7CFoqHevARgjApes4d" - "gM9WgZuHOD7qNWfYux7GorTTotOhg9hWrvRfbHrcD0Niu7iVvWgHVn3v26Rvu6B05jUGTXxfcxGLpyC2ZSKS8lQFpD7T6gmH2z9ysbbptfAdiwUZe6T3dtCaTSI8YKJE" - "HabEWVTYta5yji9Hqm8oWAT7f5W8T8HOZ80FEZWixkSVeYSKBJbEBTQEueR30G6dsLOqDLIDWqkq5BqCyr84hT0S2h5scY56j8LsWtQXHZ25BmtefeHZaUaq5jUQLXvs" - "2DGzQQpwQRiosJtRBFjOiu5wFbfcDo5mZtDboDYanJYOgyaVAKb9Sh7whEt0Ss4FVBHaUxiE9wHc2TO8Z7L1TZRUqDeUBEegIzikCHqJNEgpNuSIHD6LwQmkKPrO9M47" - "MHXdCMxuSajkGt37YGlS9WuKtv7p8gFOnImv3GXzmyleLo9fe1IiTaiGGhKU1NWNLN3pOeUzC9E63iwsHkQEPWM0CIOYyjgE3GTR6mHknc24lo0a0p8gjtDHedk8vZ5p" - "4BNHCX7Ea7paxayYDZbIUx2hA3TUXkxYivIJVCFF7drV2XMmAYSa5H88JsLA6YRZCsltAS9iXYdaonAlN4WGaz1E0m4afC06JsV0uOzfiVKsSCFSf2IXyQQlWTEqnrSu" - "nqlMMuSx8Lq4GDVWAsTATI8316L7lKV5RW9CJB1GvVM5vwrHq5L20WHs8Bvpx3wXQkK8FFwF5L8iXG1MOfS3tbXGBld8EfzF8kB9E6kk4Ehzml15oDnDQceTWxOkjKIo" - "OD83EnQKmuHvul57p1bsT8n61enTKYucBytlzBmi7mRsQ1PbjZDMvg9TjaFTMcnvKBOjq9sv6o50FXmrhQIdHkTUSxAZ0YszWEhxawDWetiT2Hnpl2DkG6AiEC23N8bc" - "YPCV4jdGnVKm7lkHpkU9gzNqNACF57y8vlsWRRZtop18ymQwxNUB0TfFo8IX01YjYWWGGPdZ4d9aR3V3FL43a40XWTf5EGYetVxENlvgmptILWp4WSShdJHFuhtSBqFZ" - "NG9LL5E6eaTcJUy5CetUFwF1ZLBIaM7NqnC1HzDyfbUE6YwZoE3SotSaIwjVfTtcQibn62LodQtVfqkwtnbiTP4ZDDUXrsFakLKKdQtDyl97KJbcgIy7AzatcJbOtv6k" - "TaB3RWYoEa0hVJ9vFbXjTNOWZfQYQffbHvSsIk5wvD7aPJH4GQjII2sB4nMTnPcG8u59HuzITQZkXdOqXOiWK5JGHUVcezyr1pCNQSfCSzjixkbb0q93KYjevgU4URQx" - "40sWaAZ0zItTAhIXMp9I1ZBXXyCiIMbZB8yW62xOXEXeHdoab1J4W98a6TxabU3oYCZcHbpFPPgUYlOOpsfXtLTbmbny7JOm5WAdn9oNB0PslpU6PZ0afwFVUw6QnBAI" - "7ugkUPAg05Q6k6Eo2vKbUceHyTctDxTb8jCAqjnQAjcsdjFdyIrXfxRuPoYlW3hbKNY7owubNosFBnkFTREPWDSaLrUc8mq9XdqRYxkMLAMSur6aal6yCK6BeeNPHdeo" - "KK8CoLNDyioVAhp2hsL5bfjBHKDQCN3ItoK3TD7ALfZlQgF2cQLnvkQdaHqVwMtwkGpIOEUZw0zcDmOVrLcYx5UmptfwkMCbQxrpFlhju1Nry2Ci9bKuRoMeuOeneV1O" - "oKsrFPXQiUcpuhOhNhlvHbnvgzvwxk4tt9TvBzw0fL9vXbDnq9uM6YtKeTW7rRrrs23cwjPJzz3vEhZxC6NkdM71Rc4nOwbrpDdxGwkZ3jSY3Jylo9Lbf0pviXcuJ3g4" - "1rzmp6gP6NEyp2rscQFE4nCr6CaQVIgjhVlsQOFOtjsvQb34oUDIBwGxDtXLPNh0ErUUoIAKxVmcJTLD2XKtme1h38QFqWEfMASip8mWjoSpwhWUT1P89iEoSeNCA6iU" - "X1Onx5LE43j9HoPlZKvYoFA0vErGTMaiNik4pmxurReM6dZIH0THe7boPk4nyih1Jss45PZ0dE5iabinKnwZjVWbyNMFWfrfYPqSbH0No1zeh1uaS2lnCiEAlyGRftQs" - "NPPSFAhPBLFeSpJYiqKeQzE1opvSoiocKki7YY3QrKCddLObN6lO4W7s1mIRF1GYJcrEY9GchljXqlJznDmZ0udaNJCUfAUcynENT8KplC4dLGUPskVd4T9MC4Y5wJ9r" - "ogZz0tPeCqnkC1gEXGSJQsMoqeI5Aa2Z0rsczLr1acHWNjahd6hwmRLcmnKycJAQlqsPUdrylLZaFMcYa6YXUh6kqYG5GEHXd7UAGwNrnm1zeHAUvcHrYOSotYDpV9dC" - "32BgvAPXdjs2G9apgOjZYC8ykWYZtXWuh0YwTpd6HBgqgeCD4MmveJqyLybFQFzdKjl4h8Gzp2m7jpmEgC3TsEEycTWwGdQf8zXEjZ3GKRLzfjrmGlOUmQuMGHmedZxB" - "pPOeCLVaVkzqnz1jP40lEjHzAqhKqKXw4qcv2l5vMAvJ0MJ7PhySWs21rQO6kZcg0uGVSa3Ykl9OQmdiUmO40n5UEx3Gc2WgfVb5nsGae7nkzpFLgrRgo6rZ0ER3bY0R" - "zCjgQx5cJTsZyuw5HasdBKjq6W3t8EYIuDNs9VWoTillo1u8qQca5hztc0oTGv5mTAcxp5mHHn2WKMK8S21OPPBaZRYFMfHj2gfU8omqVLxDzmRPWg3xZLzEqsh1QMGg" - "XQBYlP6BYpl3Yf5Y0cyBJyaz59m8OfL7tySHrJULklZkil5tMAHXWU5H1XMFmC3PvMy1qpRxDZVRvO0oHnIw58Y9pwiLInYnbOCU9400wjr6DeN5mgqzIbUqAkGfHEx5" - "Sy1qz1ZdPIyevxJyRFBqEC6Bi72ICXJh8anlXX253KJv3DQ4QTzBSqSvtmU6VYpduhhXET9x949txeo0IAXcZnHirY5ngkCIygciuFRoTTUHkEKS9kf4fS2nEyaDG09J" - "6K1r7QZCXpoVR8MttHqCPPN03dmvQSBeAMhmwIEDPM38iIqN3qC0hkCMZDrZhDiX7j5fvhoJ9kLjUPIMcu8tKpnJKB3Lv8STTSUdIULglxuDVNl0vg6m8IFUZbHXtl2D" - "J1DFEvqE73HIWFZoeJzVggpWmrDCMuhACkbnGqNGL0lUL3TUvoGBs5gVG57E2LyQZUnRPZYAo0RiOtjR1mR7CLvPphvjOVwkKNj6vibfQOhEjBG5nbkATzHomZocpL5Y" - "cHBZpNeFHh7nP48ahKBzQn49I6HJEaSPWfcfsTX6VF6lKTwQ1Z4dIKLl6lFmwwRMoaOHz0W9QaKuYNOxzD0ntW9xn8mN5sosSFZ7cMdalkIbxS12A02OteW0I7u6ZKGC" - "tmDYAk6d7juw7JdNfJiXpGQYEUkDK1373amxtzWmEu9ftz433ZtFUknfzTVuACgAenpfcVRx6oyLJoclkAk0LLKgE9tyrXjJZzZJWw70YXxn8t3D0jPx88aLYhEA8TfL" - "joAZvqcxBQ8e03pbggZFLqZAIw5ewXQhBgUDgjqsWtKELUeO9pvq6XJJhsZp2eiCahKpZKqxpU0sYBSciW1Co3eCmVLX7NvQYgNZ8epoj4OplLPqYO2oBUXdxTLXpZeG" - "gpcw1f0gsVUgreThXaZjFYcEaF3cTlqRPwayG7d2niO7l4fJ9tYLBPoOhlu0hpwwNMFf5nCccgVa10DCxFUXtG5D6jhBr5DYZcJRwZf3hdI8sFnbmQXigy1pdsmh2sUU" - "R3V09UrxqUwqO31GT7v6wCoRAqmAeNS7WeL8sYztP51JuwOewATVSNNUYU92xVpWgi1gc7zcUSnheJ4lOdTM41Q6Cl47aiTE4WhmbuW1Fgwr7p8SDOYJefMG7qFX9EBZ" - "N4L7tra75NO5qNNAj1l5RqUHvLwfP67vKkAmWdqEQOV2yofT13Hrw3bJrqB9Sh17AJD3jvSOst3sKj4UNkSJRVGJLky6t6XJyPfu9SgCqcybD8GH3A4tcotpKmUyeWaV" - "3Yb49gsjlVWNr118iAFYAtu8LuYIpv0tCDlNmoGnaOr7MYg6AhACoQGhC61W5fh0Rl8EmUYeQwr5n6FsRGANqCUlTzABzvhJeZU8R4yPzK1EKYpsNbuB0OAfjWQ4QfXo" - "3PYVwMTyxFiMTEPF7JExpQ0A0kiHocZCzBZNee4Wbcosp0V9cqlB5X2Mizv87MVqN5qeAlNv8bF3tkUTT9ny25jiWz3piarNZZriHt2Jjwr6drhDmCyAMtOzfTpWgbIo" - "4O2QBp55VldQdVAIfZaruZfYsfT0m3klv0ijQQrSXfmEonDPr9LSi5ex1k0Pt2I42sW7tO3PfHI2ess8NhKTFwaeKbhhJH4cjYWuxETfAT1hcFDp9gFVNv61hZIpPC7B" - "6Re48nr4bugJvEqjP8MgmDwCuP4wPv8whciQ8xnbDANV9atiIp6nILVVPHy6iy8aeNJ5cU8vGHKEfglaW1PF3BhNBoDtbhi2ygT7nRWncGwlpXnrPsnwBuEk1XC1d9wE" - "XSkcpTLsb3jUCod6ahdBtMBWg30E0TmP7daELf3o8SWrOXUHoCad3y9q1El10fCCDfzw7uamJ00EXaLHDNaIOckLowZUttmM1TXTyrk8V330gmOIs3pS6zF2fMgbXfdk" - "Rd7m3ocqHWKxCINsXgZlt7a5LsoulsUktOELiWnNvLJ86qo3BmYk0Ok0gGqnU4WHvVMsiCU3gEmxOxenUWlHM6JSzrk6iW0PYDjEwQvMlT4Msom4GSLDQV2oxpbvWpIg" - "xLQjt8WS6pKhDMdSMXMYproxprmKdxHdsTooFhDuIHtNZQCbqhSvAZLZ7YaVo0JMWVZHMi76eAAjPI6h1BJOmMcuywnhIlMPcSScLtu5NvwQI854m3BUSmnxwlTPrknt" - "AVoDe7cEI45aGbgaXW7ZPPSJ0lYc2OvyaTKJob2OBh8zdnA3ERIBgRXC4cNImoSvxzMBiO7BDf5zroBqceOXxtaEc7y4D7GrekVAEwIjH7DMkg7AXdBrAXKdr97HIj"; -} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/Source/TestImpactTestProcessLargeText.h b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/Source/TestImpactTestProcessLargeText.h deleted file mode 100644 index faaabae2d3..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/Source/TestImpactTestProcessLargeText.h +++ /dev/null @@ -1,19 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#pragma once - -namespace TestImpact -{ - // Large text blob in the form of a string literal so the app does not require the Az FileIO and Application environment - extern const char* const LongText; -} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/Source/TestImpactTestProcessMain.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/Source/TestImpactTestProcessMain.cpp deleted file mode 100644 index 5c46ee525e..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/Source/TestImpactTestProcessMain.cpp +++ /dev/null @@ -1,19 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include "TestImpactTestProcess.h" - -int main(int argc, char* argv[]) -{ - TestImpact::TestProcess process(argc, argv); - return process.MainFunc(); -} diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/testimpactframework_testprocess_files.cmake b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/testimpactframework_testprocess_files.cmake deleted file mode 100644 index a06f1fa706..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestProcess/Code/testimpactframework_testprocess_files.cmake +++ /dev/null @@ -1,18 +0,0 @@ -# -# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or -# its licensors. -# -# For complete copyright and license terms please see the LICENSE at the root of this -# distribution (the "License"). All use of this software is governed by the License, -# or, if provided, by the license below or the license accompanying this file. Do not -# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# - -set(FILES - Source/TestImpactTestProcessMain.cpp - Source/TestImpactTestProcess.cpp - Source/TestImpactTestProcess.h - Source/TestImpactTestProcessLargeText.cpp - Source/TestImpactTestProcessLargeText.h -) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetA/CMakeLists.txt b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetA/CMakeLists.txt deleted file mode 100644 index 8298bb7123..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetA/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -# -# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or -# its licensors. -# -# For complete copyright and license terms please see the LICENSE at the root of this -# distribution (the "License"). All use of this software is governed by the License, -# or, if provided, by the license below or the license accompanying this file. Do not -# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# - -add_subdirectory(Code) \ No newline at end of file diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetA/Code/CMakeLists.txt b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetA/Code/CMakeLists.txt deleted file mode 100644 index 7748581284..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetA/Code/CMakeLists.txt +++ /dev/null @@ -1,28 +0,0 @@ -# -# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or -# its licensors. -# -# For complete copyright and license terms please see the LICENSE at the root of this -# distribution (the "License"). All use of this software is governed by the License, -# or, if provided, by the license below or the license accompanying this file. Do not -# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# - -################################################################################ -# Tests -################################################################################ - -ly_add_target( - NAME TestImpact.TestTargetA.Tests ${PAL_TRAIT_TEST_TARGET_TYPE} - NAMESPACE AZ - FILES_CMAKE - testimpactframework_testtargeta_tests_files.cmake - INCLUDE_DIRECTORIES - PRIVATE - Tests - BUILD_DEPENDENCIES - PRIVATE - AZ::AzTestShared - AZ::AzTest -) \ No newline at end of file diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetA/Code/Tests/TestImpactTestTargetA.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetA/Code/Tests/TestImpactTestTargetA.cpp deleted file mode 100644 index 9598b88b98..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetA/Code/Tests/TestImpactTestTargetA.cpp +++ /dev/null @@ -1,73 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include - -namespace UnitTest -{ - class TestFixture - : public AllocatorsTestFixture - { - }; - - TEST(TestCase, Test1_WillPass) - { - SUCCEED(); - } - - TEST(TestCase, Test2_WillPass) - { - SUCCEED(); - } - - TEST(TestCase, Test3_WillPass) - { - SUCCEED(); - } - - TEST(TestCase, Test4_WillPass) - { - SUCCEED(); - } - - TEST(TestCase, Test5_WillPass) - { - SUCCEED(); - } - - TEST(TestCase, Test6_WillPass) - { - SUCCEED(); - } - - TEST(TestCase, Test7_WillFail) - { - FAIL(); - } - - TEST_F(TestFixture, Test1_WillPass) - { - SUCCEED(); - } - - TEST_F(TestFixture, Test2_WillPass) - { - SUCCEED(); - } - - TEST_F(TestFixture, Test3_WillPass) - { - SUCCEED(); - } -} // namespace UnitTest - -AZ_UNIT_TEST_HOOK(DEFAULT_UNIT_TEST_ENV); diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetA/Code/testimpactframework_testtargeta_tests_files.cmake b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetA/Code/testimpactframework_testtargeta_tests_files.cmake deleted file mode 100644 index 32c8746c9d..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetA/Code/testimpactframework_testtargeta_tests_files.cmake +++ /dev/null @@ -1,14 +0,0 @@ -# -# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or -# its licensors. -# -# For complete copyright and license terms please see the LICENSE at the root of this -# distribution (the "License"). All use of this software is governed by the License, -# or, if provided, by the license below or the license accompanying this file. Do not -# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# - -set(FILES - Tests/TestImpactTestTargetA.cpp -) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetB/CMakeLists.txt b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetB/CMakeLists.txt deleted file mode 100644 index 8298bb7123..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetB/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -# -# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or -# its licensors. -# -# For complete copyright and license terms please see the LICENSE at the root of this -# distribution (the "License"). All use of this software is governed by the License, -# or, if provided, by the license below or the license accompanying this file. Do not -# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# - -add_subdirectory(Code) \ No newline at end of file diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetB/Code/CMakeLists.txt b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetB/Code/CMakeLists.txt deleted file mode 100644 index 57b55fb9f4..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetB/Code/CMakeLists.txt +++ /dev/null @@ -1,29 +0,0 @@ -# -# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or -# its licensors. -# -# For complete copyright and license terms please see the LICENSE at the root of this -# distribution (the "License"). All use of this software is governed by the License, -# or, if provided, by the license below or the license accompanying this file. Do not -# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# - -################################################################################ -# Tests -################################################################################ - -ly_add_target( - NAME TestImpact.TestTargetB.Tests ${PAL_TRAIT_TEST_TARGET_TYPE} - NAMESPACE AZ - FILES_CMAKE - testimpactframework_testtargetb_tests_files.cmake - INCLUDE_DIRECTORIES - PRIVATE - Tests - BUILD_DEPENDENCIES - PRIVATE - AZ::AzCore - AZ::AzTestShared - AZ::AzTest -) \ No newline at end of file diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetB/Code/Tests/TestImpactTestTargetB.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetB/Code/Tests/TestImpactTestTargetB.cpp deleted file mode 100644 index be58b4fa17..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetB/Code/Tests/TestImpactTestTargetB.cpp +++ /dev/null @@ -1,78 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include -#include - -namespace UnitTest -{ - class TestFixture - : public AllocatorsTestFixture - { - }; - - class TestFixtureWithParams - : public AllocatorsTestFixture - , public ::testing::WithParamInterface> - { - }; - - TEST(TestCase, Test1_WillPass) - { - SUCCEED(); - } - - TEST(TestCase, Test2_WillPass) - { - SUCCEED(); - } - - TEST(TestCase, Test3_WillPass) - { - SUCCEED(); - } - - TEST_F(TestFixture, Test1_WillPass) - { - SUCCEED(); - } - - TEST_P(TestFixtureWithParams, Test1_WillPass) - { - SUCCEED(); - } - - TEST_P(TestFixtureWithParams, Test2_WillPass) - { - SUCCEED(); - } - - INSTANTIATE_TEST_CASE_P( - PermutationA, - TestFixtureWithParams, - ::testing::Combine( - ::testing::Values(1, 2, 4), - ::testing::Values(3, 5, 7), - ::testing::Values(-0.0f, 0.0f, 1.0f) - )); - - INSTANTIATE_TEST_CASE_P( - , - TestFixtureWithParams, - ::testing::Combine( - ::testing::Values(8, 16, 32), - ::testing::Values(9, 13, 17), - ::testing::Values(-10.0f, 0.05f, 10.0f) - )); -} // namespace UnitTest - -AZ_UNIT_TEST_HOOK(DEFAULT_UNIT_TEST_ENV); diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetB/Code/testimpactframework_testtargetb_tests_files.cmake b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetB/Code/testimpactframework_testtargetb_tests_files.cmake deleted file mode 100644 index 949074a5af..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetB/Code/testimpactframework_testtargetb_tests_files.cmake +++ /dev/null @@ -1,14 +0,0 @@ -# -# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or -# its licensors. -# -# For complete copyright and license terms please see the LICENSE at the root of this -# distribution (the "License"). All use of this software is governed by the License, -# or, if provided, by the license below or the license accompanying this file. Do not -# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# - -set(FILES - Tests/TestImpactTestTargetB.cpp -) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetC/CMakeLists.txt b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetC/CMakeLists.txt deleted file mode 100644 index 8298bb7123..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetC/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -# -# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or -# its licensors. -# -# For complete copyright and license terms please see the LICENSE at the root of this -# distribution (the "License"). All use of this software is governed by the License, -# or, if provided, by the license below or the license accompanying this file. Do not -# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# - -add_subdirectory(Code) \ No newline at end of file diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetC/Code/CMakeLists.txt b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetC/Code/CMakeLists.txt deleted file mode 100644 index 47862f80df..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetC/Code/CMakeLists.txt +++ /dev/null @@ -1,29 +0,0 @@ -# -# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or -# its licensors. -# -# For complete copyright and license terms please see the LICENSE at the root of this -# distribution (the "License"). All use of this software is governed by the License, -# or, if provided, by the license below or the license accompanying this file. Do not -# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# - -################################################################################ -# Tests -################################################################################ - -ly_add_target( - NAME TestImpact.TestTargetC.Tests ${PAL_TRAIT_TEST_TARGET_TYPE} - NAMESPACE AZ - FILES_CMAKE - testimpactframework_testtargetc_tests_files.cmake - INCLUDE_DIRECTORIES - PRIVATE - Tests - BUILD_DEPENDENCIES - PRIVATE - AZ::AzCore - AZ::AzTestShared - AZ::AzTest -) \ No newline at end of file diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetC/Code/Tests/TestImpactTestTargetC.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetC/Code/Tests/TestImpactTestTargetC.cpp deleted file mode 100644 index 48f0ffe9dc..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetC/Code/Tests/TestImpactTestTargetC.cpp +++ /dev/null @@ -1,63 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include -#include - -namespace UnitTest -{ - class TestFixture - : public AllocatorsTestFixture - { - }; - - template - class TestFixtureWithTypes - : public AllocatorsTestFixture - { - }; - - using TestTypes = testing::Types; - TYPED_TEST_CASE(TestFixtureWithTypes, TestTypes); - - TEST_F(TestFixture, Test1_WillPass) - { - SUCCEED(); - } - - TEST_F(TestFixture, Test2_WillPass) - { - SUCCEED(); - } - - TYPED_TEST(TestFixtureWithTypes, Test1_WillPass) - { - SUCCEED(); - } - - TYPED_TEST(TestFixtureWithTypes, Test2_WillPass) - { - SUCCEED(); - } - - TYPED_TEST(TestFixtureWithTypes, Test3_WillPass) - { - SUCCEED(); - } - - TYPED_TEST(TestFixtureWithTypes, Test4_WillPass) - { - SUCCEED(); - } -} // namespace UnitTest - -AZ_UNIT_TEST_HOOK(DEFAULT_UNIT_TEST_ENV); diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetC/Code/testimpactframework_testtargetc_tests_files.cmake b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetC/Code/testimpactframework_testtargetc_tests_files.cmake deleted file mode 100644 index fa5b4f6e9a..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetC/Code/testimpactframework_testtargetc_tests_files.cmake +++ /dev/null @@ -1,14 +0,0 @@ -# -# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or -# its licensors. -# -# For complete copyright and license terms please see the LICENSE at the root of this -# distribution (the "License"). All use of this software is governed by the License, -# or, if provided, by the license below or the license accompanying this file. Do not -# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# - -set(FILES - Tests/TestImpactTestTargetC.cpp -) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetD/CMakeLists.txt b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetD/CMakeLists.txt deleted file mode 100644 index 8298bb7123..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetD/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -# -# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or -# its licensors. -# -# For complete copyright and license terms please see the LICENSE at the root of this -# distribution (the "License"). All use of this software is governed by the License, -# or, if provided, by the license below or the license accompanying this file. Do not -# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# - -add_subdirectory(Code) \ No newline at end of file diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetD/Code/CMakeLists.txt b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetD/Code/CMakeLists.txt deleted file mode 100644 index bf821ac0d6..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetD/Code/CMakeLists.txt +++ /dev/null @@ -1,29 +0,0 @@ -# -# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or -# its licensors. -# -# For complete copyright and license terms please see the LICENSE at the root of this -# distribution (the "License"). All use of this software is governed by the License, -# or, if provided, by the license below or the license accompanying this file. Do not -# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# - -################################################################################ -# Tests -################################################################################ - -ly_add_target( - NAME TestImpact.TestTargetD.Tests ${PAL_TRAIT_TEST_TARGET_TYPE} - NAMESPACE AZ - FILES_CMAKE - testimpactframework_testtargetd_tests_files.cmake - INCLUDE_DIRECTORIES - PRIVATE - Tests - BUILD_DEPENDENCIES - PRIVATE - AZ::AzCore - AZ::AzTestShared - AZ::AzTest -) \ No newline at end of file diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetD/Code/Tests/TestImpactTestTargetD.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetD/Code/Tests/TestImpactTestTargetD.cpp deleted file mode 100644 index bd27b51fd9..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetD/Code/Tests/TestImpactTestTargetD.cpp +++ /dev/null @@ -1,188 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include -#include - -namespace UnitTest -{ - class TestFixture1 - : public AllocatorsTestFixture - { - }; - - class DISABLED_TestFixture2 - : public AllocatorsTestFixture - { - }; - - template - class TestFixtureWithTypes1 - : public AllocatorsTestFixture - { - }; - - template - class DISABLED_TestFixtureWithTypes2 - : public AllocatorsTestFixture - { - }; - - class TestFixtureWithParams1 - : public AllocatorsTestFixture - , public ::testing::WithParamInterface> - { - }; - - class DISABLED_TestFixtureWithParams2 - : public AllocatorsTestFixture - , public ::testing::WithParamInterface> - { - }; - - using TestTypes = testing::Types; - TYPED_TEST_CASE(TestFixtureWithTypes1, TestTypes); - TYPED_TEST_CASE(DISABLED_TestFixtureWithTypes2, TestTypes); - - TEST(TestCase, Test1_WillPass) - { - SUCCEED(); - } - - TEST(TestCase, DISABLED_Test2_WillPass) - { - SUCCEED(); - } - - TEST(TestCase, Test3_WillPass) - { - SUCCEED(); - } - - TEST(TestCase, Test4_WillPass) - { - SUCCEED(); - } - - TEST(TestCase, Test5_WillPass) - { - SUCCEED(); - } - - TEST_F(TestFixture1, Test1_WillPass) - { - SUCCEED(); - } - - TEST_F(TestFixture1, Test2_WillPass) - { - SUCCEED(); - } - - TEST_F(DISABLED_TestFixture2, Test1_WillPass) - { - SUCCEED(); - } - - TEST_F(DISABLED_TestFixture2, Test2_WillPass) - { - SUCCEED(); - } - - TEST_P(TestFixtureWithParams1, Test1_WillPass) - { - SUCCEED(); - } - - TEST_P(TestFixtureWithParams1, DISABLED_Test2_WillPass) - { - SUCCEED(); - } - - TEST_P(DISABLED_TestFixtureWithParams2, Test1_WillPass) - { - SUCCEED(); - } - - TEST_P(DISABLED_TestFixtureWithParams2, DISABLED_Test2_WillPass) - { - SUCCEED(); - } - - INSTANTIATE_TEST_CASE_P( - PermutationA, - TestFixtureWithParams1, - ::testing::Combine( - ::testing::Values(1, 2, 4), - ::testing::Values(3, 5, 7), - ::testing::Values(-0.0f, 0.0f, 1.0f) - )); - - INSTANTIATE_TEST_CASE_P( - , - TestFixtureWithParams1, - ::testing::Combine( - ::testing::Values(8, 16, 32), - ::testing::Values(9, 13, 17), - ::testing::Values(-10.0f, 0.05f, 10.0f) - )); - - INSTANTIATE_TEST_CASE_P( - PermutationA, - DISABLED_TestFixtureWithParams2, - ::testing::Combine( - ::testing::Values(1, 2, 4), - ::testing::Values(3, 5, 7), - ::testing::Values(-0.0f, 0.0f, 1.0f) - )); - - INSTANTIATE_TEST_CASE_P( - , - DISABLED_TestFixtureWithParams2, - ::testing::Combine( - ::testing::Values(8, 16, 32), - ::testing::Values(9, 13, 17), - ::testing::Values(-10.0f, 0.05f, 10.0f) - )); - - TYPED_TEST(TestFixtureWithTypes1, Test1_WillPass) - { - SUCCEED(); - } - - TYPED_TEST(TestFixtureWithTypes1, DISABLED_Test2_WillPass) - { - SUCCEED(); - } - - TYPED_TEST(TestFixtureWithTypes1, Test3_WillPass) - { - SUCCEED(); - } - - TYPED_TEST(DISABLED_TestFixtureWithTypes2, Test1_WillPass) - { - SUCCEED(); - } - - TYPED_TEST(DISABLED_TestFixtureWithTypes2, DISABLED_Test2_WillPass) - { - SUCCEED(); - } - - TYPED_TEST(DISABLED_TestFixtureWithTypes2, Test3_WillPass) - { - SUCCEED(); - } -} // namespace UnitTest - -AZ_UNIT_TEST_HOOK(DEFAULT_UNIT_TEST_ENV); diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetD/Code/testimpactframework_testtargetd_tests_files.cmake b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetD/Code/testimpactframework_testtargetd_tests_files.cmake deleted file mode 100644 index ebb5a422c7..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestTargetD/Code/testimpactframework_testtargetd_tests_files.cmake +++ /dev/null @@ -1,14 +0,0 @@ -# -# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or -# its licensors. -# -# For complete copyright and license terms please see the LICENSE at the root of this -# distribution (the "License"). All use of this software is governed by the License, -# or, if provided, by the license below or the license accompanying this file. Do not -# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# - -set(FILES - Tests/TestImpactTestTargetD.cpp -) From 1cef72ab8d19d0f8533ed793f6600447ad415662 Mon Sep 17 00:00:00 2001 From: John Date: Thu, 10 Jun 2021 17:32:16 +0100 Subject: [PATCH 093/102] Fix uncommented tests --- .../TestImpactFramework/Runtime/Code/CMakeLists.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/CMakeLists.txt b/Code/Tools/TestImpactFramework/Runtime/Code/CMakeLists.txt index bcfeeacb33..e89dfe60d7 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/CMakeLists.txt +++ b/Code/Tools/TestImpactFramework/Runtime/Code/CMakeLists.txt @@ -12,12 +12,6 @@ ly_get_list_relative_pal_filename(pal_dir ${CMAKE_CURRENT_LIST_DIR}/Source/Platform/${PAL_PLATFORM_NAME}) ly_get_list_relative_pal_filename(common_dir ${CMAKE_CURRENT_LIST_DIR}/Source/Platform/Common) -add_subdirectory(Tests/TestProcess) -add_subdirectory(Tests/TestTargetA) -add_subdirectory(Tests/TestTargetB) -add_subdirectory(Tests/TestTargetC) -add_subdirectory(Tests/TestTargetD) - ly_add_target( NAME TestImpact.Runtime.Static STATIC NAMESPACE AZ @@ -41,6 +35,12 @@ ly_add_target( ################################################################################ # Disabled: SPEC-7246 +#add_subdirectory(Tests/TestProcess) +#add_subdirectory(Tests/TestTargetA) +#add_subdirectory(Tests/TestTargetB) +#add_subdirectory(Tests/TestTargetC) +#add_subdirectory(Tests/TestTargetD) +# #ly_add_target( # NAME TestImpact.Runtime.Tests ${PAL_TRAIT_TEST_TARGET_TYPE} # NAMESPACE AZ From baab0cb43be02f6200616ad07659aa5bec2052cc Mon Sep 17 00:00:00 2001 From: John Date: Tue, 15 Jun 2021 17:33:57 +0100 Subject: [PATCH 094/102] Disable EntityIdGeneration test --- Code/Framework/AzCore/Tests/Components.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Code/Framework/AzCore/Tests/Components.cpp b/Code/Framework/AzCore/Tests/Components.cpp index 7801f69375..1d173abbab 100644 --- a/Code/Framework/AzCore/Tests/Components.cpp +++ b/Code/Framework/AzCore/Tests/Components.cpp @@ -1555,6 +1555,8 @@ namespace UnitTest } } + // Temporary disabled. This will be re-enabled in the short term upon completion of SPEC-7384 and + // fixed in the long term upon completion of SPEC-4849 TEST_F(Components, EntityIdGeneration) { // Generate 1 million ids across 100 threads, and ensure that none collide From 58ad67dc44ac410e64965ff2f34e397e07ad0865 Mon Sep 17 00:00:00 2001 From: John Date: Tue, 15 Jun 2021 17:44:00 +0100 Subject: [PATCH 095/102] Disable EntityIdGeneration test --- Code/Framework/AzCore/Tests/Components.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Code/Framework/AzCore/Tests/Components.cpp b/Code/Framework/AzCore/Tests/Components.cpp index 1d173abbab..9d9bd46d3d 100644 --- a/Code/Framework/AzCore/Tests/Components.cpp +++ b/Code/Framework/AzCore/Tests/Components.cpp @@ -1557,7 +1557,7 @@ namespace UnitTest // Temporary disabled. This will be re-enabled in the short term upon completion of SPEC-7384 and // fixed in the long term upon completion of SPEC-4849 - TEST_F(Components, EntityIdGeneration) + TEST_F(Components, DISABLED_EntityIdGeneration) { // Generate 1 million ids across 100 threads, and ensure that none collide AZStd::concurrent_unordered_set entityIds; From 17ba9f5c4feb423249c4579ae88b5061315bd31d Mon Sep 17 00:00:00 2001 From: John Date: Wed, 16 Jun 2021 10:09:02 +0100 Subject: [PATCH 096/102] Force non-gating at script level --- scripts/build/TestImpactAnalysis/tiaf_driver.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/scripts/build/TestImpactAnalysis/tiaf_driver.py b/scripts/build/TestImpactAnalysis/tiaf_driver.py index f452b9738f..a21f502a2a 100644 --- a/scripts/build/TestImpactAnalysis/tiaf_driver.py +++ b/scripts/build/TestImpactAnalysis/tiaf_driver.py @@ -54,8 +54,13 @@ def parse_args(): return args if __name__ == "__main__": - args = parse_args() - tiaf = TestImpact(args.config, args.pipeline, args.dst_commit) - return_code = tiaf.run(args.suite, args.test_failure_policy, args.safe_mode, args.test_timeout, args.global_timeout) - sys.exit(return_code) - \ No newline at end of file + try: + args = parse_args() + tiaf = TestImpact(args.config, args.pipeline, args.dst_commit) + return_code = tiaf.run(args.suite, args.test_failure_policy, args.safe_mode, args.test_timeout, args.global_timeout) + # Non-gating will be removed from this script and handled at the job level in SPEC-7413 + #sys.exit(return_code) + sys.exit(0) + except: + # Non-gating will be removed from this script and handled at the job level in SPEC-7413 + sys.exit(0) \ No newline at end of file From 0e6a266e89d22b65a4961971eb2554b6f3e6a460 Mon Sep 17 00:00:00 2001 From: John Date: Wed, 16 Jun 2021 11:47:17 +0100 Subject: [PATCH 097/102] Remove frontend console tests --- .../TestImpactCommandLineOptionsTest.cpp | 1436 ----------------- .../Code/Tests/TestImpactConsoleMainTest.cpp | 0 ...actConsoleTestSequenceEventHandlerTest.cpp | 42 - ...estImpactFrontendConsoleStaticTestMain.cpp | 34 - ...tImpactRuntimeConfigurationFactoryTest.cpp | 27 - ..._frontend_console_static_tests_files.cmake | 5 - 6 files changed, 1544 deletions(-) delete mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/Code/Tests/TestImpactCommandLineOptionsTest.cpp delete mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/Code/Tests/TestImpactConsoleMainTest.cpp delete mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/Code/Tests/TestImpactConsoleTestSequenceEventHandlerTest.cpp delete mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/Code/Tests/TestImpactFrontendConsoleStaticTestMain.cpp delete mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/Code/Tests/TestImpactRuntimeConfigurationFactoryTest.cpp diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Code/Tests/TestImpactCommandLineOptionsTest.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Code/Tests/TestImpactCommandLineOptionsTest.cpp deleted file mode 100644 index cd02f5c5c7..0000000000 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Code/Tests/TestImpactCommandLineOptionsTest.cpp +++ /dev/null @@ -1,1436 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include -#include - -#include -#include -#include -#include - -namespace UnitTest -{ - class CommandLineOptionsTestFixture - : public AllocatorsTestFixture - { - public: - void SetUp() override - { - AllocatorsTestFixture::SetUp(); - m_args.push_back("program.exe"); - } - protected: - void InitOptions(); - - AZStd::unique_ptr m_options; - AZStd::vector m_args; - }; - - void CommandLineOptionsTestFixture::InitOptions() - { - m_options = AZStd::make_unique(m_args.size(), const_cast(m_args.data())); - } - - TEST_F(CommandLineOptionsTestFixture, CheckEmptyArgs_ExpectDefaultValues) - { - InitOptions(); - EXPECT_EQ(m_options->GetConfigurationFile(), LY_TEST_IMPACT_DEFAULT_CONFIG_FILE); - EXPECT_EQ(m_options->GetFailedTestCoveragePolicy(), TestImpact::Policy::FailedTestCoverage::Keep); - EXPECT_EQ(m_options->GetExecutionFailurePolicy(), TestImpact::Policy::ExecutionFailure::Continue); - EXPECT_FALSE(m_options->GetGlobalTimeout().has_value()); - EXPECT_FALSE(m_options->GetTestTargetTimeout().has_value()); - EXPECT_FALSE(m_options->GetMaxConcurrency().has_value()); - EXPECT_FALSE(m_options->HasOutputChangeList()); - EXPECT_EQ(m_options->GetTargetOutputCapture(), TestImpact::Policy::TargetOutputCapture::None); - EXPECT_EQ(m_options->GetTestFailurePolicy(), TestImpact::Policy::TestFailure::Abort); - EXPECT_EQ(m_options->GetIntegrityFailurePolicy(), TestImpact::Policy::IntegrityFailure::Abort); - EXPECT_EQ(m_options->GetTestPrioritizationPolicy(), TestImpact::Policy::TestPrioritization::None); - EXPECT_EQ(m_options->GetTestSequenceType(), TestImpact::TestSequenceType::None); - EXPECT_EQ(m_options->GetTestShardingPolicy(), TestImpact::Policy::TestSharding::Never); - EXPECT_FALSE(m_options->HasChangeListFile()); - EXPECT_FALSE(m_options->GetChangeListFile().has_value()); - EXPECT_FALSE(m_options->HasSafeMode()); - EXPECT_EQ(m_options->GetSuiteFilter(), TestImpact::SuiteType::Main); - } - - TEST_F(CommandLineOptionsTestFixture, ConfigurationFileHasEmptyPath_ExpectCommandLineOptionsException) - { - m_args.push_back("-config"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(CommandLineOptionsTestFixture, ConfigurationFileHasSpecifiedPath_ExpectPath) - { - m_args.push_back("-config"); - m_args.push_back("Foo\\Bar"); - InitOptions(); - EXPECT_STREQ(m_options->GetConfigurationFile().c_str(), "Foo\\Bar"); - } - - TEST_F(CommandLineOptionsTestFixture, ConfigurationFileHasMultiplePaths_ExpectCommandLineOptionsException) - { - m_args.push_back("-config"); - m_args.push_back("value1,value2"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - // - - TEST_F(CommandLineOptionsTestFixture, UnifiedDiffFileHasEmptyPath_ExpectCommandLineOptionsException) - { - m_args.push_back("-changelist"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(CommandLineOptionsTestFixture, UnifiedDiffFileHasSpecifiedPath_ExpectPath) - { - m_args.push_back("-changelist"); - m_args.push_back("Foo\\Bar"); - InitOptions(); - EXPECT_TRUE(m_options->HasChangeListFile()); - EXPECT_STREQ(m_options->GetChangeListFile()->c_str(), "Foo\\Bar"); - } - - TEST_F(CommandLineOptionsTestFixture, UnifiedDiffFileHasMultiplePaths_ExpectCommandLineOptionsException) - { - m_args.push_back("-changelist"); - m_args.push_back("value1,value2"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - // - - TEST_F(CommandLineOptionsTestFixture, OutputChangeListHasEmptyOption_ExpectCommandLineOptionsException) - { - m_args.push_back("-ochangelist"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(CommandLineOptionsTestFixture, OutputChangeListHasMultipleValues_ExpectCommandLineOptionsException) - { - m_args.push_back("-ochangelist"); - m_args.push_back("value1,value2,value3"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - // - - TEST_F(CommandLineOptionsTestFixture, TestSequenceTypeHasEmptyOption_ExpectCommandLineOptionsException) - { - m_args.push_back("-sequence"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(CommandLineOptionsTestFixture, TestSequenceTypeHasNoneOption_ExpectNoneTestSequenceType) - { - m_args.push_back("-sequence"); - m_args.push_back("none"); - InitOptions(); - EXPECT_EQ(m_options->GetTestSequenceType(), TestImpact::TestSequenceType::None); - } - - TEST_F(CommandLineOptionsTestFixture, TestSequenceTypeHasSeedOption_ExpectSeedTestSequenceType) - { - m_args.push_back("-sequence"); - m_args.push_back("seed"); - InitOptions(); - EXPECT_EQ(m_options->GetTestSequenceType(), TestImpact::TestSequenceType::Seed); - } - - TEST_F(CommandLineOptionsTestFixture, TestSequenceTypeHasRegularOption_ExpectRegularTestSequenceType) - { - m_args.push_back("-sequence"); - m_args.push_back("regular"); - InitOptions(); - EXPECT_EQ(m_options->GetTestSequenceType(), TestImpact::TestSequenceType::Regular); - } - - TEST_F(CommandLineOptionsTestFixture, TestSequenceTypeHasImpactAnalysisOption_ExpectImpactAnalysisTestSequenceType) - { - m_args.push_back("-sequence"); - m_args.push_back("tia"); - InitOptions(); - EXPECT_EQ(m_options->GetTestSequenceType(), TestImpact::TestSequenceType::ImpactAnalysis); - } - - TEST_F(CommandLineOptionsTestFixture, TestSequenceTypeHasImpactAnalysisNoWriteOption_ExpectImpactAnalysisNoWriteTestSequenceType) - { - m_args.push_back("-sequence"); - m_args.push_back("tianowrite"); - InitOptions(); - EXPECT_EQ(m_options->GetTestSequenceType(), TestImpact::TestSequenceType::ImpactAnalysisNoWrite); - } - - TEST_F(CommandLineOptionsTestFixture, TestSequenceTypeHasSafeImpactAnalysisOption_ExpectSafeImpactAnalysisTestSequenceType) - { - m_args.push_back("-sequence"); - m_args.push_back("tiaorseed"); - InitOptions(); - EXPECT_EQ(m_options->GetTestSequenceType(), TestImpact::TestSequenceType::ImpactAnalysisOrSeed); - } - - TEST_F(CommandLineOptionsTestFixture, TestSequenceTypeHasInvalidOption_ExpectCommandLineOptionsException) - { - m_args.push_back("-sequence"); - m_args.push_back("foo"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(CommandLineOptionsTestFixture, TestSequenceTypeHasMultipleValues_ExpectCommandLineOptionsException) - { - m_args.push_back("-sequence"); - m_args.push_back("seed,tia"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - // - - TEST_F(CommandLineOptionsTestFixture, TestPrioritizationPolicyHasEmptyOption_ExpectCommandLineOptionsException) - { - m_args.push_back("-ppolicy"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(CommandLineOptionsTestFixture, TestPrioritizationPolicyHasNoneOption_ExpectNoneTestPrioritizationPolicy) - { - m_args.push_back("-ppolicy"); - m_args.push_back("none"); - InitOptions(); - EXPECT_EQ(m_options->GetTestPrioritizationPolicy(), TestImpact::Policy::TestPrioritization::None); - } - - TEST_F(CommandLineOptionsTestFixture, TestPrioritizationPolicyHasDependencyLocalityOption_ExpectDependencyLocalityTestPrioritizationPolicy) - { - m_args.push_back("-ppolicy"); - m_args.push_back("locality"); - InitOptions(); - EXPECT_EQ(m_options->GetTestPrioritizationPolicy(), TestImpact::Policy::TestPrioritization::DependencyLocality); - } - - TEST_F(CommandLineOptionsTestFixture, TestPrioritizationPolicyInvalidOption_ExpectCommandLineOptionsException) - { - m_args.push_back("-ppolicy"); - m_args.push_back("none,locality"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - // - - TEST_F(CommandLineOptionsTestFixture, ExecutionFailurePolicyHasEmptyOption_ExpectCommandLineOptionsException) - { - m_args.push_back("-ppolicy"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(CommandLineOptionsTestFixture, ExecutionFailurePolicyHasAbortOption_ExpectAbortExecutionFailurePolicy) - { - m_args.push_back("-epolicy"); - m_args.push_back("abort"); - InitOptions(); - EXPECT_EQ(m_options->GetExecutionFailurePolicy(), TestImpact::Policy::ExecutionFailure::Abort); - } - - TEST_F(CommandLineOptionsTestFixture, ExecutionFailurePolicyHasContinueOption_ExpectContinueExecutionFailurePolicy) - { - m_args.push_back("-epolicy"); - m_args.push_back("continue"); - InitOptions(); - EXPECT_EQ(m_options->GetExecutionFailurePolicy(), TestImpact::Policy::ExecutionFailure::Continue); - } - - TEST_F(CommandLineOptionsTestFixture, ExecutionFailurePolicyHasIgnoreOption_ExpectIgnoreExecutionFailurePolicy) - { - m_args.push_back("-epolicy"); - m_args.push_back("ignore"); - InitOptions(); - EXPECT_EQ(m_options->GetExecutionFailurePolicy(), TestImpact::Policy::ExecutionFailure::Ignore); - } - - TEST_F(CommandLineOptionsTestFixture, ExecutionFailurePolicyHasInvalidOption_ExpectCommandLineOptionsException) - { - m_args.push_back("-epolicy"); - m_args.push_back("foo"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(CommandLineOptionsTestFixture, ExecutionFailurePolicyHasMultipleValues_ExpectCommandLineOptionsException) - { - m_args.push_back("-epolicy"); - m_args.push_back("abort,ingore"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - // - - TEST_F(CommandLineOptionsTestFixture, ExecutionFailureDraftingPolicyHasEmptyOption_ExpectCommandLineOptionsException) - { - m_args.push_back("-rexecfailures"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(CommandLineOptionsTestFixture, FailedTestCoveragePolicyHasKeepOption_ExpectKeepFailedTestCoveragePolicy) - { - m_args.push_back("-cpolicy"); - m_args.push_back("keep"); - InitOptions(); - EXPECT_EQ(m_options->GetFailedTestCoveragePolicy(), TestImpact::Policy::FailedTestCoverage::Keep); - } - - TEST_F(CommandLineOptionsTestFixture, FailedTestCoveragePolicyHasDiscardOption_ExpectDiscardFailedTestCoveragePolicy) - { - m_args.push_back("-cpolicy"); - m_args.push_back("discard"); - InitOptions(); - EXPECT_EQ(m_options->GetFailedTestCoveragePolicy(), TestImpact::Policy::FailedTestCoverage::Discard); - } - - TEST_F(CommandLineOptionsTestFixture, FailedTestCoveragePolicyInvalidOption_ExpectCommandLineOptionsException) - { - m_args.push_back("-cpolicy"); - m_args.push_back("foo"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(CommandLineOptionsTestFixture, FailedTestCoveragePolicyPolicyHasMultipleValues_ExpectCommandLineOptionsException) - { - m_args.push_back("--cpolicy"); - m_args.push_back("keep,discard"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - // - - TEST_F(CommandLineOptionsTestFixture, TestFailurePolicyHasEmptyOption_ExpectCommandLineOptionsException) - { - m_args.push_back("-fpolicy"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(CommandLineOptionsTestFixture, TestFailurePolicyHasAbortOption_ExpectAbortTestFailurePolicy) - { - m_args.push_back("-fpolicy"); - m_args.push_back("abort"); - InitOptions(); - EXPECT_EQ(m_options->GetTestFailurePolicy(), TestImpact::Policy::TestFailure::Abort); - } - - TEST_F(CommandLineOptionsTestFixture, TestFailurePolicyHasContinueOption_ExpectContinueTestFailurePolicy) - { - m_args.push_back("-fpolicy"); - m_args.push_back("continue"); - InitOptions(); - EXPECT_EQ(m_options->GetTestFailurePolicy(), TestImpact::Policy::TestFailure::Continue); - } - - TEST_F(CommandLineOptionsTestFixture, TestFailurePolicyInvalidOption_ExpectCommandLineOptionsException) - { - m_args.push_back("-fpolicy"); - m_args.push_back("foo"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(CommandLineOptionsTestFixture, TestFailurePolicyHasMultipeValues_ExpectCommandLineOptionsException) - { - m_args.push_back("-fpolicy"); - m_args.push_back("abort,continue"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - // - - TEST_F(CommandLineOptionsTestFixture, IntegrityFailurePolicyHasEmptyOption_ExpectCommandLineOptionsException) - { - m_args.push_back("-ipolicy"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(CommandLineOptionsTestFixture, IntegrityFailurePolicyHasAbortOption_ExpectAbortIntegrityFailurePolicy) - { - m_args.push_back("-ipolicy"); - m_args.push_back("abort"); - InitOptions(); - EXPECT_EQ(m_options->GetIntegrityFailurePolicy(), TestImpact::Policy::IntegrityFailure::Abort); - } - - TEST_F(CommandLineOptionsTestFixture, IntegrityFailurePolicyHasContinueOption_ExpectContinueIntegrityFailurePolicy) - { - m_args.push_back("-ipolicy"); - m_args.push_back("continue"); - InitOptions(); - EXPECT_EQ(m_options->GetIntegrityFailurePolicy(), TestImpact::Policy::IntegrityFailure::Continue); - } - - TEST_F(CommandLineOptionsTestFixture, IntegrityFailurePolicyInvalidOption_ExpectCommandLineOptionsException) - { - m_args.push_back("-ipolicy"); - m_args.push_back("foo"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(CommandLineOptionsTestFixture, IntegrityFailurePolicyHasMultipeValues_ExpectCommandLineOptionsException) - { - m_args.push_back("-ipolicy"); - m_args.push_back("abort,continue"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - // - - TEST_F(CommandLineOptionsTestFixture, TestShardingHasEmptyOption_ExpectCommandLineOptionsException) - { - m_args.push_back("-shard"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(CommandLineOptionsTestFixture, TestShardingHasOnOption_ExpectOnTestSharding) - { - m_args.push_back("-shard"); - m_args.push_back("on"); - InitOptions(); - EXPECT_EQ(m_options->GetTestShardingPolicy(), TestImpact::Policy::TestSharding::Always); - } - - TEST_F(CommandLineOptionsTestFixture, TestShardingHasOffOption_ExpectOffTestSharding) - { - m_args.push_back("-shard"); - m_args.push_back("off"); - InitOptions(); - EXPECT_EQ(m_options->GetTestShardingPolicy(), TestImpact::Policy::TestSharding::Never); - } - - TEST_F(CommandLineOptionsTestFixture, TestShardingInvalidOption_ExpectCommandLineOptionsException) - { - m_args.push_back("-shard"); - m_args.push_back("foo"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(CommandLineOptionsTestFixture, TestShardingHasMultipeValues_ExpectCommandLineOptionsException) - { - m_args.push_back("-shard"); - m_args.push_back("on,off"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - // - - TEST_F(CommandLineOptionsTestFixture, TargetOutputCaptureHasEmptyOption_ExpectCommandLineOptionsException) - { - m_args.push_back("-targetout"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(CommandLineOptionsTestFixture, TargetOutputCaptureHasStdOutOption_ExpectStdOutTargetOutputCapture) - { - m_args.push_back("-targetout"); - m_args.push_back("stdout"); - InitOptions(); - EXPECT_EQ(m_options->GetTargetOutputCapture(), TestImpact::Policy::TargetOutputCapture::StdOut); - } - - TEST_F(CommandLineOptionsTestFixture, TargetOutputCaptureHasFileOption_ExpectFileTargetOutputCapture) - { - m_args.push_back("-targetout"); - m_args.push_back("file"); - InitOptions(); - EXPECT_EQ(m_options->GetTargetOutputCapture(), TestImpact::Policy::TargetOutputCapture::File); - } - - TEST_F(CommandLineOptionsTestFixture, TargetOutputCaptureHasStdOutAndFileOption_ExpectStdOutAndFileTargetOutputCapture) - { - m_args.push_back("-targetout"); - m_args.push_back("stdout,file"); - InitOptions(); - EXPECT_EQ(m_options->GetTargetOutputCapture(), TestImpact::Policy::TargetOutputCapture::StdOutAndFile); - } - - TEST_F(CommandLineOptionsTestFixture, TargetOutputCaptureInvalidOption_ExpectCommandLineOptionsException) - { - m_args.push_back("-targetout"); - m_args.push_back("foo"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(CommandLineOptionsTestFixture, TargetOutputCaptureHasExcessValues_ExpectCommandLineOptionsException) - { - m_args.push_back("-targetout"); - m_args.push_back("stdout,file,stdout"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - // - - TEST_F(CommandLineOptionsTestFixture, MaxConcurrencyHasEmptyOption_ExpectCommandLineOptionsException) - { - m_args.push_back("-maxconcurrency"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(CommandLineOptionsTestFixture, MaxConcurrencyHasInRangeOptions_ExpectInRangeMaxConcurrency) - { - m_args.push_back("-maxconcurrency"); - m_args.push_back("10"); - InitOptions(); - EXPECT_EQ(m_options->GetMaxConcurrency(), 10); - } - - TEST_F(CommandLineOptionsTestFixture, MaxConcurrencyHasOutOfRangeOption_ExpectCommandLineOptionsException) - { - m_args.push_back("-maxconcurrency"); - m_args.push_back("-1"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(CommandLineOptionsTestFixture, MaxConcurrencyInvalidOption_ExpectCommandLineOptionsException) - { - m_args.push_back("-maxconcurrency"); - m_args.push_back("foo"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(CommandLineOptionsTestFixture, MaxConcurrencyHasMultipeValues_ExpectCommandLineOptionsException) - { - m_args.push_back("-maxconcurrency"); - m_args.push_back("10,20"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - // - - TEST_F(CommandLineOptionsTestFixture, TestTargetTimeoutHasEmptyOption_ExpectCommandLineOptionsException) - { - m_args.push_back("-ttimeout"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(CommandLineOptionsTestFixture, TestTargetTimeoutHasInRangeOptions_ExpectInRangeTestTargetTimeout) - { - m_args.push_back("-ttimeout"); - m_args.push_back("10"); - InitOptions(); - EXPECT_EQ(m_options->GetTestTargetTimeout(), AZStd::chrono::seconds(10)); - } - - TEST_F(CommandLineOptionsTestFixture, TestTargetTimeoutHasOutOfRangeOption_ExpectCommandLineOptionsException) - { - m_args.push_back("-ttimeout"); - m_args.push_back("-1"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(CommandLineOptionsTestFixture, TestTargetTimeoutInvalidOption_ExpectCommandLineOptionsException) - { - m_args.push_back("-ttimeout"); - m_args.push_back("foo"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(CommandLineOptionsTestFixture, TestTargetTimeoutHasMultipeValues_ExpectCommandLineOptionsException) - { - m_args.push_back("-ttimeout"); - m_args.push_back("10,20"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - // - - TEST_F(CommandLineOptionsTestFixture, GlobalTimeoutHasEmptyOption_ExpectCommandLineOptionsException) - { - m_args.push_back("-gtimeout"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(CommandLineOptionsTestFixture, GlobalTimeoutHasInRangeOptions_ExpectInRangeGlobalTimeout) - { - m_args.push_back("-gtimeout"); - m_args.push_back("10"); - InitOptions(); - EXPECT_EQ(m_options->GetGlobalTimeout(), AZStd::chrono::seconds(10)); - } - - TEST_F(CommandLineOptionsTestFixture, GlobalTimeoutHasOutOfRangeOption_ExpectCommandLineOptionsException) - { - m_args.push_back("-gtimeout"); - m_args.push_back("-1"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(CommandLineOptionsTestFixture, GlobalTimeoutInvalidOption_ExpectCommandLineOptionsException) - { - m_args.push_back("-gtimeout"); - m_args.push_back("foo"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(CommandLineOptionsTestFixture, GlobalTimeoutHasMultipeValues_ExpectCommandLineOptionsException) - { - m_args.push_back("-gtimeout"); - m_args.push_back("10,20"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - // - - TEST_F(CommandLineOptionsTestFixture, SafeModeHasEmptyOption_ExpectCommandLineOptionsException) - { - m_args.push_back("-safemode"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(CommandLineOptionsTestFixture, SafeModeHasOnOption_ExpectOnSafeMode) - { - m_args.push_back("-safemode"); - m_args.push_back("on"); - InitOptions(); - EXPECT_TRUE(m_options->HasSafeMode()); - } - - TEST_F(CommandLineOptionsTestFixture, SafeModeHasOffOption_ExpectOffSafeMode) - { - m_args.push_back("-safemode"); - m_args.push_back("off"); - InitOptions(); - EXPECT_FALSE(m_options->HasSafeMode()); - } - - TEST_F(CommandLineOptionsTestFixture, SafeModeInvalidOption_ExpectCommandLineOptionsException) - { - m_args.push_back("-safemode"); - m_args.push_back("foo"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(CommandLineOptionsTestFixture, SafeModeHasMultipeValues_ExpectCommandLineOptionsException) - { - m_args.push_back("-safemode"); - m_args.push_back("on,off"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(CommandLineOptionsTestFixture, SuiteFilterEmptyOption_ExpectCommandLineOptionsException) - { - m_args.push_back("-suite"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(CommandLineOptionsTestFixture, SuiteFilterMultipleOptions_ExpectCommandLineOptionsException) - { - m_args.push_back("-suite"); - m_args.push_back("periodic,smoke"); - - try - { - InitOptions(); - - // Do not expect the command line options construction to succeed - FAIL(); - } catch ([[maybe_unused]] const TestImpact::CommandLineOptionsException& e) - { - // Expect a command line options to be thrown - SUCCEED(); - } catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(CommandLineOptionsTestFixture, SuitesFilterMainOption_ExpectMainSuiteFilter) - { - m_args.push_back("-suites"); - m_args.push_back("main"); - InitOptions(); - EXPECT_EQ(m_options->GetSuiteFilter(), TestImpact::SuiteType::Main); - } - - TEST_F(CommandLineOptionsTestFixture, SuitesFilterPeriodicOption_ExpectPeriodicSuiteFilter) - { - m_args.push_back("-suites"); - m_args.push_back("periodic"); - InitOptions(); - EXPECT_EQ(m_options->GetSuiteFilter(), TestImpact::SuiteType::Periodic); - } - - TEST_F(CommandLineOptionsTestFixture, SuitesFilterSandboxOption_ExpectSandboxSuiteFilter) - { - m_args.push_back("-suites"); - m_args.push_back("sandbox"); - InitOptions(); - EXPECT_EQ(m_options->GetSuiteFilter(), TestImpact::SuiteType::Sandbox); - } -} diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Code/Tests/TestImpactConsoleMainTest.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Code/Tests/TestImpactConsoleMainTest.cpp deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Code/Tests/TestImpactConsoleTestSequenceEventHandlerTest.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Code/Tests/TestImpactConsoleTestSequenceEventHandlerTest.cpp deleted file mode 100644 index 52422daa31..0000000000 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Code/Tests/TestImpactConsoleTestSequenceEventHandlerTest.cpp +++ /dev/null @@ -1,42 +0,0 @@ -///* -// * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or -// * its licensors. -// * -// * For complete copyright and license terms please see the LICENSE at the root of this -// * distribution (the "License"). All use of this software is governed by the License, -// * or, if provided, by the license below or the license accompanying this file. Do not -// * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, -// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// * -// */ -// -//#include -//#include -// -//#include -// -//#include -//#include -//#include -//#include -// -//namespace UnitTest -//{ -// class ConsoleTestSequenceTestFixture -// : public AllocatorsTestFixture -// { -// }; -// -// TEST_F(ConsoleTestSequenceTestFixture, CheckEmptyArgs_ExpectDefaultValues) -// { -// AZStd::unordered_set suites = { "PERIODIC","MAIN" }; -// TestImpact::Console::TestSequenceEventHandler seq(&suites); -// AZStd::vector selectedTests = { "Test1", "Test2", "Test3", "Test4", "Test5" }; -// seq.operator()(TestImpact::Client::TestRunSelection(selectedTests, {"Test6"})); -// -// for (auto i = 0; i < selectedTests.size(); i++) -// { -// seq.operator()(TestImpact::Client::TestRun(selectedTests[i], (TestImpact::Client::TestRunResult)i, AZStd::chrono::milliseconds(i))); -// } -// } -//} diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Code/Tests/TestImpactFrontendConsoleStaticTestMain.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Code/Tests/TestImpactFrontendConsoleStaticTestMain.cpp deleted file mode 100644 index 9e128f90af..0000000000 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Code/Tests/TestImpactFrontendConsoleStaticTestMain.cpp +++ /dev/null @@ -1,34 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include -#include -#include - -class TestImpactTestEnvironment - : public AZ::Test::ITestEnvironment -{ -protected: - void SetupEnvironment() override - { - AZ::AllocatorInstance::Create(); - AZ::AllocatorInstance::Create(); - } - - void TeardownEnvironment() override - { - AZ::AllocatorInstance::Destroy(); - AZ::AllocatorInstance::Destroy(); - } -}; - -AZ_UNIT_TEST_HOOK(new TestImpactTestEnvironment); diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Code/Tests/TestImpactRuntimeConfigurationFactoryTest.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Code/Tests/TestImpactRuntimeConfigurationFactoryTest.cpp deleted file mode 100644 index 4b1b3f162e..0000000000 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Code/Tests/TestImpactRuntimeConfigurationFactoryTest.cpp +++ /dev/null @@ -1,27 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include - -#include - -#include - -namespace UnitTest -{ - //TEST(FOO, BAR) - //{ - // std::ifstream configFile("C:\\o3de_TIF_Feature\\windows_vs2019\\bin\\debug\\tiaf.debug.json"); - // AZStd::string configData((std::istreambuf_iterator(configFile)), std::istreambuf_iterator()); - // const auto configuration = TestImpact::ConfigurationFactory(configData); - //} -} diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Code/testimpactframework_frontend_console_static_tests_files.cmake b/Code/Tools/TestImpactFramework/Frontend/Console/Code/testimpactframework_frontend_console_static_tests_files.cmake index 4d8e65b2cc..5714be5dfb 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Code/testimpactframework_frontend_console_static_tests_files.cmake +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Code/testimpactframework_frontend_console_static_tests_files.cmake @@ -10,9 +10,4 @@ # set(FILES - Tests/TestImpactCommandLineOptionsTest.cpp - Tests/TestImpactRuntimeConfigurationFactoryTest.cpp - Tests/TestImpactFrontendConsoleStaticTestMain.cpp - Tests/TestImpactConsoleMainTest.cpp - Tests/TestImpactConsoleTestSequenceEventHandlerTest.cpp ) From b1508b522fd2d1e31301d7bb251b26dc6eae0ccd Mon Sep 17 00:00:00 2001 From: John Date: Wed, 16 Jun 2021 13:25:26 +0100 Subject: [PATCH 098/102] Add build output folder to tiaf driver --- scripts/build/Platform/Windows/build_config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build/Platform/Windows/build_config.json b/scripts/build/Platform/Windows/build_config.json index 81100eead7..9201e13a80 100644 --- a/scripts/build/Platform/Windows/build_config.json +++ b/scripts/build/Platform/Windows/build_config.json @@ -88,7 +88,7 @@ "COMMAND": "python_windows.cmd", "PARAMETERS": { "SCRIPT_PATH": "scripts/build/TestImpactAnalysis/tiaf_driver.py", - "SCRIPT_PARAMETERS": "--testFailurePolicy abort --safeMode --suite main --pipeline !PIPELINE_NAME! --destCommit !CHANGE_ID! --config \"!OUTPUT_DIRECTORY!\\TestImpactFramework\\persistent\\tiaf.profile.json\"" + "SCRIPT_PARAMETERS": "--testFailurePolicy abort --safeMode --suite main --pipeline !PIPELINE_NAME! --destCommit !CHANGE_ID! --config \"build\\windows_vs2019\\bin\\TestImpactFramework\\persistent\\tiaf.profile.json\"" } }, "debug_vs2019": { From e7e68d3ba3a90fab8ad019ba4ef2b85d9904d6f5 Mon Sep 17 00:00:00 2001 From: John Date: Wed, 16 Jun 2021 16:21:49 +0100 Subject: [PATCH 099/102] Set test failure policy to continue --- scripts/build/Platform/Windows/build_config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build/Platform/Windows/build_config.json b/scripts/build/Platform/Windows/build_config.json index 9201e13a80..db4e6b8fde 100644 --- a/scripts/build/Platform/Windows/build_config.json +++ b/scripts/build/Platform/Windows/build_config.json @@ -88,7 +88,7 @@ "COMMAND": "python_windows.cmd", "PARAMETERS": { "SCRIPT_PATH": "scripts/build/TestImpactAnalysis/tiaf_driver.py", - "SCRIPT_PARAMETERS": "--testFailurePolicy abort --safeMode --suite main --pipeline !PIPELINE_NAME! --destCommit !CHANGE_ID! --config \"build\\windows_vs2019\\bin\\TestImpactFramework\\persistent\\tiaf.profile.json\"" + "SCRIPT_PARAMETERS": "--testFailurePolicy abort --testFailurePolicy=continue --suite main --pipeline !PIPELINE_NAME! --destCommit !CHANGE_ID! --config \"build\\windows_vs2019\\bin\\TestImpactFramework\\persistent\\tiaf.profile.json\"" } }, "debug_vs2019": { From a1c6cb80a06364dca3b3121843c879dac762406a Mon Sep 17 00:00:00 2001 From: John Date: Wed, 16 Jun 2021 17:14:45 +0100 Subject: [PATCH 100/102] Fix double test failure policy --- scripts/build/Platform/Windows/build_config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build/Platform/Windows/build_config.json b/scripts/build/Platform/Windows/build_config.json index db4e6b8fde..e52a3c2d53 100644 --- a/scripts/build/Platform/Windows/build_config.json +++ b/scripts/build/Platform/Windows/build_config.json @@ -88,7 +88,7 @@ "COMMAND": "python_windows.cmd", "PARAMETERS": { "SCRIPT_PATH": "scripts/build/TestImpactAnalysis/tiaf_driver.py", - "SCRIPT_PARAMETERS": "--testFailurePolicy abort --testFailurePolicy=continue --suite main --pipeline !PIPELINE_NAME! --destCommit !CHANGE_ID! --config \"build\\windows_vs2019\\bin\\TestImpactFramework\\persistent\\tiaf.profile.json\"" + "SCRIPT_PARAMETERS": "--testFailurePolicy=continue --suite main --pipeline !PIPELINE_NAME! --destCommit !CHANGE_ID! --config \"build\\windows_vs2019\\bin\\TestImpactFramework\\persistent\\tiaf.profile.json\"" } }, "debug_vs2019": { From 8e9ffc458cf84e8a8c5a7389b44e456d7163ebab Mon Sep 17 00:00:00 2001 From: John Date: Thu, 17 Jun 2021 13:51:59 +0100 Subject: [PATCH 101/102] Fix PythonBindingsExampleTest.Application_Run_Fails --- Code/Tools/PythonBindingsExample/tests/ApplicationTests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Code/Tools/PythonBindingsExample/tests/ApplicationTests.cpp b/Code/Tools/PythonBindingsExample/tests/ApplicationTests.cpp index e9ea13fb82..a463a5216d 100644 --- a/Code/Tools/PythonBindingsExample/tests/ApplicationTests.cpp +++ b/Code/Tools/PythonBindingsExample/tests/ApplicationTests.cpp @@ -65,7 +65,7 @@ namespace PythonBindingsExample TEST_F(PythonBindingsExampleTest, Application_Run_Fails) { - EXPECT_FALSE(s_application->Run()); + EXPECT_TRUE(s_application->Run()); } TEST_F(PythonBindingsExampleTest, Application_RunWithParameters_Works) From 43b1d0444d93d967f41da003843d7402317740fd Mon Sep 17 00:00:00 2001 From: John Date: Thu, 17 Jun 2021 19:47:01 +0100 Subject: [PATCH 102/102] Fix PythonBindingsExampleTest name --- Code/Tools/PythonBindingsExample/tests/ApplicationTests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Code/Tools/PythonBindingsExample/tests/ApplicationTests.cpp b/Code/Tools/PythonBindingsExample/tests/ApplicationTests.cpp index a463a5216d..a57b35fb7e 100644 --- a/Code/Tools/PythonBindingsExample/tests/ApplicationTests.cpp +++ b/Code/Tools/PythonBindingsExample/tests/ApplicationTests.cpp @@ -63,7 +63,7 @@ namespace PythonBindingsExample AZStd::unique_ptr PythonBindingsExampleTest::s_application; - TEST_F(PythonBindingsExampleTest, Application_Run_Fails) + TEST_F(PythonBindingsExampleTest, Application_Run_Succeeds) { EXPECT_TRUE(s_application->Run()); }