From ac42a9a748fdb889136e5da378273711caa51f4a Mon Sep 17 00:00:00 2001 From: jonawals Date: Fri, 4 Jun 2021 19:33:26 +0100 Subject: [PATCH 1/2] Changes for read-only runs and new test suites --- .../Source/TestImpactCommandLineOptions.cpp | 55 +++----- .../Source/TestImpactCommandLineOptions.h | 9 +- .../Code/Source/TestImpactConsoleMain.cpp | 83 ++++++++---- .../TestImpactRuntimeConfigurationFactory.cpp | 21 ++- .../TestImpactConfiguration.h | 3 +- .../TestImpactFramework/TestImpactRuntime.h | 10 +- .../TestImpactTestSequence.h | 35 +++++ .../TestImpactTestTargetMetaMapFactory.cpp | 49 ++++--- .../TestImpactTestTargetMetaMapFactory.h | 6 +- .../TestImpactDynamicDependencyMap.cpp | 6 +- .../Runtime/Code/Source/TestImpactRuntime.cpp | 127 ++++++++++-------- .../Code/Source/TestImpactRuntimeUtils.cpp | 7 +- .../Code/Source/TestImpactRuntimeUtils.h | 1 + .../testimpactframework_runtime_files.cmake | 92 ++++++++----- ...timpactframework_runtime_tests_files.cmake | 38 ------ cmake/LYTestWrappers.cmake | 50 +++++-- .../ConsoleFrontendConfig.in | 23 +++- .../LYTestImpactFramework.cmake | 115 ++++++++++------ 18 files changed, 449 insertions(+), 281 deletions(-) delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_tests_files.cmake diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp index a6eb2e886e..5ff070601a 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp @@ -36,21 +36,21 @@ namespace TestImpact MaxConcurrency, TestTargetTimeout, GlobalTimeout, - SuitesFilter, + SuiteFilter, SafeMode, // Values None, Seed, Regular, ImpactAnalysis, + ImpactAnalysisNoWrite, ImpactAnalysisOrSeed, Locality, Abort, Continue, Ignore, StdOut, - File, - AllSuites + File }; constexpr const char* OptionKeys[] = @@ -70,21 +70,21 @@ namespace TestImpact "maxconcurrency", "ttimeout", "gtimeout", - "suites", + "suite", "safemode", // Values "none", "seed", "regular", "tia", + "tianowrite", "tiaorseed", "locality", "abort", "continue", "ignore", "stdout", - "file", - "*" + "file" }; RepoPath ParseConfigurationFile(const AZ::CommandLine& cmd) @@ -110,6 +110,7 @@ namespace TestImpact {OptionKeys[Seed], TestSequenceType::Seed}, {OptionKeys[Regular], TestSequenceType::Regular}, {OptionKeys[ImpactAnalysis], TestSequenceType::ImpactAnalysis}, + {OptionKeys[ImpactAnalysisNoWrite], TestSequenceType::ImpactAnalysisNoWrite}, {OptionKeys[ImpactAnalysisOrSeed], TestSequenceType::ImpactAnalysisOrSeed} }; @@ -250,32 +251,16 @@ namespace TestImpact return ParseOnOffOption(OptionKeys[SafeMode], states, cmd).value_or(false); } - AZStd::unordered_set ParseSuitesFilter(const AZ::CommandLine& cmd) + SuiteType ParseSuiteFilter(const AZ::CommandLine& cmd) { - AZStd::unordered_set suitesFilter; - if (const auto numSwitchValues = cmd.GetNumSwitchValues(OptionKeys[SuitesFilter]); - numSwitchValues) + const AZStd::vector> states = { - for (auto i = 0; i < numSwitchValues; i++) - { - const auto value = cmd.GetSwitchValue(OptionKeys[SuitesFilter], i); - AZ_TestImpact_Eval(!value.empty(), CommandLineOptionsException, "Suites option value is empty"); - if (value == OptionKeys[AllSuites]) - { - AZ_TestImpact_Eval( - suitesFilter.empty(), CommandLineOptionsException, "The * suite cannot be used with other suites"); - } + {GetSuiteTypeName(SuiteType::Main), SuiteType::Main}, + {GetSuiteTypeName(SuiteType::Periodic), SuiteType::Periodic}, + {GetSuiteTypeName(SuiteType::Sandbox), SuiteType::Sandbox} + }; - suitesFilter.insert(value); - } - } - - if (suitesFilter.find(OptionKeys[AllSuites]) != suitesFilter.end()) - { - return {}; - } - - return suitesFilter; + return ParseMultiStateOption(OptionKeys[SuiteFilter], states, cmd).value_or(SuiteType::Main); } } @@ -299,7 +284,7 @@ namespace TestImpact m_testTargetTimeout = ParseTestTargetTimeout(cmd); m_globalTimeout = ParseGlobalTimeout(cmd); m_safeMode = ParseSafeMode(cmd); - m_suitesFilter = ParseSuitesFilter(cmd); + m_suiteFilter = ParseSuiteFilter(cmd); } bool CommandLineOptions::HasChangeListFile() const @@ -382,9 +367,9 @@ namespace TestImpact return m_globalTimeout; } - const AZStd::unordered_set& CommandLineOptions::GetSuitesFilter() const + SuiteType CommandLineOptions::GetSuiteFilter() const { - return m_suitesFilter; + return m_suiteFilter; } AZStd::string CommandLineOptions::GetCommandLineUsageString() @@ -452,11 +437,7 @@ namespace TestImpact " -maxconcurrency= The maximum number of concurrent test targets/shards to be in flight at \n" " any given moment.\n" " -ochangelist= Outputs the change list used for test selection.\n" - " -suites= The test suites to select from for this test sequence (multiple values are \n" - " allowed). The suite all has special significance and will allow tests from \n" - " any suite to be selected, however this particular suite is mutually exclusive\n" - " with other suite. Note: this option is only applicable to the regular sequence\n" - " and, if safe mode is enables, the tia and tiaorseed sequences."; + " -suite= The test suite to select from for this test sequence."; return help; } diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.h b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.h index 73961b5fcc..99b1b98dba 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.h +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.h @@ -18,7 +18,6 @@ #include #include #include -#include namespace TestImpact { @@ -29,6 +28,8 @@ namespace TestImpact Seed, //!< Removes any prior coverage data and runs all test targets with instrumentation to reseed the data from scratch. Regular, //!< Runs all of the test targets without any instrumentation to generate coverage data (any prior coverage data is left intact). ImpactAnalysis, //!< Uses any prior coverage data to run the instrumented subset of selected tests (if no prior coverage data a regular run is performed instead). + ImpactAnalysisNoWrite, //!< Uses any prior coverage data to run the uninstrumented subset of selected tests (if no prior coverage data a regular run is performed instead). + //!< The coverage data is not updated with the subset of selected tests. ImpactAnalysisOrSeed //!< Uses any prior coverage data to run the instrumented subset of selected tests (if no prior coverage data a seed run is performed instead). }; @@ -87,8 +88,8 @@ namespace TestImpact //! Returns the global test sequence timeout to use (if any). const AZStd::optional& GetGlobalTimeout() const; - //! Returns the filter for test suites that will be allowed to be run. - const AZStd::unordered_set& GetSuitesFilter() const; + //! Returns the filter for test suite that will be allowed to be run. + SuiteType GetSuiteFilter() const; private: RepoPath m_configurationFile; @@ -105,7 +106,7 @@ namespace TestImpact AZStd::optional m_maxConcurrency; AZStd::optional m_testTargetTimeout; AZStd::optional m_globalTimeout; - AZStd::unordered_set m_suitesFilter; + SuiteType m_suiteFilter; bool m_safeMode = false; }; } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleMain.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleMain.cpp index 84e16c2332..fab6509244 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleMain.cpp +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleMain.cpp @@ -86,6 +86,8 @@ namespace TestImpact Runtime& runtime, const AZStd::optional& changeList) { + // Even though it is possible for a regular run to be selected (see below) which does not actually require a change list, + // consider any impact analysis sequence type without a change list to be an error AZ_TestImpact_Eval( changeList.has_value(), CommandLineOptionsException, @@ -94,39 +96,72 @@ namespace TestImpact TestSequenceResult result = TestSequenceResult::Failure; if (options.HasSafeMode()) { - auto [selectedResult, discardedResult] = runtime.SafeImpactAnalysisTestSequence( - changeList.value(), - options.GetSuitesFilter(), - options.GetTestPrioritizationPolicy(), - options.GetTestTargetTimeout(), - options.GetGlobalTimeout(), - AZStd::ref(sequenceEventHandler), - AZStd::ref(sequenceEventHandler), - AZStd::ref(sequenceEventHandler)); + if (options.GetTestSequenceType() == TestSequenceType::ImpactAnalysis) + { + auto [selectedResult, discardedResult] = runtime.SafeImpactAnalysisTestSequence( + changeList.value(), + options.GetTestPrioritizationPolicy(), + options.GetTestTargetTimeout(), + options.GetGlobalTimeout(), + AZStd::ref(sequenceEventHandler), + AZStd::ref(sequenceEventHandler), + AZStd::ref(sequenceEventHandler)); - // Handling the possible timeout and failure permutations of the selected and discarded test results is splitting hairs - // so apply the following, admittedly arbitrary, rules to determine what the composite test sequence result should be - if (selectedResult == TestSequenceResult::Success && discardedResult == TestSequenceResult::Success) - { - // Trivial case: both sequences succeeded - result = TestSequenceResult::Success; + // Handling the possible timeout and failure permutations of the selected and discarded test results is splitting hairs + // so apply the following, admittedly arbitrary, rules to determine what the composite test sequence result should be + if (selectedResult == TestSequenceResult::Success && discardedResult == TestSequenceResult::Success) + { + // Trivial case: both sequences succeeded + result = TestSequenceResult::Success; + } + else if (selectedResult == TestSequenceResult::Failure || discardedResult == TestSequenceResult::Failure) + { + // One sequence failed whilst the other sequence either succeeded or timed out + result = TestSequenceResult::Failure; + } + else + { + // One sequence timed out whilst the other sequence succeeded or both sequences timed out + result = TestSequenceResult::Timeout; + } } - else if (selectedResult == TestSequenceResult::Failure || discardedResult == TestSequenceResult::Failure) + else if (options.GetTestSequenceType() == TestSequenceType::ImpactAnalysisNoWrite) { - // One sequence failed whilst the other sequence either succeeded or timed out - result = TestSequenceResult::Failure; + // A no-write impact analysis sequence with safe mode enabled is functionally identical to a regular sequence type + // due to a) the selected tests being run without instrumentation and b) the discarded tests also being run without + // instrumentation + result = runtime.RegularTestSequence( + options.GetTestTargetTimeout(), + options.GetGlobalTimeout(), + AZStd::ref(sequenceEventHandler), + AZStd::ref(sequenceEventHandler), + AZStd::ref(sequenceEventHandler)); } else { - // One sequence timed out whilst the other sequence succeeded or both sequences timed out - result = TestSequenceResult::Timeout; + throw(Exception("Unexpected sequence type")); } } else { + Policy::DynamicDependencyMap dynamicDependencyMapPolicy; + if (options.GetTestSequenceType() == TestSequenceType::ImpactAnalysis) + { + dynamicDependencyMapPolicy = Policy::DynamicDependencyMap::Update; + } + else if (options.GetTestSequenceType() == TestSequenceType::ImpactAnalysisNoWrite) + { + dynamicDependencyMapPolicy = Policy::DynamicDependencyMap::Discard; + } + else + { + throw(Exception("Unexpected sequence type")); + } + result = runtime.ImpactAnalysisTestSequence( changeList.value(), options.GetTestPrioritizationPolicy(), + dynamicDependencyMapPolicy, options.GetTestTargetTimeout(), options.GetGlobalTimeout(), AZStd::ref(sequenceEventHandler), @@ -164,9 +199,11 @@ namespace TestImpact // As of now, there are no other non-test operations other than printing a change list so getting this far is considered an error AZ_TestImpact_Eval(options.GetTestSequenceType() != TestSequenceType::None, CommandLineOptionsException, "No action specified"); - std::cout << "Constructing in-memory model of source tree and test coverage, this may take a moment...\n"; + std::cout << "Constructing in-memory model of source tree and test coverage for test suite "; + std::cout << GetSuiteTypeName(options.GetSuiteFilter()).c_str() << ", this may take a moment...\n"; Runtime runtime( RuntimeConfigurationFactory(ReadFileContents(options.GetConfigurationFile())), + options.GetSuiteFilter(), options.GetExecutionFailurePolicy(), options.GetExecutionFailureDraftingPolicy(), options.GetTestFailurePolicy(), @@ -184,14 +221,13 @@ namespace TestImpact std::cout << "Test impact analysis data for this repository was not found, seed or regular sequence fallbacks will be used.\n"; } - TestSequenceEventHandler sequenceEventHandler(&options.GetSuitesFilter()); + TestSequenceEventHandler sequenceEventHandler(options.GetSuiteFilter()); switch (const auto type = options.GetTestSequenceType()) { case TestSequenceType::Regular: { const auto result = runtime.RegularTestSequence( - options.GetSuitesFilter(), options.GetTestTargetTimeout(), options.GetGlobalTimeout(), AZStd::ref(sequenceEventHandler), @@ -211,6 +247,7 @@ namespace TestImpact return GetReturnCodeForTestSequenceResult(result); } + case TestSequenceType::ImpactAnalysisNoWrite: case TestSequenceType::ImpactAnalysis: { return WrappedImpactAnalysisTestSequence(sequenceEventHandler, options, runtime, changeList); diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactRuntimeConfigurationFactory.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactRuntimeConfigurationFactory.cpp index 9db99d3c2e..6ef1ce4172 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactRuntimeConfigurationFactory.cpp +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactRuntimeConfigurationFactory.cpp @@ -30,7 +30,7 @@ namespace TestImpact "relative_paths", "artifact_dir", "enumeration_cache_dir", - "test_impact_data_file", + "test_impact_data_files", "temp", "active", "target_sources", @@ -75,7 +75,7 @@ namespace TestImpact RelativePaths, ArtifactDir, EnumerationCacheDir, - TestImpactDataFile, + TestImpactDataFiles, TempWorkspace, ActiveWorkspace, TargetSources, @@ -144,6 +144,19 @@ namespace TestImpact return tempWorkspaceConfig; } + AZStd::array ParseTestImpactAnalysisDataFiles(const RepoPath& root, const rapidjson::Value& sparTIAFile) + { + AZStd::array sparTIAFiles; + sparTIAFiles[static_cast(SuiteType::Main)] = + GetAbsPathFromRelPath(root, sparTIAFile[GetSuiteTypeName(SuiteType::Main).c_str()].GetString()); + sparTIAFiles[static_cast(SuiteType::Periodic)] = + GetAbsPathFromRelPath(root, sparTIAFile[GetSuiteTypeName(SuiteType::Periodic).c_str()].GetString()); + sparTIAFiles[static_cast(SuiteType::Sandbox)] = + GetAbsPathFromRelPath(root, sparTIAFile[GetSuiteTypeName(SuiteType::Sandbox).c_str()].GetString()); + + return sparTIAFiles; + } + WorkspaceConfig::Active ParseActiveWorkspaceConfig(const rapidjson::Value& activeWorkspace) { WorkspaceConfig::Active activeWorkspaceConfig; @@ -151,8 +164,8 @@ namespace TestImpact activeWorkspaceConfig.m_root = activeWorkspace[Config::Keys[Config::Root]].GetString(); activeWorkspaceConfig.m_enumerationCacheDirectory = GetAbsPathFromRelPath(activeWorkspaceConfig.m_root, relativePaths[Config::Keys[Config::EnumerationCacheDir]].GetString()); - activeWorkspaceConfig.m_sparTIAFile - = GetAbsPathFromRelPath(activeWorkspaceConfig.m_root, relativePaths[Config::Keys[Config::TestImpactDataFile]].GetString()); + activeWorkspaceConfig.m_sparTIAFiles = + ParseTestImpactAnalysisDataFiles(activeWorkspaceConfig.m_root, relativePaths[Config::Keys[Config::TestImpactDataFiles]]); return activeWorkspaceConfig; } diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactConfiguration.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactConfiguration.h index 91d748edfe..ee73ac572d 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactConfiguration.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactConfiguration.h @@ -16,6 +16,7 @@ #include #include +#include #include namespace TestImpact @@ -46,8 +47,8 @@ namespace TestImpact struct Active { RepoPath m_root; //!< Path to the persistent workspace tracked by the repository. - RepoPath m_sparTIAFile; //!< Path to the test impact analysis data. RepoPath m_enumerationCacheDirectory; //!< Path to the test enumerations cache. + AZStd::array m_sparTIAFiles; //!< Paths to the test impact analysis data files for each test suite. }; Temp m_temp; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h index 266bd5ef7a..ae1f78c5c7 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h @@ -92,6 +92,7 @@ namespace TestImpact public: //! Constructs a runtime with the specified configuration and policies. //! @param config The configuration used for this runtime instance. + //! @param suiteFilter The test suite for which the coverage data and test selection will draw from. //! @param executionFailurePolicy Determines how to handle test targets that fail to execute. //! @param executionFailureDraftingPolicy Determines how test targets that previously failed to execute are drafted into subsequent test sequences. //! @param testFailurePolicy Determines how to handle test targets that report test failures. @@ -99,6 +100,7 @@ namespace TestImpact //! @param testShardingPolicy Determines how to handle test targets that have opted in to test sharding. Runtime( RuntimeConfig&& config, + SuiteType suiteFilter, Policy::ExecutionFailure executionFailurePolicy, Policy::ExecutionFailureDrafting executionFailureDraftingPolicy, Policy::TestFailure testFailurePolicy, @@ -110,7 +112,6 @@ namespace TestImpact ~Runtime(); //! Runs a test sequence where all tests with a matching suite in the suite filter and also not on the excluded list are selected. - //! @param suitesFilter The test suites that will be included in the test selection. //! @param testTargetTimeout The maximum duration individual test targets may be in flight for (infinite if empty). //! @param globalTimeout The maximum duration the entire test sequence may run for (infinite if empty). //! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests. @@ -118,7 +119,6 @@ namespace TestImpact //! @param testRunCompleteCallback The client function to be called after an individual test run has completed. //! @returns TestSequenceResult RegularTestSequence( - const AZStd::unordered_set suitesFilter, AZStd::optional testTargetTimeout, AZStd::optional globalTimeout, AZStd::optional testSequenceStartCallback, @@ -128,6 +128,7 @@ namespace TestImpact //! Runs a test sequence where tests are selected according to test impact analysis so long as they are not on the excluded list. //! @param changeList The change list used to determine the tests to select. //! @param testPrioritizationPolicy Determines how selected tests will be prioritized. + //! @param dynamicDependencyMapPolicy The policy to determine how the coverage data of produced by test sequences is used to update the dynamic dependency map. //! @param testTargetTimeout The maximum duration individual test targets may be in flight for (infinite if empty). //! @param globalTimeout The maximum duration the entire test sequence may run for (infinite if empty). //! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests. @@ -137,6 +138,7 @@ namespace TestImpact TestSequenceResult ImpactAnalysisTestSequence( const ChangeList& changeList, Policy::TestPrioritization testPrioritizationPolicy, + Policy::DynamicDependencyMap dynamicDependencyMapPolicy, AZStd::optional testTargetTimeout, AZStd::optional globalTimeout, AZStd::optional testSequenceStartCallback, @@ -145,7 +147,6 @@ namespace TestImpact //! Runs a test sequence as per the ImpactAnalysisTestSequence where the tests not selected are also run (albeit without instrumentation). //! @param changeList The change list used to determine the tests to select. - //! @param suitesFilter The test suites that will be included in the test selection. //! @param testPrioritizationPolicy Determines how selected tests will be prioritized. //! @param testTargetTimeout The maximum duration individual test targets may be in flight for (infinite if empty). //! @param globalTimeout The maximum duration the entire test sequence may run for (infinite if empty). @@ -155,7 +156,6 @@ namespace TestImpact //! @returns AZStd::pair SafeImpactAnalysisTestSequence( const ChangeList& changeList, - const AZStd::unordered_set suitesFilter, Policy::TestPrioritization testPrioritizationPolicy, AZStd::optional testTargetTimeout, AZStd::optional globalTimeout, @@ -207,6 +207,8 @@ namespace TestImpact void UpdateAndSerializeDynamicDependencyMap(const SourceCoveringTestsList& sourceCoverageTestsList); RuntimeConfig m_config; + SuiteType m_suiteFilter; + RepoPath m_sparTIAFile; Policy::ExecutionFailure m_executionFailurePolicy; Policy::ExecutionFailureDrafting m_executionFailureDraftingPolicy; Policy::TestFailure m_testFailurePolicy; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactTestSequence.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactTestSequence.h index 594c997c45..4e8908cf3f 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactTestSequence.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactTestSequence.h @@ -12,6 +12,10 @@ #pragma once +#include + +#include + namespace TestImpact { namespace Policy @@ -55,6 +59,13 @@ namespace TestImpact Continue //!< Continue the test sequence and report the test failures after the run. }; + //! Policy for updating the dynamic dependency map with the coverage data of produced by test sequences. + enum class DynamicDependencyMap + { + Discard, //!< Discard the coverage data produced by test sequences. + Update //!< Update the dynamic dependency map with the coverage data produced by test sequences. + }; + //! Policy for sharding test targets that have been marked for test sharding. enum class TestSharding { @@ -82,6 +93,30 @@ namespace TestImpact TestInterleaved //!< Tests are interlaced across shards agnostic of fixtures (fastest but prone to inter-test dependency problems). }; + //! Test suite types to select from. + enum class SuiteType : AZ::u8 + { + Main = 0, + Periodic, + Sandbox + }; + + //! User-friendly names for the test suite types. + inline AZStd::string GetSuiteTypeName(SuiteType suiteType) + { + switch (suiteType) + { + case SuiteType::Main: + return "main"; + case SuiteType::Periodic: + return "periodic"; + case SuiteType::Sandbox: + return "sandbox"; + default: + throw(RuntimeException("Unexpected suite type")); + } + } + //! Result of a test sequence that was run. enum class TestSequenceResult { diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestTargetMetaMapFactory.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestTargetMetaMapFactory.cpp index 8bbb5200bb..4b247086ed 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestTargetMetaMapFactory.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestTargetMetaMapFactory.cpp @@ -19,7 +19,7 @@ namespace TestImpact { - TestTargetMetaMap TestTargetMetaMapFactory(const AZStd::string& masterTestListData) + TestTargetMetaMap TestTargetMetaMapFactory(const AZStd::string& masterTestListData, SuiteType suiteType) { // Keys for pertinent JSON node and attribute names constexpr const char* Keys[] = @@ -27,6 +27,7 @@ namespace TestImpact "google", "test", "tests", + "suites", "suite", "launch_method", "test_runner", @@ -41,6 +42,7 @@ namespace TestImpact GoogleKey, TestKey, TestsKey, + TestSuitesKey, SuiteKey, LaunchMethodKey, TestRunnerKey, @@ -64,26 +66,35 @@ namespace TestImpact for (const auto& test : tests) { TestTargetMeta testMeta; - testMeta.m_suite = test[Keys[SuiteKey]].GetString(); - testMeta.m_customArgs = test[Keys[CommandKey]].GetString(); - testMeta.m_timeout = AZStd::chrono::seconds{ test[Keys[TimeoutKey]].GetUint() }; + const auto testSuites = test[Keys[TestSuitesKey]].GetArray(); + for (const auto& suite : testSuites) + { + // Check to see if this test target has the suite we're looking for + if (const auto suiteName = suite[Keys[SuiteKey]].GetString(); + strcmp(GetSuiteTypeName(suiteType).c_str(), suiteName) == 0) + { + testMeta.m_suite = suiteName; + testMeta.m_customArgs = suite[Keys[CommandKey]].GetString(); + testMeta.m_timeout = AZStd::chrono::seconds{ suite[Keys[TimeoutKey]].GetUint() }; + if (const auto buildTypeString = test[Keys[LaunchMethodKey]].GetString(); strcmp(buildTypeString, Keys[TestRunnerKey]) == 0) + { + testMeta.m_launchMethod = LaunchMethod::TestRunner; + } + else if (strcmp(buildTypeString, Keys[StandAloneKey]) == 0) + { + testMeta.m_launchMethod = LaunchMethod::StandAlone; + } + else + { + throw(ArtifactException("Unexpected test build type")); + } - if (const auto buildTypeString = test[Keys[LaunchMethodKey]].GetString(); strcmp(buildTypeString, Keys[TestRunnerKey]) == 0) - { - testMeta.m_launchMethod = LaunchMethod::TestRunner; + AZStd::string name = test[Keys[NameKey]].GetString(); + AZ_TestImpact_Eval(!name.empty(), ArtifactException, "Test name field cannot be empty"); + testMetas.emplace(AZStd::move(name), AZStd::move(testMeta)); + break; + } } - else if (strcmp(buildTypeString, Keys[StandAloneKey]) == 0) - { - testMeta.m_launchMethod = LaunchMethod::StandAlone; - } - else - { - throw(ArtifactException("Unexpected test build type")); - } - - AZStd::string name = test[Keys[NameKey]].GetString(); - AZ_TestImpact_Eval(!name.empty(), ArtifactException, "Test name field cannot be empty"); - testMetas.emplace(AZStd::move(name), AZStd::move(testMeta)); } // If there's no tests in the repo then something is seriously wrong diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestTargetMetaMapFactory.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestTargetMetaMapFactory.h index 039a9e89e2..b08babf528 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestTargetMetaMapFactory.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestTargetMetaMapFactory.h @@ -12,14 +12,16 @@ #pragma once +#include #include #include namespace TestImpact { - //! Constructs a list of test target meta-data artifacts from the specified master test list data. + //! Constructs a list of test target meta-data artifacts of the specified suite type from the specified master test list data. //! @param masterTestListData The raw master test list data in JSON format. + //! @param suiteType The suite type to select the target meta-data artifacts from. //! @return The constructed list of test target meta-data artifacts. - TestTargetMetaMap TestTargetMetaMapFactory(const AZStd::string& masterTestListData); + TestTargetMetaMap TestTargetMetaMapFactory(const AZStd::string& masterTestListData, SuiteType suiteType); } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp index 826fa0a87d..ece5a7c13b 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp @@ -371,11 +371,11 @@ namespace TestImpact { if (sourceDependency->GetNumCoveringTestTargets()) { - AZ_Warning( - "File Update", false, AZStd::string::format("Source file %s is potentially an orphan (used by build targets " + AZ_Printf( + "File Update", AZStd::string::format("Source file '%s' is potentially an orphan (used by build targets " "without explicitly being added to the build system, e.g. an include directive pulling in a header from the " "repository). Running the covering tests for this file with instrumentation will confirm whether or nor this " - "is the case", updatedFile.c_str()).c_str()); + "is the case.\n", updatedFile.c_str()).c_str()); updateDependencies.emplace_back(AZStd::move(*sourceDependency)); coverageToDelete.push_back(updatedFile); diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp index 83808f4ed2..2331c76880 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp @@ -72,6 +72,7 @@ namespace TestImpact Runtime::Runtime( RuntimeConfig&& config, + SuiteType suiteFilter, Policy::ExecutionFailure executionFailurePolicy, Policy::ExecutionFailureDrafting executionFailureDraftingPolicy, Policy::TestFailure testFailurePolicy, @@ -80,6 +81,7 @@ namespace TestImpact Policy::TargetOutputCapture targetOutputCapture, AZStd::optional maxConcurrency) : m_config(AZStd::move(config)) + , m_suiteFilter(suiteFilter) , m_executionFailurePolicy(executionFailurePolicy) , m_executionFailureDraftingPolicy(executionFailureDraftingPolicy) , m_testFailurePolicy(testFailurePolicy) @@ -89,7 +91,7 @@ namespace TestImpact , m_maxConcurrency(maxConcurrency.value_or(AZStd::thread::hardware_concurrency())) { // Construct the dynamic dependency map from the build target descriptors - m_dynamicDependencyMap = ConstructDynamicDependencyMap(m_config.m_buildTargetDescriptor, m_config.m_testTargetMeta); + m_dynamicDependencyMap = ConstructDynamicDependencyMap(suiteFilter, m_config.m_buildTargetDescriptor, m_config.m_testTargetMeta); // Construct the test selector and prioritizer from the dependency graph data (NOTE: currently not implemented) m_testSelectorAndPrioritizer = AZStd::make_unique(m_dynamicDependencyMap.get(), DependencyGraphDataMap{}); @@ -110,7 +112,8 @@ namespace TestImpact try { // Populate the dynamic dependency map with the existing source coverage data (if any) - const auto tiaDataRaw = ReadFileContents(m_config.m_workspace.m_active.m_sparTIAFile); + m_sparTIAFile = m_config.m_workspace.m_active.m_sparTIAFiles[static_cast(m_suiteFilter)].String(); + const auto tiaDataRaw = ReadFileContents(m_sparTIAFile); const auto tiaData = DeserializeSourceCoveringTestsList(tiaDataRaw); if (tiaData.GetNumSources()) { @@ -118,13 +121,17 @@ namespace TestImpact m_hasImpactAnalysisData = true; // Enumerate new test targets - m_testEngine->UpdateEnumerationCache( - m_dynamicDependencyMap->GetNotCoveringTests(), - Policy::ExecutionFailure::Ignore, - Policy::TestFailure::Continue, - AZStd::nullopt, - AZStd::nullopt, - AZStd::nullopt); + const auto testTargetsWithNoEnumeration = m_dynamicDependencyMap->GetNotCoveringTests(); + if (!testTargetsWithNoEnumeration.empty()) + { + m_testEngine->UpdateEnumerationCache( + testTargetsWithNoEnumeration, + Policy::ExecutionFailure::Ignore, + Policy::TestFailure::Continue, + AZStd::nullopt, + AZStd::nullopt, + AZStd::nullopt); + } } } catch (const DependencyException& e) @@ -136,8 +143,10 @@ namespace TestImpact } catch ([[maybe_unused]]const Exception& e) { - AZ_Printf("No test impact analysis data found at %s", m_config.m_workspace.m_active.m_sparTIAFile.c_str()); - } + AZ_Printf("TestImpactRuntime", + AZStd::string::format( + "No test impact analysis data found for suite '%s' at %s", GetSuiteTypeName(m_suiteFilter).c_str(), m_sparTIAFile.c_str()).c_str()); + } } Runtime::~Runtime() = default; @@ -168,13 +177,16 @@ namespace TestImpact addMutatedTestTargetsToEnumerationList(changeDependencyList.GetDeleteSourceDependencies()); // Enumerate the mutated test targets to ensure their enumeration caches are up to date - m_testEngine->UpdateEnumerationCache( - testTargets, - Policy::ExecutionFailure::Ignore, - Policy::TestFailure::Continue, - AZStd::nullopt, - AZStd::nullopt, - AZStd::nullopt); + if (!testTargets.empty()) + { + m_testEngine->UpdateEnumerationCache( + testTargets, + Policy::ExecutionFailure::Ignore, + Policy::TestFailure::Continue, + AZStd::nullopt, + AZStd::nullopt, + AZStd::nullopt); + } } AZStd::pair, AZStd::vector> Runtime::SelectCoveringTestTargetsAndUpdateEnumerationCache( @@ -233,7 +245,7 @@ namespace TestImpact void Runtime::ClearDynamicDependencyMapAndRemoveExistingFile() { - DeleteFile(m_config.m_workspace.m_active.m_sparTIAFile); + DeleteFile(m_sparTIAFile); m_dynamicDependencyMap->ClearAllSourceCoverage(); } @@ -247,12 +259,11 @@ namespace TestImpact m_dynamicDependencyMap->ReplaceSourceCoverage(sourceCoverageTestsList); const auto sparTIA = m_dynamicDependencyMap->ExportSourceCoverage(); const auto sparTIAData = SerializeSourceCoveringTestsList(sparTIA); - WriteFileContents(sparTIAData, m_config.m_workspace.m_active.m_sparTIAFile); + WriteFileContents(sparTIAData, m_sparTIAFile); m_hasImpactAnalysisData = true; } TestSequenceResult Runtime::RegularTestSequence( - const AZStd::unordered_set suitesFilter, AZStd::optional testTargetTimeout, AZStd::optional globalTimeout, AZStd::optional testSequenceStartCallback, @@ -268,21 +279,7 @@ namespace TestImpact { if (!m_testTargetExcludeList.contains(&testTarget)) { - if (suitesFilter.empty()) - { - // Suite filter is empty, all tests that are not on the excluded list are included - includedTestTargets.push_back(&testTarget); - } - else if(suitesFilter.contains(testTarget.GetSuite())) - { - // Test target belonging to a suite in the suite filter are included, provided that are not on the exclude list - includedTestTargets.push_back(&testTarget); - } - else - { - // Test target not belonging to a suite in the suite filter are excluded - excludedTestTargets.push_back(&testTarget); - } + includedTestTargets.push_back(&testTarget); } else { @@ -318,6 +315,7 @@ namespace TestImpact TestSequenceResult Runtime::ImpactAnalysisTestSequence( const ChangeList& changeList, Policy::TestPrioritization testPrioritizationPolicy, + Policy::DynamicDependencyMap dynamicDependencyMapPolicy, AZStd::optional testTargetTimeout, AZStd::optional globalTimeout, AZStd::optional testSequenceStartCallback, @@ -338,30 +336,51 @@ namespace TestImpact ExtractTestTargetNames(draftedTestTargets)); } - const auto [result, testJobs] = m_testEngine->InstrumentedRun( - includedSelectedTestTargets, - m_testShardingPolicy, - m_executionFailurePolicy, - Policy::IntegrityFailure::Continue, - m_testFailurePolicy, - m_targetOutputCapture, - testTargetTimeout, - globalTimeout, - TestRunCompleteCallbackHandler(testCompleteCallback)); - - UpdateAndSerializeDynamicDependencyMap(CreateSourceCoveringTestFromTestCoverages(testJobs, m_config.m_repo.m_root)); - - if (testSequenceEndCallback.has_value()) + if (dynamicDependencyMapPolicy == Policy::DynamicDependencyMap::Update) { - (*testSequenceEndCallback)(GenerateSequenceFailureReport(testJobs), timer.Elapsed()); - } + const auto [result, testJobs] = m_testEngine->InstrumentedRun( + includedSelectedTestTargets, + m_testShardingPolicy, + m_executionFailurePolicy, + Policy::IntegrityFailure::Continue, + m_testFailurePolicy, + m_targetOutputCapture, + testTargetTimeout, + globalTimeout, + TestRunCompleteCallbackHandler(testCompleteCallback)); - return result; + UpdateAndSerializeDynamicDependencyMap(CreateSourceCoveringTestFromTestCoverages(testJobs, m_config.m_repo.m_root)); + + if (testSequenceEndCallback.has_value()) + { + (*testSequenceEndCallback)(GenerateSequenceFailureReport(testJobs), timer.Elapsed()); + } + + return result; + } + else + { + const auto [result, testJobs] = m_testEngine->RegularRun( + includedSelectedTestTargets, + m_testShardingPolicy, + m_executionFailurePolicy, + m_testFailurePolicy, + m_targetOutputCapture, + testTargetTimeout, + globalTimeout, + TestRunCompleteCallbackHandler(testCompleteCallback)); + + if (testSequenceEndCallback.has_value()) + { + (*testSequenceEndCallback)(GenerateSequenceFailureReport(testJobs), timer.Elapsed()); + } + + return result; + } } AZStd::pair Runtime::SafeImpactAnalysisTestSequence( const ChangeList& changeList, - const AZStd::unordered_set suitesFilter, Policy::TestPrioritization testPrioritizationPolicy, AZStd::optional testTargetTimeout, AZStd::optional globalTimeout, diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.cpp index 87a402bd6a..47c0e83233 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.cpp @@ -22,10 +22,10 @@ namespace TestImpact { - TestTargetMetaMap ReadTestTargetMetaMapFile(const RepoPath& testTargetMetaConfigFile) + TestTargetMetaMap ReadTestTargetMetaMapFile(SuiteType suiteFilter, const RepoPath& testTargetMetaConfigFile) { const auto masterTestListData = ReadFileContents(testTargetMetaConfigFile); - return TestTargetMetaMapFactory(masterTestListData); + return TestTargetMetaMapFactory(masterTestListData, suiteFilter); } AZStd::vector ReadBuildTargetDescriptorFiles(const BuildTargetDescriptorConfig& buildTargetDescriptorConfig) @@ -46,10 +46,11 @@ namespace TestImpact } AZStd::unique_ptr ConstructDynamicDependencyMap( + SuiteType suiteFilter, const BuildTargetDescriptorConfig& buildTargetDescriptorConfig, const TestTargetMetaConfig& testTargetMetaConfig) { - auto testTargetmetaMap = ReadTestTargetMetaMapFile(testTargetMetaConfig.m_metaFile); + auto testTargetmetaMap = ReadTestTargetMetaMapFile(suiteFilter, testTargetMetaConfig.m_metaFile); auto buildTargetDescriptors = ReadBuildTargetDescriptorFiles(buildTargetDescriptorConfig); auto buildTargets = CompileTargetDescriptors(AZStd::move(buildTargetDescriptors), AZStd::move(testTargetmetaMap)); auto&& [productionTargets, testTargets] = buildTargets; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.h index 56775b1c58..943b0cbec4 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.h @@ -30,6 +30,7 @@ namespace TestImpact { //! Construct a dynamic dependency map from the build target descriptors and test target metas. AZStd::unique_ptr ConstructDynamicDependencyMap( + SuiteType suiteFilter, const BuildTargetDescriptorConfig& buildTargetDescriptorConfig, const TestTargetMetaConfig& testTargetMetaConfig); diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_files.cmake b/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_files.cmake index da3693433e..2053a1ea5e 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_files.cmake +++ b/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_files.cmake @@ -10,15 +10,23 @@ # set(FILES - Include/TestImpactFramework/TestImpactBitwise.h - Include/TestImpactFramework/TestImpactCallback.h Include/TestImpactFramework/TestImpactException.h - Include/TestImpactFramework/TestImpactFrameworkPath.h + Include/TestImpactFramework/TestImpactRepoPath.h + Include/TestImpactFramework/TestImpactRuntime.h + Include/TestImpactFramework/TestImpactRuntimeException.h + Include/TestImpactFramework/TestImpactConfiguration.h + Include/TestImpactFramework/TestImpactConfigurationException.h + Include/TestImpactFramework/TestImpactChangelist.h + Include/TestImpactFramework/TestImpactChangelistSerializer.h + Include/TestImpactFramework/TestImpactChangelistException.h + Include/TestImpactFramework/TestImpactTestSequence.h + Include/TestImpactFramework/TestImpactClientTestSelection.h + Include/TestImpactFramework/TestImpactClientTestRun.h + Include/TestImpactFramework/TestImpactClientFailureReport.h + Include/TestImpactFramework/TestImpactFileUtils.h Source/Artifact/TestImpactArtifactException.h Source/Artifact/Factory/TestImpactBuildTargetDescriptorFactory.cpp Source/Artifact/Factory/TestImpactBuildTargetDescriptorFactory.h - Source/Artifact/Factory/TestImpactChangeListFactory.cpp - Source/Artifact/Factory/TestImpactChangeListFactory.h Source/Artifact/Factory/TestImpactTestEnumerationSuiteFactory.cpp Source/Artifact/Factory/TestImpactTestEnumerationSuiteFactory.h Source/Artifact/Factory/TestImpactTestRunSuiteFactory.cpp @@ -27,6 +35,8 @@ set(FILES Source/Artifact/Factory/TestImpactTestTargetMetaMapFactory.h Source/Artifact/Factory/TestImpactModuleCoverageFactory.cpp Source/Artifact/Factory/TestImpactModuleCoverageFactory.h + Source/Artifact/Factory/TestImpactDependencyGraphDataFactory.cpp + Source/Artifact/Factory/TestImpactDependencyGraphDataFactory.h Source/Artifact/Static/TestImpactBuildTargetDescriptor.cpp Source/Artifact/Static/TestImpactBuildTargetDescriptor.h Source/Artifact/Static/TestImpactTargetDescriptorCompiler.cpp @@ -37,7 +47,6 @@ set(FILES Source/Artifact/Static/TestImpactTestTargetDescriptor.cpp Source/Artifact/Static/TestImpactTestTargetDescriptor.h Source/Artifact/Static/TestImpactDependencyGraphData.h - Source/Artifact/Dynamic/TestImpactChangelist.h Source/Artifact/Dynamic/TestImpactTestEnumerationSuite.h Source/Artifact/Dynamic/TestImpactTestRunSuite.h Source/Artifact/Dynamic/TestImpactTestSuite.h @@ -50,6 +59,8 @@ set(FILES Source/Process/TestImpactProcessLauncher.h Source/Process/JobRunner/TestImpactProcessJob.h Source/Process/JobRunner/TestImpactProcessJobInfo.h + Source/Process/JobRunner/TestImpactProcessJobMeta.cpp + Source/Process/JobRunner/TestImpactProcessJobMeta.h Source/Process/JobRunner/TestImpactProcessJobRunner.h Source/Process/Scheduler/TestImpactProcessScheduler.cpp Source/Process/Scheduler/TestImpactProcessScheduler.h @@ -64,6 +75,8 @@ set(FILES Source/Dependency/TestImpactTestSelectorAndPrioritizer.cpp Source/Dependency/TestImpactSourceCoveringTestsList.h Source/Dependency/TestImpactSourceCoveringTestsList.cpp + Source/Dependency/TestImpactSourceCoveringTestsSerializer.cpp + Source/Dependency/TestImpactSourceCoveringTestsSerializer.h Source/Target/TestImpactBuildTarget.cpp Source/Target/TestImpactBuildTarget.h Source/Target/TestImpactBuildTargetList.h @@ -74,29 +87,48 @@ set(FILES Source/Target/TestImpactTestTarget.cpp Source/Target/TestImpactTestTarget.h Source/Target/TestImpactTestTargetList.h - Source/Test/Enumeration/TestImpactTestEnumeration.h - Source/Test/Enumeration/TestImpactTestEnumerationException.h - Source/Test/Enumeration/TestImpactTestEnumerationSerializer.cpp - Source/Test/Enumeration/TestImpactTestEnumerationSerializer.h - Source/Test/Enumeration/TestImpactTestEnumerator.cpp - Source/Test/Enumeration/TestImpactTestEnumerator.h - Source/Test/Run/TestImpactTestRunSerializer.cpp - Source/Test/Run/TestImpactTestRunSerializer.h - Source/Test/Run/TestImpactTestRunner.cpp - Source/Test/Run/TestImpactTestRunner.h - Source/Test/Run/TestImpactInstrumentedTestRunner.cpp - Source/Test/Run/TestImpactInstrumentedTestRunner.h - Source/Test/Run/TestImpactTestRun.cpp - Source/Test/Run/TestImpactTestRun.h - Source/Test/Run/TestImpactTestRunJobData.cpp - Source/Test/Run/TestImpactTestRunJobData.h - Source/Test/Run/TestImpactTestCoverage.cpp - Source/Test/Run/TestImpactTestCoverage.h - Source/Test/Run/TestImpactTestRunException.h - Source/Test/Job/TestImpactTestJobRunner.h - Source/Test/Job/TestImpactTestJobException.h - Source/Test/Job/TestImpactTestJobCommon.h - Source/Test/TestImpactTestSuiteContainer.h + Source/TestEngine/Enumeration/TestImpactTestEnumeration.h + Source/TestEngine/Enumeration/TestImpactTestEnumerationSerializer.cpp + Source/TestEngine/Enumeration/TestImpactTestEnumerationSerializer.h + Source/TestEngine/Enumeration/TestImpactTestEnumerator.cpp + Source/TestEngine/Enumeration/TestImpactTestEnumerator.h + Source/TestEngine/Run/TestImpactTestRunSerializer.cpp + Source/TestEngine/Run/TestImpactTestRunSerializer.h + Source/TestEngine/Run/TestImpactTestRunner.cpp + Source/TestEngine/Run/TestImpactTestRunner.h + Source/TestEngine/Run/TestImpactInstrumentedTestRunner.cpp + Source/TestEngine/Run/TestImpactInstrumentedTestRunner.h + Source/TestEngine/Run/TestImpactTestRun.cpp + Source/TestEngine/Run/TestImpactTestRun.h + Source/TestEngine/Run/TestImpactTestRunJobData.cpp + Source/TestEngine/Run/TestImpactTestRunJobData.h + Source/TestEngine/Run/TestImpactTestCoverage.cpp + Source/TestEngine/Run/TestImpactTestCoverage.h + Source/TestEngine/JobRunner/TestImpactTestJobRunner.h + Source/TestEngine/JobRunner/TestImpactTestJobInfoGenerator.cpp + Source/TestEngine/JobRunner/TestImpactTestJobInfoGenerator.h + Source/TestEngine/JobRunner/TestImpactTestTargetExtension.h + Source/TestEngine/TestImpactTestEngineJobFailure.cpp + Source/TestEngine/TestImpactTestEngineJobFailure.h + Source/TestEngine/TestImpactTestSuiteContainer.h + Source/TestEngine/TestImpactTestEngine.cpp + Source/TestEngine/TestImpactTestEngine.h + Source/TestEngine/TestImpactTestEngineJob.cpp + Source/TestEngine/TestImpactTestEngineJob.h + Source/TestEngine/TestImpactTestEngineEnumeration.cpp + Source/TestEngine/TestImpactTestEngineEnumeration.h + Source/TestEngine/TestImpactTestEngineRegularRun.cpp + Source/TestEngine/TestImpactTestEngineRegularRun.h + Source/TestEngine/TestImpactTestEngineInstrumentedRun.cpp + Source/TestEngine/TestImpactTestEngineInstrumentedRun.h + Source/TestEngine/TestImpactTestEngineException.h Source/TestImpactException.cpp - Source/TestImpactFrameworkPath.cpp + Source/TestImpactRuntime.cpp + Source/TestImpactRuntimeUtils.cpp + Source/TestImpactRuntimeUtils.h + Source/TestImpactClientTestSelection.cpp + Source/TestImpactClientTestRun.cpp + Source/TestImpactClientFailureReport.cpp + Source/TestImpactChangeListSerializer.cpp + Source/TestImpactRepoPath.cpp ) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_tests_files.cmake b/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_tests_files.cmake deleted file mode 100644 index 13f788c37b..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_tests_files.cmake +++ /dev/null @@ -1,38 +0,0 @@ -# -# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or -# its licensors. -# -# For complete copyright and license terms please see the LICENSE at the root of this -# distribution (the "License"). All use of this software is governed by the License, -# or, if provided, by the license below or the license accompanying this file. Do not -# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# - -set(FILES - Tests/Artifact/TestImpactTargetDescriptorCompilerTest.cpp - Tests/Artifact/TestImpactBuildTargetDescriptorFactoryTest.cpp - Tests/Artifact/TestImpactModuleCoverageFactoryTest.cpp - Tests/Artifact/TestImpactChangeListFactoryTest.cpp - Tests/Artifact/TestImpactTestEnumerationSuiteFactoryTest.cpp - Tests/Artifact/TestImpactTestRunSuiteFactoryTest.cpp - Tests/Artifact/TestImpactTestTargetMetaMapFactoryTest.cpp - - Tests/Process/TestImpactProcessSchedulerTest.cpp - - Tests/Process/TestImpactProcessTest.cpp - Tests/Target/TestImpactBuildTargetTest.cpp - Tests/TestImpactExceptionTest.cpp - Tests/TestImpactFrameworkPathTest.cpp - Tests/Test/TestImpactTestEnumeratorTest.cpp - Tests/Test/TestImpactTestEumerationSerializerTest.cpp - Tests/Test/TestImpactTestRunSerializerTest.cpp - Tests/Test/TestImpactTestRunnerTest.cpp - Tests/Test/TestImpactInstrumentedTestRunnerTest.cpp - Tests/Test/TestImpactTestCoverageTest.cpp - Tests/TestImpactTestJobRunnerCommon.h - Tests/TestImpactTestMain.cpp - Tests/TestImpactTestUtils.cpp - Tests/TestImpactTestUtils.h - -) diff --git a/cmake/LYTestWrappers.cmake b/cmake/LYTestWrappers.cmake index 5b7e1c766b..3308c4411a 100644 --- a/cmake/LYTestWrappers.cmake +++ b/cmake/LYTestWrappers.cmake @@ -241,12 +241,18 @@ function(ly_add_test) endif() - # Store the test so we can walk through all of them in LYTestImpactFramework.cmake - set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS ${ly_add_test_NAME}) - set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_test_NAME}_TEST_SUITE ${ly_add_test_TEST_SUITE}) - set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_test_NAME}_TEST_LIBRARY ${ly_add_test_TEST_LIBRARY}) - set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_test_NAME}_TEST_TIMEOUT ${ly_add_test_TIMEOUT}) - + # Check to see whether or not this test target has been stored in the global list for walking by the test impact analysis framework + get_property(all_tests GLOBAL PROPERTY LY_ALL_TESTS) + if(NOT "${LY_ALL_TESTS_TARGET_NAME}" IN_LIST all_tests) + # This is the first reference to this test target so add it to the global list + set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS ${LY_ALL_TESTS_TARGET_NAME}) + set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${LY_ALL_TESTS_TARGET_NAME}_TEST_LIBRARY ${ly_add_test_TEST_LIBRARY}) + endif() + # Add the test suite and timeout value to the test target params + set(LY_TEST_PARAMS "${LY_TEST_PARAMS}#${ly_add_test_TEST_SUITE}") + set(LY_TEST_PARAMS "${LY_TEST_PARAMS}#${ly_add_test_TIMEOUT}") + # Store the params for this test target + set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${LY_ALL_TESTS_TARGET_NAME}_PARAMS ${LY_TEST_PARAMS}) endfunction() #! ly_add_pytest: registers target PyTest-based test with CTest @@ -288,6 +294,11 @@ function(ly_add_pytest) string(REPLACE "::" "_" pytest_report_directory "${PYTEST_XML_OUTPUT_DIR}/${ly_add_pytest_NAME}.xml") + # Set the name of the current test target for storage in the global list + set(LY_ALL_TESTS_TARGET_NAME ${ly_add_pytest_NAME}) + # Add the script path to the test target params + set(LY_TEST_PARAMS "${ly_add_pytest_PATH}") + ly_add_test( NAME ${ly_add_pytest_NAME} TEST_SUITE ${ly_add_pytest_TEST_SUITE} @@ -298,7 +309,6 @@ function(ly_add_pytest) ${ly_add_pytest_UNPARSED_ARGUMENTS} ) - set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_pytest_NAME}_SCRIPT_PATH ${ly_add_pytest_PATH}) set_tests_properties(${LY_ADDED_TEST_NAME} PROPERTIES RUN_SERIAL "${ly_add_pytest_TEST_SERIAL}") endfunction() @@ -341,6 +351,10 @@ function(ly_add_editor_python_test) file(REAL_PATH ${ly_add_editor_python_test_TEST_PROJECT} project_real_path BASE_DIRECTORY ${LY_ROOT_FOLDER}) + # Set the name of the current test target for storage in the global list + set(LY_ALL_TESTS_TARGET_NAME ${ly_add_editor_python_test_NAME}) + # Add the script path to the test target params + set(LY_TEST_PARAMS "${ly_add_editor_python_test_PATH}") # Run test via the run_epbtest.cmake script. # Parameters used are explained in run_epbtest.cmake. ly_add_test( @@ -362,8 +376,6 @@ function(ly_add_editor_python_test) TIMEOUT ${ly_add_editor_python_test_TIMEOUT} COMPONENT ${ly_add_editor_python_test_COMPONENT} ) - - set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_editor_python_test_NAME}_SCRIPT_PATH ${ly_add_editor_python_test_PATH}) set_tests_properties(${LY_ADDED_TEST_NAME} PROPERTIES RUN_SERIAL "${ly_add_editor_python_test_TEST_SERIAL}") endfunction() @@ -431,18 +443,23 @@ function(ly_add_googletest) set(full_test_command $ $ AzRunUnitTests) # Add AzTestRunner as a build dependency ly_add_dependencies(${build_target} AZ::AzTestRunner) + # Start the test target params and dd the command runner command # Ideally, we would populate the full command procedurally but the generator expressions won't be expanded by the time we need this data - set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_googletest_NAME}_TEST_COMMAND "AzRunUnitTests") + set(LY_TEST_PARAMS "AzRunUnitTests") else() set(full_test_command ${ly_add_googletest_TEST_COMMAND}) # Remove the generator expressions so we are left with the argument(s) required to run unit tests for executable targets string(REPLACE ";" "" stripped_test_command ${full_test_command}) string(GENEX_STRIP ${stripped_test_command} stripped_test_command) - set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_googletest_NAME}_TEST_COMMAND ${stripped_test_command}) + # Start the test target params and dd the command runner command + set(LY_TEST_PARAMS "${stripped_test_command}") endif() string(REPLACE "::" "_" report_directory "${GTEST_XML_OUTPUT_DIR}/${ly_add_googletest_NAME}.xml") + # Set the name of the current test target for storage in the global list + set(LY_ALL_TESTS_TARGET_NAME ${target_name}) + # Invoke the lower level ly_add_test command to add the actual ctest and setup the test labels to add_dependencies on the target ly_add_test( NAME ${ly_add_googletest_NAME} @@ -520,16 +537,21 @@ function(ly_add_googlebenchmark) # If command is not supplied attempts, uses the AzTestRunner to run googlebenchmarks on the supplied TARGET set(full_test_command $ $ AzRunBenchmarks ${output_format_args}) + # Start the test target params and dd the command runner command # Ideally, we would populate the full command procedurally but the generator expressions won't be expanded by the time we need this data - set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_googlebenchmark_NAME}_TEST_COMMAND "AzRunUnitTests") + set(LY_TEST_PARAMS "AzRunUnitTests") else() set(full_test_command ${ly_add_googlebenchmark_TEST_COMMAND}) # Remove the generator expressions so we are left with the argument(s) required to run unit tests for executable targets string(REPLACE ";" "" stripped_test_command ${full_test_command}) string(GENEX_STRIP ${stripped_test_command} stripped_test_command) - set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_googletest_NAME}_TEST_COMMAND ${stripped_test_command}) + # Start the test target params and dd the command runner command + set(LY_TEST_PARAMS "${stripped_test_command}") endif() - + + # Set the name of the current test target for storage in the global list + set(LY_ALL_TESTS_TARGET_NAME ${ly_add_googlebenchmark_NAME}) + # Set the name of the current test target for storage in the global list ly_add_test( NAME ${ly_add_googlebenchmark_NAME} TEST_REQUIRES ${ly_add_googlebenchmark_TEST_REQUIRES} diff --git a/cmake/TestImpactFramework/ConsoleFrontendConfig.in b/cmake/TestImpactFramework/ConsoleFrontendConfig.in index f6af3d801d..357499771c 100644 --- a/cmake/TestImpactFramework/ConsoleFrontendConfig.in +++ b/cmake/TestImpactFramework/ConsoleFrontendConfig.in @@ -4,7 +4,8 @@ "timestamp": "${timestamp}" }, "repo": { - "root": "${repo_dir}" + "root": "${repo_dir}", + "tiaf_bin": "${tiaf_bin}" }, "workspace": { "temp": { @@ -13,11 +14,23 @@ "artifact_dir": "RuntimeArtifact" } }, - "persistent": { - "root": "${persistent_dir}", + "active": { + "root": "${active_dir}", "relative_paths": { - "test_impact_data_file": "TestImpactData.spartia", - "enumeration_cache_dir": "EnumerationCache" + "test_impact_data_files": { + "main": "TestImpactData.main.spartia", + "periodic": "TestImpactData.periodic.spartia", + "sandbox": "TestImpactData.sandbox.spartia" + }, + "enumeration_cache_dir": "EnumerationCache", + "last_build_target_list_file": "LastRunBuildTargets.json" + } + }, + "historic": { + "root": "${historic_dir}", + "relative_paths": { + "last_run_hash_file": "last_run.hash", + "last_build_target_list_file": "LastRunBuildTargets.json" } } }, diff --git a/cmake/TestImpactFramework/LYTestImpactFramework.cmake b/cmake/TestImpactFramework/LYTestImpactFramework.cmake index 3081a4c89c..54dadbd8ad 100644 --- a/cmake/TestImpactFramework/LYTestImpactFramework.cmake +++ b/cmake/TestImpactFramework/LYTestImpactFramework.cmake @@ -89,7 +89,7 @@ function(ly_test_impact_get_test_launch_method TARGET_NAME LAUNCH_METHOD) elseif("${target_type}" STREQUAL "EXECUTABLE") set(${LAUNCH_METHOD} "stand_alone" PARENT_SCOPE) else() - message(FATAL_ERROR "Cannot deduce test target launch method for the target type ${target_type}") + message(FATAL_ERROR "Cannot deduce test target launch method for the target ${TARGET_NAME} with type ${target_type}") endif() endfunction() @@ -131,33 +131,50 @@ function(ly_test_impact_extract_python_test COMPOSITE_TEST TEST_NAME) set(${TEST_NAME} ${test_name} PARENT_SCOPE) endfunction() -#! ly_test_impact_extract_google_test_params: extracts the google test name and command parameters. +#! ly_test_impact_extract_google_test_params: extracts the suites for the given google test. # # \arg:COMPOSITE_TEST test in the form 'namespace::test' +# \arg:COMPOSITE_SUITES composite list of suites for this target # \arg:TEST_NAME name of test -# \arg:TEST_COMMAND optional command arguments to run the test -function(ly_test_impact_extract_google_test_params COMPOSITE_TEST TEST_NAME TEST_COMMAND) - get_property(test_command GLOBAL PROPERTY LY_ALL_TESTS_${COMPOSITE_TEST}_TEST_COMMAND) +# \arg:TEST_SUITES extracted list of suites for this target in JSON format +function(ly_test_impact_extract_google_test_params COMPOSITE_TEST COMPOSITE_SUITES TEST_NAME TEST_SUITES) # Namespace and test are mandatory string(REPLACE "::" ";" test_components ${COMPOSITE_TEST}) list(LENGTH test_components num_test_components) if(num_test_components LESS 2) message(FATAL_ERROR "The test ${test_components} appears to have been specified without a namespace, i.e.:\ly_add_googletest/benchmark(NAME ${test_components})\nInstead of (perhaps):\ly_add_googletest/benchmark(NAME Gem::${test_components})\nPlease add the missing namespace before proceeding.") endif() - + list(GET test_components 0 test_namespace) list(GET test_components 1 test_name) set(${TEST_NAMESPACE} ${test_namespace} PARENT_SCOPE) set(${TEST_NAME} ${test_name} PARENT_SCOPE) - set(${TEST_COMMAND} ${test_command} PARENT_SCOPE) + + set(test_suites "") + foreach(composite_suite ${COMPOSITE_SUITES}) + # Command, suite, timeout + string(REPLACE "#" ";" suite_components ${composite_suite}) + list(LENGTH suite_components num_suite_components) + if(num_suite_components LESS 3) + message(FATAL_ERROR "The suite components ${composite_suite} are required to be in the following format: command#suite#string.") + endif() + list(GET suite_components 0 test_command) + list(GET suite_components 1 test_suite) + list(GET suite_components 2 test_timeout) + set(suite_params "{ \"suite\": \"${test_suite}\", \"command\": \"${test_command}\", \"timeout\": ${test_timeout} }") + list(APPEND test_suites "${suite_params}") + endforeach() + string(REPLACE ";" ", " test_suites "${test_suites}") + set(${TEST_SUITES} ${test_suites} PARENT_SCOPE) endfunction() #! ly_test_impact_extract_python_test_params: extracts the python test name and relative script path parameters. # # \arg:COMPOSITE_TEST test in form 'namespace::test' or 'test' +# \arg:COMPOSITE_SUITES composite list of suites for this target # \arg:TEST_NAME name of test -# \arg:SCRIPT_PATH name of test -function(ly_test_impact_extract_python_test_params COMPOSITE_TEST TEST_NAME SCRIPT_PATH) +# \arg:TEST_SUITES extracted list of suites for this target in JSON format +function(ly_test_impact_extract_python_test_params COMPOSITE_TEST COMPOSITE_SUITES TEST_NAME TEST_SUITES) get_property(script_path GLOBAL PROPERTY LY_ALL_TESTS_${COMPOSITE_TEST}_SCRIPT_PATH) # namespace is optional, in which case this component will be simply the test name @@ -169,15 +186,30 @@ function(ly_test_impact_extract_python_test_params COMPOSITE_TEST TEST_NAME SCRI set(test_name ${test_components}) endif() - # Get python script path relative to repo root - ly_test_impact_rebase_file_to_repo_root( - ${script_path} - script_path - ${LY_ROOT_FOLDER} - ) - set(${TEST_NAME} ${test_name} PARENT_SCOPE) - set(${SCRIPT_PATH} ${script_path} PARENT_SCOPE) + + set(test_suites "") + foreach(composite_suite ${COMPOSITE_SUITES}) + # Script path, suite, timeout + string(REPLACE "#" ";" suite_components ${composite_suite}) + list(LENGTH suite_components num_suite_components) + if(num_suite_components LESS 3) + message(FATAL_ERROR "The suite components ${composite_suite} are required to be in the following format: script_path#suite#string.") + endif() + list(GET suite_components 0 script_path) + list(GET suite_components 1 test_suite) + list(GET suite_components 2 test_timeout) + # Get python script path relative to repo root + ly_test_impact_rebase_file_to_repo_root( + ${script_path} + script_path + ${LY_ROOT_FOLDER} + ) + set(suite_params "{ \"suite\": \"${test_suite}\", \"script\": \"${script_path}\", \"timeout\": ${test_timeout} }") + list(APPEND test_suites "${suite_params}") + endforeach() + string(REPLACE ";" ", " test_suites "${test_suites}") + set(${TEST_SUITES} ${test_suites} PARENT_SCOPE) endfunction() #! ly_test_impact_write_test_enumeration_file: exports the master test lists to file. @@ -185,7 +217,6 @@ endfunction() # \arg:TEST_ENUMERATION_TEMPLATE_FILE path to test enumeration template file function(ly_test_impact_write_test_enumeration_file TEST_ENUMERATION_TEMPLATE_FILE) get_property(LY_ALL_TESTS GLOBAL PROPERTY LY_ALL_TESTS) - # Enumerated tests for each type set(google_tests "") set(google_benchmarks "") @@ -196,29 +227,28 @@ function(ly_test_impact_write_test_enumeration_file TEST_ENUMERATION_TEMPLATE_FI # Walk the test list foreach(test ${LY_ALL_TESTS}) message(TRACE "Parsing ${test}") + get_property(test_params GLOBAL PROPERTY LY_ALL_TESTS_${test}_PARAMS) get_property(test_type GLOBAL PROPERTY LY_ALL_TESTS_${test}_TEST_LIBRARY) - get_property(test_suite GLOBAL PROPERTY LY_ALL_TESTS_${test}_TEST_SUITE) - get_property(test_timeout GLOBAL PROPERTY LY_ALL_TESTS_${test}_TEST_TIMEOUT) if("${test_type}" STREQUAL "pytest") # Python tests - ly_test_impact_extract_python_test_params(${test} test_name script_path) - list(APPEND python_tests " { \"name\": \"${test_name}\", \"suite\": \"${test_suite}\", \"script\": \"${script_path}\", \"timeout\":${test_timeout} }") + ly_test_impact_extract_python_test_params(${test} "${test_params}" test_name test_suites) + list(APPEND python_tests " { \"name\": \"${test_name}\", \"suites\": [${test_suites}] }") elseif("${test_type}" STREQUAL "pytest_editor") - # Python editor tests - ly_test_impact_extract_python_test_params(${test} test_name script_path) - list(APPEND python_editor_tests " { \"name\": \"${test_name}\", \"suite\": \"${test_suite}\", \"script\": \"${script_path}\", \"timeout\":${test_timeout} }") + # Python editor tests + ly_test_impact_extract_python_test_params(${test} "${test_params}" test_name test_suites) + list(APPEND python_editor_tests " { \"name\": \"${test_name}\", \"suites\": [${test_suites}] }") elseif("${test_type}" STREQUAL "googletest") # Google tests - ly_test_impact_extract_google_test_params(${test} test_name test_command) - ly_test_impact_get_test_launch_method(${test_name} launch_method) - list(APPEND google_tests " { \"name\": \"${test_name}\", \"suite\": \"${test_suite}\", \"command\": \"${test_command}\", \"timeout\":${test_timeout}, \"launch_method\": \"${launch_method}\" }") + ly_test_impact_extract_google_test_params(${test} "${test_params}" test_name test_suites) + ly_test_impact_get_test_launch_method(${test} launch_method) + list(APPEND google_tests " { \"name\": \"${test_name}\", \"launch_method\": \"${launch_method}\", \"suites\": [${test_suites}] }") elseif("${test_type}" STREQUAL "googlebenchmark") # Google benchmarks - ly_test_impact_extract_google_test_params(${test} test_name test_command) - list(APPEND google_benchmarks " { \"name\": \"${test_name}\", \"suite\": \"${test_suite}\", \"command\": \"${test_command}\", \"timeout\":${test_timeout} }") + ly_test_impact_extract_google_test_params(${test} "${test_params}" test_name test_suites) + list(APPEND google_benchmarks " { \"name\": \"${test_name}\", \"launch_method\": \"${launch_method}\", \"suites\": [${test_suites}] }") else() message("${test_name} is of unknown type (TEST_LIBRARY property is empty)") - list(APPEND unknown_tests " { \"name\": \"${test}\" }") + list(APPEND unknown_tests " { \"name\": \"${test}\", \"type\": \"${test_type}\" }") endif() endforeach() @@ -330,8 +360,11 @@ function(ly_test_impact_write_config_file CONFIG_TEMPLATE_FILE PERSISTENT_DATA_D # Temp dir set(temp_dir "${LY_TEST_IMPACT_TEMP_DIR}") - # Persistent dir - set(persistent_dir "${PERSISTENT_DATA_DIR}") + # Active persistent data dir + set(active_dir "${PERSISTENT_DATA_DIR}/active") + + # Historic persistent data dir + set(historic_dir "${PERSISTENT_DATA_DIR}/historic") # Source to target mappings dir set(source_target_mapping_dir "${LY_TEST_IMPACT_SOURCE_TARGET_MAPPING_DIR}") @@ -341,6 +374,9 @@ function(ly_test_impact_write_config_file CONFIG_TEMPLATE_FILE PERSISTENT_DATA_D # Build dependency artifact dir set(target_dependency_dir "${LY_TEST_IMPACT_TARGET_DEPENDENCY_DIR}") + + # Test impact analysis framework binary + set(tiaf_bin "$") # Substitute config file template with above vars file(READ "${CONFIG_TEMPLATE_FILE}" config_file) @@ -348,9 +384,13 @@ function(ly_test_impact_write_config_file CONFIG_TEMPLATE_FILE PERSISTENT_DATA_D # Write out entire config contents to a file in the build directory of the test impact framework console target file(GENERATE - OUTPUT "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/$/$.$.json" + OUTPUT "${PERSISTENT_DATA_DIR}/$.$.json" CONTENT ${config_file} ) + + # Set the above config file as the default config file to use for the test impact framework console target + target_compile_definitions(${LY_TEST_IMPACT_CONSOLE_STATIC_TARGET} PUBLIC "LY_TEST_IMPACT_DEFAULT_CONFIG_FILE=\"${PERSISTENT_DATA_DIR}/$.$.json\"") + message(DEBUG "Test impact framework post steps complete") endfunction() #! ly_test_impact_post_step: runs the post steps to be executed after all other cmake scripts have been executed. @@ -360,8 +400,7 @@ function(ly_test_impact_post_step) endif() # Directory per build config for persistent test impact data (to be checked in) - set(persistent_data_dir "${LY_ROOT_FOLDER}/Tests/test_impact_framework/${CMAKE_SYSTEM_NAME}/$") - + set(persistent_data_dir "${LY_TEST_IMPACT_WORKING_DIR}/persistent") # Directory for binaries built for this profile set(bin_dir "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/$") @@ -388,8 +427,4 @@ function(ly_test_impact_post_step) # Copy over the graphviz options file for the build dependency graphs message(DEBUG "Test impact framework config file written") file(COPY "cmake/TestImpactFramework/CMakeGraphVizOptions.cmake" DESTINATION ${CMAKE_BINARY_DIR}) - - # Set the above config file as the default config file to use for the test impact framework console target - target_compile_definitions(${LY_TEST_IMPACT_CONSOLE_STATIC_TARGET} PUBLIC "LY_TEST_IMPACT_DEFAULT_CONFIG_FILE=\"${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/$/$.$.json\"") - message(DEBUG "Test impact framework post steps complete") endfunction() From b00f56909027667fe99fc6e713175f91f401c128 Mon Sep 17 00:00:00 2001 From: jonawals Date: Sun, 6 Jun 2021 19:09:24 +0100 Subject: [PATCH 2/2] Address PR comments --- .../Code/Source/TestImpactConsoleMain.cpp | 2 +- cmake/LYTestWrappers.cmake | 32 ++++++++++--------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleMain.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleMain.cpp index fab6509244..4853c6da15 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleMain.cpp +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleMain.cpp @@ -121,7 +121,7 @@ namespace TestImpact } else { - // One sequence timed out whilst the other sequence succeeded or both sequences timed out + // One or both sequences timed out or failed result = TestSequenceResult::Timeout; } } diff --git a/cmake/LYTestWrappers.cmake b/cmake/LYTestWrappers.cmake index 3308c4411a..eebc281d06 100644 --- a/cmake/LYTestWrappers.cmake +++ b/cmake/LYTestWrappers.cmake @@ -71,6 +71,7 @@ endfunction() #! ly_add_test: Adds a new RUN_TEST using for the specified target using the supplied command # # \arg:NAME - Name to for the test run target +# \arg:PARENT_NAME(optional) - Name of the parent test run target (if this is a subsequent call to specify a suite) # \arg:TEST_REQUIRES(optional) - List of system resources that are required to run this test. # Only available option is "gpu" # \arg:TEST_SUITE(optional) - "smoke" or "periodic" or "sandbox" - prevents the test from running normally @@ -94,7 +95,7 @@ endfunction() # sets LY_ADDED_TEST_NAME to the fully qualified name of the test, in parent scope function(ly_add_test) set(options EXCLUDE_TEST_RUN_TARGET_FROM_IDE) - set(one_value_args NAME TEST_LIBRARY TEST_SUITE TIMEOUT) + set(one_value_args NAME PARENT_NAME TEST_LIBRARY TEST_SUITE TIMEOUT) set(multi_value_args TEST_REQUIRES TEST_COMMAND NON_IDE_PARAMS RUNTIME_DEPENDENCIES COMPONENT LABELS) # note that we dont use TEST_LIBRARY here, but PAL files might so do not remove! @@ -241,18 +242,24 @@ function(ly_add_test) endif() + if(NOT ly_add_test_PARENT_NAME) + set(test_target ${ly_add_test_NAME}) + else() + set(test_target ${ly_add_test_PARENT_NAME}) + endif() + # Check to see whether or not this test target has been stored in the global list for walking by the test impact analysis framework get_property(all_tests GLOBAL PROPERTY LY_ALL_TESTS) - if(NOT "${LY_ALL_TESTS_TARGET_NAME}" IN_LIST all_tests) + if(NOT "${test_target}" IN_LIST all_tests) # This is the first reference to this test target so add it to the global list - set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS ${LY_ALL_TESTS_TARGET_NAME}) - set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${LY_ALL_TESTS_TARGET_NAME}_TEST_LIBRARY ${ly_add_test_TEST_LIBRARY}) + set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS ${test_target}) + set_property(GLOBAL PROPERTY LY_ALL_TESTS_${test_target}_TEST_LIBRARY ${ly_add_test_TEST_LIBRARY}) endif() # Add the test suite and timeout value to the test target params set(LY_TEST_PARAMS "${LY_TEST_PARAMS}#${ly_add_test_TEST_SUITE}") set(LY_TEST_PARAMS "${LY_TEST_PARAMS}#${ly_add_test_TIMEOUT}") # Store the params for this test target - set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${LY_ALL_TESTS_TARGET_NAME}_PARAMS ${LY_TEST_PARAMS}) + set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${test_target}_PARAMS ${LY_TEST_PARAMS}) endfunction() #! ly_add_pytest: registers target PyTest-based test with CTest @@ -294,13 +301,12 @@ function(ly_add_pytest) string(REPLACE "::" "_" pytest_report_directory "${PYTEST_XML_OUTPUT_DIR}/${ly_add_pytest_NAME}.xml") - # Set the name of the current test target for storage in the global list - set(LY_ALL_TESTS_TARGET_NAME ${ly_add_pytest_NAME}) # Add the script path to the test target params set(LY_TEST_PARAMS "${ly_add_pytest_PATH}") ly_add_test( NAME ${ly_add_pytest_NAME} + PARENT_NAME ${ly_add_pytest_NAME} TEST_SUITE ${ly_add_pytest_TEST_SUITE} LABELS FRAMEWORK_pytest TEST_COMMAND ${LY_PYTEST_EXECUTABLE} ${ly_add_pytest_PATH} ${ly_add_pytest_EXTRA_ARGS} --junitxml=${pytest_report_directory} ${custom_marks_args} @@ -351,14 +357,13 @@ function(ly_add_editor_python_test) file(REAL_PATH ${ly_add_editor_python_test_TEST_PROJECT} project_real_path BASE_DIRECTORY ${LY_ROOT_FOLDER}) - # Set the name of the current test target for storage in the global list - set(LY_ALL_TESTS_TARGET_NAME ${ly_add_editor_python_test_NAME}) # Add the script path to the test target params set(LY_TEST_PARAMS "${ly_add_editor_python_test_PATH}") # Run test via the run_epbtest.cmake script. # Parameters used are explained in run_epbtest.cmake. ly_add_test( NAME ${ly_add_editor_python_test_NAME} + PARENT_NAME ${ly_add_editor_python_test_NAME} TEST_REQUIRES ${ly_add_editor_python_test_TEST_REQUIRES} TEST_COMMAND ${CMAKE_COMMAND} -DCMD_ARG_TEST_PROJECT=${project_real_path} @@ -457,12 +462,10 @@ function(ly_add_googletest) string(REPLACE "::" "_" report_directory "${GTEST_XML_OUTPUT_DIR}/${ly_add_googletest_NAME}.xml") - # Set the name of the current test target for storage in the global list - set(LY_ALL_TESTS_TARGET_NAME ${target_name}) - # Invoke the lower level ly_add_test command to add the actual ctest and setup the test labels to add_dependencies on the target ly_add_test( NAME ${ly_add_googletest_NAME} + PARENT_NAME ${target_name} TEST_SUITE ${ly_add_googletest_TEST_SUITE} LABELS FRAMEWORK_googletest TEST_COMMAND ${full_test_command} --gtest_output=xml:${report_directory} ${LY_GOOGLETEST_EXTRA_PARAMS} @@ -548,12 +551,11 @@ function(ly_add_googlebenchmark) # Start the test target params and dd the command runner command set(LY_TEST_PARAMS "${stripped_test_command}") endif() - - # Set the name of the current test target for storage in the global list - set(LY_ALL_TESTS_TARGET_NAME ${ly_add_googlebenchmark_NAME}) + # Set the name of the current test target for storage in the global list ly_add_test( NAME ${ly_add_googlebenchmark_NAME} + PARENT_NAME ${ly_add_googlebenchmark_NAME} TEST_REQUIRES ${ly_add_googlebenchmark_TEST_REQUIRES} TEST_COMMAND ${full_test_command} ${LY_GOOGLETEST_EXTRA_PARAMS} TEST_SUITE "benchmark"