birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #82 from aws-lumberyard-dev/TIF/Runtime_TestChanges

Browse Source 
Changes for read-only runs and new test suites
This commit is contained in:
jonawals
2021-06-06 19:10:31 +01:00
committed by GitHub
parent 7eb6e0e7d5 b00f569090
commit 50b568e1f6
18 changed files with 450 additions and 280 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp
+18 -37
View File
@@ -36,21 +36,21 @@ namespace TestImpact
MaxConcurrency,
TestTargetTimeout,
GlobalTimeout,
SuitesFilter,
SuiteFilter,
SafeMode,
// Values
None,
Seed,
Regular,
ImpactAnalysis,
ImpactAnalysisNoWrite,
ImpactAnalysisOrSeed,
Locality,
Abort,
Continue,
Ignore,
StdOut,
File,
AllSuites
File
};
constexpr const char* OptionKeys[] =
@@ -70,21 +70,21 @@ namespace TestImpact
"maxconcurrency",
"ttimeout",
"gtimeout",
"suites",
"suite",
"safemode",
// Values
"none",
"seed",
"regular",
"tia",
"tianowrite",
"tiaorseed",
"locality",
"abort",
"continue",
"ignore",
"stdout",
"file",
"*"
"file"
};
RepoPath ParseConfigurationFile(const AZ::CommandLine& cmd)
@@ -110,6 +110,7 @@ namespace TestImpact
{OptionKeys[Seed], TestSequenceType::Seed},
{OptionKeys[Regular], TestSequenceType::Regular},
{OptionKeys[ImpactAnalysis], TestSequenceType::ImpactAnalysis},
{OptionKeys[ImpactAnalysisNoWrite], TestSequenceType::ImpactAnalysisNoWrite},
{OptionKeys[ImpactAnalysisOrSeed], TestSequenceType::ImpactAnalysisOrSeed}
};
@@ -250,32 +251,16 @@ namespace TestImpact
return ParseOnOffOption(OptionKeys[SafeMode], states, cmd).value_or(false);
}
AZStd::unordered_set<AZStd::string> ParseSuitesFilter(const AZ::CommandLine& cmd)
SuiteType ParseSuiteFilter(const AZ::CommandLine& cmd)
{
AZStd::unordered_set<AZStd::string> suitesFilter;
if (const auto numSwitchValues = cmd.GetNumSwitchValues(OptionKeys[SuitesFilter]);
numSwitchValues)
const AZStd::vector<AZStd::pair<AZStd::string, SuiteType>> states =
{
for (auto i = 0; i < numSwitchValues; i++)
{
const auto value = cmd.GetSwitchValue(OptionKeys[SuitesFilter], i);
AZ_TestImpact_Eval(!value.empty(), CommandLineOptionsException, "Suites option value is empty");
if (value == OptionKeys[AllSuites])
{
AZ_TestImpact_Eval(
suitesFilter.empty(), CommandLineOptionsException, "The * suite cannot be used with other suites");
}
{GetSuiteTypeName(SuiteType::Main), SuiteType::Main},
{GetSuiteTypeName(SuiteType::Periodic), SuiteType::Periodic},
{GetSuiteTypeName(SuiteType::Sandbox), SuiteType::Sandbox}
};
suitesFilter.insert(value);
}
}
if (suitesFilter.find(OptionKeys[AllSuites]) != suitesFilter.end())
{
return {};
}
return suitesFilter;
return ParseMultiStateOption(OptionKeys[SuiteFilter], states, cmd).value_or(SuiteType::Main);
}
}
@@ -299,7 +284,7 @@ namespace TestImpact
m_testTargetTimeout = ParseTestTargetTimeout(cmd);
m_globalTimeout = ParseGlobalTimeout(cmd);
m_safeMode = ParseSafeMode(cmd);
m_suitesFilter = ParseSuitesFilter(cmd);
m_suiteFilter = ParseSuiteFilter(cmd);
}
bool CommandLineOptions::HasChangeListFile() const
@@ -382,9 +367,9 @@ namespace TestImpact
return m_globalTimeout;
}
const AZStd::unordered_set<AZStd::string>& CommandLineOptions::GetSuitesFilter() const
SuiteType CommandLineOptions::GetSuiteFilter() const
{
return m_suitesFilter;
return m_suiteFilter;
}
AZStd::string CommandLineOptions::GetCommandLineUsageString()
@@ -452,11 +437,7 @@ namespace TestImpact
" -maxconcurrency=<number> The maximum number of concurrent test targets/shards to be in flight at \n"
" any given moment.\n"
" -ochangelist=<on,off> Outputs the change list used for test selection.\n"
" -suites=<names> The test suites to select from for this test sequence (multiple values are \n"
" allowed). The suite all has special significance and will allow tests from \n"
" any suite to be selected, however this particular suite is mutually exclusive\n"
" with other suite. Note: this option is only applicable to the regular sequence\n"
" and, if safe mode is enables, the tia and tiaorseed sequences.";
" -suite=<main, periodic, sandbox> The test suite to select from for this test sequence.";
return help;
}
Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.h
+5 -4
View File
@@ -18,7 +18,6 @@
#include <AzCore/std/string/string.h>
#include <AzCore/std/optional.h>
#include <AzCore/std/containers/vector.h>
#include <AzCore/std/containers/unordered_set.h>
namespace TestImpact
{
@@ -29,6 +28,8 @@ namespace TestImpact
Seed, //!< Removes any prior coverage data and runs all test targets with instrumentation to reseed the data from scratch.
Regular, //!< Runs all of the test targets without any instrumentation to generate coverage data (any prior coverage data is left intact).
ImpactAnalysis, //!< Uses any prior coverage data to run the instrumented subset of selected tests (if no prior coverage data a regular run is performed instead).
ImpactAnalysisNoWrite, //!< Uses any prior coverage data to run the uninstrumented subset of selected tests (if no prior coverage data a regular run is performed instead).
//!< The coverage data is not updated with the subset of selected tests.
ImpactAnalysisOrSeed //!< Uses any prior coverage data to run the instrumented subset of selected tests (if no prior coverage data a seed run is performed instead).
};
@@ -87,8 +88,8 @@ namespace TestImpact
//! Returns the global test sequence timeout to use (if any).
const AZStd::optional<AZStd::chrono::milliseconds>& GetGlobalTimeout() const;
//! Returns the filter for test suites that will be allowed to be run.
const AZStd::unordered_set<AZStd::string>& GetSuitesFilter() const;
//! Returns the filter for test suite that will be allowed to be run.
SuiteType GetSuiteFilter() const;
private:
RepoPath m_configurationFile;
@@ -105,7 +106,7 @@ namespace TestImpact
AZStd::optional<size_t> m_maxConcurrency;
AZStd::optional<AZStd::chrono::milliseconds> m_testTargetTimeout;
AZStd::optional<AZStd::chrono::milliseconds> m_globalTimeout;
AZStd::unordered_set<AZStd::string> m_suitesFilter;
SuiteType m_suiteFilter;
bool m_safeMode = false;
};
} // namespace TestImpact
Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleMain.cpp
+60 -23
View File
@@ -86,6 +86,8 @@ namespace TestImpact
Runtime& runtime,
const AZStd::optional<ChangeList>& changeList)
{
// Even though it is possible for a regular run to be selected (see below) which does not actually require a change list,
// consider any impact analysis sequence type without a change list to be an error
AZ_TestImpact_Eval(
changeList.has_value(),
CommandLineOptionsException,
@@ -94,39 +96,72 @@ namespace TestImpact
TestSequenceResult result = TestSequenceResult::Failure;
if (options.HasSafeMode())
{
auto [selectedResult, discardedResult] = runtime.SafeImpactAnalysisTestSequence(
changeList.value(),
options.GetSuitesFilter(),
options.GetTestPrioritizationPolicy(),
options.GetTestTargetTimeout(),
options.GetGlobalTimeout(),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler));
if (options.GetTestSequenceType() == TestSequenceType::ImpactAnalysis)
{
auto [selectedResult, discardedResult] = runtime.SafeImpactAnalysisTestSequence(
changeList.value(),
options.GetTestPrioritizationPolicy(),
options.GetTestTargetTimeout(),
options.GetGlobalTimeout(),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler));
// Handling the possible timeout and failure permutations of the selected and discarded test results is splitting hairs
// so apply the following, admittedly arbitrary, rules to determine what the composite test sequence result should be
if (selectedResult == TestSequenceResult::Success && discardedResult == TestSequenceResult::Success)
{
// Trivial case: both sequences succeeded
result = TestSequenceResult::Success;
// Handling the possible timeout and failure permutations of the selected and discarded test results is splitting hairs
// so apply the following, admittedly arbitrary, rules to determine what the composite test sequence result should be
if (selectedResult == TestSequenceResult::Success && discardedResult == TestSequenceResult::Success)
{
// Trivial case: both sequences succeeded
result = TestSequenceResult::Success;
}
else if (selectedResult == TestSequenceResult::Failure || discardedResult == TestSequenceResult::Failure)
{
// One sequence failed whilst the other sequence either succeeded or timed out
result = TestSequenceResult::Failure;
}
else
{
// One or both sequences timed out or failed
result = TestSequenceResult::Timeout;
}
}
else if (selectedResult == TestSequenceResult::Failure || discardedResult == TestSequenceResult::Failure)
else if (options.GetTestSequenceType() == TestSequenceType::ImpactAnalysisNoWrite)
{
// One sequence failed whilst the other sequence either succeeded or timed out
result = TestSequenceResult::Failure;
// A no-write impact analysis sequence with safe mode enabled is functionally identical to a regular sequence type
// due to a) the selected tests being run without instrumentation and b) the discarded tests also being run without
// instrumentation
result = runtime.RegularTestSequence(
options.GetTestTargetTimeout(),
options.GetGlobalTimeout(),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler));
}
else
{
// One sequence timed out whilst the other sequence succeeded or both sequences timed out
result = TestSequenceResult::Timeout;
throw(Exception("Unexpected sequence type"));
}
}
else
{
Policy::DynamicDependencyMap dynamicDependencyMapPolicy;
if (options.GetTestSequenceType() == TestSequenceType::ImpactAnalysis)
{
dynamicDependencyMapPolicy = Policy::DynamicDependencyMap::Update;
}
else if (options.GetTestSequenceType() == TestSequenceType::ImpactAnalysisNoWrite)
{
dynamicDependencyMapPolicy = Policy::DynamicDependencyMap::Discard;
}
else
{
throw(Exception("Unexpected sequence type"));
}
result = runtime.ImpactAnalysisTestSequence(
changeList.value(),
options.GetTestPrioritizationPolicy(),
dynamicDependencyMapPolicy,
options.GetTestTargetTimeout(),
options.GetGlobalTimeout(),
AZStd::ref(sequenceEventHandler),
@@ -164,9 +199,11 @@ namespace TestImpact
// As of now, there are no other non-test operations other than printing a change list so getting this far is considered an error
AZ_TestImpact_Eval(options.GetTestSequenceType() != TestSequenceType::None, CommandLineOptionsException, "No action specified");
std::cout << "Constructing in-memory model of source tree and test coverage, this may take a moment...\n";
std::cout << "Constructing in-memory model of source tree and test coverage for test suite ";
std::cout << GetSuiteTypeName(options.GetSuiteFilter()).c_str() << ", this may take a moment...\n";
Runtime runtime(
RuntimeConfigurationFactory(ReadFileContents<CommandLineOptionsException>(options.GetConfigurationFile())),
options.GetSuiteFilter(),
options.GetExecutionFailurePolicy(),
options.GetExecutionFailureDraftingPolicy(),
options.GetTestFailurePolicy(),
@@ -184,14 +221,13 @@ namespace TestImpact
std::cout << "Test impact analysis data for this repository was not found, seed or regular sequence fallbacks will be used.\n";
}
TestSequenceEventHandler sequenceEventHandler(&options.GetSuitesFilter());
TestSequenceEventHandler sequenceEventHandler(options.GetSuiteFilter());
switch (const auto type = options.GetTestSequenceType())
{
case TestSequenceType::Regular:
{
const auto result = runtime.RegularTestSequence(
options.GetSuitesFilter(),
options.GetTestTargetTimeout(),
options.GetGlobalTimeout(),
AZStd::ref(sequenceEventHandler),
@@ -211,6 +247,7 @@ namespace TestImpact
return GetReturnCodeForTestSequenceResult(result);
}
case TestSequenceType::ImpactAnalysisNoWrite:
case TestSequenceType::ImpactAnalysis:
{
return WrappedImpactAnalysisTestSequence(sequenceEventHandler, options, runtime, changeList);
Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactRuntimeConfigurationFactory.cpp
+17 -4
View File
@@ -30,7 +30,7 @@ namespace TestImpact
"relative_paths",
"artifact_dir",
"enumeration_cache_dir",
"test_impact_data_file",
"test_impact_data_files",
"temp",
"active",
"target_sources",
@@ -75,7 +75,7 @@ namespace TestImpact
RelativePaths,
ArtifactDir,
EnumerationCacheDir,
TestImpactDataFile,
TestImpactDataFiles,
TempWorkspace,
ActiveWorkspace,
TargetSources,
@@ -144,6 +144,19 @@ namespace TestImpact
return tempWorkspaceConfig;
}
AZStd::array<RepoPath, 3> ParseTestImpactAnalysisDataFiles(const RepoPath& root, const rapidjson::Value& sparTIAFile)
{
AZStd::array<RepoPath, 3> sparTIAFiles;
sparTIAFiles[static_cast<size_t>(SuiteType::Main)] =
GetAbsPathFromRelPath(root, sparTIAFile[GetSuiteTypeName(SuiteType::Main).c_str()].GetString());
sparTIAFiles[static_cast<size_t>(SuiteType::Periodic)] =
GetAbsPathFromRelPath(root, sparTIAFile[GetSuiteTypeName(SuiteType::Periodic).c_str()].GetString());
sparTIAFiles[static_cast<size_t>(SuiteType::Sandbox)] =
GetAbsPathFromRelPath(root, sparTIAFile[GetSuiteTypeName(SuiteType::Sandbox).c_str()].GetString());
return sparTIAFiles;
}
WorkspaceConfig::Active ParseActiveWorkspaceConfig(const rapidjson::Value& activeWorkspace)
{
WorkspaceConfig::Active activeWorkspaceConfig;
@@ -151,8 +164,8 @@ namespace TestImpact
activeWorkspaceConfig.m_root = activeWorkspace[Config::Keys[Config::Root]].GetString();
activeWorkspaceConfig.m_enumerationCacheDirectory
= GetAbsPathFromRelPath(activeWorkspaceConfig.m_root, relativePaths[Config::Keys[Config::EnumerationCacheDir]].GetString());
activeWorkspaceConfig.m_sparTIAFile
= GetAbsPathFromRelPath(activeWorkspaceConfig.m_root, relativePaths[Config::Keys[Config::TestImpactDataFile]].GetString());
activeWorkspaceConfig.m_sparTIAFiles =
ParseTestImpactAnalysisDataFiles(activeWorkspaceConfig.m_root, relativePaths[Config::Keys[Config::TestImpactDataFiles]]);
return activeWorkspaceConfig;
}
Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactConfiguration.h
+2 -1
View File
@@ -16,6 +16,7 @@
#include <TestImpactFramework/TestImpactRepoPath.h>
#include <AzCore/std/containers/vector.h>
#include <AzCore/std/containers/array.h>
#include <AzCore/std/string/string.h>
namespace TestImpact
@@ -46,8 +47,8 @@ namespace TestImpact
struct Active
{
RepoPath m_root; //!< Path to the persistent workspace tracked by the repository.
RepoPath m_sparTIAFile; //!< Path to the test impact analysis data.
RepoPath m_enumerationCacheDirectory; //!< Path to the test enumerations cache.
AZStd::array<RepoPath, 3> m_sparTIAFiles; //!< Paths to the test impact analysis data files for each test suite.
};
Temp m_temp;
Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h
+6 -4
View File
@@ -92,6 +92,7 @@ namespace TestImpact
public:
//! Constructs a runtime with the specified configuration and policies.
//! @param config The configuration used for this runtime instance.
//! @param suiteFilter The test suite for which the coverage data and test selection will draw from.
//! @param executionFailurePolicy Determines how to handle test targets that fail to execute.
//! @param executionFailureDraftingPolicy Determines how test targets that previously failed to execute are drafted into subsequent test sequences.
//! @param testFailurePolicy Determines how to handle test targets that report test failures.
@@ -99,6 +100,7 @@ namespace TestImpact
//! @param testShardingPolicy Determines how to handle test targets that have opted in to test sharding.
Runtime(
RuntimeConfig&& config,
SuiteType suiteFilter,
Policy::ExecutionFailure executionFailurePolicy,
Policy::ExecutionFailureDrafting executionFailureDraftingPolicy,
Policy::TestFailure testFailurePolicy,
@@ -110,7 +112,6 @@ namespace TestImpact
~Runtime();
//! Runs a test sequence where all tests with a matching suite in the suite filter and also not on the excluded list are selected.
//! @param suitesFilter The test suites that will be included in the test selection.
//! @param testTargetTimeout The maximum duration individual test targets may be in flight for (infinite if empty).
//! @param globalTimeout The maximum duration the entire test sequence may run for (infinite if empty).
//! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests.
@@ -118,7 +119,6 @@ namespace TestImpact
//! @param testRunCompleteCallback The client function to be called after an individual test run has completed.
//! @returns
TestSequenceResult RegularTestSequence(
const AZStd::unordered_set<AZStd::string> suitesFilter,
AZStd::optional<AZStd::chrono::milliseconds> testTargetTimeout,
AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
AZStd::optional<TestSequenceStartCallback> testSequenceStartCallback,
@@ -128,6 +128,7 @@ namespace TestImpact
//! Runs a test sequence where tests are selected according to test impact analysis so long as they are not on the excluded list.
//! @param changeList The change list used to determine the tests to select.
//! @param testPrioritizationPolicy Determines how selected tests will be prioritized.
//! @param dynamicDependencyMapPolicy The policy to determine how the coverage data of produced by test sequences is used to update the dynamic dependency map.
//! @param testTargetTimeout The maximum duration individual test targets may be in flight for (infinite if empty).
//! @param globalTimeout The maximum duration the entire test sequence may run for (infinite if empty).
//! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests.
@@ -137,6 +138,7 @@ namespace TestImpact
TestSequenceResult ImpactAnalysisTestSequence(
const ChangeList& changeList,
Policy::TestPrioritization testPrioritizationPolicy,
Policy::DynamicDependencyMap dynamicDependencyMapPolicy,
AZStd::optional<AZStd::chrono::milliseconds> testTargetTimeout,
AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
AZStd::optional<ImpactAnalysisTestSequenceStartCallback> testSequenceStartCallback,
@@ -145,7 +147,6 @@ namespace TestImpact
//! Runs a test sequence as per the ImpactAnalysisTestSequence where the tests not selected are also run (albeit without instrumentation).
//! @param changeList The change list used to determine the tests to select.
//! @param suitesFilter The test suites that will be included in the test selection.
//! @param testPrioritizationPolicy Determines how selected tests will be prioritized.
//! @param testTargetTimeout The maximum duration individual test targets may be in flight for (infinite if empty).
//! @param globalTimeout The maximum duration the entire test sequence may run for (infinite if empty).
@@ -155,7 +156,6 @@ namespace TestImpact
//! @returns
AZStd::pair<TestSequenceResult, TestSequenceResult> SafeImpactAnalysisTestSequence(
const ChangeList& changeList,
const AZStd::unordered_set<AZStd::string> suitesFilter,
Policy::TestPrioritization testPrioritizationPolicy,
AZStd::optional<AZStd::chrono::milliseconds> testTargetTimeout,
AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
@@ -207,6 +207,8 @@ namespace TestImpact
void UpdateAndSerializeDynamicDependencyMap(const SourceCoveringTestsList& sourceCoverageTestsList);
RuntimeConfig m_config;
SuiteType m_suiteFilter;
RepoPath m_sparTIAFile;
Policy::ExecutionFailure m_executionFailurePolicy;
Policy::ExecutionFailureDrafting m_executionFailureDraftingPolicy;
Policy::TestFailure m_testFailurePolicy;
Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactTestSequence.h
+35
View File
@@ -12,6 +12,10 @@
#pragma once
#include <TestImpactFramework/TestImpactRuntimeException.h>
#include <AzCore/std/containers/array.h>
namespace TestImpact
{
namespace Policy
@@ -55,6 +59,13 @@ namespace TestImpact
Continue //!< Continue the test sequence and report the test failures after the run.
};
//! Policy for updating the dynamic dependency map with the coverage data of produced by test sequences.
enum class DynamicDependencyMap
{
Discard, //!< Discard the coverage data produced by test sequences.
Update //!< Update the dynamic dependency map with the coverage data produced by test sequences.
};
//! Policy for sharding test targets that have been marked for test sharding.
enum class TestSharding
{
@@ -82,6 +93,30 @@ namespace TestImpact
TestInterleaved //!< Tests are interlaced across shards agnostic of fixtures (fastest but prone to inter-test dependency problems).
};
//! Test suite types to select from.
enum class SuiteType : AZ::u8
{
Main = 0,
Periodic,
Sandbox
};
//! User-friendly names for the test suite types.
inline AZStd::string GetSuiteTypeName(SuiteType suiteType)
{
switch (suiteType)
{
case SuiteType::Main:
return "main";
case SuiteType::Periodic:
return "periodic";
case SuiteType::Sandbox:
return "sandbox";
default:
throw(RuntimeException("Unexpected suite type"));
}
}
//! Result of a test sequence that was run.
enum class TestSequenceResult
{
Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestTargetMetaMapFactory.cpp
+30 -19
View File
@@ -19,7 +19,7 @@
namespace TestImpact
{
TestTargetMetaMap TestTargetMetaMapFactory(const AZStd::string& masterTestListData)
TestTargetMetaMap TestTargetMetaMapFactory(const AZStd::string& masterTestListData, SuiteType suiteType)
{
// Keys for pertinent JSON node and attribute names
constexpr const char* Keys[] =
@@ -27,6 +27,7 @@ namespace TestImpact
"google",
"test",
"tests",
"suites",
"suite",
"launch_method",
"test_runner",
@@ -41,6 +42,7 @@ namespace TestImpact
GoogleKey,
TestKey,
TestsKey,
TestSuitesKey,
SuiteKey,
LaunchMethodKey,
TestRunnerKey,
@@ -64,26 +66,35 @@ namespace TestImpact
for (const auto& test : tests)
{
TestTargetMeta testMeta;
testMeta.m_suite = test[Keys[SuiteKey]].GetString();
testMeta.m_customArgs = test[Keys[CommandKey]].GetString();
testMeta.m_timeout = AZStd::chrono::seconds{ test[Keys[TimeoutKey]].GetUint() };
const auto testSuites = test[Keys[TestSuitesKey]].GetArray();
for (const auto& suite : testSuites)
{
// Check to see if this test target has the suite we're looking for
if (const auto suiteName = suite[Keys[SuiteKey]].GetString();
strcmp(GetSuiteTypeName(suiteType).c_str(), suiteName) == 0)
{
testMeta.m_suite = suiteName;
testMeta.m_customArgs = suite[Keys[CommandKey]].GetString();
testMeta.m_timeout = AZStd::chrono::seconds{ suite[Keys[TimeoutKey]].GetUint() };
if (const auto buildTypeString = test[Keys[LaunchMethodKey]].GetString(); strcmp(buildTypeString, Keys[TestRunnerKey]) == 0)
{
testMeta.m_launchMethod = LaunchMethod::TestRunner;
}
else if (strcmp(buildTypeString, Keys[StandAloneKey]) == 0)
{
testMeta.m_launchMethod = LaunchMethod::StandAlone;
}
else
{
throw(ArtifactException("Unexpected test build type"));
}
if (const auto buildTypeString = test[Keys[LaunchMethodKey]].GetString(); strcmp(buildTypeString, Keys[TestRunnerKey]) == 0)
{
testMeta.m_launchMethod = LaunchMethod::TestRunner;
AZStd::string name = test[Keys[NameKey]].GetString();
AZ_TestImpact_Eval(!name.empty(), ArtifactException, "Test name field cannot be empty");
testMetas.emplace(AZStd::move(name), AZStd::move(testMeta));
break;
}
}
else if (strcmp(buildTypeString, Keys[StandAloneKey]) == 0)
{
testMeta.m_launchMethod = LaunchMethod::StandAlone;
}
else
{
throw(ArtifactException("Unexpected test build type"));
}
AZStd::string name = test[Keys[NameKey]].GetString();
AZ_TestImpact_Eval(!name.empty(), ArtifactException, "Test name field cannot be empty");
testMetas.emplace(AZStd::move(name), AZStd::move(testMeta));
}
// If there's no tests in the repo then something is seriously wrong
Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestTargetMetaMapFactory.h
+4 -2
View File
@@ -12,14 +12,16 @@
#pragma once
#include <TestImpactFramework/TestImpactTestSequence.h>
#include <Artifact/Static/TestImpactTestTargetMeta.h>
#include <AzCore/std/containers/vector.h>
namespace TestImpact
{
//! Constructs a list of test target meta-data artifacts from the specified master test list data.
//! Constructs a list of test target meta-data artifacts of the specified suite type from the specified master test list data.
//! @param masterTestListData The raw master test list data in JSON format.
//! @param suiteType The suite type to select the target meta-data artifacts from.
//! @return The constructed list of test target meta-data artifacts.
TestTargetMetaMap TestTargetMetaMapFactory(const AZStd::string& masterTestListData);
TestTargetMetaMap TestTargetMetaMapFactory(const AZStd::string& masterTestListData, SuiteType suiteType);
} // namespace TestImpact
Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp
+3 -3
View File
@@ -371,11 +371,11 @@ namespace TestImpact
{
if (sourceDependency->GetNumCoveringTestTargets())
{
AZ_Warning(
"File Update", false, AZStd::string::format("Source file %s is potentially an orphan (used by build targets "
AZ_Printf(
"File Update", AZStd::string::format("Source file '%s' is potentially an orphan (used by build targets "
"without explicitly being added to the build system, e.g. an include directive pulling in a header from the "
"repository). Running the covering tests for this file with instrumentation will confirm whether or nor this "
"is the case", updatedFile.c_str()).c_str());
"is the case.\n", updatedFile.c_str()).c_str());
updateDependencies.emplace_back(AZStd::move(*sourceDependency));
coverageToDelete.push_back(updatedFile);
Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp
+73 -54
View File
@@ -72,6 +72,7 @@ namespace TestImpact
Runtime::Runtime(
RuntimeConfig&& config,
SuiteType suiteFilter,
Policy::ExecutionFailure executionFailurePolicy,
Policy::ExecutionFailureDrafting executionFailureDraftingPolicy,
Policy::TestFailure testFailurePolicy,
@@ -80,6 +81,7 @@ namespace TestImpact
Policy::TargetOutputCapture targetOutputCapture,
AZStd::optional<size_t> maxConcurrency)
: m_config(AZStd::move(config))
, m_suiteFilter(suiteFilter)
, m_executionFailurePolicy(executionFailurePolicy)
, m_executionFailureDraftingPolicy(executionFailureDraftingPolicy)
, m_testFailurePolicy(testFailurePolicy)
@@ -89,7 +91,7 @@ namespace TestImpact
, m_maxConcurrency(maxConcurrency.value_or(AZStd::thread::hardware_concurrency()))
{
// Construct the dynamic dependency map from the build target descriptors
m_dynamicDependencyMap = ConstructDynamicDependencyMap(m_config.m_buildTargetDescriptor, m_config.m_testTargetMeta);
m_dynamicDependencyMap = ConstructDynamicDependencyMap(suiteFilter, m_config.m_buildTargetDescriptor, m_config.m_testTargetMeta);
// Construct the test selector and prioritizer from the dependency graph data (NOTE: currently not implemented)
m_testSelectorAndPrioritizer = AZStd::make_unique<TestSelectorAndPrioritizer>(m_dynamicDependencyMap.get(), DependencyGraphDataMap{});
@@ -110,7 +112,8 @@ namespace TestImpact
try
{
// Populate the dynamic dependency map with the existing source coverage data (if any)
const auto tiaDataRaw = ReadFileContents<Exception>(m_config.m_workspace.m_active.m_sparTIAFile);
m_sparTIAFile = m_config.m_workspace.m_active.m_sparTIAFiles[static_cast<size_t>(m_suiteFilter)].String();
const auto tiaDataRaw = ReadFileContents<Exception>(m_sparTIAFile);
const auto tiaData = DeserializeSourceCoveringTestsList(tiaDataRaw);
if (tiaData.GetNumSources())
{
@@ -118,13 +121,17 @@ namespace TestImpact
m_hasImpactAnalysisData = true;
// Enumerate new test targets
m_testEngine->UpdateEnumerationCache(
m_dynamicDependencyMap->GetNotCoveringTests(),
Policy::ExecutionFailure::Ignore,
Policy::TestFailure::Continue,
AZStd::nullopt,
AZStd::nullopt,
AZStd::nullopt);
const auto testTargetsWithNoEnumeration = m_dynamicDependencyMap->GetNotCoveringTests();
if (!testTargetsWithNoEnumeration.empty())
{
m_testEngine->UpdateEnumerationCache(
testTargetsWithNoEnumeration,
Policy::ExecutionFailure::Ignore,
Policy::TestFailure::Continue,
AZStd::nullopt,
AZStd::nullopt,
AZStd::nullopt);
}
}
}
catch (const DependencyException& e)
@@ -136,8 +143,10 @@ namespace TestImpact
}
catch ([[maybe_unused]]const Exception& e)
{
AZ_Printf("No test impact analysis data found at %s", m_config.m_workspace.m_active.m_sparTIAFile.c_str());
}
AZ_Printf("TestImpactRuntime",
AZStd::string::format(
"No test impact analysis data found for suite '%s' at %s", GetSuiteTypeName(m_suiteFilter).c_str(), m_sparTIAFile.c_str()).c_str());
}
}
Runtime::~Runtime() = default;
@@ -168,13 +177,16 @@ namespace TestImpact
addMutatedTestTargetsToEnumerationList(changeDependencyList.GetDeleteSourceDependencies());
// Enumerate the mutated test targets to ensure their enumeration caches are up to date
m_testEngine->UpdateEnumerationCache(
testTargets,
Policy::ExecutionFailure::Ignore,
Policy::TestFailure::Continue,
AZStd::nullopt,
AZStd::nullopt,
AZStd::nullopt);
if (!testTargets.empty())
{
m_testEngine->UpdateEnumerationCache(
testTargets,
Policy::ExecutionFailure::Ignore,
Policy::TestFailure::Continue,
AZStd::nullopt,
AZStd::nullopt,
AZStd::nullopt);
}
}
AZStd::pair<AZStd::vector<const TestTarget*>, AZStd::vector<const TestTarget*>> Runtime::SelectCoveringTestTargetsAndUpdateEnumerationCache(
@@ -233,7 +245,7 @@ namespace TestImpact
void Runtime::ClearDynamicDependencyMapAndRemoveExistingFile()
{
DeleteFile(m_config.m_workspace.m_active.m_sparTIAFile);
DeleteFile(m_sparTIAFile);
m_dynamicDependencyMap->ClearAllSourceCoverage();
}
@@ -247,12 +259,11 @@ namespace TestImpact
m_dynamicDependencyMap->ReplaceSourceCoverage(sourceCoverageTestsList);
const auto sparTIA = m_dynamicDependencyMap->ExportSourceCoverage();
const auto sparTIAData = SerializeSourceCoveringTestsList(sparTIA);
WriteFileContents<RuntimeException>(sparTIAData, m_config.m_workspace.m_active.m_sparTIAFile);
WriteFileContents<RuntimeException>(sparTIAData, m_sparTIAFile);
m_hasImpactAnalysisData = true;
}
TestSequenceResult Runtime::RegularTestSequence(
const AZStd::unordered_set<AZStd::string> suitesFilter,
AZStd::optional<AZStd::chrono::milliseconds> testTargetTimeout,
AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
AZStd::optional<TestSequenceStartCallback> testSequenceStartCallback,
@@ -268,21 +279,7 @@ namespace TestImpact
{
if (!m_testTargetExcludeList.contains(&testTarget))
{
if (suitesFilter.empty())
{
// Suite filter is empty, all tests that are not on the excluded list are included
includedTestTargets.push_back(&testTarget);
}
else if(suitesFilter.contains(testTarget.GetSuite()))
{
// Test target belonging to a suite in the suite filter are included, provided that are not on the exclude list
includedTestTargets.push_back(&testTarget);
}
else
{
// Test target not belonging to a suite in the suite filter are excluded
excludedTestTargets.push_back(&testTarget);
}
includedTestTargets.push_back(&testTarget);
}
else
{
@@ -318,6 +315,7 @@ namespace TestImpact
TestSequenceResult Runtime::ImpactAnalysisTestSequence(
const ChangeList& changeList,
Policy::TestPrioritization testPrioritizationPolicy,
Policy::DynamicDependencyMap dynamicDependencyMapPolicy,
AZStd::optional<AZStd::chrono::milliseconds> testTargetTimeout,
AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
AZStd::optional<ImpactAnalysisTestSequenceStartCallback> testSequenceStartCallback,
@@ -338,30 +336,51 @@ namespace TestImpact
ExtractTestTargetNames(draftedTestTargets));
}
const auto [result, testJobs] = m_testEngine->InstrumentedRun(
includedSelectedTestTargets,
m_testShardingPolicy,
m_executionFailurePolicy,
Policy::IntegrityFailure::Continue,
m_testFailurePolicy,
m_targetOutputCapture,
testTargetTimeout,
globalTimeout,
TestRunCompleteCallbackHandler(testCompleteCallback));
UpdateAndSerializeDynamicDependencyMap(CreateSourceCoveringTestFromTestCoverages(testJobs, m_config.m_repo.m_root));
if (testSequenceEndCallback.has_value())
if (dynamicDependencyMapPolicy == Policy::DynamicDependencyMap::Update)
{
(*testSequenceEndCallback)(GenerateSequenceFailureReport(testJobs), timer.Elapsed());
}
const auto [result, testJobs] = m_testEngine->InstrumentedRun(
includedSelectedTestTargets,
m_testShardingPolicy,
m_executionFailurePolicy,
Policy::IntegrityFailure::Continue,
m_testFailurePolicy,
m_targetOutputCapture,
testTargetTimeout,
globalTimeout,
TestRunCompleteCallbackHandler(testCompleteCallback));
return result;
UpdateAndSerializeDynamicDependencyMap(CreateSourceCoveringTestFromTestCoverages(testJobs, m_config.m_repo.m_root));
if (testSequenceEndCallback.has_value())
{
(*testSequenceEndCallback)(GenerateSequenceFailureReport(testJobs), timer.Elapsed());
}
return result;
}
else
{
const auto [result, testJobs] = m_testEngine->RegularRun(
includedSelectedTestTargets,
m_testShardingPolicy,
m_executionFailurePolicy,
m_testFailurePolicy,
m_targetOutputCapture,
testTargetTimeout,
globalTimeout,
TestRunCompleteCallbackHandler(testCompleteCallback));
if (testSequenceEndCallback.has_value())
{
(*testSequenceEndCallback)(GenerateSequenceFailureReport(testJobs), timer.Elapsed());
}
return result;
}
}
AZStd::pair<TestSequenceResult, TestSequenceResult> Runtime::SafeImpactAnalysisTestSequence(
const ChangeList& changeList,
const AZStd::unordered_set<AZStd::string> suitesFilter,
Policy::TestPrioritization testPrioritizationPolicy,
AZStd::optional<AZStd::chrono::milliseconds> testTargetTimeout,
AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.cpp
+4 -3
View File
@@ -22,10 +22,10 @@
namespace TestImpact
{
TestTargetMetaMap ReadTestTargetMetaMapFile(const RepoPath& testTargetMetaConfigFile)
TestTargetMetaMap ReadTestTargetMetaMapFile(SuiteType suiteFilter, const RepoPath& testTargetMetaConfigFile)
{
const auto masterTestListData = ReadFileContents<RuntimeException>(testTargetMetaConfigFile);
return TestTargetMetaMapFactory(masterTestListData);
return TestTargetMetaMapFactory(masterTestListData, suiteFilter);
}
AZStd::vector<TestImpact::BuildTargetDescriptor> ReadBuildTargetDescriptorFiles(const BuildTargetDescriptorConfig& buildTargetDescriptorConfig)
@@ -46,10 +46,11 @@ namespace TestImpact
}
AZStd::unique_ptr<TestImpact::DynamicDependencyMap> ConstructDynamicDependencyMap(
SuiteType suiteFilter,
const BuildTargetDescriptorConfig& buildTargetDescriptorConfig,
const TestTargetMetaConfig& testTargetMetaConfig)
{
auto testTargetmetaMap = ReadTestTargetMetaMapFile(testTargetMetaConfig.m_metaFile);
auto testTargetmetaMap = ReadTestTargetMetaMapFile(suiteFilter, testTargetMetaConfig.m_metaFile);
auto buildTargetDescriptors = ReadBuildTargetDescriptorFiles(buildTargetDescriptorConfig);
auto buildTargets = CompileTargetDescriptors(AZStd::move(buildTargetDescriptors), AZStd::move(testTargetmetaMap));
auto&& [productionTargets, testTargets] = buildTargets;
Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.h
+1
View File
@@ -30,6 +30,7 @@ namespace TestImpact
{
//! Construct a dynamic dependency map from the build target descriptors and test target metas.
AZStd::unique_ptr<TestImpact::DynamicDependencyMap> ConstructDynamicDependencyMap(
SuiteType suiteFilter,
const BuildTargetDescriptorConfig& buildTargetDescriptorConfig,
const TestTargetMetaConfig& testTargetMetaConfig);
Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_files.cmake
+62 -30
View File
@@ -10,15 +10,23 @@
#
set(FILES
Include/TestImpactFramework/TestImpactBitwise.h
Include/TestImpactFramework/TestImpactCallback.h
Include/TestImpactFramework/TestImpactException.h
Include/TestImpactFramework/TestImpactFrameworkPath.h
Include/TestImpactFramework/TestImpactRepoPath.h
Include/TestImpactFramework/TestImpactRuntime.h
Include/TestImpactFramework/TestImpactRuntimeException.h
Include/TestImpactFramework/TestImpactConfiguration.h
Include/TestImpactFramework/TestImpactConfigurationException.h
Include/TestImpactFramework/TestImpactChangelist.h
Include/TestImpactFramework/TestImpactChangelistSerializer.h
Include/TestImpactFramework/TestImpactChangelistException.h
Include/TestImpactFramework/TestImpactTestSequence.h
Include/TestImpactFramework/TestImpactClientTestSelection.h
Include/TestImpactFramework/TestImpactClientTestRun.h
Include/TestImpactFramework/TestImpactClientFailureReport.h
Include/TestImpactFramework/TestImpactFileUtils.h
Source/Artifact/TestImpactArtifactException.h
Source/Artifact/Factory/TestImpactBuildTargetDescriptorFactory.cpp
Source/Artifact/Factory/TestImpactBuildTargetDescriptorFactory.h
Source/Artifact/Factory/TestImpactChangeListFactory.cpp
Source/Artifact/Factory/TestImpactChangeListFactory.h
Source/Artifact/Factory/TestImpactTestEnumerationSuiteFactory.cpp
Source/Artifact/Factory/TestImpactTestEnumerationSuiteFactory.h
Source/Artifact/Factory/TestImpactTestRunSuiteFactory.cpp
@@ -27,6 +35,8 @@ set(FILES
Source/Artifact/Factory/TestImpactTestTargetMetaMapFactory.h
Source/Artifact/Factory/TestImpactModuleCoverageFactory.cpp
Source/Artifact/Factory/TestImpactModuleCoverageFactory.h
Source/Artifact/Factory/TestImpactDependencyGraphDataFactory.cpp
Source/Artifact/Factory/TestImpactDependencyGraphDataFactory.h
Source/Artifact/Static/TestImpactBuildTargetDescriptor.cpp
Source/Artifact/Static/TestImpactBuildTargetDescriptor.h
Source/Artifact/Static/TestImpactTargetDescriptorCompiler.cpp
@@ -37,7 +47,6 @@ set(FILES
Source/Artifact/Static/TestImpactTestTargetDescriptor.cpp
Source/Artifact/Static/TestImpactTestTargetDescriptor.h
Source/Artifact/Static/TestImpactDependencyGraphData.h
Source/Artifact/Dynamic/TestImpactChangelist.h
Source/Artifact/Dynamic/TestImpactTestEnumerationSuite.h
Source/Artifact/Dynamic/TestImpactTestRunSuite.h
Source/Artifact/Dynamic/TestImpactTestSuite.h
@@ -50,6 +59,8 @@ set(FILES
Source/Process/TestImpactProcessLauncher.h
Source/Process/JobRunner/TestImpactProcessJob.h
Source/Process/JobRunner/TestImpactProcessJobInfo.h
Source/Process/JobRunner/TestImpactProcessJobMeta.cpp
Source/Process/JobRunner/TestImpactProcessJobMeta.h
Source/Process/JobRunner/TestImpactProcessJobRunner.h
Source/Process/Scheduler/TestImpactProcessScheduler.cpp
Source/Process/Scheduler/TestImpactProcessScheduler.h
@@ -64,6 +75,8 @@ set(FILES
Source/Dependency/TestImpactTestSelectorAndPrioritizer.cpp
Source/Dependency/TestImpactSourceCoveringTestsList.h
Source/Dependency/TestImpactSourceCoveringTestsList.cpp
Source/Dependency/TestImpactSourceCoveringTestsSerializer.cpp
Source/Dependency/TestImpactSourceCoveringTestsSerializer.h
Source/Target/TestImpactBuildTarget.cpp
Source/Target/TestImpactBuildTarget.h
Source/Target/TestImpactBuildTargetList.h
@@ -74,29 +87,48 @@ set(FILES
Source/Target/TestImpactTestTarget.cpp
Source/Target/TestImpactTestTarget.h
Source/Target/TestImpactTestTargetList.h
Source/Test/Enumeration/TestImpactTestEnumeration.h
Source/Test/Enumeration/TestImpactTestEnumerationException.h
Source/Test/Enumeration/TestImpactTestEnumerationSerializer.cpp
Source/Test/Enumeration/TestImpactTestEnumerationSerializer.h
Source/Test/Enumeration/TestImpactTestEnumerator.cpp
Source/Test/Enumeration/TestImpactTestEnumerator.h
Source/Test/Run/TestImpactTestRunSerializer.cpp
Source/Test/Run/TestImpactTestRunSerializer.h
Source/Test/Run/TestImpactTestRunner.cpp
Source/Test/Run/TestImpactTestRunner.h
Source/Test/Run/TestImpactInstrumentedTestRunner.cpp
Source/Test/Run/TestImpactInstrumentedTestRunner.h
Source/Test/Run/TestImpactTestRun.cpp
Source/Test/Run/TestImpactTestRun.h
Source/Test/Run/TestImpactTestRunJobData.cpp
Source/Test/Run/TestImpactTestRunJobData.h
Source/Test/Run/TestImpactTestCoverage.cpp
Source/Test/Run/TestImpactTestCoverage.h
Source/Test/Run/TestImpactTestRunException.h
Source/Test/Job/TestImpactTestJobRunner.h
Source/Test/Job/TestImpactTestJobException.h
Source/Test/Job/TestImpactTestJobCommon.h
Source/Test/TestImpactTestSuiteContainer.h
Source/TestEngine/Enumeration/TestImpactTestEnumeration.h
Source/TestEngine/Enumeration/TestImpactTestEnumerationSerializer.cpp
Source/TestEngine/Enumeration/TestImpactTestEnumerationSerializer.h
Source/TestEngine/Enumeration/TestImpactTestEnumerator.cpp
Source/TestEngine/Enumeration/TestImpactTestEnumerator.h
Source/TestEngine/Run/TestImpactTestRunSerializer.cpp
Source/TestEngine/Run/TestImpactTestRunSerializer.h
Source/TestEngine/Run/TestImpactTestRunner.cpp
Source/TestEngine/Run/TestImpactTestRunner.h
Source/TestEngine/Run/TestImpactInstrumentedTestRunner.cpp
Source/TestEngine/Run/TestImpactInstrumentedTestRunner.h
Source/TestEngine/Run/TestImpactTestRun.cpp
Source/TestEngine/Run/TestImpactTestRun.h
Source/TestEngine/Run/TestImpactTestRunJobData.cpp
Source/TestEngine/Run/TestImpactTestRunJobData.h
Source/TestEngine/Run/TestImpactTestCoverage.cpp
Source/TestEngine/Run/TestImpactTestCoverage.h
Source/TestEngine/JobRunner/TestImpactTestJobRunner.h
Source/TestEngine/JobRunner/TestImpactTestJobInfoGenerator.cpp
Source/TestEngine/JobRunner/TestImpactTestJobInfoGenerator.h
Source/TestEngine/JobRunner/TestImpactTestTargetExtension.h
Source/TestEngine/TestImpactTestEngineJobFailure.cpp
Source/TestEngine/TestImpactTestEngineJobFailure.h
Source/TestEngine/TestImpactTestSuiteContainer.h
Source/TestEngine/TestImpactTestEngine.cpp
Source/TestEngine/TestImpactTestEngine.h
Source/TestEngine/TestImpactTestEngineJob.cpp
Source/TestEngine/TestImpactTestEngineJob.h
Source/TestEngine/TestImpactTestEngineEnumeration.cpp
Source/TestEngine/TestImpactTestEngineEnumeration.h
Source/TestEngine/TestImpactTestEngineRegularRun.cpp
Source/TestEngine/TestImpactTestEngineRegularRun.h
Source/TestEngine/TestImpactTestEngineInstrumentedRun.cpp
Source/TestEngine/TestImpactTestEngineInstrumentedRun.h
Source/TestEngine/TestImpactTestEngineException.h
Source/TestImpactException.cpp
Source/TestImpactFrameworkPath.cpp
Source/TestImpactRuntime.cpp
Source/TestImpactRuntimeUtils.cpp
Source/TestImpactRuntimeUtils.h
Source/TestImpactClientTestSelection.cpp
Source/TestImpactClientTestRun.cpp
Source/TestImpactClientFailureReport.cpp
Source/TestImpactChangeListSerializer.cpp
Source/TestImpactRepoPath.cpp
)
Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_tests_files.cmake
-38
View File
@@ -1,38 +0,0 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
set(FILES
Tests/Artifact/TestImpactTargetDescriptorCompilerTest.cpp
Tests/Artifact/TestImpactBuildTargetDescriptorFactoryTest.cpp
Tests/Artifact/TestImpactModuleCoverageFactoryTest.cpp
Tests/Artifact/TestImpactChangeListFactoryTest.cpp
Tests/Artifact/TestImpactTestEnumerationSuiteFactoryTest.cpp
Tests/Artifact/TestImpactTestRunSuiteFactoryTest.cpp
Tests/Artifact/TestImpactTestTargetMetaMapFactoryTest.cpp
Tests/Process/TestImpactProcessSchedulerTest.cpp
Tests/Process/TestImpactProcessTest.cpp
Tests/Target/TestImpactBuildTargetTest.cpp
Tests/TestImpactExceptionTest.cpp
Tests/TestImpactFrameworkPathTest.cpp
Tests/Test/TestImpactTestEnumeratorTest.cpp
Tests/Test/TestImpactTestEumerationSerializerTest.cpp
Tests/Test/TestImpactTestRunSerializerTest.cpp
Tests/Test/TestImpactTestRunnerTest.cpp
Tests/Test/TestImpactInstrumentedTestRunnerTest.cpp
Tests/Test/TestImpactTestCoverageTest.cpp
Tests/TestImpactTestJobRunnerCommon.h
Tests/TestImpactTestMain.cpp
Tests/TestImpactTestUtils.cpp
Tests/TestImpactTestUtils.h
)
cmake/LYTestWrappers.cmake
+37 -13
View File
@@ -71,6 +71,7 @@ endfunction()
#! ly_add_test: Adds a new RUN_TEST using for the specified target using the supplied command
#
# \arg:NAME - Name to for the test run target
# \arg:PARENT_NAME(optional) - Name of the parent test run target (if this is a subsequent call to specify a suite)
# \arg:TEST_REQUIRES(optional) - List of system resources that are required to run this test.
# Only available option is "gpu"
# \arg:TEST_SUITE(optional) - "smoke" or "periodic" or "sandbox" - prevents the test from running normally
@@ -94,7 +95,7 @@ endfunction()
# sets LY_ADDED_TEST_NAME to the fully qualified name of the test, in parent scope
function(ly_add_test)
set(options EXCLUDE_TEST_RUN_TARGET_FROM_IDE)
set(one_value_args NAME TEST_LIBRARY TEST_SUITE TIMEOUT)
set(one_value_args NAME PARENT_NAME TEST_LIBRARY TEST_SUITE TIMEOUT)
set(multi_value_args TEST_REQUIRES TEST_COMMAND NON_IDE_PARAMS RUNTIME_DEPENDENCIES COMPONENT LABELS)
# note that we dont use TEST_LIBRARY here, but PAL files might so do not remove!
@@ -241,12 +242,24 @@ function(ly_add_test)
endif()
# Store the test so we can walk through all of them in LYTestImpactFramework.cmake
set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS ${ly_add_test_NAME})
set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_test_NAME}_TEST_SUITE ${ly_add_test_TEST_SUITE})
set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_test_NAME}_TEST_LIBRARY ${ly_add_test_TEST_LIBRARY})
set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_test_NAME}_TEST_TIMEOUT ${ly_add_test_TIMEOUT})
if(NOT ly_add_test_PARENT_NAME)
set(test_target ${ly_add_test_NAME})
else()
set(test_target ${ly_add_test_PARENT_NAME})
endif()
# Check to see whether or not this test target has been stored in the global list for walking by the test impact analysis framework
get_property(all_tests GLOBAL PROPERTY LY_ALL_TESTS)
if(NOT "${test_target}" IN_LIST all_tests)
# This is the first reference to this test target so add it to the global list
set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS ${test_target})
set_property(GLOBAL PROPERTY LY_ALL_TESTS_${test_target}_TEST_LIBRARY ${ly_add_test_TEST_LIBRARY})
endif()
# Add the test suite and timeout value to the test target params
set(LY_TEST_PARAMS "${LY_TEST_PARAMS}#${ly_add_test_TEST_SUITE}")
set(LY_TEST_PARAMS "${LY_TEST_PARAMS}#${ly_add_test_TIMEOUT}")
# Store the params for this test target
set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${test_target}_PARAMS ${LY_TEST_PARAMS})
endfunction()
#! ly_add_pytest: registers target PyTest-based test with CTest
@@ -288,8 +301,12 @@ function(ly_add_pytest)
string(REPLACE "::" "_" pytest_report_directory "${PYTEST_XML_OUTPUT_DIR}/${ly_add_pytest_NAME}.xml")
# Add the script path to the test target params
set(LY_TEST_PARAMS "${ly_add_pytest_PATH}")
ly_add_test(
NAME ${ly_add_pytest_NAME}
PARENT_NAME ${ly_add_pytest_NAME}
TEST_SUITE ${ly_add_pytest_TEST_SUITE}
LABELS FRAMEWORK_pytest
TEST_COMMAND ${LY_PYTEST_EXECUTABLE} ${ly_add_pytest_PATH} ${ly_add_pytest_EXTRA_ARGS} --junitxml=${pytest_report_directory} ${custom_marks_args}
@@ -298,7 +315,6 @@ function(ly_add_pytest)
${ly_add_pytest_UNPARSED_ARGUMENTS}
)
set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_pytest_NAME}_SCRIPT_PATH ${ly_add_pytest_PATH})
set_tests_properties(${LY_ADDED_TEST_NAME} PROPERTIES RUN_SERIAL "${ly_add_pytest_TEST_SERIAL}")
endfunction()
@@ -341,10 +357,13 @@ function(ly_add_editor_python_test)
file(REAL_PATH ${ly_add_editor_python_test_TEST_PROJECT} project_real_path BASE_DIRECTORY ${LY_ROOT_FOLDER})
# Add the script path to the test target params
set(LY_TEST_PARAMS "${ly_add_editor_python_test_PATH}")
# Run test via the run_epbtest.cmake script.
# Parameters used are explained in run_epbtest.cmake.
ly_add_test(
NAME ${ly_add_editor_python_test_NAME}
PARENT_NAME ${ly_add_editor_python_test_NAME}
TEST_REQUIRES ${ly_add_editor_python_test_TEST_REQUIRES}
TEST_COMMAND ${CMAKE_COMMAND}
-DCMD_ARG_TEST_PROJECT=${project_real_path}
@@ -362,8 +381,6 @@ function(ly_add_editor_python_test)
TIMEOUT ${ly_add_editor_python_test_TIMEOUT}
COMPONENT ${ly_add_editor_python_test_COMPONENT}
)
set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_editor_python_test_NAME}_SCRIPT_PATH ${ly_add_editor_python_test_PATH})
set_tests_properties(${LY_ADDED_TEST_NAME} PROPERTIES RUN_SERIAL "${ly_add_editor_python_test_TEST_SERIAL}")
endfunction()
@@ -431,14 +448,16 @@ function(ly_add_googletest)
set(full_test_command $<TARGET_FILE:AZ::AzTestRunner> $<TARGET_FILE:${build_target}> AzRunUnitTests)
# Add AzTestRunner as a build dependency
ly_add_dependencies(${build_target} AZ::AzTestRunner)
# Start the test target params and dd the command runner command
# Ideally, we would populate the full command procedurally but the generator expressions won't be expanded by the time we need this data
set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_googletest_NAME}_TEST_COMMAND "AzRunUnitTests")
set(LY_TEST_PARAMS "AzRunUnitTests")
else()
set(full_test_command ${ly_add_googletest_TEST_COMMAND})
# Remove the generator expressions so we are left with the argument(s) required to run unit tests for executable targets
string(REPLACE ";" "" stripped_test_command ${full_test_command})
string(GENEX_STRIP ${stripped_test_command} stripped_test_command)
set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_googletest_NAME}_TEST_COMMAND ${stripped_test_command})
# Start the test target params and dd the command runner command
set(LY_TEST_PARAMS "${stripped_test_command}")
endif()
string(REPLACE "::" "_" report_directory "${GTEST_XML_OUTPUT_DIR}/${ly_add_googletest_NAME}.xml")
@@ -446,6 +465,7 @@ function(ly_add_googletest)
# Invoke the lower level ly_add_test command to add the actual ctest and setup the test labels to add_dependencies on the target
ly_add_test(
NAME ${ly_add_googletest_NAME}
PARENT_NAME ${target_name}
TEST_SUITE ${ly_add_googletest_TEST_SUITE}
LABELS FRAMEWORK_googletest
TEST_COMMAND ${full_test_command} --gtest_output=xml:${report_directory} ${LY_GOOGLETEST_EXTRA_PARAMS}
@@ -520,18 +540,22 @@ function(ly_add_googlebenchmark)
# If command is not supplied attempts, uses the AzTestRunner to run googlebenchmarks on the supplied TARGET
set(full_test_command $<TARGET_FILE:AZ::AzTestRunner> $<TARGET_FILE:${build_target}> AzRunBenchmarks ${output_format_args})
# Start the test target params and dd the command runner command
# Ideally, we would populate the full command procedurally but the generator expressions won't be expanded by the time we need this data
set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_googlebenchmark_NAME}_TEST_COMMAND "AzRunUnitTests")
set(LY_TEST_PARAMS "AzRunUnitTests")
else()
set(full_test_command ${ly_add_googlebenchmark_TEST_COMMAND})
# Remove the generator expressions so we are left with the argument(s) required to run unit tests for executable targets
string(REPLACE ";" "" stripped_test_command ${full_test_command})
string(GENEX_STRIP ${stripped_test_command} stripped_test_command)
set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_googletest_NAME}_TEST_COMMAND ${stripped_test_command})
# Start the test target params and dd the command runner command
set(LY_TEST_PARAMS "${stripped_test_command}")
endif()
# Set the name of the current test target for storage in the global list
ly_add_test(
NAME ${ly_add_googlebenchmark_NAME}
PARENT_NAME ${ly_add_googlebenchmark_NAME}
TEST_REQUIRES ${ly_add_googlebenchmark_TEST_REQUIRES}
TEST_COMMAND ${full_test_command} ${LY_GOOGLETEST_EXTRA_PARAMS}
TEST_SUITE "benchmark"
cmake/TestImpactFramework/ConsoleFrontendConfig.in
+18 -5
View File
@@ -4,7 +4,8 @@
"timestamp": "${timestamp}"
},
"repo": {
"root": "${repo_dir}"
"root": "${repo_dir}",
"tiaf_bin": "${tiaf_bin}"
},
"workspace": {
"temp": {
@@ -13,11 +14,23 @@
"artifact_dir": "RuntimeArtifact"
}
},
"persistent": {
"root": "${persistent_dir}",
"active": {
"root": "${active_dir}",
"relative_paths": {
"test_impact_data_file": "TestImpactData.spartia",
"enumeration_cache_dir": "EnumerationCache"
"test_impact_data_files": {
"main": "TestImpactData.main.spartia",
"periodic": "TestImpactData.periodic.spartia",
"sandbox": "TestImpactData.sandbox.spartia"
},
"enumeration_cache_dir": "EnumerationCache",
"last_build_target_list_file": "LastRunBuildTargets.json"
}
},
"historic": {
"root": "${historic_dir}",
"relative_paths": {
"last_run_hash_file": "last_run.hash",
"last_build_target_list_file": "LastRunBuildTargets.json"
}
}
},
cmake/TestImpactFramework/LYTestImpactFramework.cmake
+75 -40
View File
@@ -89,7 +89,7 @@ function(ly_test_impact_get_test_launch_method TARGET_NAME LAUNCH_METHOD)
elseif("${target_type}" STREQUAL "EXECUTABLE")
set(${LAUNCH_METHOD} "stand_alone" PARENT_SCOPE)
else()
message(FATAL_ERROR "Cannot deduce test target launch method for the target type ${target_type}")
message(FATAL_ERROR "Cannot deduce test target launch method for the target ${TARGET_NAME} with type ${target_type}")
endif()
endfunction()
@@ -131,33 +131,50 @@ function(ly_test_impact_extract_python_test COMPOSITE_TEST TEST_NAME)
set(${TEST_NAME} ${test_name} PARENT_SCOPE)
endfunction()
#! ly_test_impact_extract_google_test_params: extracts the google test name and command parameters.
#! ly_test_impact_extract_google_test_params: extracts the suites for the given google test.
#
# \arg:COMPOSITE_TEST test in the form 'namespace::test'
# \arg:COMPOSITE_SUITES composite list of suites for this target
# \arg:TEST_NAME name of test
# \arg:TEST_COMMAND optional command arguments to run the test
function(ly_test_impact_extract_google_test_params COMPOSITE_TEST TEST_NAME TEST_COMMAND)
get_property(test_command GLOBAL PROPERTY LY_ALL_TESTS_${COMPOSITE_TEST}_TEST_COMMAND)
# \arg:TEST_SUITES extracted list of suites for this target in JSON format
function(ly_test_impact_extract_google_test_params COMPOSITE_TEST COMPOSITE_SUITES TEST_NAME TEST_SUITES)
# Namespace and test are mandatory
string(REPLACE "::" ";" test_components ${COMPOSITE_TEST})
list(LENGTH test_components num_test_components)
if(num_test_components LESS 2)
message(FATAL_ERROR "The test ${test_components} appears to have been specified without a namespace, i.e.:\ly_add_googletest/benchmark(NAME ${test_components})\nInstead of (perhaps):\ly_add_googletest/benchmark(NAME Gem::${test_components})\nPlease add the missing namespace before proceeding.")
endif()
list(GET test_components 0 test_namespace)
list(GET test_components 1 test_name)
set(${TEST_NAMESPACE} ${test_namespace} PARENT_SCOPE)
set(${TEST_NAME} ${test_name} PARENT_SCOPE)
set(${TEST_COMMAND} ${test_command} PARENT_SCOPE)
set(test_suites "")
foreach(composite_suite ${COMPOSITE_SUITES})
# Command, suite, timeout
string(REPLACE "#" ";" suite_components ${composite_suite})
list(LENGTH suite_components num_suite_components)
if(num_suite_components LESS 3)
message(FATAL_ERROR "The suite components ${composite_suite} are required to be in the following format: command#suite#string.")
endif()
list(GET suite_components 0 test_command)
list(GET suite_components 1 test_suite)
list(GET suite_components 2 test_timeout)
set(suite_params "{ \"suite\": \"${test_suite}\", \"command\": \"${test_command}\", \"timeout\": ${test_timeout} }")
list(APPEND test_suites "${suite_params}")
endforeach()
string(REPLACE ";" ", " test_suites "${test_suites}")
set(${TEST_SUITES} ${test_suites} PARENT_SCOPE)
endfunction()
#! ly_test_impact_extract_python_test_params: extracts the python test name and relative script path parameters.
#
# \arg:COMPOSITE_TEST test in form 'namespace::test' or 'test'
# \arg:COMPOSITE_SUITES composite list of suites for this target
# \arg:TEST_NAME name of test
# \arg:SCRIPT_PATH name of test
function(ly_test_impact_extract_python_test_params COMPOSITE_TEST TEST_NAME SCRIPT_PATH)
# \arg:TEST_SUITES extracted list of suites for this target in JSON format
function(ly_test_impact_extract_python_test_params COMPOSITE_TEST COMPOSITE_SUITES TEST_NAME TEST_SUITES)
get_property(script_path GLOBAL PROPERTY LY_ALL_TESTS_${COMPOSITE_TEST}_SCRIPT_PATH)
# namespace is optional, in which case this component will be simply the test name
@@ -169,15 +186,30 @@ function(ly_test_impact_extract_python_test_params COMPOSITE_TEST TEST_NAME SCRI
set(test_name ${test_components})
endif()
# Get python script path relative to repo root
ly_test_impact_rebase_file_to_repo_root(
${script_path}
script_path
${LY_ROOT_FOLDER}
)
set(${TEST_NAME} ${test_name} PARENT_SCOPE)
set(${SCRIPT_PATH} ${script_path} PARENT_SCOPE)
set(test_suites "")
foreach(composite_suite ${COMPOSITE_SUITES})
# Script path, suite, timeout
string(REPLACE "#" ";" suite_components ${composite_suite})
list(LENGTH suite_components num_suite_components)
if(num_suite_components LESS 3)
message(FATAL_ERROR "The suite components ${composite_suite} are required to be in the following format: script_path#suite#string.")
endif()
list(GET suite_components 0 script_path)
list(GET suite_components 1 test_suite)
list(GET suite_components 2 test_timeout)
# Get python script path relative to repo root
ly_test_impact_rebase_file_to_repo_root(
${script_path}
script_path
${LY_ROOT_FOLDER}
)
set(suite_params "{ \"suite\": \"${test_suite}\", \"script\": \"${script_path}\", \"timeout\": ${test_timeout} }")
list(APPEND test_suites "${suite_params}")
endforeach()
string(REPLACE ";" ", " test_suites "${test_suites}")
set(${TEST_SUITES} ${test_suites} PARENT_SCOPE)
endfunction()
#! ly_test_impact_write_test_enumeration_file: exports the master test lists to file.
@@ -185,7 +217,6 @@ endfunction()
# \arg:TEST_ENUMERATION_TEMPLATE_FILE path to test enumeration template file
function(ly_test_impact_write_test_enumeration_file TEST_ENUMERATION_TEMPLATE_FILE)
get_property(LY_ALL_TESTS GLOBAL PROPERTY LY_ALL_TESTS)
# Enumerated tests for each type
set(google_tests "")
set(google_benchmarks "")
@@ -196,29 +227,28 @@ function(ly_test_impact_write_test_enumeration_file TEST_ENUMERATION_TEMPLATE_FI
# Walk the test list
foreach(test ${LY_ALL_TESTS})
message(TRACE "Parsing ${test}")
get_property(test_params GLOBAL PROPERTY LY_ALL_TESTS_${test}_PARAMS)
get_property(test_type GLOBAL PROPERTY LY_ALL_TESTS_${test}_TEST_LIBRARY)
get_property(test_suite GLOBAL PROPERTY LY_ALL_TESTS_${test}_TEST_SUITE)
get_property(test_timeout GLOBAL PROPERTY LY_ALL_TESTS_${test}_TEST_TIMEOUT)
if("${test_type}" STREQUAL "pytest")
# Python tests
ly_test_impact_extract_python_test_params(${test} test_name script_path)
list(APPEND python_tests " { \"name\": \"${test_name}\", \"suite\": \"${test_suite}\", \"script\": \"${script_path}\", \"timeout\":${test_timeout} }")
ly_test_impact_extract_python_test_params(${test} "${test_params}" test_name test_suites)
list(APPEND python_tests " { \"name\": \"${test_name}\", \"suites\": [${test_suites}] }")
elseif("${test_type}" STREQUAL "pytest_editor")
# Python editor tests
ly_test_impact_extract_python_test_params(${test} test_name script_path)
list(APPEND python_editor_tests " { \"name\": \"${test_name}\", \"suite\": \"${test_suite}\", \"script\": \"${script_path}\", \"timeout\":${test_timeout} }")
# Python editor tests
ly_test_impact_extract_python_test_params(${test} "${test_params}" test_name test_suites)
list(APPEND python_editor_tests " { \"name\": \"${test_name}\", \"suites\": [${test_suites}] }")
elseif("${test_type}" STREQUAL "googletest")
# Google tests
ly_test_impact_extract_google_test_params(${test} test_name test_command)
ly_test_impact_get_test_launch_method(${test_name} launch_method)
list(APPEND google_tests " { \"name\": \"${test_name}\", \"suite\": \"${test_suite}\", \"command\": \"${test_command}\", \"timeout\":${test_timeout}, \"launch_method\": \"${launch_method}\" }")
ly_test_impact_extract_google_test_params(${test} "${test_params}" test_name test_suites)
ly_test_impact_get_test_launch_method(${test} launch_method)
list(APPEND google_tests " { \"name\": \"${test_name}\", \"launch_method\": \"${launch_method}\", \"suites\": [${test_suites}] }")
elseif("${test_type}" STREQUAL "googlebenchmark")
# Google benchmarks
ly_test_impact_extract_google_test_params(${test} test_name test_command)
list(APPEND google_benchmarks " { \"name\": \"${test_name}\", \"suite\": \"${test_suite}\", \"command\": \"${test_command}\", \"timeout\":${test_timeout} }")
ly_test_impact_extract_google_test_params(${test} "${test_params}" test_name test_suites)
list(APPEND google_benchmarks " { \"name\": \"${test_name}\", \"launch_method\": \"${launch_method}\", \"suites\": [${test_suites}] }")
else()
message("${test_name} is of unknown type (TEST_LIBRARY property is empty)")
list(APPEND unknown_tests " { \"name\": \"${test}\" }")
list(APPEND unknown_tests " { \"name\": \"${test}\", \"type\": \"${test_type}\" }")
endif()
endforeach()
@@ -330,8 +360,11 @@ function(ly_test_impact_write_config_file CONFIG_TEMPLATE_FILE PERSISTENT_DATA_D
# Temp dir
set(temp_dir "${LY_TEST_IMPACT_TEMP_DIR}")
# Persistent dir
set(persistent_dir "${PERSISTENT_DATA_DIR}")
# Active persistent data dir
set(active_dir "${PERSISTENT_DATA_DIR}/active")
# Historic persistent data dir
set(historic_dir "${PERSISTENT_DATA_DIR}/historic")
# Source to target mappings dir
set(source_target_mapping_dir "${LY_TEST_IMPACT_SOURCE_TARGET_MAPPING_DIR}")
@@ -341,6 +374,9 @@ function(ly_test_impact_write_config_file CONFIG_TEMPLATE_FILE PERSISTENT_DATA_D
# Build dependency artifact dir
set(target_dependency_dir "${LY_TEST_IMPACT_TARGET_DEPENDENCY_DIR}")
# Test impact analysis framework binary
set(tiaf_bin "$<TARGET_FILE:${LY_TEST_IMPACT_CONSOLE_TARGET}>")
# Substitute config file template with above vars
file(READ "${CONFIG_TEMPLATE_FILE}" config_file)
@@ -348,9 +384,13 @@ function(ly_test_impact_write_config_file CONFIG_TEMPLATE_FILE PERSISTENT_DATA_D
# Write out entire config contents to a file in the build directory of the test impact framework console target
file(GENERATE
OUTPUT "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/$<CONFIG>/$<TARGET_FILE_BASE_NAME:${LY_TEST_IMPACT_CONSOLE_TARGET}>.$<CONFIG>.json"
OUTPUT "${PERSISTENT_DATA_DIR}/$<TARGET_FILE_BASE_NAME:${LY_TEST_IMPACT_CONSOLE_TARGET}>.$<CONFIG>.json"
CONTENT ${config_file}
)
# Set the above config file as the default config file to use for the test impact framework console target
target_compile_definitions(${LY_TEST_IMPACT_CONSOLE_STATIC_TARGET} PUBLIC "LY_TEST_IMPACT_DEFAULT_CONFIG_FILE=\"${PERSISTENT_DATA_DIR}/$<TARGET_FILE_BASE_NAME:${LY_TEST_IMPACT_CONSOLE_TARGET}>.$<CONFIG>.json\"")
message(DEBUG "Test impact framework post steps complete")
endfunction()
#! ly_test_impact_post_step: runs the post steps to be executed after all other cmake scripts have been executed.
@@ -360,8 +400,7 @@ function(ly_test_impact_post_step)
endif()
# Directory per build config for persistent test impact data (to be checked in)
set(persistent_data_dir "${LY_ROOT_FOLDER}/Tests/test_impact_framework/${CMAKE_SYSTEM_NAME}/$<CONFIG>")
set(persistent_data_dir "${LY_TEST_IMPACT_WORKING_DIR}/persistent")
# Directory for binaries built for this profile
set(bin_dir "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/$<CONFIG>")
@@ -388,8 +427,4 @@ function(ly_test_impact_post_step)
# Copy over the graphviz options file for the build dependency graphs
message(DEBUG "Test impact framework config file written")
file(COPY "cmake/TestImpactFramework/CMakeGraphVizOptions.cmake" DESTINATION ${CMAKE_BINARY_DIR})
# Set the above config file as the default config file to use for the test impact framework console target
target_compile_definitions(${LY_TEST_IMPACT_CONSOLE_STATIC_TARGET} PUBLIC "LY_TEST_IMPACT_DEFAULT_CONFIG_FILE=\"${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/$<CONFIG>/$<TARGET_FILE_BASE_NAME:${LY_TEST_IMPACT_CONSOLE_TARGET}>.$<CONFIG>.json\"")
message(DEBUG "Test impact framework post steps complete")
endfunction()