birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'TIF/Runtime' into TIF/Jenkins

Browse Source
This commit is contained in:
jonawals
2021-06-08 13:17:47 +01:00
parent c9aaadea51 f54679bd7f
commit 3b3b43413d
28 changed files with 1156 additions and 454 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp
+97 -105
View File
@@ -28,7 +28,7 @@ namespace TestImpact
Sequence,
TestPrioritizationPolicy,
ExecutionFailurePolicy,
ExecutionFailureDraftingPolicy,
FailedTestCoveragePolicy,
TestFailurePolicy,
IntegrityFailurePolicy,
TestShardingPolicy,
@@ -36,13 +36,14 @@ namespace TestImpact
MaxConcurrency,
TestTargetTimeout,
GlobalTimeout,
SuitesFilter,
SuiteFilter,
SafeMode,
// Values
None,
Seed,
Regular,
ImpactAnalysis,
ImpactAnalysisNoWrite,
ImpactAnalysisOrSeed,
Locality,
Abort,
@@ -50,7 +51,8 @@ namespace TestImpact
Ignore,
StdOut,
File,
AllSuites
Discard,
Keep
};
constexpr const char* OptionKeys[] =
@@ -62,7 +64,7 @@ namespace TestImpact
"sequence",
"ppolicy",
"epolicy",
"rexecfailures",
"cpolicy",
"fpolicy",
"ipolicy",
"shard",
@@ -70,13 +72,14 @@ namespace TestImpact
"maxconcurrency",
"ttimeout",
"gtimeout",
"suites",
"suite",
"safemode",
// Values
"none",
"seed",
"regular",
"tia",
"tianowrite",
"tiaorseed",
"locality",
"abort",
@@ -84,7 +87,8 @@ namespace TestImpact
"ignore",
"stdout",
"file",
"*"
"discard",
"keep"
};
RepoPath ParseConfigurationFile(const AZ::CommandLine& cmd)
@@ -110,6 +114,7 @@ namespace TestImpact
{OptionKeys[Seed], TestSequenceType::Seed},
{OptionKeys[Regular], TestSequenceType::Regular},
{OptionKeys[ImpactAnalysis], TestSequenceType::ImpactAnalysis},
{OptionKeys[ImpactAnalysisNoWrite], TestSequenceType::ImpactAnalysisNoWrite},
{OptionKeys[ImpactAnalysisOrSeed], TestSequenceType::ImpactAnalysisOrSeed}
};
@@ -138,15 +143,15 @@ namespace TestImpact
return ParseMultiStateOption(OptionKeys[ExecutionFailurePolicy], states, cmd).value_or(Policy::ExecutionFailure::Continue);
}
Policy::ExecutionFailureDrafting ParseExecutionFailureDraftingPolicy(const AZ::CommandLine& cmd)
Policy::FailedTestCoverage ParseFailedTestCoveragePolicy(const AZ::CommandLine& cmd)
{
const BinaryStateValue<Policy::ExecutionFailureDrafting> states =
const AZStd::vector<AZStd::pair<AZStd::string, Policy::FailedTestCoverage>> states =
{
Policy::ExecutionFailureDrafting::Never,
Policy::ExecutionFailureDrafting::Always
{OptionKeys[Discard], Policy::FailedTestCoverage::Discard},
{OptionKeys[Keep], Policy::FailedTestCoverage::Keep}
};
return ParseOnOffOption(OptionKeys[ExecutionFailureDraftingPolicy], states, cmd).value_or(Policy::ExecutionFailureDrafting::Always);
return ParseMultiStateOption(OptionKeys[FailedTestCoveragePolicy], states, cmd).value_or(Policy::FailedTestCoverage::Keep);
}
Policy::TestFailure ParseTestFailurePolicy(const AZ::CommandLine& cmd)
@@ -250,32 +255,16 @@ namespace TestImpact
return ParseOnOffOption(OptionKeys[SafeMode], states, cmd).value_or(false);
}
AZStd::unordered_set<AZStd::string> ParseSuitesFilter(const AZ::CommandLine& cmd)
SuiteType ParseSuiteFilter(const AZ::CommandLine& cmd)
{
AZStd::unordered_set<AZStd::string> suitesFilter;
if (const auto numSwitchValues = cmd.GetNumSwitchValues(OptionKeys[SuitesFilter]);
numSwitchValues)
const AZStd::vector<AZStd::pair<AZStd::string, SuiteType>> states =
{
for (auto i = 0; i < numSwitchValues; i++)
{
const auto value = cmd.GetSwitchValue(OptionKeys[SuitesFilter], i);
AZ_TestImpact_Eval(!value.empty(), CommandLineOptionsException, "Suites option value is empty");
if (value == OptionKeys[AllSuites])
{
AZ_TestImpact_Eval(
suitesFilter.empty(), CommandLineOptionsException, "The * suite cannot be used with other suites");
}
{GetSuiteTypeName(SuiteType::Main), SuiteType::Main},
{GetSuiteTypeName(SuiteType::Periodic), SuiteType::Periodic},
{GetSuiteTypeName(SuiteType::Sandbox), SuiteType::Sandbox}
};
suitesFilter.insert(value);
}
}
if (suitesFilter.find(OptionKeys[AllSuites]) != suitesFilter.end())
{
return {};
}
return suitesFilter;
return ParseMultiStateOption(OptionKeys[SuiteFilter], states, cmd).value_or(SuiteType::Main);
}
}
@@ -290,7 +279,7 @@ namespace TestImpact
m_testSequenceType = ParseTestSequenceType(cmd);
m_testPrioritizationPolicy = ParseTestPrioritizationPolicy(cmd);
m_executionFailurePolicy = ParseExecutionFailurePolicy(cmd);
m_executionFailureDraftingPolicy = ParseExecutionFailureDraftingPolicy(cmd);
m_failedTestCoveragePolicy = ParseFailedTestCoveragePolicy(cmd);
m_testFailurePolicy = ParseTestFailurePolicy(cmd);
m_integrityFailurePolicy = ParseIntegrityFailurePolicy(cmd);
m_testShardingPolicy = ParseTestShardingPolicy(cmd);
@@ -299,7 +288,7 @@ namespace TestImpact
m_testTargetTimeout = ParseTestTargetTimeout(cmd);
m_globalTimeout = ParseGlobalTimeout(cmd);
m_safeMode = ParseSafeMode(cmd);
m_suitesFilter = ParseSuitesFilter(cmd);
m_suiteFilter = ParseSuiteFilter(cmd);
}
bool CommandLineOptions::HasChangeListFile() const
@@ -342,9 +331,9 @@ namespace TestImpact
return m_executionFailurePolicy;
}
Policy::ExecutionFailureDrafting CommandLineOptions::GetExecutionFailureDraftingPolicy() const
Policy::FailedTestCoverage CommandLineOptions::GetFailedTestCoveragePolicy() const
{
return m_executionFailureDraftingPolicy;
return m_failedTestCoveragePolicy;
}
Policy::TestFailure CommandLineOptions::GetTestFailurePolicy() const
@@ -382,9 +371,9 @@ namespace TestImpact
return m_globalTimeout;
}
const AZStd::unordered_set<AZStd::string>& CommandLineOptions::GetSuitesFilter() const
SuiteType CommandLineOptions::GetSuiteFilter() const
{
return m_suitesFilter;
return m_suiteFilter;
}
AZStd::string CommandLineOptions::GetCommandLineUsageString()
@@ -392,71 +381,74 @@ namespace TestImpact
AZStd::string help =
"usage: tiaf [options]\n"
" options:\n"
" -config=<filename> Path to the configuration file for the TIAF runtime (default: \n"
" <tiaf binay build dir>.<tiaf binary build type>.json).\n"
" -changelist=<filename> Path to the JSON of source file changes to perform test impact \n"
" analysis on.\n"
" -gtimeout=<seconds> Global timeout value to terminate the entire test sequence should it \n"
" be exceeded.\n"
" -ttimeout=<seconds> Timeout value to terminate individual test targets should it be \n"
" exceeded.\n"
" -sequence=<none, seed, regular, tia, tiaorseed> The type of test sequence to perform, where none runs no tests and\n"
" will report a all tests successful, seed removes any prior coverage \n"
" data and runs all test targets with instrumentation to reseed the \n"
" data from scratch, regular runs all of the test targets without any \n"
" instrumentation to generate coverage data(any prior coverage data is \n"
" left intact), tia uses any prior coverage data to run the instrumented \n"
" subset of selected tests(if no prior coverage data a regular run is \n"
" performed instead) and tiaorseed uses any prior coverage data to run \n"
" the instrumented subset of selected tests(if no prior coverage data a \n"
" seed run is performed instead).\n"
" -safemode=<on,off> Flag to specify a safe mode sequence where the set of unselected \n"
" tests is run without instrumentation after the set of selected \n"
" instrumented tests is run (this has the effect of ensuring all \n"
" tests are run regardless).\n"
" -shard=<on,off> Break any test targets with a sharding policy into the number of \n"
" shards according to the maximum concurrency value.\n"
" -rexecfailures=<on,off> Attempt to execute test targets that previously failed to execute.\n"
" -targetout=<sdtout, file> Capture of individual test run stdout, where stdout will capture \n"
" each individual test target's stdout and output each one to stdout \n"
" and file will capture each individual test target's stdout and output \n"
" each one individually to a file (multiple values are accepted).\n"
" -epolicy=<abort, continue, ignore> Policy for handling test execution failure (test targets could not be \n"
" launched due to the binary not being built, incorrect paths, etc.), \n"
" where abort will abort the entire test sequence upon the first test\n"
" target execution failureand report a failure(along with the return \n"
" code of the test target that failed to launch), continue will continue \n"
" with the test sequence in the event of test target execution failures\n"
" and treat the test targets that failed to launch as as test failures\n"
" (along with the return codes of the test targets that failed to \n"
" launch), ignore will continue with the test sequence in the event of \n"
" test target execution failuresand treat the test targets that failed\n"
" to launch as as test passes(along with the return codes of the test \n"
" targets that failed to launch).\n"
" -fpolicy <abort, continue> Policy for handling test failures (test targets report failing tests), \n"
" where abort will abort the entire test sequenceupon the first test \n"
" failureand report a failure and continue will continue with the test\n"
" sequence in the event of test failuresand report the test failures.\n"
" -ipolicy=<abort, seed, rerun> Policy for handling coverage data integrity failures, where abort will \n"
" abort the test sequenceand report a failure, seed will attempt another \n"
" sequence using the seed sequence type, otherwise will abort and report \n"
" a failure (this option has no effect for regularand seed sequence \n"
" types) and rerun will attempt another sequence using the regular \n"
" sequence type, otherwise will abortand report a failure(this option has \n"
" no effect for regular sequence type).\n"
" -ppolicy=<none, locality> Policy for prioritizing selected test targets, where none will not \n"
" attempt any test target prioritization and locality will attempt to \n"
" prioritize test targets according to the locality of their covering \n"
" production targets in the dependency graph(if no dependency graph data \n"
" available, no prioritization will occur).\n"
" -maxconcurrency=<number> The maximum number of concurrent test targets/shards to be in flight at \n"
" any given moment.\n"
" -ochangelist=<on,off> Outputs the change list used for test selection.\n"
" -suites=<names> The test suites to select from for this test sequence (multiple values are \n"
" allowed). The suite all has special significance and will allow tests from \n"
" any suite to be selected, however this particular suite is mutually exclusive\n"
" with other suite. Note: this option is only applicable to the regular sequence\n"
" and, if safe mode is enables, the tia and tiaorseed sequences.";
" -config=<filename> Path to the configuration file for the TIAF runtime (default: \n"
" <tiaf binay build dir>.<tiaf binary build type>.json).\n"
" -changelist=<filename> Path to the JSON of source file changes to perform test impact \n"
" analysis on.\n"
" -gtimeout=<seconds> Global timeout value to terminate the entire test sequence should it \n"
" be exceeded.\n"
" -ttimeout=<seconds> Timeout value to terminate individual test targets should it be \n"
" exceeded.\n"
" -sequence=<none, seed, regular, tia, tianowrite, tiaorseed> The type of test sequence to perform, where 'none' runs no tests and\n"
" will report a all tests successful, 'seed' removes any prior coverage \n"
" data and runs all test targets with instrumentation to reseed the \n"
" data from scratch, 'regular' runs all of the test targets without any \n"
" instrumentation to generate coverage data(any prior coverage data is \n"
" left intact), 'tia' uses any prior coverage data to run the instrumented \n"
" subset of selected tests(if no prior coverage data a regular run is \n"
" performed instead), 'tianowrite' uses any prior coverage data to run the \n"
" uninstrumented subset of selected tests (if no prior coverage data a \n"
" regular run is performed instead). The coverage data is not updated with \n"
" the subset of selected tests and 'tiaorseed' uses any prior coverage data \n"
" to run the instrumented subset of selected tests (if no prior coverage \n"
" data a seed run is performed instead).\n"
" -safemode=<on,off> Flag to specify a safe mode sequence where the set of unselected \n"
" tests is run without instrumentation after the set of selected \n"
" instrumented tests is run (this has the effect of ensuring all \n"
" tests are run regardless).\n"
" -shard=<on,off> Break any test targets with a sharding policy into the number of \n"
" shards according to the maximum concurrency value.\n"
" -cpolicy=<remove, keep> Policy for handling the coverage data of failing tests, where 'discard' \n"
" will discard the coverage data produced by the failing tests, causing \n"
" them to be drafted into future test runs and 'keep' will keep any existing \n"
" coverage data and update the coverage data for failed tests that produce \n"
" coverage.\n"
" -targetout=<sdtout, file> Capture of individual test run stdout, where 'stdout' will capture \n"
" each individual test target's stdout and output each one to stdout \n"
" and 'file' will capture each individual test target's stdout and output \n"
" each one individually to a file (multiple values are accepted).\n"
" -epolicy=<abort, continue, ignore> Policy for handling test execution failure (test targets could not be \n"
" launched due to the binary not being built, incorrect paths, etc.), \n"
" where 'abort' will abort the entire test sequence upon the first test\n"
" target execution failure and report a failure(along with the return \n"
" code of the test target that failed to launch), 'continue' will continue \n"
" with the test sequence in the event of test target execution failures\n"
" and treat the test targets that failed to launch as test failures\n"
" (along with the return codes of the test targets that failed to \n"
" launch), 'ignore' will continue with the test sequence in the event of \n"
" test target execution failures and treat the test targets that failed\n"
" to launch as test passes(along with the return codes of the test \n"
" targets that failed to launch).\n"
" -fpolicy <abort, continue> Policy for handling test failures (test targets report failing tests), \n"
" where 'abort' will abort the entire test sequence upon the first test \n"
" failure and report a failure and 'continue' will continue with the test\n"
" sequence in the event of test failures and report the test failures.\n"
" -ipolicy=<abort, seed, rerun> Policy for handling coverage data integrity failures, where 'abort' will \n"
" abort the test sequence and report a failure, 'seed' will attempt another \n"
" sequence using the seed sequence type, otherwise will abort and report \n"
" a failure (this option has no effect for regular and seed sequence \n"
" types) and 'rerun' will attempt another sequence using the regular \n"
" sequence type, otherwise will abort and report a failure(this option has \n"
" no effect for regular sequence type).\n"
" -ppolicy=<none, locality> Policy for prioritizing selected test targets, where 'none' will not \n"
" attempt any test target prioritization and 'locality' will attempt to \n"
" prioritize test targets according to the locality of their covering \n"
" production targets in the dependency graph(if no dependency graph data \n"
" available, no prioritization will occur).\n"
" -maxconcurrency=<number> The maximum number of concurrent test targets/shards to be in flight at \n"
" any given moment.\n"
" -ochangelist=<on,off> Outputs the change list used for test selection.\n"
" -suite=<main, periodic, sandbox> The test suite to select from for this test sequence.";
return help;
}
Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.h
+8 -7
View File
@@ -18,7 +18,6 @@
#include <AzCore/std/string/string.h>
#include <AzCore/std/optional.h>
#include <AzCore/std/containers/vector.h>
#include <AzCore/std/containers/unordered_set.h>
namespace TestImpact
{
@@ -29,6 +28,8 @@ namespace TestImpact
Seed, //!< Removes any prior coverage data and runs all test targets with instrumentation to reseed the data from scratch.
Regular, //!< Runs all of the test targets without any instrumentation to generate coverage data (any prior coverage data is left intact).
ImpactAnalysis, //!< Uses any prior coverage data to run the instrumented subset of selected tests (if no prior coverage data a regular run is performed instead).
ImpactAnalysisNoWrite, //!< Uses any prior coverage data to run the uninstrumented subset of selected tests (if no prior coverage data a regular run is performed instead).
//!< The coverage data is not updated with the subset of selected tests.
ImpactAnalysisOrSeed //!< Uses any prior coverage data to run the instrumented subset of selected tests (if no prior coverage data a seed run is performed instead).
};
@@ -63,8 +64,8 @@ namespace TestImpact
//! Returns the test execution failure policy to use.
Policy::ExecutionFailure GetExecutionFailurePolicy() const;
//! Returns the test historic test execution failure drafting policy to use.
Policy::ExecutionFailureDrafting GetExecutionFailureDraftingPolicy() const;
//! Returns failed test coverage drafting policy to use.
Policy::FailedTestCoverage GetFailedTestCoveragePolicy() const;
//! Returns the test failure policy to use.
Policy::TestFailure GetTestFailurePolicy() const;
@@ -87,8 +88,8 @@ namespace TestImpact
//! Returns the global test sequence timeout to use (if any).
const AZStd::optional<AZStd::chrono::milliseconds>& GetGlobalTimeout() const;
//! Returns the filter for test suites that will be allowed to be run.
const AZStd::unordered_set<AZStd::string>& GetSuitesFilter() const;
//! Returns the filter for test suite that will be allowed to be run.
SuiteType GetSuiteFilter() const;
private:
RepoPath m_configurationFile;
@@ -97,7 +98,7 @@ namespace TestImpact
TestSequenceType m_testSequenceType;
Policy::TestPrioritization m_testPrioritizationPolicy = Policy::TestPrioritization::None;
Policy::ExecutionFailure m_executionFailurePolicy = Policy::ExecutionFailure::Continue;
Policy::ExecutionFailureDrafting m_executionFailureDraftingPolicy = Policy::ExecutionFailureDrafting::Always;
Policy::FailedTestCoverage m_failedTestCoveragePolicy = Policy::FailedTestCoverage::Keep;
Policy::TestFailure m_testFailurePolicy = Policy::TestFailure::Abort;
Policy::IntegrityFailure m_integrityFailurePolicy = Policy::IntegrityFailure::Abort;
Policy::TestSharding m_testShardingPolicy = Policy::TestSharding::Never;
@@ -105,7 +106,7 @@ namespace TestImpact
AZStd::optional<size_t> m_maxConcurrency;
AZStd::optional<AZStd::chrono::milliseconds> m_testTargetTimeout;
AZStd::optional<AZStd::chrono::milliseconds> m_globalTimeout;
AZStd::unordered_set<AZStd::string> m_suitesFilter;
SuiteType m_suiteFilter;
bool m_safeMode = false;
};
} // namespace TestImpact
Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleMain.cpp
+61 -24
View File
@@ -86,6 +86,8 @@ namespace TestImpact
Runtime& runtime,
const AZStd::optional<ChangeList>& changeList)
{
// Even though it is possible for a regular run to be selected (see below) which does not actually require a change list,
// consider any impact analysis sequence type without a change list to be an error
AZ_TestImpact_Eval(
changeList.has_value(),
CommandLineOptionsException,
@@ -94,39 +96,72 @@ namespace TestImpact
TestSequenceResult result = TestSequenceResult::Failure;
if (options.HasSafeMode())
{
auto [selectedResult, discardedResult] = runtime.SafeImpactAnalysisTestSequence(
changeList.value(),
options.GetSuitesFilter(),
options.GetTestPrioritizationPolicy(),
options.GetTestTargetTimeout(),
options.GetGlobalTimeout(),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler));
if (options.GetTestSequenceType() == TestSequenceType::ImpactAnalysis)
{
auto [selectedResult, discardedResult] = runtime.SafeImpactAnalysisTestSequence(
changeList.value(),
options.GetTestPrioritizationPolicy(),
options.GetTestTargetTimeout(),
options.GetGlobalTimeout(),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler));
// Handling the possible timeout and failure permutations of the selected and discarded test results is splitting hairs
// so apply the following, admittedly arbitrary, rules to determine what the composite test sequence result should be
if (selectedResult == TestSequenceResult::Success && discardedResult == TestSequenceResult::Success)
{
// Trivial case: both sequences succeeded
result = TestSequenceResult::Success;
// Handling the possible timeout and failure permutations of the selected and discarded test results is splitting hairs
// so apply the following, admittedly arbitrary, rules to determine what the composite test sequence result should be
if (selectedResult == TestSequenceResult::Success && discardedResult == TestSequenceResult::Success)
{
// Trivial case: both sequences succeeded
result = TestSequenceResult::Success;
}
else if (selectedResult == TestSequenceResult::Failure || discardedResult == TestSequenceResult::Failure)
{
// One sequence failed whilst the other sequence either succeeded or timed out
result = TestSequenceResult::Failure;
}
else
{
// One or both sequences timed out or failed
result = TestSequenceResult::Timeout;
}
}
else if (selectedResult == TestSequenceResult::Failure || discardedResult == TestSequenceResult::Failure)
else if (options.GetTestSequenceType() == TestSequenceType::ImpactAnalysisNoWrite)
{
// One sequence failed whilst the other sequence either succeeded or timed out
result = TestSequenceResult::Failure;
// A no-write impact analysis sequence with safe mode enabled is functionally identical to a regular sequence type
// due to a) the selected tests being run without instrumentation and b) the discarded tests also being run without
// instrumentation
result = runtime.RegularTestSequence(
options.GetTestTargetTimeout(),
options.GetGlobalTimeout(),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler));
}
else
{
// One sequence timed out whilst the other sequence succeeded or both sequences timed out
result = TestSequenceResult::Timeout;
throw(Exception("Unexpected sequence type"));
}
}
else
{
Policy::DynamicDependencyMap dynamicDependencyMapPolicy;
if (options.GetTestSequenceType() == TestSequenceType::ImpactAnalysis)
{
dynamicDependencyMapPolicy = Policy::DynamicDependencyMap::Update;
}
else if (options.GetTestSequenceType() == TestSequenceType::ImpactAnalysisNoWrite)
{
dynamicDependencyMapPolicy = Policy::DynamicDependencyMap::Discard;
}
else
{
throw(Exception("Unexpected sequence type"));
}
result = runtime.ImpactAnalysisTestSequence(
changeList.value(),
options.GetTestPrioritizationPolicy(),
dynamicDependencyMapPolicy,
options.GetTestTargetTimeout(),
options.GetGlobalTimeout(),
AZStd::ref(sequenceEventHandler),
@@ -164,11 +199,13 @@ namespace TestImpact
// As of now, there are no other non-test operations other than printing a change list so getting this far is considered an error
AZ_TestImpact_Eval(options.GetTestSequenceType() != TestSequenceType::None, CommandLineOptionsException, "No action specified");
std::cout << "Constructing in-memory model of source tree and test coverage, this may take a moment...\n";
std::cout << "Constructing in-memory model of source tree and test coverage for test suite ";
std::cout << GetSuiteTypeName(options.GetSuiteFilter()).c_str() << ", this may take a moment...\n";
Runtime runtime(
RuntimeConfigurationFactory(ReadFileContents<CommandLineOptionsException>(options.GetConfigurationFile())),
options.GetSuiteFilter(),
options.GetExecutionFailurePolicy(),
options.GetExecutionFailureDraftingPolicy(),
options.GetFailedTestCoveragePolicy(),
options.GetTestFailurePolicy(),
options.GetIntegrityFailurePolicy(),
options.GetTestShardingPolicy(),
@@ -184,14 +221,13 @@ namespace TestImpact
std::cout << "Test impact analysis data for this repository was not found, seed or regular sequence fallbacks will be used.\n";
}
TestSequenceEventHandler sequenceEventHandler(&options.GetSuitesFilter());
TestSequenceEventHandler sequenceEventHandler(options.GetSuiteFilter());
switch (const auto type = options.GetTestSequenceType())
{
case TestSequenceType::Regular:
{
const auto result = runtime.RegularTestSequence(
options.GetSuitesFilter(),
options.GetTestTargetTimeout(),
options.GetGlobalTimeout(),
AZStd::ref(sequenceEventHandler),
@@ -211,6 +247,7 @@ namespace TestImpact
return GetReturnCodeForTestSequenceResult(result);
}
case TestSequenceType::ImpactAnalysisNoWrite:
case TestSequenceType::ImpactAnalysis:
{
return WrappedImpactAnalysisTestSequence(sequenceEventHandler, options, runtime, changeList);
Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleTestSequenceEventHandler.cpp
+234
View File
@@ -0,0 +1,234 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <TestImpactConsoleTestSequenceEventHandler.h>
#include <TestImpactConsoleUtils.h>
#include <iostream>
namespace TestImpact
{
namespace Console
{
namespace Output
{
void TestSuiteFilter(SuiteType filter)
{
std::cout << "Test suite filter: " << GetSuiteTypeName(filter).c_str() << "\n";
}
void ImpactAnalysisTestSelection(size_t numSelectedTests, size_t numDiscardedTests, size_t numExcludedTests, size_t numDraftedTests)
{
const float totalTests = numSelectedTests + numDiscardedTests;
const float saving = (1.0 - (numSelectedTests / totalTests)) * 100.0f;
std::cout << numSelectedTests << " tests selected, " << numDiscardedTests << " tests discarded (" << saving << "% test saving)\n";
std::cout << "Of which " << numExcludedTests << " tests have been excluded and " << numDraftedTests << " tests have been drafted.\n";
}
void FailureReport(const Client::SequenceFailure& failureReport, AZStd::chrono::milliseconds duration)
{
std::cout << "Sequence completed in " << (duration.count() / 1000.f) << "s with";
if (!failureReport.GetExecutionFailures().empty() ||
!failureReport.GetTestRunFailures().empty() ||
!failureReport.GetTimedOutTests().empty() ||
!failureReport.GetUnexecutedTests().empty())
{
std::cout << ":\n";
std::cout << SetColor(Foreground::White, Background::Red).c_str()
<< failureReport.GetTestRunFailures().size()
<< ResetColor().c_str() << " test failures\n";
std::cout << SetColor(Foreground::White, Background::Red).c_str()
<< failureReport.GetExecutionFailures().size()
<< ResetColor().c_str() << " execution failures\n";
std::cout << SetColor(Foreground::White, Background::Red).c_str()
<< failureReport.GetTimedOutTests().size()
<< ResetColor().c_str() << " test timeouts\n";
std::cout << SetColor(Foreground::White, Background::Red).c_str()
<< failureReport.GetUnexecutedTests().size()
<< ResetColor().c_str() << " unexecuted tests\n";
if (!failureReport.GetTestRunFailures().empty())
{
std::cout << "\nTest failures:\n";
for (const auto& testRunFailure : failureReport.GetTestRunFailures())
{
std::cout << " " << testRunFailure.GetTargetName().c_str();
for (const auto& testCaseFailure : testRunFailure.GetTestCaseFailures())
{
std::cout << "." << testCaseFailure.GetName().c_str();
for (const auto& testFailure : testCaseFailure.GetTestFailures())
{
std::cout << "." << testFailure.GetName().c_str() << "\n";
}
}
}
}
if (!failureReport.GetExecutionFailures().empty())
{
std::cout << "\nExecution failures:\n";
for (const auto& executionFailure : failureReport.GetExecutionFailures())
{
std::cout << " " << executionFailure.GetTargetName().c_str() << "\n";
std::cout << executionFailure.GetCommandString().c_str() << "\n";
}
}
if (!failureReport.GetTimedOutTests().empty())
{
std::cout << "\nTimed out tests:\n";
for (const auto& testTimeout : failureReport.GetTimedOutTests())
{
std::cout << " " << testTimeout.GetTargetName().c_str() << "\n";
}
}
if (!failureReport.GetUnexecutedTests().empty())
{
std::cout << "\nUnexecuted tests:\n";
for (const auto& unexecutedTest : failureReport.GetUnexecutedTests())
{
std::cout << " " << unexecutedTest.GetTargetName().c_str() << "\n";
}
}
}
else
{
std::cout << SetColor(Foreground::White, Background::Green).c_str() << " \100% passes!\n" << ResetColor().c_str();
}
}
}
TestSequenceEventHandler::TestSequenceEventHandler(SuiteType suiteFilter)
: m_suiteFilter(suiteFilter)
{
}
// TestSequenceStartCallback
void TestSequenceEventHandler::operator()(Client::TestRunSelection&& selectedTests)
{
ClearState();
m_numTests = selectedTests.GetNumIncludedTestRuns();
Output::TestSuiteFilter(m_suiteFilter);
std::cout << selectedTests.GetNumIncludedTestRuns() << " tests selected, " << selectedTests.GetNumExcludedTestRuns() << " excluded.\n";
}
// ImpactAnalysisTestSequenceStartCallback
void TestSequenceEventHandler::operator()(
Client::TestRunSelection&& selectedTests,
AZStd::vector<AZStd::string>&& discardedTests,
AZStd::vector<AZStd::string>&& draftedTests)
{
ClearState();
m_numTests = selectedTests.GetNumIncludedTestRuns() + draftedTests.size();
Output::TestSuiteFilter(m_suiteFilter);
Output::ImpactAnalysisTestSelection(
selectedTests.GetTotalNumTests(), discardedTests.size(), selectedTests.GetNumExcludedTestRuns(), draftedTests.size());
}
// SafeImpactAnalysisTestSequenceStartCallback
void TestSequenceEventHandler::operator()(
Client::TestRunSelection&& selectedTests,
Client::TestRunSelection&& discardedTests,
AZStd::vector<AZStd::string>&& draftedTests)
{
ClearState();
m_numTests = selectedTests.GetNumIncludedTestRuns() + draftedTests.size();
Output::TestSuiteFilter(m_suiteFilter);
Output::ImpactAnalysisTestSelection(
selectedTests.GetTotalNumTests(),
discardedTests.GetTotalNumTests(),
selectedTests.GetNumExcludedTestRuns() + discardedTests.GetNumExcludedTestRuns(),
draftedTests.size());
}
// TestSequenceCompleteCallback
void TestSequenceEventHandler::operator()(
Client::SequenceFailure&& failureReport,
AZStd::chrono::milliseconds duration)
{
Output::FailureReport(failureReport, duration);
std::cout << "Updating and serializing the test impact analysis data, this may take a moment...\n";
}
// SafeTestSequenceCompleteCallback
void TestSequenceEventHandler::operator()(
Client::SequenceFailure&& selectedFailureReport,
Client::SequenceFailure&& discardedFailureReport,
AZStd::chrono::milliseconds selectedDuration,
AZStd::chrono::milliseconds discaredDuration)
{
std::cout << "Selected test run:\n";
Output::FailureReport(selectedFailureReport, selectedDuration);
std::cout << "Discarded test run:\n";
Output::FailureReport(discardedFailureReport, discaredDuration);
std::cout << "Updating and serializing the test impact analysis data, this may take a moment...\n";
}
// TestRunCompleteCallback
void TestSequenceEventHandler::operator()([[maybe_unused]] Client::TestRun&& test)
{
m_numTestsComplete++;
const auto progress = AZStd::string::format("(%03u/%03u)", m_numTestsComplete, m_numTests, test.GetTargetName().c_str());
AZStd::string result;
switch (test.GetResult())
{
case Client::TestRunResult::AllTestsPass:
{
result = SetColorForString(Foreground::White, Background::Green, "PASS");
break;
}
case Client::TestRunResult::FailedToExecute:
{
result = SetColorForString(Foreground::White, Background::Red, "EXEC");
break;
}
case Client::TestRunResult::NotRun:
{
result = SetColorForString(Foreground::White, Background::Yellow, "SKIP");
break;
}
case Client::TestRunResult::TestFailures:
{
result = SetColorForString(Foreground::White, Background::Red, "FAIL");
break;
}
case Client::TestRunResult::Timeout:
{
result = SetColorForString(Foreground::White, Background::Magenta, "TIME");
break;
}
}
std::cout << progress.c_str() << " " << result.c_str() << " " << test.GetTargetName().c_str() << " (" << (test.GetDuration().count() / 1000.f) << "s)\n";
}
void TestSequenceEventHandler::ClearState()
{
m_numTests = 0;
m_numTestsComplete = 0;
}
} // namespace Console
} // namespace TestImpact
Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleTestSequenceEventHandler.h
+72
View File
@@ -0,0 +1,72 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <TestImpactFramework/TestImpactTestSequence.h>
#include <TestImpactFramework/TestImpactClientTestSelection.h>
#include <TestImpactFramework/TestImpactClientFailureReport.h>
#include <TestImpactFramework/TestImpactClientTestRun.h>
#include <AzCore/std/chrono/chrono.h>
#include <AzCore/std/string/string.h>
#include <AzCore/std/containers/unordered_set.h>
#pragma once
namespace TestImpact
{
namespace Console
{
//! Event handler for all test sequence types.
class TestSequenceEventHandler
{
public:
explicit TestSequenceEventHandler(SuiteType suiteFilter);
//! TestSequenceStartCallback.
void operator()(Client::TestRunSelection&& selectedTests);
//! ImpactAnalysisTestSequenceStartCallback.
void operator()(
Client::TestRunSelection&& selectedTests,
AZStd::vector<AZStd::string>&& discardedTests,
AZStd::vector<AZStd::string>&& draftedTests);
//! SafeImpactAnalysisTestSequenceStartCallback.
void operator()(
Client::TestRunSelection&& selectedTests,
Client::TestRunSelection&& discardedTests,
AZStd::vector<AZStd::string>&& draftedTests);
//! TestSequenceCompleteCallback.
void operator()(
Client::SequenceFailure&& failureReport,
AZStd::chrono::milliseconds duration);
//! SafeTestSequenceCompleteCallback.
void operator()(
Client::SequenceFailure&& selectedFailureReport,
Client::SequenceFailure&& discardedFailureReport,
AZStd::chrono::milliseconds selectedDuration,
AZStd::chrono::milliseconds discaredDuration);
//! TestRunCompleteCallback.
void operator()(Client::TestRun&& test);
private:
void ClearState();
SuiteType m_suiteFilter;
size_t m_numTests = 0;
size_t m_numTestsComplete = 0;
};
} // namespace Console
} // namespace TestImpact
Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleUtils.cpp
+34
View File
@@ -0,0 +1,34 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <TestImpactConsoleUtils.h>
namespace TestImpact
{
namespace Console
{
AZStd::string SetColor(Foreground foreground, Background background)
{
return AZStd::string::format("\033[%u;%um", aznumeric_cast<uint32_t>(foreground), aznumeric_cast<uint32_t>(background));
}
AZStd::string SetColorForString(Foreground foreground, Background background, const AZStd::string& str)
{
return AZStd::string::format("%s%s%s", SetColor(foreground, background).c_str(), str.c_str(), ResetColor().c_str());
}
AZStd::string ResetColor()
{
return "\033[0m";
}
} // namespace Console
} // namespace TestImpact
Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleUtils.h
+56
View File
@@ -0,0 +1,56 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <AzCore/std/string/string.h>
namespace TestImpact
{
namespace Console
{
//! The set of available foreground colors.
enum class Foreground
{
Black = 30,
Red,
Green,
Yellow,
Blue,
Magenta,
Cyan,
White
};
//! The set of available background colors.
enum class Background
{
Black = 40,
Red,
Green,
Yellow,
Blue,
Magenta,
Cyan,
White
};
//! Returns a string to be used to set the specified foreground and background color.
AZStd::string SetColor(Foreground foreground, Background background);
//! Returns a string with the specified string set to the specified foreground and background color followed by a color reset.
AZStd::string SetColorForString(Foreground foreground, Background background, const AZStd::string& str);
//! Returns a string to be used to reset the color back to white foreground on black background.
AZStd::string ResetColor();
} // namespace Console
} // namespace TestImpact
Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactRuntimeConfigurationFactory.cpp
+17 -4
View File
@@ -30,7 +30,7 @@ namespace TestImpact
"relative_paths",
"artifact_dir",
"enumeration_cache_dir",
"test_impact_data_file",
"test_impact_data_files",
"temp",
"active",
"target_sources",
@@ -75,7 +75,7 @@ namespace TestImpact
RelativePaths,
ArtifactDir,
EnumerationCacheDir,
TestImpactDataFile,
TestImpactDataFiles,
TempWorkspace,
ActiveWorkspace,
TargetSources,
@@ -144,6 +144,19 @@ namespace TestImpact
return tempWorkspaceConfig;
}
AZStd::array<RepoPath, 3> ParseTestImpactAnalysisDataFiles(const RepoPath& root, const rapidjson::Value& sparTIAFile)
{
AZStd::array<RepoPath, 3> sparTIAFiles;
sparTIAFiles[static_cast<size_t>(SuiteType::Main)] =
GetAbsPathFromRelPath(root, sparTIAFile[GetSuiteTypeName(SuiteType::Main).c_str()].GetString());
sparTIAFiles[static_cast<size_t>(SuiteType::Periodic)] =
GetAbsPathFromRelPath(root, sparTIAFile[GetSuiteTypeName(SuiteType::Periodic).c_str()].GetString());
sparTIAFiles[static_cast<size_t>(SuiteType::Sandbox)] =
GetAbsPathFromRelPath(root, sparTIAFile[GetSuiteTypeName(SuiteType::Sandbox).c_str()].GetString());
return sparTIAFiles;
}
WorkspaceConfig::Active ParseActiveWorkspaceConfig(const rapidjson::Value& activeWorkspace)
{
WorkspaceConfig::Active activeWorkspaceConfig;
@@ -151,8 +164,8 @@ namespace TestImpact
activeWorkspaceConfig.m_root = activeWorkspace[Config::Keys[Config::Root]].GetString();
activeWorkspaceConfig.m_enumerationCacheDirectory
= GetAbsPathFromRelPath(activeWorkspaceConfig.m_root, relativePaths[Config::Keys[Config::EnumerationCacheDir]].GetString());
activeWorkspaceConfig.m_sparTIAFile
= GetAbsPathFromRelPath(activeWorkspaceConfig.m_root, relativePaths[Config::Keys[Config::TestImpactDataFile]].GetString());
activeWorkspaceConfig.m_sparTIAFiles =
ParseTestImpactAnalysisDataFiles(activeWorkspaceConfig.m_root, relativePaths[Config::Keys[Config::TestImpactDataFiles]]);
return activeWorkspaceConfig;
}
Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactConfiguration.h
+2 -1
View File
@@ -16,6 +16,7 @@
#include <TestImpactFramework/TestImpactRepoPath.h>
#include <AzCore/std/containers/vector.h>
#include <AzCore/std/containers/array.h>
#include <AzCore/std/string/string.h>
namespace TestImpact
@@ -46,8 +47,8 @@ namespace TestImpact
struct Active
{
RepoPath m_root; //!< Path to the persistent workspace tracked by the repository.
RepoPath m_sparTIAFile; //!< Path to the test impact analysis data.
RepoPath m_enumerationCacheDirectory; //!< Path to the test enumerations cache.
AZStd::array<RepoPath, 3> m_sparTIAFiles; //!< Paths to the test impact analysis data files for each test suite.
};
Temp m_temp;
Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h
+16 -8
View File
@@ -35,6 +35,7 @@ namespace TestImpact
class TestEngine;
class TestTarget;
class SourceCoveringTestsList;
class TestEngineInstrumentedRun;
//! Callback for a test sequence that isn't using test impact analysis to determine selected tests.
//! @param tests The tests that will be run for this sequence.
@@ -80,7 +81,8 @@ namespace TestImpact
using SafeTestSequenceCompleteCallback = AZStd::function<void(
Client::SequenceFailure&& selectedFailureReport,
Client::SequenceFailure&& discardedFailureReport,
AZStd::chrono::milliseconds duration)>;
AZStd::chrono::milliseconds selectedDuration,
AZStd::chrono::milliseconds discardedDuration)>;
//! Callback for test runs that have completed for any reason.
//! @param selectedTests The test that has completed.
@@ -92,6 +94,7 @@ namespace TestImpact
public:
//! Constructs a runtime with the specified configuration and policies.
//! @param config The configuration used for this runtime instance.
//! @param suiteFilter The test suite for which the coverage data and test selection will draw from.
//! @param executionFailurePolicy Determines how to handle test targets that fail to execute.
//! @param executionFailureDraftingPolicy Determines how test targets that previously failed to execute are drafted into subsequent test sequences.
//! @param testFailurePolicy Determines how to handle test targets that report test failures.
@@ -99,8 +102,9 @@ namespace TestImpact
//! @param testShardingPolicy Determines how to handle test targets that have opted in to test sharding.
Runtime(
RuntimeConfig&& config,
SuiteType suiteFilter,
Policy::ExecutionFailure executionFailurePolicy,
Policy::ExecutionFailureDrafting executionFailureDraftingPolicy,
Policy::FailedTestCoverage failedTestCoveragePolicy,
Policy::TestFailure testFailurePolicy,
Policy::IntegrityFailure integrationFailurePolicy,
Policy::TestSharding testShardingPolicy,
@@ -110,7 +114,6 @@ namespace TestImpact
~Runtime();
//! Runs a test sequence where all tests with a matching suite in the suite filter and also not on the excluded list are selected.
//! @param suitesFilter The test suites that will be included in the test selection.
//! @param testTargetTimeout The maximum duration individual test targets may be in flight for (infinite if empty).
//! @param globalTimeout The maximum duration the entire test sequence may run for (infinite if empty).
//! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests.
@@ -118,7 +121,6 @@ namespace TestImpact
//! @param testRunCompleteCallback The client function to be called after an individual test run has completed.
//! @returns
TestSequenceResult RegularTestSequence(
const AZStd::unordered_set<AZStd::string> suitesFilter,
AZStd::optional<AZStd::chrono::milliseconds> testTargetTimeout,
AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
AZStd::optional<TestSequenceStartCallback> testSequenceStartCallback,
@@ -128,6 +130,7 @@ namespace TestImpact
//! Runs a test sequence where tests are selected according to test impact analysis so long as they are not on the excluded list.
//! @param changeList The change list used to determine the tests to select.
//! @param testPrioritizationPolicy Determines how selected tests will be prioritized.
//! @param dynamicDependencyMapPolicy The policy to determine how the coverage data of produced by test sequences is used to update the dynamic dependency map.
//! @param testTargetTimeout The maximum duration individual test targets may be in flight for (infinite if empty).
//! @param globalTimeout The maximum duration the entire test sequence may run for (infinite if empty).
//! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests.
@@ -137,6 +140,7 @@ namespace TestImpact
TestSequenceResult ImpactAnalysisTestSequence(
const ChangeList& changeList,
Policy::TestPrioritization testPrioritizationPolicy,
Policy::DynamicDependencyMap dynamicDependencyMapPolicy,
AZStd::optional<AZStd::chrono::milliseconds> testTargetTimeout,
AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
AZStd::optional<ImpactAnalysisTestSequenceStartCallback> testSequenceStartCallback,
@@ -145,7 +149,6 @@ namespace TestImpact
//! Runs a test sequence as per the ImpactAnalysisTestSequence where the tests not selected are also run (albeit without instrumentation).
//! @param changeList The change list used to determine the tests to select.
//! @param suitesFilter The test suites that will be included in the test selection.
//! @param testPrioritizationPolicy Determines how selected tests will be prioritized.
//! @param testTargetTimeout The maximum duration individual test targets may be in flight for (infinite if empty).
//! @param globalTimeout The maximum duration the entire test sequence may run for (infinite if empty).
@@ -155,7 +158,6 @@ namespace TestImpact
//! @returns
AZStd::pair<TestSequenceResult, TestSequenceResult> SafeImpactAnalysisTestSequence(
const ChangeList& changeList,
const AZStd::unordered_set<AZStd::string> suitesFilter,
Policy::TestPrioritization testPrioritizationPolicy,
AZStd::optional<AZStd::chrono::milliseconds> testTargetTimeout,
AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
@@ -200,15 +202,21 @@ namespace TestImpact
AZStd::pair<AZStd::vector<const TestTarget*>, AZStd::vector<const TestTarget*>> SelectTestTargetsByExcludeList(
AZStd::vector<const TestTarget*> testTargets) const;
//! Prunes the existing coverage for the specified jobs and creates the consolidated source covering tests list from the
//! test engine instrumented run jobs.
SourceCoveringTestsList CreateSourceCoveringTestFromTestCoverages(const AZStd::vector<TestEngineInstrumentedRun>& jobs);
//! Prepares the dynamic dependency map for a seed update by clearing all existing data and deleting the file that will be serialized.
void ClearDynamicDependencyMapAndRemoveExistingFile();
//! Updates the dynamic dependency map and serializes the entire map to disk.
void UpdateAndSerializeDynamicDependencyMap(const SourceCoveringTestsList& sourceCoverageTestsList);
void UpdateAndSerializeDynamicDependencyMap(const AZStd::vector<TestEngineInstrumentedRun>& jobs);
RuntimeConfig m_config;
SuiteType m_suiteFilter;
RepoPath m_sparTIAFile;
Policy::ExecutionFailure m_executionFailurePolicy;
Policy::ExecutionFailureDrafting m_executionFailureDraftingPolicy;
Policy::FailedTestCoverage m_failedTestCoveragePolicy;
Policy::TestFailure m_testFailurePolicy;
Policy::IntegrityFailure m_integrationFailurePolicy;
Policy::TestSharding m_testShardingPolicy;
Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactTestSequence.h
+39 -4
View File
@@ -12,6 +12,10 @@
#pragma once
#include <TestImpactFramework/TestImpactRuntimeException.h>
#include <AzCore/std/containers/array.h>
namespace TestImpact
{
namespace Policy
@@ -27,11 +31,11 @@ namespace TestImpact
Ignore //!< Continue the test sequence and ignore the execution failures.
};
//! Policy for reattempting the execution of test targets that failed to execute in previous runs.
enum class ExecutionFailureDrafting
//! Policy for handling the coverage data of failed tests targets (both test that failed to execute and tests that ran but failed).
enum class FailedTestCoverage
{
Never, //!< Do not attempt to execute historic execution failures.
Always //!< Reattempt the exectution of historic execution failures.
Discard, //!< Discard the coverage data produced by the failing tests, causing them to be drafted into future test runs.
Keep //!< Keep any existing coverage data and update the coverage data for failed test targetss that produce coverage.
};
//! Policy for prioritizing selected tests.
@@ -55,6 +59,13 @@ namespace TestImpact
Continue //!< Continue the test sequence and report the test failures after the run.
};
//! Policy for updating the dynamic dependency map with the coverage data of produced by test sequences.
enum class DynamicDependencyMap
{
Discard, //!< Discard the coverage data produced by test sequences.
Update //!< Update the dynamic dependency map with the coverage data produced by test sequences.
};
//! Policy for sharding test targets that have been marked for test sharding.
enum class TestSharding
{
@@ -82,6 +93,30 @@ namespace TestImpact
TestInterleaved //!< Tests are interlaced across shards agnostic of fixtures (fastest but prone to inter-test dependency problems).
};
//! Test suite types to select from.
enum class SuiteType : AZ::u8
{
Main = 0,
Periodic,
Sandbox
};
//! User-friendly names for the test suite types.
inline AZStd::string GetSuiteTypeName(SuiteType suiteType)
{
switch (suiteType)
{
case SuiteType::Main:
return "main";
case SuiteType::Periodic:
return "periodic";
case SuiteType::Sandbox:
return "sandbox";
default:
throw(RuntimeException("Unexpected suite type"));
}
}
//! Result of a test sequence that was run.
enum class TestSequenceResult
{
Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestTargetMetaMapFactory.cpp
+30 -19
View File
@@ -19,7 +19,7 @@
namespace TestImpact
{
TestTargetMetaMap TestTargetMetaMapFactory(const AZStd::string& masterTestListData)
TestTargetMetaMap TestTargetMetaMapFactory(const AZStd::string& masterTestListData, SuiteType suiteType)
{
// Keys for pertinent JSON node and attribute names
constexpr const char* Keys[] =
@@ -27,6 +27,7 @@ namespace TestImpact
"google",
"test",
"tests",
"suites",
"suite",
"launch_method",
"test_runner",
@@ -41,6 +42,7 @@ namespace TestImpact
GoogleKey,
TestKey,
TestsKey,
TestSuitesKey,
SuiteKey,
LaunchMethodKey,
TestRunnerKey,
@@ -64,26 +66,35 @@ namespace TestImpact
for (const auto& test : tests)
{
TestTargetMeta testMeta;
testMeta.m_suite = test[Keys[SuiteKey]].GetString();
testMeta.m_customArgs = test[Keys[CommandKey]].GetString();
testMeta.m_timeout = AZStd::chrono::seconds{ test[Keys[TimeoutKey]].GetUint() };
const auto testSuites = test[Keys[TestSuitesKey]].GetArray();
for (const auto& suite : testSuites)
{
// Check to see if this test target has the suite we're looking for
if (const auto suiteName = suite[Keys[SuiteKey]].GetString();
strcmp(GetSuiteTypeName(suiteType).c_str(), suiteName) == 0)
{
testMeta.m_suite = suiteName;
testMeta.m_customArgs = suite[Keys[CommandKey]].GetString();
testMeta.m_timeout = AZStd::chrono::seconds{ suite[Keys[TimeoutKey]].GetUint() };
if (const auto buildTypeString = test[Keys[LaunchMethodKey]].GetString(); strcmp(buildTypeString, Keys[TestRunnerKey]) == 0)
{
testMeta.m_launchMethod = LaunchMethod::TestRunner;
}
else if (strcmp(buildTypeString, Keys[StandAloneKey]) == 0)
{
testMeta.m_launchMethod = LaunchMethod::StandAlone;
}
else
{
throw(ArtifactException("Unexpected test build type"));
}
if (const auto buildTypeString = test[Keys[LaunchMethodKey]].GetString(); strcmp(buildTypeString, Keys[TestRunnerKey]) == 0)
{
testMeta.m_launchMethod = LaunchMethod::TestRunner;
AZStd::string name = test[Keys[NameKey]].GetString();
AZ_TestImpact_Eval(!name.empty(), ArtifactException, "Test name field cannot be empty");
testMetas.emplace(AZStd::move(name), AZStd::move(testMeta));
break;
}
}
else if (strcmp(buildTypeString, Keys[StandAloneKey]) == 0)
{
testMeta.m_launchMethod = LaunchMethod::StandAlone;
}
else
{
throw(ArtifactException("Unexpected test build type"));
}
AZStd::string name = test[Keys[NameKey]].GetString();
AZ_TestImpact_Eval(!name.empty(), ArtifactException, "Test name field cannot be empty");
testMetas.emplace(AZStd::move(name), AZStd::move(testMeta));
}
// If there's no tests in the repo then something is seriously wrong
Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactTestTargetMetaMapFactory.h
+4 -2
View File
@@ -12,14 +12,16 @@
#pragma once
#include <TestImpactFramework/TestImpactTestSequence.h>
#include <Artifact/Static/TestImpactTestTargetMeta.h>
#include <AzCore/std/containers/vector.h>
namespace TestImpact
{
//! Constructs a list of test target meta-data artifacts from the specified master test list data.
//! Constructs a list of test target meta-data artifacts of the specified suite type from the specified master test list data.
//! @param masterTestListData The raw master test list data in JSON format.
//! @param suiteType The suite type to select the target meta-data artifacts from.
//! @return The constructed list of test target meta-data artifacts.
TestTargetMetaMap TestTargetMetaMapFactory(const AZStd::string& masterTestListData);
TestTargetMetaMap TestTargetMetaMapFactory(const AZStd::string& masterTestListData, SuiteType suiteType);
} // namespace TestImpact
Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp
+69 -22
View File
@@ -56,7 +56,7 @@ namespace TestImpact
for (const auto& target : m_testTargets.GetTargets())
{
mapBuildTargetSources(&target);
m_testTargetSourceCoverageCount[&target] = 0;
m_testTargetSourceCoverage[&target] = {};
}
}
@@ -133,8 +133,9 @@ namespace TestImpact
return buildTarget;
}
void DynamicDependencyMap::ReplaceSourceCoverage(const SourceCoveringTestsList& sourceCoverageDelta)
void DynamicDependencyMap::ReplaceSourceCoverageInternal(const SourceCoveringTestsList& sourceCoverageDelta, bool pruneIfNoParentsOrCoverage)
{
AZStd::vector<AZStd::string> killList;
for (const auto& sourceCoverage : sourceCoverageDelta.GetCoverage())
{
// Autogen input files are not compiled sources and thus supplying coverage data for them makes no sense
@@ -144,20 +145,35 @@ namespace TestImpact
sourceCoverage.GetPath().c_str()).c_str());
auto [sourceDependencyIt, inserted] = m_sourceDependencyMap.insert(sourceCoverage.GetPath().String());
auto& [key, sourceDependency] = *sourceDependencyIt;
auto& [source, sourceDependency] = *sourceDependencyIt;
// Knock down the source coverage count for the test targets and clear any existing coverage for the delta
// Before we can replace the coverage for this source dependency, we must:
// 1. Remove the source from the test target covering sources map
// 2. Prune the covered targets for the parent test target(s) of this source dependency
// 3. Clear any existing coverage for the delta
// 1.
for (const auto& testTarget : sourceDependency.m_coveringTestTargets)
{
if (auto coveringTestTargetIt = m_testTargetSourceCoverageCount.find(testTarget);
coveringTestTargetIt != m_testTargetSourceCoverageCount.end())
if (auto coveringTestTargetIt = m_testTargetSourceCoverage.find(testTarget);
coveringTestTargetIt != m_testTargetSourceCoverage.end())
{
if (coveringTestTargetIt->second > 0)
{
coveringTestTargetIt->second--;
}
coveringTestTargetIt->second.erase(source);
}
}
// 2.
// This step is prohibitively expensive as it requires iterating over all of the sources of the build targets covered by
// the parent test targets of this source dependency to ensure that this is in fact the last source being cleared and thus
// it can be determined that the parent test target is no longer covering the given build target
//
// The implications of this are that multiple calls to the test selector and priritizor's SelectTestTargets method will end
// up pulling in more test targets than needed for newly-created production sources until the next time the dynamic dependency
// map reconstructed, however until this use case materializes the implications described will not be addressed
// 3.
sourceDependency.m_coveringTestTargets.clear();
// Update the dependency with any new coverage data
@@ -169,8 +185,8 @@ namespace TestImpact
// Source to covering test target mapping
sourceDependency.m_coveringTestTargets.insert(testTarget);
// Test target covering sources count
m_testTargetSourceCoverageCount[testTarget]++;
// Add the source to the test target covering sources map
m_testTargetSourceCoverage[testTarget].insert(source);
// Build target to covering test target mapping
for (const auto& parentTarget : sourceDependency.m_parentTargets)
@@ -186,13 +202,18 @@ namespace TestImpact
}
// If the new coverage data results in a parentless and coverageless entry, consider it a dead entry and remove accordingly
if (sourceDependency.m_coveringTestTargets.empty() && sourceDependency.m_parentTargets.empty())
if (sourceDependency.m_coveringTestTargets.empty() && sourceDependency.m_parentTargets.empty() && pruneIfNoParentsOrCoverage)
{
m_sourceDependencyMap.erase(sourceDependencyIt);
}
}
}
void DynamicDependencyMap::ReplaceSourceCoverage(const SourceCoveringTestsList& sourceCoverageDelta)
{
ReplaceSourceCoverageInternal(sourceCoverageDelta, true);
}
void DynamicDependencyMap::ClearSourceCoverage(const AZStd::vector<RepoPath>& paths)
{
for (const auto& path : paths)
@@ -215,9 +236,14 @@ namespace TestImpact
void DynamicDependencyMap::ClearAllSourceCoverage()
{
for (const auto& [path, coverage] : m_sourceDependencyMap)
for (auto it = m_sourceDependencyMap.begin(); it != m_sourceDependencyMap.end(); ++it)
{
ReplaceSourceCoverage(SourceCoveringTestsList(AZStd::vector<SourceCoveringTests>{ SourceCoveringTests(RepoPath(path)) }));
const auto& [path, coverage] = *it;
ReplaceSourceCoverageInternal(SourceCoveringTestsList(AZStd::vector<SourceCoveringTests>{ SourceCoveringTests(RepoPath(path)) }), false);
if (coverage.m_coveringTestTargets.empty() && coverage.m_parentTargets.empty())
{
it = m_sourceDependencyMap.erase(it);
}
}
}
@@ -371,11 +397,11 @@ namespace TestImpact
{
if (sourceDependency->GetNumCoveringTestTargets())
{
AZ_Warning(
"File Update", false, AZStd::string::format("Source file %s is potentially an orphan (used by build targets "
AZ_Printf(
"File Update", AZStd::string::format("Source file '%s' is potentially an orphan (used by build targets "
"without explicitly being added to the build system, e.g. an include directive pulling in a header from the "
"repository). Running the covering tests for this file with instrumentation will confirm whether or nor this "
"is the case", updatedFile.c_str()).c_str());
"is the case.\n", updatedFile.c_str()).c_str());
updateDependencies.emplace_back(AZStd::move(*sourceDependency));
coverageToDelete.push_back(updatedFile);
@@ -429,12 +455,33 @@ namespace TestImpact
return ChangeDependencyList(AZStd::move(createDependencies), AZStd::move(updateDependencies), AZStd::move(deleteDependencies));
}
void DynamicDependencyMap::RemoveTestTargetFromSourceCoverage(const TestTarget* testTarget)
{
if (const auto& it = m_testTargetSourceCoverage.find(testTarget);
it != m_testTargetSourceCoverage.end())
{
for (const auto& source : it->second)
{
const auto sourceDependency = m_sourceDependencyMap.find(source);
AZ_TestImpact_Eval(
sourceDependency != m_sourceDependencyMap.end(),
DependencyException,
AZStd::string::format("Test target '%s' has covering source '%s' yet cannot be found in the dependency map",
testTarget->GetName().c_str(), source.c_str()));
sourceDependency->second.m_coveringTestTargets.erase(testTarget);
}
m_testTargetSourceCoverage.erase(testTarget);
}
}
AZStd::vector<const TestTarget*> DynamicDependencyMap::GetCoveringTests() const
{
AZStd::vector<const TestTarget*> covering;
for (const auto& [testTarget, coveringSources] : m_testTargetSourceCoverageCount)
for (const auto& [testTarget, coveringSources] : m_testTargetSourceCoverage)
{
if (coveringSources > 0)
if (!coveringSources.empty())
{
covering.push_back(testTarget);
}
@@ -446,9 +493,9 @@ namespace TestImpact
AZStd::vector<const TestTarget*> DynamicDependencyMap::GetNotCoveringTests() const
{
AZStd::vector<const TestTarget*> notCovering;
for(const auto& [testTarget, coveringSources] : m_testTargetSourceCoverageCount)
for(const auto& [testTarget, coveringSources] : m_testTargetSourceCoverage)
{
if(coveringSources == 0)
if (coveringSources.empty())
{
notCovering.push_back(testTarget);
}
Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.h
+14 -4
View File
@@ -81,7 +81,6 @@ namespace TestImpact
SourceDependency GetSourceDependencyOrThrow(const RepoPath& path) const;
//! Replaces the source coverage of the specified sources with the specified source coverage.
//! @note The covering targets for the parent test target(s) will not be pruned if those covering targets are removed.
//! @param sourceCoverageDelta The source coverage delta to replace in the dependency map.
void ReplaceSourceCoverage(const SourceCoveringTestsList& sourceCoverageDelta);
@@ -100,6 +99,9 @@ namespace TestImpact
//! @returns The change list as resolved to the appropriate source dependencies.
[[nodiscard]] ChangeDependencyList ApplyAndResoveChangeList(const ChangeList& changeList);
//! Removes the specified test target from all source coverage.
void RemoveTestTargetFromSourceCoverage(const TestTarget* testTarget);
//! Returns the test targets that cover one or more sources in the repository.
AZStd::vector<const TestTarget*> GetCoveringTests() const;
@@ -107,6 +109,13 @@ namespace TestImpact
AZStd::vector<const TestTarget*> GetNotCoveringTests() const;
private:
//! Internal handler for ReplaceSourceCoverage where the pruning of parentless and coverageless source depenencies after the
//! source coverage has been replaced must be explicitly stated.
//! @note The covered targets for the source dependency's parent test target(s) will not be pruned if those covering targets are removed.
//! @param sourceCoverageDelta The source coverage delta to replace in the dependency map.
//! @param pruneIfNoParentsOrCoverage Flag to specify whether or not newly parentless and coverageless dependencies will be removed.
void ReplaceSourceCoverageInternal(const SourceCoveringTestsList& sourceCoverageDelta, bool pruneIfNoParentsOrCoverage);
//! Clears the source coverage of the specified sources.
//! @note The covering targets for the parent test target(s) will not be pruned if those covering targets are removed.
void ClearSourceCoverage(const AZStd::vector<RepoPath>& paths);
@@ -120,13 +129,14 @@ namespace TestImpact
//! The dependency map of sources to their parent build targets and covering test targets.
AZStd::unordered_map<AZStd::string, DependencyData> m_sourceDependencyMap;
//! Map of all test targets and the sources they cover.
AZStd::unordered_map<const TestTarget*, AZStd::unordered_set<AZStd::string>> m_testTargetSourceCoverage;
//! The map of build targets and their covering test targets.
//! @note As per the note for ReplaceSourceCoverageInternal, this map is currently not pruned when source coverage is replaced.
AZStd::unordered_map<const BuildTarget*, AZStd::unordered_set<const TestTarget*>> m_buildTargetCoverage;
//! Mapping of autogen input sources to their generated output sources.
AZStd::unordered_map<AZStd::string, AZStd::vector<AZStd::string>> m_autogenInputToOutputMap;
//! Number of sources that each test target in the repository covers.
AZStd::unordered_map<const TestTarget*, size_t> m_testTargetSourceCoverageCount;
};
} // namespace TestImpact
Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.cpp
+1 -1
View File
@@ -73,7 +73,7 @@ namespace TestImpact
}
catch (const TestEngineException& e)
{
AZ_Printf("Enumerate", "Enumeration cache error: %s", e.what());
AZ_Printf("Enumerate", AZStd::string::format("Enumeration cache error: %s\n", e.what()).c_str());
DeleteFile(jobInfo->GetCache()->m_file);
}
Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.cpp
+1 -1
View File
@@ -72,7 +72,7 @@ namespace TestImpact
}
catch (const Exception& e)
{
AZ_Printf("RunInstrumentedTests", e.what());
AZ_Printf("RunInstrumentedTests", AZStd::string::format("%s\n", e.what()).c_str());
runs[jobId] = AZStd::nullopt;
}
}
Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunner.cpp
+1 -1
View File
@@ -46,7 +46,7 @@ namespace TestImpact
}
catch (const Exception& e)
{
AZ_Printf("RunTests", e.what());
AZ_Printf("RunTests", AZStd::string::format("%s\n", e.what()).c_str());
runs[jobId] = AZStd::nullopt;
}
}
Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.cpp
+12
View File
@@ -10,6 +10,8 @@
*
*/
#include <TestImpactFramework/TestImpactFileUtils.h>
#include <Target/TestImpactTestTarget.h>
#include <TestEngine/TestImpactTestEngineException.h>
#include <TestEngine/TestImpactTestEngine.h>
@@ -247,11 +249,17 @@ namespace TestImpact
, m_testEnumerator(AZStd::make_unique<TestEnumerator>(maxConcurrentRuns))
, m_instrumentedTestRunner(AZStd::make_unique<InstrumentedTestRunner>(maxConcurrentRuns))
, m_testRunner(AZStd::make_unique<TestRunner>(maxConcurrentRuns))
, m_artifactDir(artifactDir)
{
}
TestEngine::~TestEngine() = default;
void TestEngine::DeleteArtifactXmls() const
{
DeleteFiles(m_artifactDir, "*.xml");
}
AZStd::pair<TestSequenceResult, AZStd::vector<TestEngineEnumeration>> TestEngine::UpdateEnumerationCache(
const AZStd::vector<const TestTarget*>& testTargets,
Policy::ExecutionFailure executionFailurePolicy,
@@ -283,6 +291,8 @@ namespace TestImpact
AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
AZStd::optional<TestEngineJobCompleteCallback> callback)
{
DeleteArtifactXmls();
TestEngineJobMap<TestRunner::JobInfo::IdType> engineJobs;
const auto jobInfos = m_testJobInfoGenerator->GenerateRegularTestRunJobInfos(testTargets);
@@ -308,6 +318,8 @@ namespace TestImpact
AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
AZStd::optional<TestEngineJobCompleteCallback> callback)
{
DeleteArtifactXmls();
TestEngineJobMap<InstrumentedTestRunner::JobInfo::IdType> engineJobs;
const auto jobInfos = m_testJobInfoGenerator->GenerateInstrumentedTestRunJobInfos(testTargets, CoverageLevel::Source);
Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.h
+4
View File
@@ -118,10 +118,14 @@ namespace TestImpact
AZStd::optional<TestEngineJobCompleteCallback> callback);
private:
//! Cleans up the artifacts directory of any artifacts from previous runs.
void DeleteArtifactXmls() const;
size_t m_maxConcurrentRuns = 0;
AZStd::unique_ptr<TestJobInfoGenerator> m_testJobInfoGenerator;
AZStd::unique_ptr<TestEnumerator> m_testEnumerator;
AZStd::unique_ptr<InstrumentedTestRunner> m_instrumentedTestRunner;
AZStd::unique_ptr<TestRunner> m_testRunner;
RepoPath m_artifactDir;
};
} // namespace TestImpact
Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp
+185 -65
View File
@@ -70,18 +70,31 @@ namespace TestImpact
};
}
//! Utility for concatenating two vectors.
template<typename T>
AZStd::vector<T> ConcatenateVectors(const AZStd::vector<T>& v1, const AZStd::vector<T>& v2)
{
AZStd::vector<T> result;
result.reserve(v1.size() + v2.size());
result.insert(result.end(), v1.begin(), v1.end());
result.insert(result.end(), v2.begin(), v2.end());
return result;
}
Runtime::Runtime(
RuntimeConfig&& config,
SuiteType suiteFilter,
Policy::ExecutionFailure executionFailurePolicy,
Policy::ExecutionFailureDrafting executionFailureDraftingPolicy,
Policy::FailedTestCoverage failedTestCoveragePolicy,
Policy::TestFailure testFailurePolicy,
Policy::IntegrityFailure integrationFailurePolicy,
Policy::TestSharding testShardingPolicy,
Policy::TargetOutputCapture targetOutputCapture,
AZStd::optional<size_t> maxConcurrency)
: m_config(AZStd::move(config))
, m_suiteFilter(suiteFilter)
, m_executionFailurePolicy(executionFailurePolicy)
, m_executionFailureDraftingPolicy(executionFailureDraftingPolicy)
, m_failedTestCoveragePolicy(failedTestCoveragePolicy)
, m_testFailurePolicy(testFailurePolicy)
, m_integrationFailurePolicy(integrationFailurePolicy)
, m_testShardingPolicy(testShardingPolicy)
@@ -89,7 +102,7 @@ namespace TestImpact
, m_maxConcurrency(maxConcurrency.value_or(AZStd::thread::hardware_concurrency()))
{
// Construct the dynamic dependency map from the build target descriptors
m_dynamicDependencyMap = ConstructDynamicDependencyMap(m_config.m_buildTargetDescriptor, m_config.m_testTargetMeta);
m_dynamicDependencyMap = ConstructDynamicDependencyMap(suiteFilter, m_config.m_buildTargetDescriptor, m_config.m_testTargetMeta);
// Construct the test selector and prioritizer from the dependency graph data (NOTE: currently not implemented)
m_testSelectorAndPrioritizer = AZStd::make_unique<TestSelectorAndPrioritizer>(m_dynamicDependencyMap.get(), DependencyGraphDataMap{});
@@ -110,7 +123,8 @@ namespace TestImpact
try
{
// Populate the dynamic dependency map with the existing source coverage data (if any)
const auto tiaDataRaw = ReadFileContents<Exception>(m_config.m_workspace.m_active.m_sparTIAFile);
m_sparTIAFile = m_config.m_workspace.m_active.m_sparTIAFiles[static_cast<size_t>(m_suiteFilter)].String();
const auto tiaDataRaw = ReadFileContents<Exception>(m_sparTIAFile);
const auto tiaData = DeserializeSourceCoveringTestsList(tiaDataRaw);
if (tiaData.GetNumSources())
{
@@ -118,13 +132,17 @@ namespace TestImpact
m_hasImpactAnalysisData = true;
// Enumerate new test targets
m_testEngine->UpdateEnumerationCache(
m_dynamicDependencyMap->GetNotCoveringTests(),
Policy::ExecutionFailure::Ignore,
Policy::TestFailure::Continue,
AZStd::nullopt,
AZStd::nullopt,
AZStd::nullopt);
const auto testTargetsWithNoEnumeration = m_dynamicDependencyMap->GetNotCoveringTests();
if (!testTargetsWithNoEnumeration.empty())
{
m_testEngine->UpdateEnumerationCache(
testTargetsWithNoEnumeration,
Policy::ExecutionFailure::Ignore,
Policy::TestFailure::Continue,
AZStd::nullopt,
AZStd::nullopt,
AZStd::nullopt);
}
}
}
catch (const DependencyException& e)
@@ -136,8 +154,10 @@ namespace TestImpact
}
catch ([[maybe_unused]]const Exception& e)
{
AZ_Printf("No test impact analysis data found at %s", m_config.m_workspace.m_active.m_sparTIAFile.c_str());
}
AZ_Printf("TestImpactRuntime",
AZStd::string::format(
"No test impact analysis data found for suite '%s' at %s\n", GetSuiteTypeName(m_suiteFilter).c_str(), m_sparTIAFile.c_str()).c_str());
}
}
Runtime::~Runtime() = default;
@@ -168,13 +188,16 @@ namespace TestImpact
addMutatedTestTargetsToEnumerationList(changeDependencyList.GetDeleteSourceDependencies());
// Enumerate the mutated test targets to ensure their enumeration caches are up to date
m_testEngine->UpdateEnumerationCache(
testTargets,
Policy::ExecutionFailure::Ignore,
Policy::TestFailure::Continue,
AZStd::nullopt,
AZStd::nullopt,
AZStd::nullopt);
if (!testTargets.empty())
{
m_testEngine->UpdateEnumerationCache(
testTargets,
Policy::ExecutionFailure::Ignore,
Policy::TestFailure::Continue,
AZStd::nullopt,
AZStd::nullopt,
AZStd::nullopt);
}
}
AZStd::pair<AZStd::vector<const TestTarget*>, AZStd::vector<const TestTarget*>> Runtime::SelectCoveringTestTargetsAndUpdateEnumerationCache(
@@ -233,12 +256,78 @@ namespace TestImpact
void Runtime::ClearDynamicDependencyMapAndRemoveExistingFile()
{
DeleteFile(m_config.m_workspace.m_active.m_sparTIAFile);
m_dynamicDependencyMap->ClearAllSourceCoverage();
DeleteFile(m_sparTIAFile);
}
void Runtime::UpdateAndSerializeDynamicDependencyMap(const SourceCoveringTestsList& sourceCoverageTestsList)
SourceCoveringTestsList Runtime::CreateSourceCoveringTestFromTestCoverages(const AZStd::vector<TestEngineInstrumentedRun>& jobs)
{
AZStd::unordered_map<AZStd::string, AZStd::unordered_set<AZStd::string>> coverage;
for (const auto& job : jobs)
{
// First we must remove any existing coverage for the test target so as to not end up with source remnants from previous
// coverage that is no longer covered by this revision of the test target
m_dynamicDependencyMap->RemoveTestTargetFromSourceCoverage(job.GetTestTarget());
// Next we will update the coverage of test targets that completed (with or without failures), unless the failed test coverage
// policy dictates we should instead discard the coverage of test targets with failing tests
const auto testResult = job.GetTestResult();
if (m_failedTestCoveragePolicy == Policy::FailedTestCoverage::Discard && testResult == Client::TestRunResult::TestFailures)
{
// Discard the coverage for this job
continue;
}
if (testResult == Client::TestRunResult::AllTestsPass || testResult == Client::TestRunResult::TestFailures)
{
if (testResult == Client::TestRunResult::AllTestsPass)
{
// Passing tests should have coverage data, otherwise something is very wrong
AZ_TestImpact_Eval(
job.GetTestCoverge().has_value(),
RuntimeException,
AZStd::string::format(
"Test target '%s' completed its test run successfully but produced no coverage data",
job.GetTestTarget()->GetName().c_str()));
}
if (!job.GetTestCoverge().has_value())
{
// When a test run completes with failing tests but produces no coverage artifact that's typically a sign of the
// test aborting due to an unhandled exception, in which case ignore it and let it be picked up in the failure report
continue;
}
for (const auto& source : job.GetTestCoverge().value().GetSourcesCovered())
{
coverage[source.String()].insert(job.GetTestTarget()->GetName());
}
}
}
AZStd::vector<SourceCoveringTests> sourceCoveringTests;
sourceCoveringTests.reserve(coverage.size());
for (auto&& [source, testTargets] : coverage)
{
if (const auto sourcePath = RepoPath(source);
sourcePath.IsRelativeTo(m_config.m_repo.m_root))
{
sourceCoveringTests.push_back(
SourceCoveringTests(RepoPath(sourcePath.LexicallyRelative(m_config.m_repo.m_root)), AZStd::move(testTargets)));
}
else
{
AZ_Warning("TestImpact", false, "Ignoring source, source it outside of repo: '%s'", sourcePath.c_str());
}
}
return SourceCoveringTestsList(AZStd::move(sourceCoveringTests));
}
void Runtime::UpdateAndSerializeDynamicDependencyMap(const AZStd::vector<TestEngineInstrumentedRun>& jobs)
{
const auto sourceCoverageTestsList = CreateSourceCoveringTestFromTestCoverages(jobs);
if (!sourceCoverageTestsList.GetNumSources())
{
return;
@@ -247,12 +336,11 @@ namespace TestImpact
m_dynamicDependencyMap->ReplaceSourceCoverage(sourceCoverageTestsList);
const auto sparTIA = m_dynamicDependencyMap->ExportSourceCoverage();
const auto sparTIAData = SerializeSourceCoveringTestsList(sparTIA);
WriteFileContents<RuntimeException>(sparTIAData, m_config.m_workspace.m_active.m_sparTIAFile);
WriteFileContents<RuntimeException>(sparTIAData, m_sparTIAFile);
m_hasImpactAnalysisData = true;
}
TestSequenceResult Runtime::RegularTestSequence(
const AZStd::unordered_set<AZStd::string> suitesFilter,
AZStd::optional<AZStd::chrono::milliseconds> testTargetTimeout,
AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
AZStd::optional<TestSequenceStartCallback> testSequenceStartCallback,
@@ -268,21 +356,7 @@ namespace TestImpact
{
if (!m_testTargetExcludeList.contains(&testTarget))
{
if (suitesFilter.empty())
{
// Suite filter is empty, all tests that are not on the excluded list are included
includedTestTargets.push_back(&testTarget);
}
else if(suitesFilter.contains(testTarget.GetSuite()))
{
// Test target belonging to a suite in the suite filter are included, provided that are not on the exclude list
includedTestTargets.push_back(&testTarget);
}
else
{
// Test target not belonging to a suite in the suite filter are excluded
excludedTestTargets.push_back(&testTarget);
}
includedTestTargets.push_back(&testTarget);
}
else
{
@@ -318,6 +392,7 @@ namespace TestImpact
TestSequenceResult Runtime::ImpactAnalysisTestSequence(
const ChangeList& changeList,
Policy::TestPrioritization testPrioritizationPolicy,
Policy::DynamicDependencyMap dynamicDependencyMapPolicy,
AZStd::optional<AZStd::chrono::milliseconds> testTargetTimeout,
AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
AZStd::optional<ImpactAnalysisTestSequenceStartCallback> testSequenceStartCallback,
@@ -325,11 +400,20 @@ namespace TestImpact
AZStd::optional<TestRunCompleteCallback> testCompleteCallback)
{
Timer timer;
AZStd::vector<const TestTarget*> draftedTestTargets;
// Draft in the test targets that have no coverage entries in the dynamic dependency map
AZStd::vector<const TestTarget*> draftedTestTargets = m_dynamicDependencyMap->GetNotCoveringTests();
// The test targets that were selected for the change list by the dynamic dependency map and the test targets that were not
auto [selectedTestTargets, discardedTestTargets] = SelectCoveringTestTargetsAndUpdateEnumerationCache(changeList, testPrioritizationPolicy);
// The subset of selected test targets that are not on the configuration's exclude list and those that are
auto [includedSelectedTestTargets, excludedSelectedTestTargets] = SelectTestTargetsByExcludeList(selectedTestTargets);
// We present to the client the included selected test targets and the drafted test targets as distinct sets but internally
// we consider the concatenated set of the two the actual set of tests to run
AZStd::vector<const TestTarget*> testTargetsToRun = ConcatenateVectors(includedSelectedTestTargets, draftedTestTargets);
if (testSequenceStartCallback.has_value())
{
(*testSequenceStartCallback)(
@@ -338,30 +422,51 @@ namespace TestImpact
ExtractTestTargetNames(draftedTestTargets));
}
const auto [result, testJobs] = m_testEngine->InstrumentedRun(
includedSelectedTestTargets,
m_testShardingPolicy,
m_executionFailurePolicy,
Policy::IntegrityFailure::Continue,
m_testFailurePolicy,
m_targetOutputCapture,
testTargetTimeout,
globalTimeout,
TestRunCompleteCallbackHandler(testCompleteCallback));
UpdateAndSerializeDynamicDependencyMap(CreateSourceCoveringTestFromTestCoverages(testJobs, m_config.m_repo.m_root));
if (testSequenceEndCallback.has_value())
if (dynamicDependencyMapPolicy == Policy::DynamicDependencyMap::Update)
{
(*testSequenceEndCallback)(GenerateSequenceFailureReport(testJobs), timer.Elapsed());
}
const auto [result, testJobs] = m_testEngine->InstrumentedRun(
testTargetsToRun,
m_testShardingPolicy,
m_executionFailurePolicy,
Policy::IntegrityFailure::Continue,
m_testFailurePolicy,
m_targetOutputCapture,
testTargetTimeout,
globalTimeout,
TestRunCompleteCallbackHandler(testCompleteCallback));
return result;
UpdateAndSerializeDynamicDependencyMap(testJobs);
if (testSequenceEndCallback.has_value())
{
(*testSequenceEndCallback)(GenerateSequenceFailureReport(testJobs), timer.Elapsed());
}
return result;
}
else
{
const auto [result, testJobs] = m_testEngine->RegularRun(
testTargetsToRun,
m_testShardingPolicy,
m_executionFailurePolicy,
m_testFailurePolicy,
m_targetOutputCapture,
testTargetTimeout,
globalTimeout,
TestRunCompleteCallbackHandler(testCompleteCallback));
if (testSequenceEndCallback.has_value())
{
(*testSequenceEndCallback)(GenerateSequenceFailureReport(testJobs), timer.Elapsed());
}
return result;
}
}
AZStd::pair<TestSequenceResult, TestSequenceResult> Runtime::SafeImpactAnalysisTestSequence(
const ChangeList& changeList,
const AZStd::unordered_set<AZStd::string> suitesFilter,
Policy::TestPrioritization testPrioritizationPolicy,
AZStd::optional<AZStd::chrono::milliseconds> testTargetTimeout,
AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
@@ -370,12 +475,23 @@ namespace TestImpact
AZStd::optional<TestRunCompleteCallback> testCompleteCallback)
{
Timer timer;
AZStd::vector<const TestTarget*> draftedTestTargets;
// Draft in the test targets that have no coverage entries in the dynamic dependency map
AZStd::vector<const TestTarget*> draftedTestTargets = m_dynamicDependencyMap->GetNotCoveringTests();
// The test targets that were selected for the change list by the dynamic dependency map and the test targets that were not
auto [selectedTestTargets, discardedTestTargets] = SelectCoveringTestTargetsAndUpdateEnumerationCache(changeList, testPrioritizationPolicy);
// The subset of selected test targets that are not on the configuration's exclude list and those that are
auto [includedSelectedTestTargets, excludedSelectedTestTargets] = SelectTestTargetsByExcludeList(selectedTestTargets);
// The subset of discarded test targets that are not on the configuration's exclude list and those that are
auto [includedDiscardedTestTargets, excludedDiscardedTestTargets] = SelectTestTargetsByExcludeList(discardedTestTargets);
// We present to the client the included selected test targets and the drafted test targets as distinct sets but internally
// we consider the concatenated set of the two the actual set of tests to run
AZStd::vector<const TestTarget*> testTargetsToRun = ConcatenateVectors(includedSelectedTestTargets, draftedTestTargets);
if (testSequenceStartCallback.has_value())
{
(*testSequenceStartCallback)(
@@ -386,7 +502,7 @@ namespace TestImpact
// Impact analysis run of the selected test targets
const auto [selectedResult, selectedTestJobs] = m_testEngine->InstrumentedRun(
includedSelectedTestTargets,
testTargetsToRun,
m_testShardingPolicy,
m_executionFailurePolicy,
Policy::IntegrityFailure::Continue,
@@ -395,6 +511,8 @@ namespace TestImpact
testTargetTimeout,
globalTimeout,
TestRunCompleteCallbackHandler(testCompleteCallback));
const auto selectedDuraton = timer.Elapsed();
// Carry the remaining global sequence time over to the discarded test run
if (globalTimeout.has_value())
@@ -414,16 +532,18 @@ namespace TestImpact
globalTimeout,
TestRunCompleteCallbackHandler(testCompleteCallback));
UpdateAndSerializeDynamicDependencyMap(CreateSourceCoveringTestFromTestCoverages(selectedTestJobs, m_config.m_repo.m_root));
const auto discardedDuraton = timer.Elapsed();
if (testSequenceEndCallback.has_value())
{
(*testSequenceEndCallback)(
GenerateSequenceFailureReport(selectedTestJobs),
GenerateSequenceFailureReport(discardedTestJobs),
timer.Elapsed());
selectedDuraton,
discardedDuraton);
}
UpdateAndSerializeDynamicDependencyMap(selectedTestJobs);
return { selectedResult, discardedResult };
}
@@ -466,14 +586,14 @@ namespace TestImpact
globalTimeout,
TestRunCompleteCallbackHandler(testCompleteCallback));
ClearDynamicDependencyMapAndRemoveExistingFile();
UpdateAndSerializeDynamicDependencyMap(CreateSourceCoveringTestFromTestCoverages(testJobs, m_config.m_repo.m_root));
if (testSequenceEndCallback.has_value())
{
(*testSequenceEndCallback)(GenerateSequenceFailureReport(testJobs), timer.Elapsed());
}
ClearDynamicDependencyMapAndRemoveExistingFile();
UpdateAndSerializeDynamicDependencyMap(testJobs);
return result;
}
Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.cpp
+5 -54
View File
@@ -22,10 +22,10 @@
namespace TestImpact
{
TestTargetMetaMap ReadTestTargetMetaMapFile(const RepoPath& testTargetMetaConfigFile)
TestTargetMetaMap ReadTestTargetMetaMapFile(SuiteType suiteFilter, const RepoPath& testTargetMetaConfigFile)
{
const auto masterTestListData = ReadFileContents<RuntimeException>(testTargetMetaConfigFile);
return TestTargetMetaMapFactory(masterTestListData);
return TestTargetMetaMapFactory(masterTestListData, suiteFilter);
}
AZStd::vector<TestImpact::BuildTargetDescriptor> ReadBuildTargetDescriptorFiles(const BuildTargetDescriptorConfig& buildTargetDescriptorConfig)
@@ -46,10 +46,11 @@ namespace TestImpact
}
AZStd::unique_ptr<TestImpact::DynamicDependencyMap> ConstructDynamicDependencyMap(
SuiteType suiteFilter,
const BuildTargetDescriptorConfig& buildTargetDescriptorConfig,
const TestTargetMetaConfig& testTargetMetaConfig)
{
auto testTargetmetaMap = ReadTestTargetMetaMapFile(testTargetMetaConfig.m_metaFile);
auto testTargetmetaMap = ReadTestTargetMetaMapFile(suiteFilter, testTargetMetaConfig.m_metaFile);
auto buildTargetDescriptors = ReadBuildTargetDescriptorFiles(buildTargetDescriptorConfig);
auto buildTargets = CompileTargetDescriptors(AZStd::move(buildTargetDescriptors), AZStd::move(testTargetmetaMap));
auto&& [productionTargets, testTargets] = buildTargets;
@@ -81,54 +82,4 @@ namespace TestImpact
return testNames;
}
SourceCoveringTestsList CreateSourceCoveringTestFromTestCoverages(const AZStd::vector<TestEngineInstrumentedRun>& jobs, const RepoPath& root)
{
AZStd::unordered_map<AZStd::string, AZStd::unordered_set<AZStd::string>> coverage;
for (const auto& job : jobs)
{
if (const auto testResult = job.GetTestResult();
testResult == Client::TestRunResult::AllTestsPass || testResult == Client::TestRunResult::TestFailures)
{
if (testResult == Client::TestRunResult::AllTestsPass)
{
// Passing tests should have coverage data, otherwise something is very wrong
AZ_TestImpact_Eval(
job.GetTestCoverge().has_value(),
RuntimeException,
AZStd::string::format(
"Test target '%s' completed its test run successfully but produced no coverage data",
job.GetTestTarget()->GetName().c_str()));
}
else if (!job.GetTestCoverge().has_value())
{
// When a test run completes with failing tests but produces no coverage artifact that's typically a sign of the
// test aborting due to an unhandled exception, in which case ignore it and let it be picked up in the failure report
continue;
}
for (const auto& source : job.GetTestCoverge().value().GetSourcesCovered())
{
coverage[source.String()].insert(job.GetTestTarget()->GetName());
}
}
}
AZStd::vector<SourceCoveringTests> sourceCoveringTests;
sourceCoveringTests.reserve(coverage.size());
for (auto&& [source, testTargets] : coverage)
{
if (const auto sourcePath = RepoPath(source);
sourcePath.IsRelativeTo(root))
{
sourceCoveringTests.push_back(SourceCoveringTests(RepoPath(sourcePath.LexicallyRelative(root)), AZStd::move(testTargets)));
}
else
{
AZ_Warning("TestImpact", false, "Ignoring source, source it outside of repo: %s", sourcePath.c_str());
}
}
return SourceCoveringTestsList(AZStd::move(sourceCoveringTests));
}
}
} // namespace TestImpact
Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.h
+2 -6
View File
@@ -30,6 +30,7 @@ namespace TestImpact
{
//! Construct a dynamic dependency map from the build target descriptors and test target metas.
AZStd::unique_ptr<TestImpact::DynamicDependencyMap> ConstructDynamicDependencyMap(
SuiteType suiteFilter,
const BuildTargetDescriptorConfig& buildTargetDescriptorConfig,
const TestTargetMetaConfig& testTargetMetaConfig);
@@ -39,12 +40,7 @@ namespace TestImpact
const AZStd::vector<AZStd::string>& excludedTestTargets);
//! Extracts the name information from the specified test targets.
AZStd::vector<AZStd::string> ExtractTestTargetNames(const AZStd::vector<const TestTarget*> testTargets);
//! Creates the consolidates source covering tests list from the test engine instrumented run jobs.
SourceCoveringTestsList CreateSourceCoveringTestFromTestCoverages(
const AZStd::vector<TestEngineInstrumentedRun>& jobs,
const RepoPath& root);
AZStd::vector<AZStd::string> ExtractTestTargetNames(const AZStd::vector<const TestTarget*> testTargets);
//! Generates a test run failure report from the specified test engine job information.
//! @tparam TestJob The test engine job type.
Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_files.cmake
+62 -30
View File
@@ -10,15 +10,23 @@
#
set(FILES
Include/TestImpactFramework/TestImpactBitwise.h
Include/TestImpactFramework/TestImpactCallback.h
Include/TestImpactFramework/TestImpactException.h
Include/TestImpactFramework/TestImpactFrameworkPath.h
Include/TestImpactFramework/TestImpactRepoPath.h
Include/TestImpactFramework/TestImpactRuntime.h
Include/TestImpactFramework/TestImpactRuntimeException.h
Include/TestImpactFramework/TestImpactConfiguration.h
Include/TestImpactFramework/TestImpactConfigurationException.h
Include/TestImpactFramework/TestImpactChangelist.h
Include/TestImpactFramework/TestImpactChangelistSerializer.h
Include/TestImpactFramework/TestImpactChangelistException.h
Include/TestImpactFramework/TestImpactTestSequence.h
Include/TestImpactFramework/TestImpactClientTestSelection.h
Include/TestImpactFramework/TestImpactClientTestRun.h
Include/TestImpactFramework/TestImpactClientFailureReport.h
Include/TestImpactFramework/TestImpactFileUtils.h
Source/Artifact/TestImpactArtifactException.h
Source/Artifact/Factory/TestImpactBuildTargetDescriptorFactory.cpp
Source/Artifact/Factory/TestImpactBuildTargetDescriptorFactory.h
Source/Artifact/Factory/TestImpactChangeListFactory.cpp
Source/Artifact/Factory/TestImpactChangeListFactory.h
Source/Artifact/Factory/TestImpactTestEnumerationSuiteFactory.cpp
Source/Artifact/Factory/TestImpactTestEnumerationSuiteFactory.h
Source/Artifact/Factory/TestImpactTestRunSuiteFactory.cpp
@@ -27,6 +35,8 @@ set(FILES
Source/Artifact/Factory/TestImpactTestTargetMetaMapFactory.h
Source/Artifact/Factory/TestImpactModuleCoverageFactory.cpp
Source/Artifact/Factory/TestImpactModuleCoverageFactory.h
Source/Artifact/Factory/TestImpactDependencyGraphDataFactory.cpp
Source/Artifact/Factory/TestImpactDependencyGraphDataFactory.h
Source/Artifact/Static/TestImpactBuildTargetDescriptor.cpp
Source/Artifact/Static/TestImpactBuildTargetDescriptor.h
Source/Artifact/Static/TestImpactTargetDescriptorCompiler.cpp
@@ -37,7 +47,6 @@ set(FILES
Source/Artifact/Static/TestImpactTestTargetDescriptor.cpp
Source/Artifact/Static/TestImpactTestTargetDescriptor.h
Source/Artifact/Static/TestImpactDependencyGraphData.h
Source/Artifact/Dynamic/TestImpactChangelist.h
Source/Artifact/Dynamic/TestImpactTestEnumerationSuite.h
Source/Artifact/Dynamic/TestImpactTestRunSuite.h
Source/Artifact/Dynamic/TestImpactTestSuite.h
@@ -50,6 +59,8 @@ set(FILES
Source/Process/TestImpactProcessLauncher.h
Source/Process/JobRunner/TestImpactProcessJob.h
Source/Process/JobRunner/TestImpactProcessJobInfo.h
Source/Process/JobRunner/TestImpactProcessJobMeta.cpp
Source/Process/JobRunner/TestImpactProcessJobMeta.h
Source/Process/JobRunner/TestImpactProcessJobRunner.h
Source/Process/Scheduler/TestImpactProcessScheduler.cpp
Source/Process/Scheduler/TestImpactProcessScheduler.h
@@ -64,6 +75,8 @@ set(FILES
Source/Dependency/TestImpactTestSelectorAndPrioritizer.cpp
Source/Dependency/TestImpactSourceCoveringTestsList.h
Source/Dependency/TestImpactSourceCoveringTestsList.cpp
Source/Dependency/TestImpactSourceCoveringTestsSerializer.cpp
Source/Dependency/TestImpactSourceCoveringTestsSerializer.h
Source/Target/TestImpactBuildTarget.cpp
Source/Target/TestImpactBuildTarget.h
Source/Target/TestImpactBuildTargetList.h
@@ -74,29 +87,48 @@ set(FILES
Source/Target/TestImpactTestTarget.cpp
Source/Target/TestImpactTestTarget.h
Source/Target/TestImpactTestTargetList.h
Source/Test/Enumeration/TestImpactTestEnumeration.h
Source/Test/Enumeration/TestImpactTestEnumerationException.h
Source/Test/Enumeration/TestImpactTestEnumerationSerializer.cpp
Source/Test/Enumeration/TestImpactTestEnumerationSerializer.h
Source/Test/Enumeration/TestImpactTestEnumerator.cpp
Source/Test/Enumeration/TestImpactTestEnumerator.h
Source/Test/Run/TestImpactTestRunSerializer.cpp
Source/Test/Run/TestImpactTestRunSerializer.h
Source/Test/Run/TestImpactTestRunner.cpp
Source/Test/Run/TestImpactTestRunner.h
Source/Test/Run/TestImpactInstrumentedTestRunner.cpp
Source/Test/Run/TestImpactInstrumentedTestRunner.h
Source/Test/Run/TestImpactTestRun.cpp
Source/Test/Run/TestImpactTestRun.h
Source/Test/Run/TestImpactTestRunJobData.cpp
Source/Test/Run/TestImpactTestRunJobData.h
Source/Test/Run/TestImpactTestCoverage.cpp
Source/Test/Run/TestImpactTestCoverage.h
Source/Test/Run/TestImpactTestRunException.h
Source/Test/Job/TestImpactTestJobRunner.h
Source/Test/Job/TestImpactTestJobException.h
Source/Test/Job/TestImpactTestJobCommon.h
Source/Test/TestImpactTestSuiteContainer.h
Source/TestEngine/Enumeration/TestImpactTestEnumeration.h
Source/TestEngine/Enumeration/TestImpactTestEnumerationSerializer.cpp
Source/TestEngine/Enumeration/TestImpactTestEnumerationSerializer.h
Source/TestEngine/Enumeration/TestImpactTestEnumerator.cpp
Source/TestEngine/Enumeration/TestImpactTestEnumerator.h
Source/TestEngine/Run/TestImpactTestRunSerializer.cpp
Source/TestEngine/Run/TestImpactTestRunSerializer.h
Source/TestEngine/Run/TestImpactTestRunner.cpp
Source/TestEngine/Run/TestImpactTestRunner.h
Source/TestEngine/Run/TestImpactInstrumentedTestRunner.cpp
Source/TestEngine/Run/TestImpactInstrumentedTestRunner.h
Source/TestEngine/Run/TestImpactTestRun.cpp
Source/TestEngine/Run/TestImpactTestRun.h
Source/TestEngine/Run/TestImpactTestRunJobData.cpp
Source/TestEngine/Run/TestImpactTestRunJobData.h
Source/TestEngine/Run/TestImpactTestCoverage.cpp
Source/TestEngine/Run/TestImpactTestCoverage.h
Source/TestEngine/JobRunner/TestImpactTestJobRunner.h
Source/TestEngine/JobRunner/TestImpactTestJobInfoGenerator.cpp
Source/TestEngine/JobRunner/TestImpactTestJobInfoGenerator.h
Source/TestEngine/JobRunner/TestImpactTestTargetExtension.h
Source/TestEngine/TestImpactTestEngineJobFailure.cpp
Source/TestEngine/TestImpactTestEngineJobFailure.h
Source/TestEngine/TestImpactTestSuiteContainer.h
Source/TestEngine/TestImpactTestEngine.cpp
Source/TestEngine/TestImpactTestEngine.h
Source/TestEngine/TestImpactTestEngineJob.cpp
Source/TestEngine/TestImpactTestEngineJob.h
Source/TestEngine/TestImpactTestEngineEnumeration.cpp
Source/TestEngine/TestImpactTestEngineEnumeration.h
Source/TestEngine/TestImpactTestEngineRegularRun.cpp
Source/TestEngine/TestImpactTestEngineRegularRun.h
Source/TestEngine/TestImpactTestEngineInstrumentedRun.cpp
Source/TestEngine/TestImpactTestEngineInstrumentedRun.h
Source/TestEngine/TestImpactTestEngineException.h
Source/TestImpactException.cpp
Source/TestImpactFrameworkPath.cpp
Source/TestImpactRuntime.cpp
Source/TestImpactRuntimeUtils.cpp
Source/TestImpactRuntimeUtils.h
Source/TestImpactClientTestSelection.cpp
Source/TestImpactClientTestRun.cpp
Source/TestImpactClientFailureReport.cpp
Source/TestImpactChangeListSerializer.cpp
Source/TestImpactRepoPath.cpp
)
Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_tests_files.cmake
-38
View File
@@ -1,38 +0,0 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
set(FILES
Tests/Artifact/TestImpactTargetDescriptorCompilerTest.cpp
Tests/Artifact/TestImpactBuildTargetDescriptorFactoryTest.cpp
Tests/Artifact/TestImpactModuleCoverageFactoryTest.cpp
Tests/Artifact/TestImpactChangeListFactoryTest.cpp
Tests/Artifact/TestImpactTestEnumerationSuiteFactoryTest.cpp
Tests/Artifact/TestImpactTestRunSuiteFactoryTest.cpp
Tests/Artifact/TestImpactTestTargetMetaMapFactoryTest.cpp
Tests/Process/TestImpactProcessSchedulerTest.cpp
Tests/Process/TestImpactProcessTest.cpp
Tests/Target/TestImpactBuildTargetTest.cpp
Tests/TestImpactExceptionTest.cpp
Tests/TestImpactFrameworkPathTest.cpp
Tests/Test/TestImpactTestEnumeratorTest.cpp
Tests/Test/TestImpactTestEumerationSerializerTest.cpp
Tests/Test/TestImpactTestRunSerializerTest.cpp
Tests/Test/TestImpactTestRunnerTest.cpp
Tests/Test/TestImpactInstrumentedTestRunnerTest.cpp
Tests/Test/TestImpactTestCoverageTest.cpp
Tests/TestImpactTestJobRunnerCommon.h
Tests/TestImpactTestMain.cpp
Tests/TestImpactTestUtils.cpp
Tests/TestImpactTestUtils.h
)
cmake/LYTestWrappers.cmake
+37 -13
View File
@@ -71,6 +71,7 @@ endfunction()
#! ly_add_test: Adds a new RUN_TEST using for the specified target using the supplied command
#
# \arg:NAME - Name to for the test run target
# \arg:PARENT_NAME(optional) - Name of the parent test run target (if this is a subsequent call to specify a suite)
# \arg:TEST_REQUIRES(optional) - List of system resources that are required to run this test.
# Only available option is "gpu"
# \arg:TEST_SUITE(optional) - "smoke" or "periodic" or "sandbox" - prevents the test from running normally
@@ -94,7 +95,7 @@ endfunction()
# sets LY_ADDED_TEST_NAME to the fully qualified name of the test, in parent scope
function(ly_add_test)
set(options EXCLUDE_TEST_RUN_TARGET_FROM_IDE)
set(one_value_args NAME TEST_LIBRARY TEST_SUITE TIMEOUT)
set(one_value_args NAME PARENT_NAME TEST_LIBRARY TEST_SUITE TIMEOUT)
set(multi_value_args TEST_REQUIRES TEST_COMMAND NON_IDE_PARAMS RUNTIME_DEPENDENCIES COMPONENT LABELS)
# note that we dont use TEST_LIBRARY here, but PAL files might so do not remove!
@@ -241,12 +242,24 @@ function(ly_add_test)
endif()
# Store the test so we can walk through all of them in LYTestImpactFramework.cmake
set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS ${ly_add_test_NAME})
set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_test_NAME}_TEST_SUITE ${ly_add_test_TEST_SUITE})
set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_test_NAME}_TEST_LIBRARY ${ly_add_test_TEST_LIBRARY})
set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_test_NAME}_TEST_TIMEOUT ${ly_add_test_TIMEOUT})
if(NOT ly_add_test_PARENT_NAME)
set(test_target ${ly_add_test_NAME})
else()
set(test_target ${ly_add_test_PARENT_NAME})
endif()
# Check to see whether or not this test target has been stored in the global list for walking by the test impact analysis framework
get_property(all_tests GLOBAL PROPERTY LY_ALL_TESTS)
if(NOT "${test_target}" IN_LIST all_tests)
# This is the first reference to this test target so add it to the global list
set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS ${test_target})
set_property(GLOBAL PROPERTY LY_ALL_TESTS_${test_target}_TEST_LIBRARY ${ly_add_test_TEST_LIBRARY})
endif()
# Add the test suite and timeout value to the test target params
set(LY_TEST_PARAMS "${LY_TEST_PARAMS}#${ly_add_test_TEST_SUITE}")
set(LY_TEST_PARAMS "${LY_TEST_PARAMS}#${ly_add_test_TIMEOUT}")
# Store the params for this test target
set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${test_target}_PARAMS ${LY_TEST_PARAMS})
endfunction()
#! ly_add_pytest: registers target PyTest-based test with CTest
@@ -288,8 +301,12 @@ function(ly_add_pytest)
string(REPLACE "::" "_" pytest_report_directory "${PYTEST_XML_OUTPUT_DIR}/${ly_add_pytest_NAME}.xml")
# Add the script path to the test target params
set(LY_TEST_PARAMS "${ly_add_pytest_PATH}")
ly_add_test(
NAME ${ly_add_pytest_NAME}
PARENT_NAME ${ly_add_pytest_NAME}
TEST_SUITE ${ly_add_pytest_TEST_SUITE}
LABELS FRAMEWORK_pytest
TEST_COMMAND ${LY_PYTEST_EXECUTABLE} ${ly_add_pytest_PATH} ${ly_add_pytest_EXTRA_ARGS} --junitxml=${pytest_report_directory} ${custom_marks_args}
@@ -298,7 +315,6 @@ function(ly_add_pytest)
${ly_add_pytest_UNPARSED_ARGUMENTS}
)
set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_pytest_NAME}_SCRIPT_PATH ${ly_add_pytest_PATH})
set_tests_properties(${LY_ADDED_TEST_NAME} PROPERTIES RUN_SERIAL "${ly_add_pytest_TEST_SERIAL}")
endfunction()
@@ -341,10 +357,13 @@ function(ly_add_editor_python_test)
file(REAL_PATH ${ly_add_editor_python_test_TEST_PROJECT} project_real_path BASE_DIRECTORY ${LY_ROOT_FOLDER})
# Add the script path to the test target params
set(LY_TEST_PARAMS "${ly_add_editor_python_test_PATH}")
# Run test via the run_epbtest.cmake script.
# Parameters used are explained in run_epbtest.cmake.
ly_add_test(
NAME ${ly_add_editor_python_test_NAME}
PARENT_NAME ${ly_add_editor_python_test_NAME}
TEST_REQUIRES ${ly_add_editor_python_test_TEST_REQUIRES}
TEST_COMMAND ${CMAKE_COMMAND}
-DCMD_ARG_TEST_PROJECT=${project_real_path}
@@ -362,8 +381,6 @@ function(ly_add_editor_python_test)
TIMEOUT ${ly_add_editor_python_test_TIMEOUT}
COMPONENT ${ly_add_editor_python_test_COMPONENT}
)
set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_editor_python_test_NAME}_SCRIPT_PATH ${ly_add_editor_python_test_PATH})
set_tests_properties(${LY_ADDED_TEST_NAME} PROPERTIES RUN_SERIAL "${ly_add_editor_python_test_TEST_SERIAL}")
endfunction()
@@ -431,14 +448,16 @@ function(ly_add_googletest)
set(full_test_command $<TARGET_FILE:AZ::AzTestRunner> $<TARGET_FILE:${build_target}> AzRunUnitTests)
# Add AzTestRunner as a build dependency
ly_add_dependencies(${build_target} AZ::AzTestRunner)
# Start the test target params and dd the command runner command
# Ideally, we would populate the full command procedurally but the generator expressions won't be expanded by the time we need this data
set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_googletest_NAME}_TEST_COMMAND "AzRunUnitTests")
set(LY_TEST_PARAMS "AzRunUnitTests")
else()
set(full_test_command ${ly_add_googletest_TEST_COMMAND})
# Remove the generator expressions so we are left with the argument(s) required to run unit tests for executable targets
string(REPLACE ";" "" stripped_test_command ${full_test_command})
string(GENEX_STRIP ${stripped_test_command} stripped_test_command)
set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_googletest_NAME}_TEST_COMMAND ${stripped_test_command})
# Start the test target params and dd the command runner command
set(LY_TEST_PARAMS "${stripped_test_command}")
endif()
string(REPLACE "::" "_" report_directory "${GTEST_XML_OUTPUT_DIR}/${ly_add_googletest_NAME}.xml")
@@ -446,6 +465,7 @@ function(ly_add_googletest)
# Invoke the lower level ly_add_test command to add the actual ctest and setup the test labels to add_dependencies on the target
ly_add_test(
NAME ${ly_add_googletest_NAME}
PARENT_NAME ${target_name}
TEST_SUITE ${ly_add_googletest_TEST_SUITE}
LABELS FRAMEWORK_googletest
TEST_COMMAND ${full_test_command} --gtest_output=xml:${report_directory} ${LY_GOOGLETEST_EXTRA_PARAMS}
@@ -520,18 +540,22 @@ function(ly_add_googlebenchmark)
# If command is not supplied attempts, uses the AzTestRunner to run googlebenchmarks on the supplied TARGET
set(full_test_command $<TARGET_FILE:AZ::AzTestRunner> $<TARGET_FILE:${build_target}> AzRunBenchmarks ${output_format_args})
# Start the test target params and dd the command runner command
# Ideally, we would populate the full command procedurally but the generator expressions won't be expanded by the time we need this data
set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_googlebenchmark_NAME}_TEST_COMMAND "AzRunUnitTests")
set(LY_TEST_PARAMS "AzRunUnitTests")
else()
set(full_test_command ${ly_add_googlebenchmark_TEST_COMMAND})
# Remove the generator expressions so we are left with the argument(s) required to run unit tests for executable targets
string(REPLACE ";" "" stripped_test_command ${full_test_command})
string(GENEX_STRIP ${stripped_test_command} stripped_test_command)
set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_googletest_NAME}_TEST_COMMAND ${stripped_test_command})
# Start the test target params and dd the command runner command
set(LY_TEST_PARAMS "${stripped_test_command}")
endif()
# Set the name of the current test target for storage in the global list
ly_add_test(
NAME ${ly_add_googlebenchmark_NAME}
PARENT_NAME ${ly_add_googlebenchmark_NAME}
TEST_REQUIRES ${ly_add_googlebenchmark_TEST_REQUIRES}
TEST_COMMAND ${full_test_command} ${LY_GOOGLETEST_EXTRA_PARAMS}
TEST_SUITE "benchmark"
cmake/TestImpactFramework/ConsoleFrontendConfig.in
+18 -5
View File
@@ -4,7 +4,8 @@
"timestamp": "${timestamp}"
},
"repo": {
"root": "${repo_dir}"
"root": "${repo_dir}",
"tiaf_bin": "${tiaf_bin}"
},
"workspace": {
"temp": {
@@ -13,11 +14,23 @@
"artifact_dir": "RuntimeArtifact"
}
},
"persistent": {
"root": "${persistent_dir}",
"active": {
"root": "${active_dir}",
"relative_paths": {
"test_impact_data_file": "TestImpactData.spartia",
"enumeration_cache_dir": "EnumerationCache"
"test_impact_data_files": {
"main": "TestImpactData.main.spartia",
"periodic": "TestImpactData.periodic.spartia",
"sandbox": "TestImpactData.sandbox.spartia"
},
"enumeration_cache_dir": "EnumerationCache",
"last_build_target_list_file": "LastRunBuildTargets.json"
}
},
"historic": {
"root": "${historic_dir}",
"relative_paths": {
"last_run_hash_file": "last_run.hash",
"last_build_target_list_file": "LastRunBuildTargets.json"
}
}
},
cmake/TestImpactFramework/LYTestImpactFramework.cmake
+75 -40
View File
@@ -89,7 +89,7 @@ function(ly_test_impact_get_test_launch_method TARGET_NAME LAUNCH_METHOD)
elseif("${target_type}" STREQUAL "EXECUTABLE")
set(${LAUNCH_METHOD} "stand_alone" PARENT_SCOPE)
else()
message(FATAL_ERROR "Cannot deduce test target launch method for the target type ${target_type}")
message(FATAL_ERROR "Cannot deduce test target launch method for the target ${TARGET_NAME} with type ${target_type}")
endif()
endfunction()
@@ -131,33 +131,50 @@ function(ly_test_impact_extract_python_test COMPOSITE_TEST TEST_NAME)
set(${TEST_NAME} ${test_name} PARENT_SCOPE)
endfunction()
#! ly_test_impact_extract_google_test_params: extracts the google test name and command parameters.
#! ly_test_impact_extract_google_test_params: extracts the suites for the given google test.
#
# \arg:COMPOSITE_TEST test in the form 'namespace::test'
# \arg:COMPOSITE_SUITES composite list of suites for this target
# \arg:TEST_NAME name of test
# \arg:TEST_COMMAND optional command arguments to run the test
function(ly_test_impact_extract_google_test_params COMPOSITE_TEST TEST_NAME TEST_COMMAND)
get_property(test_command GLOBAL PROPERTY LY_ALL_TESTS_${COMPOSITE_TEST}_TEST_COMMAND)
# \arg:TEST_SUITES extracted list of suites for this target in JSON format
function(ly_test_impact_extract_google_test_params COMPOSITE_TEST COMPOSITE_SUITES TEST_NAME TEST_SUITES)
# Namespace and test are mandatory
string(REPLACE "::" ";" test_components ${COMPOSITE_TEST})
list(LENGTH test_components num_test_components)
if(num_test_components LESS 2)
message(FATAL_ERROR "The test ${test_components} appears to have been specified without a namespace, i.e.:\ly_add_googletest/benchmark(NAME ${test_components})\nInstead of (perhaps):\ly_add_googletest/benchmark(NAME Gem::${test_components})\nPlease add the missing namespace before proceeding.")
endif()
list(GET test_components 0 test_namespace)
list(GET test_components 1 test_name)
set(${TEST_NAMESPACE} ${test_namespace} PARENT_SCOPE)
set(${TEST_NAME} ${test_name} PARENT_SCOPE)
set(${TEST_COMMAND} ${test_command} PARENT_SCOPE)
set(test_suites "")
foreach(composite_suite ${COMPOSITE_SUITES})
# Command, suite, timeout
string(REPLACE "#" ";" suite_components ${composite_suite})
list(LENGTH suite_components num_suite_components)
if(num_suite_components LESS 3)
message(FATAL_ERROR "The suite components ${composite_suite} are required to be in the following format: command#suite#string.")
endif()
list(GET suite_components 0 test_command)
list(GET suite_components 1 test_suite)
list(GET suite_components 2 test_timeout)
set(suite_params "{ \"suite\": \"${test_suite}\", \"command\": \"${test_command}\", \"timeout\": ${test_timeout} }")
list(APPEND test_suites "${suite_params}")
endforeach()
string(REPLACE ";" ", " test_suites "${test_suites}")
set(${TEST_SUITES} ${test_suites} PARENT_SCOPE)
endfunction()
#! ly_test_impact_extract_python_test_params: extracts the python test name and relative script path parameters.
#
# \arg:COMPOSITE_TEST test in form 'namespace::test' or 'test'
# \arg:COMPOSITE_SUITES composite list of suites for this target
# \arg:TEST_NAME name of test
# \arg:SCRIPT_PATH name of test
function(ly_test_impact_extract_python_test_params COMPOSITE_TEST TEST_NAME SCRIPT_PATH)
# \arg:TEST_SUITES extracted list of suites for this target in JSON format
function(ly_test_impact_extract_python_test_params COMPOSITE_TEST COMPOSITE_SUITES TEST_NAME TEST_SUITES)
get_property(script_path GLOBAL PROPERTY LY_ALL_TESTS_${COMPOSITE_TEST}_SCRIPT_PATH)
# namespace is optional, in which case this component will be simply the test name
@@ -169,15 +186,30 @@ function(ly_test_impact_extract_python_test_params COMPOSITE_TEST TEST_NAME SCRI
set(test_name ${test_components})
endif()
# Get python script path relative to repo root
ly_test_impact_rebase_file_to_repo_root(
${script_path}
script_path
${LY_ROOT_FOLDER}
)
set(${TEST_NAME} ${test_name} PARENT_SCOPE)
set(${SCRIPT_PATH} ${script_path} PARENT_SCOPE)
set(test_suites "")
foreach(composite_suite ${COMPOSITE_SUITES})
# Script path, suite, timeout
string(REPLACE "#" ";" suite_components ${composite_suite})
list(LENGTH suite_components num_suite_components)
if(num_suite_components LESS 3)
message(FATAL_ERROR "The suite components ${composite_suite} are required to be in the following format: script_path#suite#string.")
endif()
list(GET suite_components 0 script_path)
list(GET suite_components 1 test_suite)
list(GET suite_components 2 test_timeout)
# Get python script path relative to repo root
ly_test_impact_rebase_file_to_repo_root(
${script_path}
script_path
${LY_ROOT_FOLDER}
)
set(suite_params "{ \"suite\": \"${test_suite}\", \"script\": \"${script_path}\", \"timeout\": ${test_timeout} }")
list(APPEND test_suites "${suite_params}")
endforeach()
string(REPLACE ";" ", " test_suites "${test_suites}")
set(${TEST_SUITES} ${test_suites} PARENT_SCOPE)
endfunction()
#! ly_test_impact_write_test_enumeration_file: exports the master test lists to file.
@@ -185,7 +217,6 @@ endfunction()
# \arg:TEST_ENUMERATION_TEMPLATE_FILE path to test enumeration template file
function(ly_test_impact_write_test_enumeration_file TEST_ENUMERATION_TEMPLATE_FILE)
get_property(LY_ALL_TESTS GLOBAL PROPERTY LY_ALL_TESTS)
# Enumerated tests for each type
set(google_tests "")
set(google_benchmarks "")
@@ -196,29 +227,28 @@ function(ly_test_impact_write_test_enumeration_file TEST_ENUMERATION_TEMPLATE_FI
# Walk the test list
foreach(test ${LY_ALL_TESTS})
message(TRACE "Parsing ${test}")
get_property(test_params GLOBAL PROPERTY LY_ALL_TESTS_${test}_PARAMS)
get_property(test_type GLOBAL PROPERTY LY_ALL_TESTS_${test}_TEST_LIBRARY)
get_property(test_suite GLOBAL PROPERTY LY_ALL_TESTS_${test}_TEST_SUITE)
get_property(test_timeout GLOBAL PROPERTY LY_ALL_TESTS_${test}_TEST_TIMEOUT)
if("${test_type}" STREQUAL "pytest")
# Python tests
ly_test_impact_extract_python_test_params(${test} test_name script_path)
list(APPEND python_tests " { \"name\": \"${test_name}\", \"suite\": \"${test_suite}\", \"script\": \"${script_path}\", \"timeout\":${test_timeout} }")
ly_test_impact_extract_python_test_params(${test} "${test_params}" test_name test_suites)
list(APPEND python_tests " { \"name\": \"${test_name}\", \"suites\": [${test_suites}] }")
elseif("${test_type}" STREQUAL "pytest_editor")
# Python editor tests
ly_test_impact_extract_python_test_params(${test} test_name script_path)
list(APPEND python_editor_tests " { \"name\": \"${test_name}\", \"suite\": \"${test_suite}\", \"script\": \"${script_path}\", \"timeout\":${test_timeout} }")
# Python editor tests
ly_test_impact_extract_python_test_params(${test} "${test_params}" test_name test_suites)
list(APPEND python_editor_tests " { \"name\": \"${test_name}\", \"suites\": [${test_suites}] }")
elseif("${test_type}" STREQUAL "googletest")
# Google tests
ly_test_impact_extract_google_test_params(${test} test_name test_command)
ly_test_impact_get_test_launch_method(${test_name} launch_method)
list(APPEND google_tests " { \"name\": \"${test_name}\", \"suite\": \"${test_suite}\", \"command\": \"${test_command}\", \"timeout\":${test_timeout}, \"launch_method\": \"${launch_method}\" }")
ly_test_impact_extract_google_test_params(${test} "${test_params}" test_name test_suites)
ly_test_impact_get_test_launch_method(${test} launch_method)
list(APPEND google_tests " { \"name\": \"${test_name}\", \"launch_method\": \"${launch_method}\", \"suites\": [${test_suites}] }")
elseif("${test_type}" STREQUAL "googlebenchmark")
# Google benchmarks
ly_test_impact_extract_google_test_params(${test} test_name test_command)
list(APPEND google_benchmarks " { \"name\": \"${test_name}\", \"suite\": \"${test_suite}\", \"command\": \"${test_command}\", \"timeout\":${test_timeout} }")
ly_test_impact_extract_google_test_params(${test} "${test_params}" test_name test_suites)
list(APPEND google_benchmarks " { \"name\": \"${test_name}\", \"launch_method\": \"${launch_method}\", \"suites\": [${test_suites}] }")
else()
message("${test_name} is of unknown type (TEST_LIBRARY property is empty)")
list(APPEND unknown_tests " { \"name\": \"${test}\" }")
list(APPEND unknown_tests " { \"name\": \"${test}\", \"type\": \"${test_type}\" }")
endif()
endforeach()
@@ -330,8 +360,11 @@ function(ly_test_impact_write_config_file CONFIG_TEMPLATE_FILE PERSISTENT_DATA_D
# Temp dir
set(temp_dir "${LY_TEST_IMPACT_TEMP_DIR}")
# Persistent dir
set(persistent_dir "${PERSISTENT_DATA_DIR}")
# Active persistent data dir
set(active_dir "${PERSISTENT_DATA_DIR}/active")
# Historic persistent data dir
set(historic_dir "${PERSISTENT_DATA_DIR}/historic")
# Source to target mappings dir
set(source_target_mapping_dir "${LY_TEST_IMPACT_SOURCE_TARGET_MAPPING_DIR}")
@@ -341,6 +374,9 @@ function(ly_test_impact_write_config_file CONFIG_TEMPLATE_FILE PERSISTENT_DATA_D
# Build dependency artifact dir
set(target_dependency_dir "${LY_TEST_IMPACT_TARGET_DEPENDENCY_DIR}")
# Test impact analysis framework binary
set(tiaf_bin "$<TARGET_FILE:${LY_TEST_IMPACT_CONSOLE_TARGET}>")
# Substitute config file template with above vars
file(READ "${CONFIG_TEMPLATE_FILE}" config_file)
@@ -348,9 +384,13 @@ function(ly_test_impact_write_config_file CONFIG_TEMPLATE_FILE PERSISTENT_DATA_D
# Write out entire config contents to a file in the build directory of the test impact framework console target
file(GENERATE
OUTPUT "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/$<CONFIG>/$<TARGET_FILE_BASE_NAME:${LY_TEST_IMPACT_CONSOLE_TARGET}>.$<CONFIG>.json"
OUTPUT "${PERSISTENT_DATA_DIR}/$<TARGET_FILE_BASE_NAME:${LY_TEST_IMPACT_CONSOLE_TARGET}>.$<CONFIG>.json"
CONTENT ${config_file}
)
# Set the above config file as the default config file to use for the test impact framework console target
target_compile_definitions(${LY_TEST_IMPACT_CONSOLE_STATIC_TARGET} PUBLIC "LY_TEST_IMPACT_DEFAULT_CONFIG_FILE=\"${PERSISTENT_DATA_DIR}/$<TARGET_FILE_BASE_NAME:${LY_TEST_IMPACT_CONSOLE_TARGET}>.$<CONFIG>.json\"")
message(DEBUG "Test impact framework post steps complete")
endfunction()
#! ly_test_impact_post_step: runs the post steps to be executed after all other cmake scripts have been executed.
@@ -360,8 +400,7 @@ function(ly_test_impact_post_step)
endif()
# Directory per build config for persistent test impact data (to be checked in)
set(persistent_data_dir "${LY_ROOT_FOLDER}/Tests/test_impact_framework/${CMAKE_SYSTEM_NAME}/$<CONFIG>")
set(persistent_data_dir "${LY_TEST_IMPACT_WORKING_DIR}/persistent")
# Directory for binaries built for this profile
set(bin_dir "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/$<CONFIG>")
@@ -388,8 +427,4 @@ function(ly_test_impact_post_step)
# Copy over the graphviz options file for the build dependency graphs
message(DEBUG "Test impact framework config file written")
file(COPY "cmake/TestImpactFramework/CMakeGraphVizOptions.cmake" DESTINATION ${CMAKE_BINARY_DIR})
# Set the above config file as the default config file to use for the test impact framework console target
target_compile_definitions(${LY_TEST_IMPACT_CONSOLE_STATIC_TARGET} PUBLIC "LY_TEST_IMPACT_DEFAULT_CONFIG_FILE=\"${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/$<CONFIG>/$<TARGET_FILE_BASE_NAME:${LY_TEST_IMPACT_CONSOLE_TARGET}>.$<CONFIG>.json\"")
message(DEBUG "Test impact framework post steps complete")
endfunction()