birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Changes for read-only runs and new test suites

Browse Source
This commit is contained in:
jonawals
2021-06-04 19:33:26 +01:00
parent 7eb6e0e7d5
commit ac42a9a748
18 changed files with 449 additions and 281 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp
+18 -37
View File
@@ -36,21 +36,21 @@ namespace TestImpact
MaxConcurrency,
TestTargetTimeout,
GlobalTimeout,
SuitesFilter,
SuiteFilter,
SafeMode,
// Values
None,
Seed,
Regular,
ImpactAnalysis,
ImpactAnalysisNoWrite,
ImpactAnalysisOrSeed,
Locality,
Abort,
Continue,
Ignore,
StdOut,
File,
AllSuites
File
};
constexpr const char* OptionKeys[] =
@@ -70,21 +70,21 @@ namespace TestImpact
"maxconcurrency",
"ttimeout",
"gtimeout",
"suites",
"suite",
"safemode",
// Values
"none",
"seed",
"regular",
"tia",
"tianowrite",
"tiaorseed",
"locality",
"abort",
"continue",
"ignore",
"stdout",
"file",
"*"
"file"
};
RepoPath ParseConfigurationFile(const AZ::CommandLine& cmd)
@@ -110,6 +110,7 @@ namespace TestImpact
{OptionKeys[Seed], TestSequenceType::Seed},
{OptionKeys[Regular], TestSequenceType::Regular},
{OptionKeys[ImpactAnalysis], TestSequenceType::ImpactAnalysis},
{OptionKeys[ImpactAnalysisNoWrite], TestSequenceType::ImpactAnalysisNoWrite},
{OptionKeys[ImpactAnalysisOrSeed], TestSequenceType::ImpactAnalysisOrSeed}
};
@@ -250,32 +251,16 @@ namespace TestImpact
return ParseOnOffOption(OptionKeys[SafeMode], states, cmd).value_or(false);
}
AZStd::unordered_set<AZStd::string> ParseSuitesFilter(const AZ::CommandLine& cmd)
SuiteType ParseSuiteFilter(const AZ::CommandLine& cmd)
{
AZStd::unordered_set<AZStd::string> suitesFilter;
if (const auto numSwitchValues = cmd.GetNumSwitchValues(OptionKeys[SuitesFilter]);
numSwitchValues)
const AZStd::vector<AZStd::pair<AZStd::string, SuiteType>> states =
{
for (auto i = 0; i < numSwitchValues; i++)
{
const auto value = cmd.GetSwitchValue(OptionKeys[SuitesFilter], i);
AZ_TestImpact_Eval(!value.empty(), CommandLineOptionsException, "Suites option value is empty");
if (value == OptionKeys[AllSuites])
{
AZ_TestImpact_Eval(
suitesFilter.empty(), CommandLineOptionsException, "The * suite cannot be used with other suites");
}
{GetSuiteTypeName(SuiteType::Main), SuiteType::Main},
{GetSuiteTypeName(SuiteType::Periodic), SuiteType::Periodic},
{GetSuiteTypeName(SuiteType::Sandbox), SuiteType::Sandbox}
};
suitesFilter.insert(value);
}
}
if (suitesFilter.find(OptionKeys[AllSuites]) != suitesFilter.end())
{
return {};
}
return suitesFilter;
return ParseMultiStateOption(OptionKeys[SuiteFilter], states, cmd).value_or(SuiteType::Main);
}
}
@@ -299,7 +284,7 @@ namespace TestImpact
m_testTargetTimeout = ParseTestTargetTimeout(cmd);
m_globalTimeout = ParseGlobalTimeout(cmd);
m_safeMode = ParseSafeMode(cmd);
m_suitesFilter = ParseSuitesFilter(cmd);
m_suiteFilter = ParseSuiteFilter(cmd);
}
bool CommandLineOptions::HasChangeListFile() const
@@ -382,9 +367,9 @@ namespace TestImpact
return m_globalTimeout;
}
const AZStd::unordered_set<AZStd::string>& CommandLineOptions::GetSuitesFilter() const
SuiteType CommandLineOptions::GetSuiteFilter() const
{
return m_suitesFilter;
return m_suiteFilter;
}
AZStd::string CommandLineOptions::GetCommandLineUsageString()
@@ -452,11 +437,7 @@ namespace TestImpact
" -maxconcurrency=<number> The maximum number of concurrent test targets/shards to be in flight at \n"
" any given moment.\n"
" -ochangelist=<on,off> Outputs the change list used for test selection.\n"
" -suites=<names> The test suites to select from for this test sequence (multiple values are \n"
" allowed). The suite all has special significance and will allow tests from \n"
" any suite to be selected, however this particular suite is mutually exclusive\n"
" with other suite. Note: this option is only applicable to the regular sequence\n"
" and, if safe mode is enables, the tia and tiaorseed sequences.";
" -suite=<main, periodic, sandbox> The test suite to select from for this test sequence.";
return help;
}
Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.h
+5 -4
View File
@@ -18,7 +18,6 @@
#include <AzCore/std/string/string.h>
#include <AzCore/std/optional.h>
#include <AzCore/std/containers/vector.h>
#include <AzCore/std/containers/unordered_set.h>
namespace TestImpact
{
@@ -29,6 +28,8 @@ namespace TestImpact
Seed, //!< Removes any prior coverage data and runs all test targets with instrumentation to reseed the data from scratch.
Regular, //!< Runs all of the test targets without any instrumentation to generate coverage data (any prior coverage data is left intact).
ImpactAnalysis, //!< Uses any prior coverage data to run the instrumented subset of selected tests (if no prior coverage data a regular run is performed instead).
ImpactAnalysisNoWrite, //!< Uses any prior coverage data to run the uninstrumented subset of selected tests (if no prior coverage data a regular run is performed instead).
//!< The coverage data is not updated with the subset of selected tests.
ImpactAnalysisOrSeed //!< Uses any prior coverage data to run the instrumented subset of selected tests (if no prior coverage data a seed run is performed instead).
};
@@ -87,8 +88,8 @@ namespace TestImpact
//! Returns the global test sequence timeout to use (if any).
const AZStd::optional<AZStd::chrono::milliseconds>& GetGlobalTimeout() const;
//! Returns the filter for test suites that will be allowed to be run.
const AZStd::unordered_set<AZStd::string>& GetSuitesFilter() const;
//! Returns the filter for test suite that will be allowed to be run.
SuiteType GetSuiteFilter() const;
private:
RepoPath m_configurationFile;
@@ -105,7 +106,7 @@ namespace TestImpact
AZStd::optional<size_t> m_maxConcurrency;
AZStd::optional<AZStd::chrono::milliseconds> m_testTargetTimeout;
AZStd::optional<AZStd::chrono::milliseconds> m_globalTimeout;
AZStd::unordered_set<AZStd::string> m_suitesFilter;
SuiteType m_suiteFilter;
bool m_safeMode = false;
};
} // namespace TestImpact
Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleMain.cpp
+60 -23
View File
@@ -86,6 +86,8 @@ namespace TestImpact
Runtime& runtime,
const AZStd::optional<ChangeList>& changeList)
{
// Even though it is possible for a regular run to be selected (see below) which does not actually require a change list,
// consider any impact analysis sequence type without a change list to be an error
AZ_TestImpact_Eval(
changeList.has_value(),
CommandLineOptionsException,
@@ -94,39 +96,72 @@ namespace TestImpact
TestSequenceResult result = TestSequenceResult::Failure;
if (options.HasSafeMode())
{
auto [selectedResult, discardedResult] = runtime.SafeImpactAnalysisTestSequence(
changeList.value(),
options.GetSuitesFilter(),
options.GetTestPrioritizationPolicy(),
options.GetTestTargetTimeout(),
options.GetGlobalTimeout(),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler));
if (options.GetTestSequenceType() == TestSequenceType::ImpactAnalysis)
{
auto [selectedResult, discardedResult] = runtime.SafeImpactAnalysisTestSequence(
changeList.value(),
options.GetTestPrioritizationPolicy(),
options.GetTestTargetTimeout(),
options.GetGlobalTimeout(),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler));
// Handling the possible timeout and failure permutations of the selected and discarded test results is splitting hairs
// so apply the following, admittedly arbitrary, rules to determine what the composite test sequence result should be
if (selectedResult == TestSequenceResult::Success && discardedResult == TestSequenceResult::Success)
{
// Trivial case: both sequences succeeded
result = TestSequenceResult::Success;
// Handling the possible timeout and failure permutations of the selected and discarded test results is splitting hairs
// so apply the following, admittedly arbitrary, rules to determine what the composite test sequence result should be
if (selectedResult == TestSequenceResult::Success && discardedResult == TestSequenceResult::Success)
{
// Trivial case: both sequences succeeded
result = TestSequenceResult::Success;
}
else if (selectedResult == TestSequenceResult::Failure || discardedResult == TestSequenceResult::Failure)
{
// One sequence failed whilst the other sequence either succeeded or timed out
result = TestSequenceResult::Failure;
}
else
{
// One sequence timed out whilst the other sequence succeeded or both sequences timed out
result = TestSequenceResult::Timeout;
}
}
else if (selectedResult == TestSequenceResult::Failure || discardedResult == TestSequenceResult::Failure)
else if (options.GetTestSequenceType() == TestSequenceType::ImpactAnalysisNoWrite)
{
// One sequence failed whilst the other sequence either succeeded or timed out
result = TestSequenceResult::Failure;
// A no-write impact analysis sequence with safe mode enabled is functionally identical to a regular sequence type
// due to a) the selected tests being run without instrumentation and b) the discarded tests also being run without
// instrumentation
result = runtime.RegularTestSequence(
options.GetTestTargetTimeout(),
options.GetGlobalTimeout(),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler));
}
else
{
// One sequence timed out whilst the other sequence succeeded or both sequences timed out
result = TestSequenceResult::Timeout;
throw(Exception("Unexpected sequence type"));
}
}
else
{
Policy::DynamicDependencyMap dynamicDependencyMapPolicy;
if (options.GetTestSequenceType() == TestSequenceType::ImpactAnalysis)
{
dynamicDependencyMapPolicy = Policy::DynamicDependencyMap::Update;
}
else if (options.GetTestSequenceType() == TestSequenceType::ImpactAnalysisNoWrite)
{
dynamicDependencyMapPolicy = Policy::DynamicDependencyMap::Discard;
}
else
{
throw(Exception("Unexpected sequence type"));
}
result = runtime.ImpactAnalysisTestSequence(
changeList.value(),
options.GetTestPrioritizationPolicy(),
dynamicDependencyMapPolicy,
options.GetTestTargetTimeout(),
options.GetGlobalTimeout(),
AZStd::ref(sequenceEventHandler),
@@ -164,9 +199,11 @@ namespace TestImpact
// As of now, there are no other non-test operations other than printing a change list so getting this far is considered an error
AZ_TestImpact_Eval(options.GetTestSequenceType() != TestSequenceType::None, CommandLineOptionsException, "No action specified");
std::cout << "Constructing in-memory model of source tree and test coverage, this may take a moment...\n";
std::cout << "Constructing in-memory model of source tree and test coverage for test suite ";
std::cout << GetSuiteTypeName(options.GetSuiteFilter()).c_str() << ", this may take a moment...\n";
Runtime runtime(
RuntimeConfigurationFactory(ReadFileContents<CommandLineOptionsException>(options.GetConfigurationFile())),
options.GetSuiteFilter(),
options.GetExecutionFailurePolicy(),
options.GetExecutionFailureDraftingPolicy(),
options.GetTestFailurePolicy(),
@@ -184,14 +221,13 @@ namespace TestImpact
std::cout << "Test impact analysis data for this repository was not found, seed or regular sequence fallbacks will be used.\n";
}
TestSequenceEventHandler sequenceEventHandler(&options.GetSuitesFilter());
TestSequenceEventHandler sequenceEventHandler(options.GetSuiteFilter());
switch (const auto type = options.GetTestSequenceType())
{
case TestSequenceType::Regular:
{
const auto result = runtime.RegularTestSequence(
options.GetSuitesFilter(),
options.GetTestTargetTimeout(),
options.GetGlobalTimeout(),
AZStd::ref(sequenceEventHandler),
@@ -211,6 +247,7 @@ namespace TestImpact
return GetReturnCodeForTestSequenceResult(result);
}
case TestSequenceType::ImpactAnalysisNoWrite:
case TestSequenceType::ImpactAnalysis:
{
return WrappedImpactAnalysisTestSequence(sequenceEventHandler, options, runtime, changeList);
Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactRuntimeConfigurationFactory.cpp
+17 -4
View File
@@ -30,7 +30,7 @@ namespace TestImpact
"relative_paths",
"artifact_dir",
"enumeration_cache_dir",
"test_impact_data_file",
"test_impact_data_files",
"temp",
"active",
"target_sources",
@@ -75,7 +75,7 @@ namespace TestImpact
RelativePaths,
ArtifactDir,
EnumerationCacheDir,
TestImpactDataFile,
TestImpactDataFiles,
TempWorkspace,
ActiveWorkspace,
TargetSources,
@@ -144,6 +144,19 @@ namespace TestImpact
return tempWorkspaceConfig;
}
AZStd::array<RepoPath, 3> ParseTestImpactAnalysisDataFiles(const RepoPath& root, const rapidjson::Value& sparTIAFile)
{
AZStd::array<RepoPath, 3> sparTIAFiles;
sparTIAFiles[static_cast<size_t>(SuiteType::Main)] =
GetAbsPathFromRelPath(root, sparTIAFile[GetSuiteTypeName(SuiteType::Main).c_str()].GetString());
sparTIAFiles[static_cast<size_t>(SuiteType::Periodic)] =
GetAbsPathFromRelPath(root, sparTIAFile[GetSuiteTypeName(SuiteType::Periodic).c_str()].GetString());
sparTIAFiles[static_cast<size_t>(SuiteType::Sandbox)] =
GetAbsPathFromRelPath(root, sparTIAFile[GetSuiteTypeName(SuiteType::Sandbox).c_str()].GetString());
return sparTIAFiles;
}
WorkspaceConfig::Active ParseActiveWorkspaceConfig(const rapidjson::Value& activeWorkspace)
{
WorkspaceConfig::Active activeWorkspaceConfig;
@@ -151,8 +164,8 @@ namespace TestImpact
activeWorkspaceConfig.m_root = activeWorkspace[Config::Keys[Config::Root]].GetString();
activeWorkspaceConfig.m_enumerationCacheDirectory
= GetAbsPathFromRelPath(activeWorkspaceConfig.m_root, relativePaths[Config::Keys[Config::EnumerationCacheDir]].GetString());
activeWorkspaceConfig.m_sparTIAFile
= GetAbsPathFromRelPath(activeWorkspaceConfig.m_root, relativePaths[Config::Keys[Config::TestImpactDataFile]].GetString());
activeWorkspaceConfig.m_sparTIAFiles =
ParseTestImpactAnalysisDataFiles(activeWorkspaceConfig.m_root, relativePaths[Config::Keys[Config::TestImpactDataFiles]]);
return activeWorkspaceConfig;
}