Merge pull request #1393 from aws-lumberyard-dev/TIF/Runtime

Tif/runtime
This commit is contained in:
jonawals
2021-06-18 11:16:13 +01:00
committed by GitHub
167 changed files with 13360 additions and 315 deletions
+3 -1
View File
@@ -1555,7 +1555,9 @@ namespace UnitTest
}
}
TEST_F(Components, EntityIdGeneration)
// Temporary disabled. This will be re-enabled in the short term upon completion of SPEC-7384 and
// fixed in the long term upon completion of SPEC-4849
TEST_F(Components, DISABLED_EntityIdGeneration)
{
// Generate 1 million ids across 100 threads, and ensure that none collide
AZStd::concurrent_unordered_set<AZ::EntityId> entityIds;
@@ -114,6 +114,6 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED)
ly_add_googletest(
NAME AZ::PythonBindingsExample.Tests
TEST_COMMAND $<TARGET_FILE:AZ::PythonBindingsExample.Tests>
TEST_COMMAND $<TARGET_FILE:AZ::PythonBindingsExample.Tests> --unittest
)
endif()
@@ -63,9 +63,9 @@ namespace PythonBindingsExample
AZStd::unique_ptr<PythonBindingsExample::Application> PythonBindingsExampleTest::s_application;
TEST_F(PythonBindingsExampleTest, Application_Run_Fails)
TEST_F(PythonBindingsExampleTest, Application_Run_Succeeds)
{
EXPECT_FALSE(s_application->Run());
EXPECT_TRUE(s_application->Run());
}
TEST_F(PythonBindingsExampleTest, Application_RunWithParameters_Works)
@@ -9,4 +9,4 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
add_subdirectory(Code)
add_subdirectory(Code)
@@ -9,8 +9,24 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
ly_add_target(
NAME TestImpact.Frontend.Console.Static STATIC
NAMESPACE AZ
FILES_CMAKE
testimpactframework_frontend_console_static_files.cmake
INCLUDE_DIRECTORIES
PUBLIC
Include
PRIVATE
Source
BUILD_DEPENDENCIES
PUBLIC
AZ::TestImpact.Runtime.Static
)
ly_add_target(
NAME TestImpact.Frontend.Console EXECUTABLE
OUTPUT_NAME tiaf
NAMESPACE AZ
FILES_CMAKE
testimpactframework_frontend_console_files.cmake
@@ -19,5 +35,31 @@ ly_add_target(
Source
BUILD_DEPENDENCIES
PRIVATE
AZ::TestImpact.Runtime.Static
AZ::TestImpact.Frontend.Console.Static
)
################################################################################
# Tests
################################################################################
# Disbled:SPEC-7246
#ly_add_target(
# NAME TestImpact.Frontend.Console.Tests ${PAL_TRAIT_TEST_TARGET_TYPE}
# NAMESPACE AZ
# FILES_CMAKE
# testimpactframework_frontend_console_static_tests_files.cmake
# INCLUDE_DIRECTORIES
# PRIVATE
# Include
# Source
# Tests
# BUILD_DEPENDENCIES
# PRIVATE
# AZ::AzTestShared
# AZ::AzTest
# AZ::TestImpact.Frontend.Console.Static
#)
#
#ly_add_googletest(
# NAME AZ::TestImpact.Frontend.Console.Static.Tests
#)
@@ -0,0 +1,35 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
namespace TestImpact
{
namespace Console
{
enum class ReturnCode : int
{
Success = 0, //!< The instigation operation(s) returned without error.
InvalidArgs, //!< The specified command line arguments were incorrect.
InvalidUnifiedDiff, //!< The specified unified diff could not be transformed into a valid change list.
InvalidConfiguration, //!< The runtime configuration is malformed.
RuntimeError, //!< The runtime encountered an error that it could not recover from.
UnhandledError, //!< The framework encountered an error that it anticipated but did not handle and could not recover from.
UnknownError, //!< An error of unknown origin was encountered that the console or runtime could not recover from.
TestFailure, //!< The test sequence had one or more test failures.
Timeout //!< The test sequence runtime exceeded the global timeout value.
};
//! Entry point for the console front end application.
[[nodiscard]] ReturnCode Main(int argc, char** argv);
} // namespace Console
} // namespace TestImpact
@@ -0,0 +1,455 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <TestImpactCommandLineOptions.h>
#include <TestImpactCommandLineOptionsUtils.h>
#include <AzCore/Settings/CommandLine.h>
namespace TestImpact
{
namespace
{
enum
{
// Options
ConfigKey,
ChangeListKey,
OutputChangeListKey,
SequenceKey,
TestPrioritizationPolicyKey,
ExecutionFailurePolicyKey,
FailedTestCoveragePolicyKey,
TestFailurePolicyKey,
IntegrityFailurePolicyKey,
TestShardingPolicyKey,
TargetOutputCaptureKey,
MaxConcurrencyKey,
TestTargetTimeoutKey,
GlobalTimeoutKey,
SuiteFilterKey,
SafeModeKey,
// Values
None,
Seed,
Regular,
ImpactAnalysis,
ImpactAnalysisNoWrite,
ImpactAnalysisOrSeed,
Locality,
Abort,
Continue,
Ignore,
StdOut,
File,
Discard,
Keep
};
constexpr const char* OptionKeys[] =
{
// Options
"config",
"changelist",
"ochangelist",
"sequence",
"ppolicy",
"epolicy",
"cpolicy",
"fpolicy",
"ipolicy",
"shard",
"targetout",
"maxconcurrency",
"ttimeout",
"gtimeout",
"suite",
"safemode",
// Values
"none",
"seed",
"regular",
"tia",
"tianowrite",
"tiaorseed",
"locality",
"abort",
"continue",
"ignore",
"stdout",
"file",
"discard",
"keep"
};
RepoPath ParseConfigurationFile(const AZ::CommandLine& cmd)
{
return ParsePathOption(OptionKeys[ConfigKey], cmd).value_or(LY_TEST_IMPACT_DEFAULT_CONFIG_FILE);
}
AZStd::optional<RepoPath> ParseChangeListFile(const AZ::CommandLine& cmd)
{
return ParsePathOption(OptionKeys[ChangeListKey], cmd);
}
bool ParseOutputChangeList(const AZ::CommandLine& cmd)
{
return ParseOnOffOption(OptionKeys[OutputChangeListKey], BinaryStateValue<bool>{ false, true }, cmd).value_or(false);
}
TestSequenceType ParseTestSequenceType(const AZ::CommandLine& cmd)
{
const AZStd::vector<AZStd::pair<AZStd::string, TestSequenceType>> states =
{
{OptionKeys[None], TestSequenceType::None},
{OptionKeys[Seed], TestSequenceType::Seed},
{OptionKeys[Regular], TestSequenceType::Regular},
{OptionKeys[ImpactAnalysis], TestSequenceType::ImpactAnalysis},
{OptionKeys[ImpactAnalysisNoWrite], TestSequenceType::ImpactAnalysisNoWrite},
{OptionKeys[ImpactAnalysisOrSeed], TestSequenceType::ImpactAnalysisOrSeed}
};
return ParseMultiStateOption(OptionKeys[SequenceKey], states, cmd).value_or(TestSequenceType::None);
}
Policy::TestPrioritization ParseTestPrioritizationPolicy(const AZ::CommandLine& cmd)
{
const BinaryStateOption<Policy::TestPrioritization> states =
{
{OptionKeys[None], Policy::TestPrioritization::None},
{OptionKeys[Locality], Policy::TestPrioritization::DependencyLocality}
};
return ParseBinaryStateOption(OptionKeys[TestPrioritizationPolicyKey], states, cmd).value_or(Policy::TestPrioritization::None);
}
Policy::ExecutionFailure ParseExecutionFailurePolicy(const AZ::CommandLine& cmd)
{
const AZStd::vector<AZStd::pair<AZStd::string, Policy::ExecutionFailure>> states =
{
{OptionKeys[Abort], Policy::ExecutionFailure::Abort},
{OptionKeys[Continue], Policy::ExecutionFailure::Continue},
{OptionKeys[Ignore], Policy::ExecutionFailure::Ignore}
};
return ParseMultiStateOption(OptionKeys[ExecutionFailurePolicyKey], states, cmd).value_or(Policy::ExecutionFailure::Continue);
}
Policy::FailedTestCoverage ParseFailedTestCoveragePolicy(const AZ::CommandLine& cmd)
{
const AZStd::vector<AZStd::pair<AZStd::string, Policy::FailedTestCoverage>> states =
{
{OptionKeys[Discard], Policy::FailedTestCoverage::Discard},
{OptionKeys[Keep], Policy::FailedTestCoverage::Keep}
};
return ParseMultiStateOption(OptionKeys[FailedTestCoveragePolicyKey], states, cmd).value_or(Policy::FailedTestCoverage::Keep);
}
Policy::TestFailure ParseTestFailurePolicy(const AZ::CommandLine& cmd)
{
const BinaryStateValue<Policy::TestFailure> states =
{
Policy::TestFailure::Abort,
Policy::TestFailure::Continue
};
return ParseAbortContinueOption(OptionKeys[TestFailurePolicyKey], states, cmd).value_or(Policy::TestFailure::Abort);
}
Policy::IntegrityFailure ParseIntegrityFailurePolicy(const AZ::CommandLine& cmd)
{
const BinaryStateValue<Policy::IntegrityFailure> states =
{
Policy::IntegrityFailure::Abort,
Policy::IntegrityFailure::Continue
};
return ParseAbortContinueOption(OptionKeys[IntegrityFailurePolicyKey], states, cmd).value_or(Policy::IntegrityFailure::Abort);
}
Policy::TestSharding ParseTestShardingPolicy(const AZ::CommandLine& cmd)
{
const BinaryStateValue<Policy::TestSharding> states =
{
Policy::TestSharding::Never,
Policy::TestSharding::Always
};
return ParseOnOffOption(OptionKeys[TestShardingPolicyKey], states, cmd).value_or(Policy::TestSharding::Never);
}
Policy::TargetOutputCapture ParseTargetOutputCapture(const AZ::CommandLine& cmd)
{
if (const auto numSwitchValues = cmd.GetNumSwitchValues(OptionKeys[TargetOutputCaptureKey]);
numSwitchValues)
{
AZ_TestImpact_Eval(
numSwitchValues <= 2, CommandLineOptionsException, "Unexpected parameters for target output capture option");
Policy::TargetOutputCapture targetOutputCapture = Policy::TargetOutputCapture::None;
for (auto i = 0; i < numSwitchValues; i++)
{
const auto option = cmd.GetSwitchValue(OptionKeys[TargetOutputCaptureKey], i);
if (option == OptionKeys[StdOut])
{
if (targetOutputCapture == Policy::TargetOutputCapture::File)
{
targetOutputCapture = Policy::TargetOutputCapture::StdOutAndFile;
}
else
{
targetOutputCapture = Policy::TargetOutputCapture::StdOut;
}
}
else if (option == OptionKeys[File])
{
if (targetOutputCapture == Policy::TargetOutputCapture::StdOut)
{
targetOutputCapture = Policy::TargetOutputCapture::StdOutAndFile;
}
else
{
targetOutputCapture = Policy::TargetOutputCapture::File;
}
}
else
{
throw CommandLineOptionsException(
AZStd::string::format("Unexpected value for target output capture option: %s", option.c_str()));
}
}
return targetOutputCapture;
}
return Policy::TargetOutputCapture::None;
}
AZStd::optional<size_t> ParseMaxConcurrency(const AZ::CommandLine& cmd)
{
return ParseUnsignedIntegerOption(OptionKeys[MaxConcurrencyKey], cmd);
}
AZStd::optional<AZStd::chrono::milliseconds> ParseTestTargetTimeout(const AZ::CommandLine& cmd)
{
return ParseSecondsOption(OptionKeys[TestTargetTimeoutKey], cmd);
}
AZStd::optional<AZStd::chrono::milliseconds> ParseGlobalTimeout(const AZ::CommandLine& cmd)
{
return ParseSecondsOption(OptionKeys[GlobalTimeoutKey], cmd);
}
bool ParseSafeMode(const AZ::CommandLine& cmd)
{
const BinaryStateValue<bool> states = { false, true };
return ParseOnOffOption(OptionKeys[SafeModeKey], states, cmd).value_or(false);
}
SuiteType ParseSuiteFilter(const AZ::CommandLine& cmd)
{
const AZStd::vector<AZStd::pair<AZStd::string, SuiteType>> states =
{
{GetSuiteTypeName(SuiteType::Main), SuiteType::Main},
{GetSuiteTypeName(SuiteType::Periodic), SuiteType::Periodic},
{GetSuiteTypeName(SuiteType::Sandbox), SuiteType::Sandbox}
};
return ParseMultiStateOption(OptionKeys[SuiteFilterKey], states, cmd).value_or(SuiteType::Main);
}
}
CommandLineOptions::CommandLineOptions(int argc, char** argv)
{
AZ::CommandLine cmd;
cmd.Parse(argc, argv);
m_configurationFile = ParseConfigurationFile(cmd);
m_changeListFile = ParseChangeListFile(cmd);
m_outputChangeList = ParseOutputChangeList(cmd);
m_testSequenceType = ParseTestSequenceType(cmd);
m_testPrioritizationPolicy = ParseTestPrioritizationPolicy(cmd);
m_executionFailurePolicy = ParseExecutionFailurePolicy(cmd);
m_failedTestCoveragePolicy = ParseFailedTestCoveragePolicy(cmd);
m_testFailurePolicy = ParseTestFailurePolicy(cmd);
m_integrityFailurePolicy = ParseIntegrityFailurePolicy(cmd);
m_testShardingPolicy = ParseTestShardingPolicy(cmd);
m_targetOutputCapture = ParseTargetOutputCapture(cmd);
m_maxConcurrency = ParseMaxConcurrency(cmd);
m_testTargetTimeout = ParseTestTargetTimeout(cmd);
m_globalTimeout = ParseGlobalTimeout(cmd);
m_safeMode = ParseSafeMode(cmd);
m_suiteFilter = ParseSuiteFilter(cmd);
}
bool CommandLineOptions::HasChangeListFile() const
{
return m_changeListFile.has_value();
}
bool CommandLineOptions::HasSafeMode() const
{
return m_safeMode;
}
const AZStd::optional<RepoPath>& CommandLineOptions::GetChangeListFile() const
{
return m_changeListFile;
}
bool CommandLineOptions::HasOutputChangeList() const
{
return m_outputChangeList;
}
const RepoPath& CommandLineOptions::GetConfigurationFile() const
{
return m_configurationFile;
}
TestSequenceType CommandLineOptions::GetTestSequenceType() const
{
return m_testSequenceType;
}
Policy::TestPrioritization CommandLineOptions::GetTestPrioritizationPolicy() const
{
return m_testPrioritizationPolicy;
}
Policy::ExecutionFailure CommandLineOptions::GetExecutionFailurePolicy() const
{
return m_executionFailurePolicy;
}
Policy::FailedTestCoverage CommandLineOptions::GetFailedTestCoveragePolicy() const
{
return m_failedTestCoveragePolicy;
}
Policy::TestFailure CommandLineOptions::GetTestFailurePolicy() const
{
return m_testFailurePolicy;
}
Policy::IntegrityFailure CommandLineOptions::GetIntegrityFailurePolicy() const
{
return m_integrityFailurePolicy;
}
Policy::TestSharding CommandLineOptions::GetTestShardingPolicy() const
{
return m_testShardingPolicy;
}
Policy::TargetOutputCapture CommandLineOptions::GetTargetOutputCapture() const
{
return m_targetOutputCapture;
}
const AZStd::optional<size_t>& CommandLineOptions::GetMaxConcurrency() const
{
return m_maxConcurrency;
}
const AZStd::optional<AZStd::chrono::milliseconds>& CommandLineOptions::GetTestTargetTimeout() const
{
return m_testTargetTimeout;
}
const AZStd::optional<AZStd::chrono::milliseconds>& CommandLineOptions::GetGlobalTimeout() const
{
return m_globalTimeout;
}
SuiteType CommandLineOptions::GetSuiteFilter() const
{
return m_suiteFilter;
}
AZStd::string CommandLineOptions::GetCommandLineUsageString()
{
AZStd::string help =
"usage: tiaf [options]\n"
" options:\n"
" -config=<filename> Path to the configuration file for the TIAF runtime (default: \n"
" <tiaf binay build dir>.<tiaf binary build type>.json).\n"
" -changelist=<filename> Path to the JSON of source file changes to perform test impact \n"
" analysis on.\n"
" -gtimeout=<seconds> Global timeout value to terminate the entire test sequence should it \n"
" be exceeded.\n"
" -ttimeout=<seconds> Timeout value to terminate individual test targets should it be \n"
" exceeded.\n"
" -sequence=<none, seed, regular, tia, tianowrite, tiaorseed> The type of test sequence to perform, where 'none' runs no tests and\n"
" will report a all tests successful, 'seed' removes any prior coverage \n"
" data and runs all test targets with instrumentation to reseed the \n"
" data from scratch, 'regular' runs all of the test targets without any \n"
" instrumentation to generate coverage data(any prior coverage data is \n"
" left intact), 'tia' uses any prior coverage data to run the instrumented \n"
" subset of selected tests(if no prior coverage data a regular run is \n"
" performed instead), 'tianowrite' uses any prior coverage data to run the \n"
" uninstrumented subset of selected tests (if no prior coverage data a \n"
" regular run is performed instead). The coverage data is not updated with \n"
" the subset of selected tests and 'tiaorseed' uses any prior coverage data \n"
" to run the instrumented subset of selected tests (if no prior coverage \n"
" data a seed run is performed instead).\n"
" -safemode=<on,off> Flag to specify a safe mode sequence where the set of unselected \n"
" tests is run without instrumentation after the set of selected \n"
" instrumented tests is run (this has the effect of ensuring all \n"
" tests are run regardless).\n"
" -shard=<on,off> Break any test targets with a sharding policy into the number of \n"
" shards according to the maximum concurrency value.\n"
" -cpolicy=<remove, keep> Policy for handling the coverage data of failing tests, where 'discard' \n"
" will discard the coverage data produced by the failing tests, causing \n"
" them to be drafted into future test runs and 'keep' will keep any existing \n"
" coverage data and update the coverage data for failed tests that produce \n"
" coverage.\n"
" -targetout=<sdtout, file> Capture of individual test run stdout, where 'stdout' will capture \n"
" each individual test target's stdout and output each one to stdout \n"
" and 'file' will capture each individual test target's stdout and output \n"
" each one individually to a file (multiple values are accepted).\n"
" -epolicy=<abort, continue, ignore> Policy for handling test execution failure (test targets could not be \n"
" launched due to the binary not being built, incorrect paths, etc.), \n"
" where 'abort' will abort the entire test sequence upon the first test\n"
" target execution failure and report a failure(along with the return \n"
" code of the test target that failed to launch), 'continue' will continue \n"
" with the test sequence in the event of test target execution failures\n"
" and treat the test targets that failed to launch as test failures\n"
" (along with the return codes of the test targets that failed to \n"
" launch), 'ignore' will continue with the test sequence in the event of \n"
" test target execution failures and treat the test targets that failed\n"
" to launch as test passes(along with the return codes of the test \n"
" targets that failed to launch).\n"
" -fpolicy <abort, continue> Policy for handling test failures (test targets report failing tests), \n"
" where 'abort' will abort the entire test sequence upon the first test \n"
" failure and report a failure and 'continue' will continue with the test\n"
" sequence in the event of test failures and report the test failures.\n"
" -ipolicy=<abort, seed, rerun> Policy for handling coverage data integrity failures, where 'abort' will \n"
" abort the test sequence and report a failure, 'seed' will attempt another \n"
" sequence using the seed sequence type, otherwise will abort and report \n"
" a failure (this option has no effect for regular and seed sequence \n"
" types) and 'rerun' will attempt another sequence using the regular \n"
" sequence type, otherwise will abort and report a failure(this option has \n"
" no effect for regular sequence type).\n"
" -ppolicy=<none, locality> Policy for prioritizing selected test targets, where 'none' will not \n"
" attempt any test target prioritization and 'locality' will attempt to \n"
" prioritize test targets according to the locality of their covering \n"
" production targets in the dependency graph(if no dependency graph data \n"
" available, no prioritization will occur).\n"
" -maxconcurrency=<number> The maximum number of concurrent test targets/shards to be in flight at \n"
" any given moment.\n"
" -ochangelist=<on,off> Outputs the change list used for test selection.\n"
" -suite=<main, periodic, sandbox> The test suite to select from for this test sequence.";
return help;
}
} // namespace TestImpact
@@ -0,0 +1,112 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <TestImpactFramework/TestImpactTestSequence.h>
#include <TestImpactFramework/TestImpactRepoPath.h>
#include <AzCore/std/string/string.h>
#include <AzCore/std/optional.h>
#include <AzCore/std/containers/vector.h>
namespace TestImpact
{
//! The type of test sequence to run.
enum class TestSequenceType
{
None, //!< Runs no tests and will report all tests successful.
Seed, //!< Removes any prior coverage data and runs all test targets with instrumentation to reseed the data from scratch.
Regular, //!< Runs all of the test targets without any instrumentation to generate coverage data (any prior coverage data is left intact).
ImpactAnalysis, //!< Uses any prior coverage data to run the instrumented subset of selected tests (if no prior coverage data a regular run is performed instead).
ImpactAnalysisNoWrite, //!< Uses any prior coverage data to run the uninstrumented subset of selected tests (if no prior coverage data a regular run is performed instead).
//!< The coverage data is not updated with the subset of selected tests.
ImpactAnalysisOrSeed //!< Uses any prior coverage data to run the instrumented subset of selected tests (if no prior coverage data a seed run is performed instead).
};
//! Representation of the command line options supplied to the console frontend application.
class CommandLineOptions
{
public:
CommandLineOptions(int argc, char** argv);
static AZStd::string GetCommandLineUsageString();
//! Returns true if a change list file path has been supplied, otherwise false.
bool HasChangeListFile() const;
//! Returns true if the safe mode option has been enabled, otherwise false.
bool HasSafeMode() const;
//! Returns true if the output change list option has been enabled, otherwise false.
bool HasOutputChangeList() const;
//! Returns the path to the runtime configuration file.
const RepoPath& GetConfigurationFile() const;
//! Returns the path to the change list file (if any).
const AZStd::optional<RepoPath>& GetChangeListFile() const;
//! Returns the test sequence type to run.
TestSequenceType GetTestSequenceType() const;
//! Returns the test prioritization policy to use.
Policy::TestPrioritization GetTestPrioritizationPolicy() const;
//! Returns the test execution failure policy to use.
Policy::ExecutionFailure GetExecutionFailurePolicy() const;
//! Returns failed test coverage drafting policy to use.
Policy::FailedTestCoverage GetFailedTestCoveragePolicy() const;
//! Returns the test failure policy to use.
Policy::TestFailure GetTestFailurePolicy() const;
//! Returns the integration failure policy to use.
Policy::IntegrityFailure GetIntegrityFailurePolicy() const;
//! Returns the test sharding policy to use.
Policy::TestSharding GetTestShardingPolicy() const;
//! Returns the test target standard output capture policy to use.
Policy::TargetOutputCapture GetTargetOutputCapture() const;
//! Returns the maximum number of test targets to be in flight at any given time.
const AZStd::optional<size_t>& GetMaxConcurrency() const;
//! Returns the individual test target timeout to use (if any).
const AZStd::optional<AZStd::chrono::milliseconds>& GetTestTargetTimeout() const;
//! Returns the global test sequence timeout to use (if any).
const AZStd::optional<AZStd::chrono::milliseconds>& GetGlobalTimeout() const;
//! Returns the filter for test suite that will be allowed to be run.
SuiteType GetSuiteFilter() const;
private:
RepoPath m_configurationFile;
AZStd::optional<RepoPath> m_changeListFile;
bool m_outputChangeList = false;
TestSequenceType m_testSequenceType;
Policy::TestPrioritization m_testPrioritizationPolicy = Policy::TestPrioritization::None;
Policy::ExecutionFailure m_executionFailurePolicy = Policy::ExecutionFailure::Continue;
Policy::FailedTestCoverage m_failedTestCoveragePolicy = Policy::FailedTestCoverage::Keep;
Policy::TestFailure m_testFailurePolicy = Policy::TestFailure::Abort;
Policy::IntegrityFailure m_integrityFailurePolicy = Policy::IntegrityFailure::Abort;
Policy::TestSharding m_testShardingPolicy = Policy::TestSharding::Never;
Policy::TargetOutputCapture m_targetOutputCapture = Policy::TargetOutputCapture::None;
AZStd::optional<size_t> m_maxConcurrency;
AZStd::optional<AZStd::chrono::milliseconds> m_testTargetTimeout;
AZStd::optional<AZStd::chrono::milliseconds> m_globalTimeout;
SuiteType m_suiteFilter;
bool m_safeMode = false;
};
} // namespace TestImpact
@@ -0,0 +1,26 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <TestImpactFramework/TestImpactException.h>
namespace TestImpact
{
//! Exception for command line options.
class CommandLineOptionsException
: public Exception
{
public:
using Exception::Exception;
};
} // namespace TestImpact
@@ -0,0 +1,79 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <TestImpactCommandLineOptionsUtils.h>
#include <AzCore/std/string/conversions.h>
namespace TestImpact
{
//! Attempts to parse a path option value.
AZStd::optional<RepoPath> ParsePathOption(const AZStd::string& optionName, const AZ::CommandLine& cmd)
{
if (const auto numSwitchValues = cmd.GetNumSwitchValues(optionName);
numSwitchValues)
{
AZ_TestImpact_Eval(
numSwitchValues == 1,
CommandLineOptionsException,
AZStd::string::format("Unexpected number of parameters for %s option", optionName.c_str()));
const auto value = cmd.GetSwitchValue(optionName, 0);
AZ_TestImpact_Eval(
!value.empty(),
CommandLineOptionsException,
AZStd::string::format("%s file option value is empty", optionName.c_str()));
return value;
}
return AZStd::nullopt;
}
//! Attempts to pass an unsigned integer option value.
AZStd::optional<size_t> ParseUnsignedIntegerOption(const AZStd::string& optionName, const AZ::CommandLine& cmd)
{
if (const auto numSwitchValues = cmd.GetNumSwitchValues(optionName);
numSwitchValues)
{
AZ_TestImpact_Eval(
numSwitchValues == 1,
CommandLineOptionsException,
AZStd::string::format("Unexpected number of parameters for %s option", optionName.c_str()));
const auto strValue = cmd.GetSwitchValue(optionName, 0);
size_t successfulParse = 0; // Will be non-zero if the parse was successful
auto value = AZStd::stoul(strValue, &successfulParse, 0);
AZ_TestImpact_Eval(
successfulParse,
CommandLineOptionsException,
AZStd::string::format("Couldn't parse unsigned integer option value: %s", strValue.c_str()));
return aznumeric_caster(value);
}
return AZStd::nullopt;
}
//! Attempts to parse an option value in seconds.
AZStd::optional<AZStd::chrono::milliseconds> ParseSecondsOption(const AZStd::string& optionName, const AZ::CommandLine& cmd)
{
if (const auto option = ParseUnsignedIntegerOption(optionName, cmd);
option.has_value())
{
return AZStd::chrono::seconds(option.value());
}
return AZStd::nullopt;
}
} // namespace TestImpact
@@ -0,0 +1,122 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <TestImpactCommandLineOptions.h>
#include <TestImpactCommandLineOptionsException.h>
#include <AzCore/Settings/CommandLine.h>
namespace TestImpact
{
//! Representation of a command line option value name and its typed value.
template<typename T>
using OptionValue = AZStd::pair<AZStd::string, T>;
//! Representation of a binary state command line option with its two values.
template<typename T>
using BinaryStateOption = AZStd::pair<OptionValue<T>, OptionValue<T>>;
//! Representation of the values for a binary state option.
template<typename T>
using BinaryStateValue = AZStd::pair<T, T>;
//! Attempts to parse the specified binary state option.
template<typename T>
AZStd::optional<T> ParseBinaryStateOption(
const AZStd::string& optionName,
const AZStd::pair<OptionValue<T>,
OptionValue<T>>& state, const AZ::CommandLine& cmd)
{
if (const auto numSwitchValues = cmd.GetNumSwitchValues(optionName);
numSwitchValues)
{
AZ_TestImpact_Eval(
numSwitchValues == 1,
CommandLineOptionsException,
AZStd::string::format("Unexpected number of parameters for %s option", optionName.c_str()));
const auto option = cmd.GetSwitchValue(optionName, 0);
if (const auto& [optionValueText, optionValue] = state.first;
option == optionValueText)
{
return optionValue;
}
if (const auto& [optionValueText, optionValue] = state.second;
option == optionValueText)
{
return optionValue;
}
throw CommandLineOptionsException(
AZStd::string::format("Unexpected value for %s option: %s", optionName.c_str(), option.c_str()));
}
return AZStd::nullopt;
}
//! Attempts to pass an arbitrarily sized state option.
template<typename T>
AZStd::optional<T> ParseMultiStateOption(
const AZStd::string& optionName,
const AZStd::vector<AZStd::pair<AZStd::string, T>>& states,
const AZ::CommandLine& cmd)
{
if (const auto numSwitchValues = cmd.GetNumSwitchValues(optionName);
numSwitchValues)
{
AZ_TestImpact_Eval(
numSwitchValues == 1,
CommandLineOptionsException,
AZStd::string::format("Unexpected number of parameters for %s option", optionName.c_str()));
const auto option = cmd.GetSwitchValue(optionName, 0);
for (const auto& state : states)
{
if (const auto& [optionValueText, optionValue] = state;
option == optionValueText)
{
return optionValue;
}
}
throw CommandLineOptionsException(
AZStd::string::format("Unexpected value for %s option: %s", optionName.c_str(), option.c_str()));
}
return AZStd::nullopt;
}
//! Attempts to pass a specialization of the binary state option where the command line values are "on" and "off".
template<typename T>
AZStd::optional<T> ParseOnOffOption(const AZStd::string& optionName, const AZStd::pair<T, T>& states, const AZ::CommandLine& cmd)
{
return ParseBinaryStateOption(optionName, BinaryStateOption<T>{ {"off", states.first}, { "on", states.second } }, cmd);
}
//! Attempts to pass a specialization of the binary state option where the command line values are "abort" and "continue".
template<typename T>
AZStd::optional<T> ParseAbortContinueOption(const AZStd::string& optionName, const AZStd::pair<T, T>& states, const AZ::CommandLine& cmd)
{
return ParseBinaryStateOption(optionName, BinaryStateOption<T>{ {"abort", states.first}, { "continue", states.second } }, cmd);
}
//! Attempts to parse a path option value.
AZStd::optional<RepoPath> ParsePathOption(const AZStd::string& optionName, const AZ::CommandLine& cmd);
//! Attempts to pass an unsigned integer option value.
AZStd::optional<size_t> ParseUnsignedIntegerOption(const AZStd::string& optionName, const AZ::CommandLine& cmd);
//! Attempts to parse an option value in seconds.
AZStd::optional<AZStd::chrono::milliseconds> ParseSecondsOption(const AZStd::string& optionName, const AZ::CommandLine& cmd);
} // namespace TestImpact
@@ -1,17 +1,29 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
int main([[maybe_unused]] int argc, [[maybe_unused]] char** argv)
#include <TestImpactFramework/TestImpactConsoleMain.h>
#include <AzCore/Memory/OSAllocator.h>
#include <AzCore/Memory/SystemAllocator.h>
int main(int argc, char** argv)
{
return 0;
}
AZ::AllocatorInstance<AZ::OSAllocator>::Create();
AZ::AllocatorInstance<AZ::SystemAllocator>::Create();
TestImpact::Console::ReturnCode returnCode = TestImpact::Console::Main(argc, argv);
AZ::AllocatorInstance<AZ::SystemAllocator>::Destroy();
AZ::AllocatorInstance<AZ::OSAllocator>::Destroy();
return static_cast<int>(returnCode);
}
@@ -0,0 +1,316 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <TestImpactFramework/TestImpactException.h>
#include <TestImpactFramework/TestImpactChangeListException.h>
#include <TestImpactFramework/TestImpactConfigurationException.h>
#include <TestImpactFramework/TestImpactRuntimeException.h>
#include <TestImpactFramework/TestImpactConsoleMain.h>
#include <TestImpactFramework/TestImpactChangeListSerializer.h>
#include <TestImpactFramework/TestImpactChangeList.h>
#include <TestImpactFramework/TestImpactRuntime.h>
#include <TestImpactFramework/TestImpactFileUtils.h>
#include <TestImpactFramework/TestImpactClientTestSelection.h>
#include <TestImpactFramework/TestImpactRuntime.h>
#include <TestImpactConsoleTestSequenceEventHandler.h>
#include <TestImpactCommandLineOptions.h>
#include <TestImpactRuntimeConfigurationFactory.h>
#include <TestImpactCommandLineOptionsException.h>
#include <AzCore/IO/SystemFile.h>
#include <AzCore/std/containers/vector.h>
#include <AzCore/std/string/string.h>
#include <iostream>
namespace TestImpact
{
namespace Console
{
//! Generates a string to be used for printing to the console for the specified change list.
AZStd::string GenerateChangeListString(const ChangeList& changeList)
{
AZStd::string output;
const auto& outputFiles = [&output](const AZStd::vector<RepoPath>& files)
{
for (const auto& file : files)
{
output += AZStd::string::format("\t%s\n", file.c_str());
}
};
output += AZStd::string::format("Created files (%u):\n", changeList.m_createdFiles.size());
outputFiles(changeList.m_createdFiles);
output += AZStd::string::format("Updated files (%u):\n", changeList.m_updatedFiles.size());
outputFiles(changeList.m_updatedFiles);
output += AZStd::string::format("Deleted files (%u):\n", changeList.m_deletedFiles.size());
outputFiles(changeList.m_deletedFiles);
return output;
}
//! Gets the appropriate console return code for the specified test sequence result.
ReturnCode GetReturnCodeForTestSequenceResult(TestSequenceResult result)
{
switch (result)
{
case TestSequenceResult::Success:
return ReturnCode::Success;
case TestSequenceResult::Failure:
return ReturnCode::TestFailure;
case TestSequenceResult::Timeout:
return ReturnCode::Timeout;
default:
std::cout << "Unexpected TestSequenceResult value: " << aznumeric_cast<size_t>(result) << std::endl;
return ReturnCode::UnknownError;
}
}
//! Wrapper around impact analysis sequences to handle the case where the safe mode option is active.
ReturnCode WrappedImpactAnalysisTestSequence(
TestSequenceEventHandler& sequenceEventHandler,
const CommandLineOptions& options,
Runtime& runtime,
const AZStd::optional<ChangeList>& changeList)
{
// Even though it is possible for a regular run to be selected (see below) which does not actually require a change list,
// consider any impact analysis sequence type without a change list to be an error
AZ_TestImpact_Eval(
changeList.has_value(),
CommandLineOptionsException,
"Expected a change list for impact analysis but none was provided");
TestSequenceResult result = TestSequenceResult::Failure;
if (options.HasSafeMode())
{
if (options.GetTestSequenceType() == TestSequenceType::ImpactAnalysis)
{
auto [selectedResult, discardedResult] = runtime.SafeImpactAnalysisTestSequence(
changeList.value(),
options.GetTestPrioritizationPolicy(),
options.GetTestTargetTimeout(),
options.GetGlobalTimeout(),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler));
// Handling the possible timeout and failure permutations of the selected and discarded test results is splitting hairs
// so apply the following, admittedly arbitrary, rules to determine what the composite test sequence result should be
if (selectedResult == TestSequenceResult::Success && discardedResult == TestSequenceResult::Success)
{
// Trivial case: both sequences succeeded
result = TestSequenceResult::Success;
}
else if (selectedResult == TestSequenceResult::Failure || discardedResult == TestSequenceResult::Failure)
{
// One sequence failed whilst the other sequence either succeeded or timed out
result = TestSequenceResult::Failure;
}
else
{
// One or both sequences timed out or failed
result = TestSequenceResult::Timeout;
}
}
else if (options.GetTestSequenceType() == TestSequenceType::ImpactAnalysisNoWrite)
{
// A no-write impact analysis sequence with safe mode enabled is functionally identical to a regular sequence type
// due to a) the selected tests being run without instrumentation and b) the discarded tests also being run without
// instrumentation
result = runtime.RegularTestSequence(
options.GetTestTargetTimeout(),
options.GetGlobalTimeout(),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler));
}
else
{
throw(Exception("Unexpected sequence type"));
}
}
else
{
Policy::DynamicDependencyMap dynamicDependencyMapPolicy;
if (options.GetTestSequenceType() == TestSequenceType::ImpactAnalysis)
{
dynamicDependencyMapPolicy = Policy::DynamicDependencyMap::Update;
}
else if (options.GetTestSequenceType() == TestSequenceType::ImpactAnalysisNoWrite)
{
dynamicDependencyMapPolicy = Policy::DynamicDependencyMap::Discard;
}
else
{
throw(Exception("Unexpected sequence type"));
}
result = runtime.ImpactAnalysisTestSequence(
changeList.value(),
options.GetTestPrioritizationPolicy(),
dynamicDependencyMapPolicy,
options.GetTestTargetTimeout(),
options.GetGlobalTimeout(),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler));
}
return GetReturnCodeForTestSequenceResult(result);
};
//! Entry point for the test impact analysis framework console front end application.
ReturnCode Main(int argc, char** argv)
{
try
{
CommandLineOptions options(argc, argv);
AZStd::optional<ChangeList> changeList;
// If we have a change list, check to see whether or not the client has requested the printing of said change list
if (options.HasChangeListFile())
{
changeList = DeserializeChangeList(ReadFileContents<CommandLineOptionsException>(*options.GetChangeListFile()));
if (options.HasOutputChangeList())
{
std::cout << "Change List:\n";
std::cout << GenerateChangeListString(*changeList).c_str();
if (options.GetTestSequenceType() == TestSequenceType::None)
{
return ReturnCode::Success;
}
}
}
// As of now, there are no other non-test operations other than printing a change list so getting this far is considered an error
AZ_TestImpact_Eval(options.GetTestSequenceType() != TestSequenceType::None, CommandLineOptionsException, "No action specified");
std::cout << "Constructing in-memory model of source tree and test coverage for test suite ";
std::cout << GetSuiteTypeName(options.GetSuiteFilter()).c_str() << ", this may take a moment...\n";
Runtime runtime(
RuntimeConfigurationFactory(ReadFileContents<CommandLineOptionsException>(options.GetConfigurationFile())),
options.GetSuiteFilter(),
options.GetExecutionFailurePolicy(),
options.GetFailedTestCoveragePolicy(),
options.GetTestFailurePolicy(),
options.GetIntegrityFailurePolicy(),
options.GetTestShardingPolicy(),
options.GetTargetOutputCapture(),
options.GetMaxConcurrency());
if (runtime.HasImpactAnalysisData())
{
std::cout << "Test impact analysis data for this repository was found.\n";
}
else
{
std::cout << "Test impact analysis data for this repository was not found, seed or regular sequence fallbacks will be used.\n";
}
TestSequenceEventHandler sequenceEventHandler(options.GetSuiteFilter());
switch (const auto type = options.GetTestSequenceType())
{
case TestSequenceType::Regular:
{
const auto result = runtime.RegularTestSequence(
options.GetTestTargetTimeout(),
options.GetGlobalTimeout(),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler));
return GetReturnCodeForTestSequenceResult(result);
}
case TestSequenceType::Seed:
{
const auto result = runtime.SeededTestSequence(
options.GetTestTargetTimeout(),
options.GetGlobalTimeout(),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler));
return GetReturnCodeForTestSequenceResult(result);
}
case TestSequenceType::ImpactAnalysisNoWrite:
case TestSequenceType::ImpactAnalysis:
{
return WrappedImpactAnalysisTestSequence(sequenceEventHandler, options, runtime, changeList);
}
case TestSequenceType::ImpactAnalysisOrSeed:
{
if (runtime.HasImpactAnalysisData())
{
return WrappedImpactAnalysisTestSequence(sequenceEventHandler, options, runtime, changeList);
}
else
{
const auto result = runtime.SeededTestSequence(
options.GetTestTargetTimeout(),
options.GetGlobalTimeout(),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler));
return GetReturnCodeForTestSequenceResult(result);
}
}
default:
std::cout << "Unexpected TestSequenceType value: " << static_cast<size_t>(type) << std::endl;
return ReturnCode::UnknownError;
}
}
catch (const CommandLineOptionsException& e)
{
std::cout << e.what() << std::endl;
std::cout << CommandLineOptions::GetCommandLineUsageString().c_str() << std::endl;
return ReturnCode::InvalidArgs;
}
catch (const ChangeListException& e)
{
std::cout << e.what() << std::endl;
return ReturnCode::InvalidUnifiedDiff;
}
catch (const ConfigurationException& e)
{
std::cout << e.what() << std::endl;
return ReturnCode::InvalidConfiguration;
}
catch (const RuntimeException& e)
{
std::cout << e.what() << std::endl;
return ReturnCode::RuntimeError;
}
catch (const Exception& e)
{
std::cout << e.what() << std::endl;
return ReturnCode::UnhandledError;
}
catch (const std::exception& e)
{
std::cout << e.what() << std::endl;
return ReturnCode::UnknownError;
}
catch (...)
{
std::cout << "An unknown error occurred" << std::endl;
return ReturnCode::UnknownError;
}
}
} // namespace Console
} // namespace TestImpact
@@ -0,0 +1,234 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <TestImpactConsoleTestSequenceEventHandler.h>
#include <TestImpactConsoleUtils.h>
#include <iostream>
namespace TestImpact
{
namespace Console
{
namespace Output
{
void TestSuiteFilter(SuiteType filter)
{
std::cout << "Test suite filter: " << GetSuiteTypeName(filter).c_str() << "\n";
}
void ImpactAnalysisTestSelection(size_t numSelectedTests, size_t numDiscardedTests, size_t numExcludedTests, size_t numDraftedTests)
{
const float totalTests = numSelectedTests + numDiscardedTests;
const float saving = (1.0 - (numSelectedTests / totalTests)) * 100.0f;
std::cout << numSelectedTests << " tests selected, " << numDiscardedTests << " tests discarded (" << saving << "% test saving)\n";
std::cout << "Of which " << numExcludedTests << " tests have been excluded and " << numDraftedTests << " tests have been drafted.\n";
}
void FailureReport(const Client::SequenceFailure& failureReport, AZStd::chrono::milliseconds duration)
{
std::cout << "Sequence completed in " << (duration.count() / 1000.f) << "s with";
if (!failureReport.GetExecutionFailures().empty() ||
!failureReport.GetTestRunFailures().empty() ||
!failureReport.GetTimedOutTests().empty() ||
!failureReport.GetUnexecutedTests().empty())
{
std::cout << ":\n";
std::cout << SetColor(Foreground::White, Background::Red).c_str()
<< failureReport.GetTestRunFailures().size()
<< ResetColor().c_str() << " test failures\n";
std::cout << SetColor(Foreground::White, Background::Red).c_str()
<< failureReport.GetExecutionFailures().size()
<< ResetColor().c_str() << " execution failures\n";
std::cout << SetColor(Foreground::White, Background::Red).c_str()
<< failureReport.GetTimedOutTests().size()
<< ResetColor().c_str() << " test timeouts\n";
std::cout << SetColor(Foreground::White, Background::Red).c_str()
<< failureReport.GetUnexecutedTests().size()
<< ResetColor().c_str() << " unexecuted tests\n";
if (!failureReport.GetTestRunFailures().empty())
{
std::cout << "\nTest failures:\n";
for (const auto& testRunFailure : failureReport.GetTestRunFailures())
{
std::cout << " " << testRunFailure.GetTargetName().c_str();
for (const auto& testCaseFailure : testRunFailure.GetTestCaseFailures())
{
std::cout << "." << testCaseFailure.GetName().c_str();
for (const auto& testFailure : testCaseFailure.GetTestFailures())
{
std::cout << "." << testFailure.GetName().c_str() << "\n";
}
}
}
}
if (!failureReport.GetExecutionFailures().empty())
{
std::cout << "\nExecution failures:\n";
for (const auto& executionFailure : failureReport.GetExecutionFailures())
{
std::cout << " " << executionFailure.GetTargetName().c_str() << "\n";
std::cout << executionFailure.GetCommandString().c_str() << "\n";
}
}
if (!failureReport.GetTimedOutTests().empty())
{
std::cout << "\nTimed out tests:\n";
for (const auto& testTimeout : failureReport.GetTimedOutTests())
{
std::cout << " " << testTimeout.GetTargetName().c_str() << "\n";
}
}
if (!failureReport.GetUnexecutedTests().empty())
{
std::cout << "\nUnexecuted tests:\n";
for (const auto& unexecutedTest : failureReport.GetUnexecutedTests())
{
std::cout << " " << unexecutedTest.GetTargetName().c_str() << "\n";
}
}
}
else
{
std::cout << SetColor(Foreground::White, Background::Green).c_str() << " \100% passes!\n" << ResetColor().c_str();
}
}
}
TestSequenceEventHandler::TestSequenceEventHandler(SuiteType suiteFilter)
: m_suiteFilter(suiteFilter)
{
}
// TestSequenceStartCallback
void TestSequenceEventHandler::operator()(Client::TestRunSelection&& selectedTests)
{
ClearState();
m_numTests = selectedTests.GetNumIncludedTestRuns();
Output::TestSuiteFilter(m_suiteFilter);
std::cout << selectedTests.GetNumIncludedTestRuns() << " tests selected, " << selectedTests.GetNumExcludedTestRuns() << " excluded.\n";
}
// ImpactAnalysisTestSequenceStartCallback
void TestSequenceEventHandler::operator()(
Client::TestRunSelection&& selectedTests,
AZStd::vector<AZStd::string>&& discardedTests,
AZStd::vector<AZStd::string>&& draftedTests)
{
ClearState();
m_numTests = selectedTests.GetNumIncludedTestRuns() + draftedTests.size();
Output::TestSuiteFilter(m_suiteFilter);
Output::ImpactAnalysisTestSelection(
selectedTests.GetTotalNumTests(), discardedTests.size(), selectedTests.GetNumExcludedTestRuns(), draftedTests.size());
}
// SafeImpactAnalysisTestSequenceStartCallback
void TestSequenceEventHandler::operator()(
Client::TestRunSelection&& selectedTests,
Client::TestRunSelection&& discardedTests,
AZStd::vector<AZStd::string>&& draftedTests)
{
ClearState();
m_numTests = selectedTests.GetNumIncludedTestRuns() + draftedTests.size();
Output::TestSuiteFilter(m_suiteFilter);
Output::ImpactAnalysisTestSelection(
selectedTests.GetTotalNumTests(),
discardedTests.GetTotalNumTests(),
selectedTests.GetNumExcludedTestRuns() + discardedTests.GetNumExcludedTestRuns(),
draftedTests.size());
}
// TestSequenceCompleteCallback
void TestSequenceEventHandler::operator()(
Client::SequenceFailure&& failureReport,
AZStd::chrono::milliseconds duration)
{
Output::FailureReport(failureReport, duration);
std::cout << "Updating and serializing the test impact analysis data, this may take a moment...\n";
}
// SafeTestSequenceCompleteCallback
void TestSequenceEventHandler::operator()(
Client::SequenceFailure&& selectedFailureReport,
Client::SequenceFailure&& discardedFailureReport,
AZStd::chrono::milliseconds selectedDuration,
AZStd::chrono::milliseconds discaredDuration)
{
std::cout << "Selected test run:\n";
Output::FailureReport(selectedFailureReport, selectedDuration);
std::cout << "Discarded test run:\n";
Output::FailureReport(discardedFailureReport, discaredDuration);
std::cout << "Updating and serializing the test impact analysis data, this may take a moment...\n";
}
// TestRunCompleteCallback
void TestSequenceEventHandler::operator()([[maybe_unused]] Client::TestRun&& test)
{
m_numTestsComplete++;
const auto progress = AZStd::string::format("(%03u/%03u)", m_numTestsComplete, m_numTests, test.GetTargetName().c_str());
AZStd::string result;
switch (test.GetResult())
{
case Client::TestRunResult::AllTestsPass:
{
result = SetColorForString(Foreground::White, Background::Green, "PASS");
break;
}
case Client::TestRunResult::FailedToExecute:
{
result = SetColorForString(Foreground::White, Background::Red, "EXEC");
break;
}
case Client::TestRunResult::NotRun:
{
result = SetColorForString(Foreground::White, Background::Yellow, "SKIP");
break;
}
case Client::TestRunResult::TestFailures:
{
result = SetColorForString(Foreground::White, Background::Red, "FAIL");
break;
}
case Client::TestRunResult::Timeout:
{
result = SetColorForString(Foreground::White, Background::Magenta, "TIME");
break;
}
}
std::cout << progress.c_str() << " " << result.c_str() << " " << test.GetTargetName().c_str() << " (" << (test.GetDuration().count() / 1000.f) << "s)\n";
}
void TestSequenceEventHandler::ClearState()
{
m_numTests = 0;
m_numTestsComplete = 0;
}
} // namespace Console
} // namespace TestImpact
@@ -0,0 +1,72 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <TestImpactFramework/TestImpactTestSequence.h>
#include <TestImpactFramework/TestImpactClientTestSelection.h>
#include <TestImpactFramework/TestImpactClientFailureReport.h>
#include <TestImpactFramework/TestImpactClientTestRun.h>
#include <AzCore/std/chrono/chrono.h>
#include <AzCore/std/string/string.h>
#include <AzCore/std/containers/unordered_set.h>
#pragma once
namespace TestImpact
{
namespace Console
{
//! Event handler for all test sequence types.
class TestSequenceEventHandler
{
public:
explicit TestSequenceEventHandler(SuiteType suiteFilter);
//! TestSequenceStartCallback.
void operator()(Client::TestRunSelection&& selectedTests);
//! ImpactAnalysisTestSequenceStartCallback.
void operator()(
Client::TestRunSelection&& selectedTests,
AZStd::vector<AZStd::string>&& discardedTests,
AZStd::vector<AZStd::string>&& draftedTests);
//! SafeImpactAnalysisTestSequenceStartCallback.
void operator()(
Client::TestRunSelection&& selectedTests,
Client::TestRunSelection&& discardedTests,
AZStd::vector<AZStd::string>&& draftedTests);
//! TestSequenceCompleteCallback.
void operator()(
Client::SequenceFailure&& failureReport,
AZStd::chrono::milliseconds duration);
//! SafeTestSequenceCompleteCallback.
void operator()(
Client::SequenceFailure&& selectedFailureReport,
Client::SequenceFailure&& discardedFailureReport,
AZStd::chrono::milliseconds selectedDuration,
AZStd::chrono::milliseconds discaredDuration);
//! TestRunCompleteCallback.
void operator()(Client::TestRun&& test);
private:
void ClearState();
SuiteType m_suiteFilter;
size_t m_numTests = 0;
size_t m_numTestsComplete = 0;
};
} // namespace Console
} // namespace TestImpact
@@ -0,0 +1,34 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <TestImpactConsoleUtils.h>
namespace TestImpact
{
namespace Console
{
AZStd::string SetColor(Foreground foreground, Background background)
{
return AZStd::string::format("\033[%u;%um", aznumeric_cast<uint32_t>(foreground), aznumeric_cast<uint32_t>(background));
}
AZStd::string SetColorForString(Foreground foreground, Background background, const AZStd::string& str)
{
return AZStd::string::format("%s%s%s", SetColor(foreground, background).c_str(), str.c_str(), ResetColor().c_str());
}
AZStd::string ResetColor()
{
return "\033[0m";
}
} // namespace Console
} // namespace TestImpact
@@ -0,0 +1,56 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <AzCore/std/string/string.h>
namespace TestImpact
{
namespace Console
{
//! The set of available foreground colors.
enum class Foreground
{
Black = 30,
Red,
Green,
Yellow,
Blue,
Magenta,
Cyan,
White
};
//! The set of available background colors.
enum class Background
{
Black = 40,
Red,
Green,
Yellow,
Blue,
Magenta,
Cyan,
White
};
//! Returns a string to be used to set the specified foreground and background color.
AZStd::string SetColor(Foreground foreground, Background background);
//! Returns a string with the specified string set to the specified foreground and background color followed by a color reset.
AZStd::string SetColorForString(Foreground foreground, Background background, const AZStd::string& str);
//! Returns a string to be used to reset the color back to white foreground on black background.
AZStd::string ResetColor();
} // namespace Console
} // namespace TestImpact
@@ -0,0 +1,306 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <TestImpactFramework/TestImpactConfigurationException.h>
#include <TestImpactRuntimeConfigurationFactory.h>
#include <AzCore/JSON/document.h>
#include <AzCore/std/functional.h>
#include <AzCore/std/optional.h>
namespace TestImpact
{
namespace Config
{
// Keys for pertinent JSON elements
constexpr const char* Keys[] =
{
"root",
"platform",
"relative_paths",
"artifact_dir",
"enumeration_cache_dir",
"test_impact_data_files",
"temp",
"active",
"target_sources",
"static",
"autogen",
"static",
"include_filters",
"input_output_pairer",
"input",
"dir",
"matchers",
"target_dependency_file",
"target_vertex",
"file",
"test_runner",
"instrumentation",
"bin",
"exclude",
"shard",
"fixture_contiguous",
"fixture_interleaved",
"test_contiguous",
"test_interleaved",
"never",
"target",
"policy",
"artifacts",
"meta",
"repo",
"workspace",
"build_target_descriptor",
"dependency_graph_data",
"test_target_meta",
"test_engine",
"target"
};
enum
{
Root = 0,
PlatformName,
RelativePaths,
ArtifactDir,
EnumerationCacheDir,
TestImpactDataFiles,
TempWorkspace,
ActiveWorkspace,
TargetSources,
StaticSources,
AutogenSources,
StaticArtifacts,
SourceIncludeFilters,
AutogenInputOutputPairer,
AutogenInputSources,
Directory,
DependencyGraphMatchers,
TargetDependencyFileMatcher,
TargetVertexMatcher,
TestTargetMetaFile,
TestRunner,
TestInstrumentation,
BinaryFile,
TargetExcludeFilter,
TestSharding,
ContinuousFixtureSharding,
InterleavedFixtureSharding,
ContinuousTestSharding,
InterleavedTestSharding,
NeverShard,
TargetName,
TestShardingPolicy,
Artifacts,
Meta,
Repository,
Workspace,
BuildTargetDescriptor,
DependencyGraphData,
TestTargetMeta,
TestEngine,
TargetConfig
};
}
//! Returns an absolute path for a path relative to the specified root.
RepoPath GetAbsPathFromRelPath(const RepoPath& root, const RepoPath& rel)
{
return root / rel;
}
ConfigMeta ParseConfigMeta(const rapidjson::Value& meta)
{
ConfigMeta configMeta;
configMeta.m_platform = meta[Config::Keys[Config::PlatformName]].GetString();
return configMeta;
}
RepoConfig ParseRepoConfig(const rapidjson::Value& repo)
{
RepoConfig repoConfig;
repoConfig.m_root = repo[Config::Keys[Config::Root]].GetString();
return repoConfig;
}
WorkspaceConfig::Temp ParseTempWorkspaceConfig(const rapidjson::Value& tempWorkspace)
{
WorkspaceConfig::Temp tempWorkspaceConfig;
tempWorkspaceConfig.m_root = tempWorkspace[Config::Keys[Config::Root]].GetString();
tempWorkspaceConfig.m_artifactDirectory =
GetAbsPathFromRelPath(
tempWorkspaceConfig.m_root, tempWorkspace[Config::Keys[Config::RelativePaths]][Config::Keys[Config::ArtifactDir]].GetString());
return tempWorkspaceConfig;
}
AZStd::array<RepoPath, 3> ParseTestImpactAnalysisDataFiles(const RepoPath& root, const rapidjson::Value& sparTIAFile)
{
AZStd::array<RepoPath, 3> sparTIAFiles;
sparTIAFiles[static_cast<size_t>(SuiteType::Main)] =
GetAbsPathFromRelPath(root, sparTIAFile[GetSuiteTypeName(SuiteType::Main).c_str()].GetString());
sparTIAFiles[static_cast<size_t>(SuiteType::Periodic)] =
GetAbsPathFromRelPath(root, sparTIAFile[GetSuiteTypeName(SuiteType::Periodic).c_str()].GetString());
sparTIAFiles[static_cast<size_t>(SuiteType::Sandbox)] =
GetAbsPathFromRelPath(root, sparTIAFile[GetSuiteTypeName(SuiteType::Sandbox).c_str()].GetString());
return sparTIAFiles;
}
WorkspaceConfig::Active ParseActiveWorkspaceConfig(const rapidjson::Value& activeWorkspace)
{
WorkspaceConfig::Active activeWorkspaceConfig;
const auto& relativePaths = activeWorkspace[Config::Keys[Config::RelativePaths]];
activeWorkspaceConfig.m_root = activeWorkspace[Config::Keys[Config::Root]].GetString();
activeWorkspaceConfig.m_enumerationCacheDirectory
= GetAbsPathFromRelPath(activeWorkspaceConfig.m_root, relativePaths[Config::Keys[Config::EnumerationCacheDir]].GetString());
activeWorkspaceConfig.m_sparTIAFiles =
ParseTestImpactAnalysisDataFiles(activeWorkspaceConfig.m_root, relativePaths[Config::Keys[Config::TestImpactDataFiles]]);
return activeWorkspaceConfig;
}
WorkspaceConfig ParseWorkspaceConfig(const rapidjson::Value& workspace)
{
WorkspaceConfig workspaceConfig;
workspaceConfig.m_temp = ParseTempWorkspaceConfig(workspace[Config::Keys[Config::TempWorkspace]]);
workspaceConfig.m_active = ParseActiveWorkspaceConfig(workspace[Config::Keys[Config::ActiveWorkspace]]);
return workspaceConfig;
}
BuildTargetDescriptorConfig ParseBuildTargetDescriptorConfig(const rapidjson::Value& buildTargetDescriptor)
{
BuildTargetDescriptorConfig buildTargetDescriptorConfig;
const auto& targetSources = buildTargetDescriptor[Config::Keys[Config::TargetSources]];
const auto& staticTargetSources = targetSources[Config::Keys[Config::StaticSources]];
const auto& autogenTargetSources = targetSources[Config::Keys[Config::AutogenSources]];
buildTargetDescriptorConfig.m_mappingDirectory = buildTargetDescriptor[Config::Keys[Config::Directory]].GetString();
const auto& staticInclusionFilters = staticTargetSources[Config::Keys[Config::SourceIncludeFilters]].GetArray();
buildTargetDescriptorConfig.m_staticInclusionFilters.reserve(staticInclusionFilters.Size());
for (const auto& staticInclusionFilter : staticInclusionFilters)
{
buildTargetDescriptorConfig.m_staticInclusionFilters.push_back(staticInclusionFilter.GetString());
}
buildTargetDescriptorConfig.m_inputOutputPairer = autogenTargetSources[Config::Keys[Config::AutogenInputOutputPairer]].GetString();
const auto& inputInclusionFilters =
autogenTargetSources[Config::Keys[Config::AutogenInputSources]][Config::Keys[Config::SourceIncludeFilters]].GetArray();
buildTargetDescriptorConfig.m_inputInclusionFilters.reserve(inputInclusionFilters.Size());
for (const auto& inputInclusionFilter : inputInclusionFilters)
{
buildTargetDescriptorConfig.m_inputInclusionFilters.push_back(inputInclusionFilter.GetString());
}
return buildTargetDescriptorConfig;
}
DependencyGraphDataConfig ParseDependencyGraphDataConfig(const rapidjson::Value& dependencyGraphData)
{
DependencyGraphDataConfig dependencyGraphDataConfig;
const auto& matchers = dependencyGraphData[Config::Keys[Config::DependencyGraphMatchers]];
dependencyGraphDataConfig.m_graphDirectory = dependencyGraphData[Config::Keys[Config::Directory]].GetString();
dependencyGraphDataConfig.m_targetDependencyFileMatcher = matchers[Config::Keys[Config::TargetDependencyFileMatcher]].GetString();
dependencyGraphDataConfig.m_targetVertexMatcher = matchers[Config::Keys[Config::TargetVertexMatcher]].GetString();
return dependencyGraphDataConfig;
}
TestTargetMetaConfig ParseTestTargetMetaConfig(const rapidjson::Value& testTargetMeta)
{
TestTargetMetaConfig testTargetMetaConfig;
testTargetMetaConfig.m_metaFile = testTargetMeta[Config::Keys[Config::TestTargetMetaFile]].GetString();
return testTargetMetaConfig;
}
TestEngineConfig ParseTestEngineConfig(const rapidjson::Value& testEngine)
{
TestEngineConfig testEngineConfig;
testEngineConfig.m_testRunner.m_binary = testEngine[Config::Keys[Config::TestRunner]][Config::Keys[Config::BinaryFile]].GetString();
testEngineConfig.m_instrumentation.m_binary = testEngine[Config::Keys[Config::TestInstrumentation]][Config::Keys[Config::BinaryFile]].GetString();
return testEngineConfig;
}
TargetConfig ParseTargetConfig(const rapidjson::Value& target)
{
TargetConfig targetConfig;
targetConfig.m_outputDirectory = target[Config::Keys[Config::Directory]].GetString();
const auto& testExcludes = target[Config::Keys[Config::TargetExcludeFilter]].GetArray();
targetConfig.m_excludedTestTargets.reserve(testExcludes.Size());
for (const auto& testExclude : testExcludes)
{
targetConfig.m_excludedTestTargets.push_back(testExclude.GetString());
}
const auto& testShards = target[Config::Keys[Config::TestSharding]].GetArray();
targetConfig.m_shardedTestTargets.reserve(testShards.Size());
for (const auto& testShard : testShards)
{
const auto getShardingConfiguration = [](const AZStd::string& config)
{
if (config == Config::Keys[Config::ContinuousFixtureSharding])
{
return ShardConfiguration::FixtureContiguous;
}
else if (config == Config::Keys[Config::InterleavedFixtureSharding])
{
return ShardConfiguration::FixtureInterleaved;
}
else if (config == Config::Keys[Config::ContinuousTestSharding])
{
return ShardConfiguration::TestContiguous;
}
else if (config == Config::Keys[Config::InterleavedTestSharding])
{
return ShardConfiguration::TestInterleaved;
}
else if (config == Config::Keys[Config::NeverShard])
{
return ShardConfiguration::Never;
}
else
{
throw ConfigurationException(AZStd::string::format("Unexpected sharding configuration: %s", config.c_str()));
}
};
TargetConfig::ShardedTarget shard;
shard.m_name = testShard[Config::Keys[Config::TargetName]].GetString();
shard.m_configuration = getShardingConfiguration(testShard[Config::Keys[Config::TestShardingPolicy]].GetString());
targetConfig.m_shardedTestTargets.push_back(AZStd::move(shard));
}
return targetConfig;
}
RuntimeConfig RuntimeConfigurationFactory(const AZStd::string& configurationData)
{
rapidjson::Document configurationFile;
if (configurationFile.Parse(configurationData.c_str()).HasParseError())
{
throw TestImpact::ConfigurationException("Could not parse runtimeConfig data, JSON has errors");
}
RuntimeConfig runtimeConfig;
const auto& staticArtifacts = configurationFile[Config::Keys[Config::Artifacts]][Config::Keys[Config::StaticArtifacts]];
runtimeConfig.m_meta = ParseConfigMeta(configurationFile[Config::Keys[Config::Meta]]);
runtimeConfig.m_repo = ParseRepoConfig(configurationFile[Config::Keys[Config::Repository]]);
runtimeConfig.m_workspace = ParseWorkspaceConfig(configurationFile[Config::Keys[Config::Workspace]]);
runtimeConfig.m_buildTargetDescriptor = ParseBuildTargetDescriptorConfig(staticArtifacts[Config::Keys[Config::BuildTargetDescriptor]]);
runtimeConfig.m_dependencyGraphData = ParseDependencyGraphDataConfig(staticArtifacts[Config::Keys[Config::DependencyGraphData]]);
runtimeConfig.m_testTargetMeta = ParseTestTargetMetaConfig(staticArtifacts[Config::Keys[Config::TestTargetMeta]]);
runtimeConfig.m_testEngine = ParseTestEngineConfig(configurationFile[Config::Keys[Config::TestEngine]]);
runtimeConfig.m_target = ParseTargetConfig(configurationFile[Config::Keys[Config::TargetConfig]]);
return runtimeConfig;
}
} // namespace TestImpact
@@ -0,0 +1,19 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <TestImpactFramework/TestImpactConfiguration.h>
namespace TestImpact
{
//! Parses the configuration data (in JSON format) and returns the constructed runtime configuration.
RuntimeConfig RuntimeConfigurationFactory(const AZStd::string& configurationData);
} // namespace TestImpact
@@ -0,0 +1,26 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
set(FILES
Include/TestImpactFramework/TestImpactConsoleMain.h
Source/TestImpactCommandLineOptions.h
Source/TestImpactCommandLineOptions.cpp
Source/TestImpactCommandLineOptionsUtils.cpp
Source/TestImpactCommandLineOptionsUtils.h
Source/TestImpactCommandLineOptionsException.h
Source/TestImpactRuntimeConfigurationFactory.h
Source/TestImpactRuntimeConfigurationFactory.cpp
Source/TestImpactConsoleMain.cpp
Source/TestImpactConsoleTestSequenceEventHandler.cpp
Source/TestImpactConsoleTestSequenceEventHandler.h
Source/TestImpactConsoleUtils.cpp
Source/TestImpactConsoleUtils.h
)
@@ -0,0 +1,13 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
set(FILES
)
@@ -9,8 +9,8 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
ly_get_list_relative_pal_filename(pal_dir ${CMAKE_CURRENT_LIST_DIR}/Source/Platform/${PAL_PLATFORM_NAME})
ly_get_list_relative_pal_filename(common_dir ${CMAKE_CURRENT_LIST_DIR}/Source/Platform/Common)
ly_add_target(
NAME TestImpact.Runtime.Static STATIC
@@ -18,12 +18,71 @@ ly_add_target(
FILES_CMAKE
testimpactframework_runtime_files.cmake
${pal_dir}/platform_${PAL_PLATFORM_NAME_LOWERCASE}_files.cmake
PLATFORM_INCLUDE_FILES
${common_dir}/${PAL_TRAIT_COMPILER_ID}/testimpactframework_${PAL_TRAIT_COMPILER_ID_LOWERCASE}.cmake
INCLUDE_DIRECTORIES
PRIVATE
Source
PUBLIC
Include
BUILD_DEPENDENCIES
Public
PUBLIC
AZ::AzCore
)
################################################################################
# Tests
################################################################################
# Disabled: SPEC-7246
#add_subdirectory(Tests/TestProcess)
#add_subdirectory(Tests/TestTargetA)
#add_subdirectory(Tests/TestTargetB)
#add_subdirectory(Tests/TestTargetC)
#add_subdirectory(Tests/TestTargetD)
#
#ly_add_target(
# NAME TestImpact.Runtime.Tests ${PAL_TRAIT_TEST_TARGET_TYPE}
# NAMESPACE AZ
# FILES_CMAKE
# testimpactframework_runtime_tests_files.cmake
# INCLUDE_DIRECTORIES
# PRIVATE
# Include
# Source
# Tests
# BUILD_DEPENDENCIES
# PRIVATE
# AZ::AzTestShared
# AZ::AzTest
# AZ::TestImpact.Runtime.Static
# RUNTIME_DEPENDENCIES
# AZ::AzTestRunner
# AZ::TestImpact.TestProcess.Console
# AZ::TestImpact.TestTargetA.Tests
# AZ::TestImpact.TestTargetB.Tests
# AZ::TestImpact.TestTargetC.Tests
# AZ::TestImpact.TestTargetD.Tests
# COMPILE_DEFINITIONS
# PRIVATE
# LY_TEST_IMPACT_AZ_TESTRUNNER_BIN="$<TARGET_FILE:AzTestRunner>"
# LY_TEST_IMPACT_TEST_PROCESS_BIN="$<TARGET_FILE:TestImpact.TestProcess.Console>"
# LY_TEST_IMPACT_TEST_TARGET_A_BIN="$<TARGET_FILE:TestImpact.TestTargetA.Tests>"
# LY_TEST_IMPACT_TEST_TARGET_B_BIN="$<TARGET_FILE:TestImpact.TestTargetB.Tests>"
# LY_TEST_IMPACT_TEST_TARGET_C_BIN="$<TARGET_FILE:TestImpact.TestTargetC.Tests>"
# LY_TEST_IMPACT_TEST_TARGET_D_BIN="$<TARGET_FILE:TestImpact.TestTargetD.Tests>"
# LY_TEST_IMPACT_TEST_TARGET_A_BASE_NAME="$<TARGET_FILE_BASE_NAME:TestImpact.TestTargetA.Tests>"
# LY_TEST_IMPACT_TEST_TARGET_B_BASE_NAME="$<TARGET_FILE_BASE_NAME:TestImpact.TestTargetB.Tests>"
# LY_TEST_IMPACT_TEST_TARGET_C_BASE_NAME="$<TARGET_FILE_BASE_NAME:TestImpact.TestTargetC.Tests>"
# LY_TEST_IMPACT_TEST_TARGET_D_BASE_NAME="$<TARGET_FILE_BASE_NAME:TestImpact.TestTargetD.Tests>"
# LY_TEST_IMPACT_TEST_TARGET_ENUMERATION_DIR="${GTEST_XML_OUTPUT_DIR}/TestImpact/Temp/Exclusive/Enum"
# LY_TEST_IMPACT_TEST_TARGET_RESULTS_DIR="${GTEST_XML_OUTPUT_DIR}/TestImpact/Temp/Exclusive/Result"
# LY_TEST_IMPACT_TEST_TARGET_COVERAGE_DIR="${GTEST_XML_OUTPUT_DIR}/TestImpact/Temp/Exclusive/Coverage"
# LY_TEST_IMPACT_INSTRUMENTATION_BIN="${LY_TEST_IMPACT_INSTRUMENTATION_BIN}"
# LY_TEST_IMPACT_MODULES_DIR="${CMAKE_BINARY_DIR}"
# LY_TEST_IMPACT_COVERAGE_SOURCES_DIR="${CMAKE_CURRENT_SOURCE_DIR}"
#)
#
#ly_add_googletest(
# NAME AZ::TestImpact.Runtime.Tests
#)
@@ -0,0 +1,28 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <TestImpactFramework/TestImpactRepoPath.h>
#include <AzCore/std/containers/vector.h>
namespace TestImpact
{
//! Representation of the file CRUD operations of a given set of source changes.
struct ChangeList
{
AZStd::vector<RepoPath> m_createdFiles; //!< Files that were newly created.
AZStd::vector<RepoPath> m_updatedFiles; //!< Files that were updated.
AZStd::vector<RepoPath> m_deletedFiles; //!< Files that were deleted.
};
} // namespace TestImpact
@@ -0,0 +1,26 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <TestImpactFramework/TestImpactException.h>
namespace TestImpact
{
//! Exception for change list operations.
class ChangeListException
: public Exception
{
public:
using Exception::Exception;
};
} // namespace TestImpact
@@ -0,0 +1,26 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <TestImpactFramework/TestImpactChangeList.h>
#include <AzCore/std/string/string.h>
namespace TestImpact
{
//! Serializes the specified change list to JSON format.
AZStd::string SerializeChangeList(const ChangeList& changeList);
//! Deserializes a change list from the specified test run data in JSON format.
ChangeList DeserializeChangeList(const AZStd::string& changeListString);
} // namespace TestImpact
@@ -0,0 +1,129 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <AzCore/std/string/string.h>
#include <AzCore/std/containers/vector.h>
namespace TestImpact
{
namespace Client
{
//! Represents a test target that failed, either due to failing to execute, completing in an abnormal state or completing with failing tests.
class TargetFailure
{
public:
TargetFailure(const AZStd::string& targetName);
//! Returns the name of the test target this failure pertains to.
const AZStd::string& GetTargetName() const;
private:
AZStd::string m_targetName;
};
//! Represents a test target that failed to execute.
class ExecutionFailure
: public TargetFailure
{
public:
ExecutionFailure(const AZStd::string& targetName, const AZStd::string& command);
//! Returns the command string used to execute this test target.
const AZStd::string& GetCommandString() const;
private:
AZStd::string m_commandString;
};
//! Represents an individual test of a test target that failed.
class TestFailure
{
public:
TestFailure(const AZStd::string& testName, const AZStd::string& errorMessage);
//! Returns the name of the test that failed.
const AZStd::string& GetName() const;
//! Returns the error message of the test that failed.
const AZStd::string& GetErrorMessage() const;
private:
AZStd::string m_name;
AZStd::string m_errorMessage;
};
//! Represents a collection of tests that failed.
//! @note Only the failing tests are included in the collection.
class TestCaseFailure
{
public:
TestCaseFailure(const AZStd::string& testCaseName, AZStd::vector<TestFailure>&& testFailures);
//! Returns the name of the test case containing the failing tests.
const AZStd::string& GetName() const;
//! Returns the collection of tests in this test case that failed.
const AZStd::vector<TestFailure>& GetTestFailures() const;
private:
AZStd::string m_name;
AZStd::vector<TestFailure> m_testFailures;
};
//! Represents a test target that launched successfully but contains failing tests.
class TestRunFailure
: public TargetFailure
{
public:
TestRunFailure(const AZStd::string& targetName, AZStd::vector<TestCaseFailure>&& testFailures);
//! Returns the total number of failing tests in this run.
size_t GetNumTestFailures() const;
//! Returns the test cases in this run containing failing tests.
const AZStd::vector<TestCaseFailure>& GetTestCaseFailures() const;
private:
AZStd::vector<TestCaseFailure> m_testCaseFailures;
size_t m_numTestFailures = 0;
};
//! Base class for reporting failing test sequences.
class SequenceFailure
{
public:
SequenceFailure(
AZStd::vector<ExecutionFailure>&& executionFailures,
AZStd::vector<TestRunFailure>&& testRunFailures,
AZStd::vector<TargetFailure>&& timedOutTests,
AZStd::vector<TargetFailure>&& unexecutedTests);
//! Returns the test targets in this sequence that failed to execute.
const AZStd::vector<ExecutionFailure>& GetExecutionFailures() const;
//! Returns the test targets that contain failing tests.
const AZStd::vector<TestRunFailure>& GetTestRunFailures() const;
//! Returns the test targets in this sequence that were terminated for exceeding their allotted runtime.
const AZStd::vector<TargetFailure>& GetTimedOutTests() const;
//! Returns the test targets in this sequence that were not executed due to the sequence terminating prematurely.
const AZStd::vector<TargetFailure>& GetUnexecutedTests() const;
private:
AZStd::vector<ExecutionFailure> m_executionFailures;
AZStd::vector<TestRunFailure> m_testRunFailures;
AZStd::vector<TargetFailure> m_timedOutTests;
AZStd::vector<TargetFailure> m_unexecutedTests;
};
} // namespace Client
} // namespace TestImpact
@@ -0,0 +1,46 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <AzCore/std/string/string.h>
#include <AzCore/std/chrono/chrono.h>
#pragma once
namespace TestImpact
{
namespace Client
{
//! Result of a test run.
enum class TestRunResult
{
NotRun, //!< The test run was not executed due to the test sequence terminating prematurely.
FailedToExecute, //!< The test run failed to execute either due to the target binary missing or incorrect arguments.
Timeout, //!< The test run timed out whilst in flight before being able to complete its run.
TestFailures, //!< The test run completed its run but there were failing tests.
AllTestsPass //!< The test run completed its run and all tests passed.
};
class TestRun
{
public:
TestRun(const AZStd::string& name, TestRunResult result, AZStd::chrono::milliseconds duration);
const AZStd::string& GetTargetName() const;
TestRunResult GetResult() const;
AZStd::chrono::milliseconds GetDuration() const;
private:
AZStd::string m_targetName;
TestRunResult m_result;
AZStd::chrono::milliseconds m_duration;
};
} // namespace Client
} // namespace TestImpact
@@ -0,0 +1,51 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <AzCore/std/string/string.h>
#include <AzCore/std/containers/vector.h>
#pragma once
namespace TestImpact
{
namespace Client
{
//! The set of test targets selected to run regardless of whether or not the test targets are to be excluded either for being on the primary exclude
//! list and/or being part of a test suite excluded from this run.
//! @note Only the included test targets will be run. The excluded test targets, although selected, will not be run.
class TestRunSelection
{
public:
TestRunSelection(const AZStd::vector<AZStd::string>& includedTests, const AZStd::vector<AZStd::string>& excludedTests);
TestRunSelection(AZStd::vector<AZStd::string>&& includedTests, AZStd::vector<AZStd::string>&& excludedTests);
//! Returns the test runs that were selected to be run and will actually be run.
const AZStd::vector<AZStd::string>& GetIncludededTestRuns() const;
//! Returns the test runs that were selected to be run but will not actually be run.
const AZStd::vector<AZStd::string>& GetExcludedTestRuns() const;
//! Returns the number of selected test runs that will be run.
size_t GetNumIncludedTestRuns() const;
//! Returns the number of selected test runs that will not be run.
size_t GetNumExcludedTestRuns() const;
//! Returns the total number of test runs selected regardless of whether or not they will actually be run.
size_t GetTotalNumTests() const;
private:
AZStd::vector<AZStd::string> m_includedTestRuns;
AZStd::vector<AZStd::string> m_excludedTestRuns;
};
} // namespace Client
} // namespace TestImpact
@@ -0,0 +1,126 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <TestImpactFramework/TestImpactTestSequence.h>
#include <TestImpactFramework/TestImpactRepoPath.h>
#include <AzCore/std/containers/vector.h>
#include <AzCore/std/containers/array.h>
#include <AzCore/std/string/string.h>
namespace TestImpact
{
//! Meta-data about the configuration.
struct ConfigMeta
{
AZStd::string m_platform; //!< The platform for which the configuration pertains to.
};
//! Repository configuration.
struct RepoConfig
{
RepoPath m_root; //!< The absolute path to the repository root.
};
//! Test impact analysis framework workspace configuration.
struct WorkspaceConfig
{
//! Temporary workspace configuration.
struct Temp
{
RepoPath m_root; //!< Path to the temporary workspace (cleaned prior to use).
RepoPath m_artifactDirectory; //!< Path to read and write runtime artifacts to and from.
};
//! Active persistent data workspace configuration.
struct Active
{
RepoPath m_root; //!< Path to the persistent workspace tracked by the repository.
RepoPath m_enumerationCacheDirectory; //!< Path to the test enumerations cache.
AZStd::array<RepoPath, 3> m_sparTIAFiles; //!< Paths to the test impact analysis data files for each test suite.
};
Temp m_temp;
Active m_active;
};
//! Build target descriptor configuration.
struct BuildTargetDescriptorConfig
{
RepoPath m_mappingDirectory; //!< Path to the source to target mapping files.
AZStd::vector<AZStd::string> m_staticInclusionFilters; //!< File extensions to include for static files.
AZStd::string m_inputOutputPairer; //!< Regex for matching autogen input files with autogen outputs files.
AZStd::vector<AZStd::string> m_inputInclusionFilters; //!< File extensions fo include for autogen input files.
};
//! Dependency graph configuration.
struct DependencyGraphDataConfig
{
RepoPath m_graphDirectory; //!< Path to the dependency graph files.
AZStd::string m_targetDependencyFileMatcher; //!< Regex for matching dependency graph files to build targets.
AZStd::string m_targetVertexMatcher; //!< Regex form matching dependency graph vertices to build targets.
};
//! Test target meta configuration.
struct TestTargetMetaConfig
{
RepoPath m_metaFile; //!< Path to the test target meta file.
};
//! Test engine configuration.
struct TestEngineConfig
{
//! Test runner configuration.
struct TestRunner
{
RepoPath m_binary; //!< Path to the test runner binary.
};
//! Test instrumentation configuration.
struct Instrumentation
{
RepoPath m_binary; //!< Path to the test instrumentation binary.
};
TestRunner m_testRunner;
Instrumentation m_instrumentation;
};
//! Build target configuration.
struct TargetConfig
{
//! Test target sharding configuration.
struct ShardedTarget
{
AZStd::string m_name; //!< Name of test target this sharding configuration applies to.
ShardConfiguration m_configuration; //!< The shard configuration to use.
};
RepoPath m_outputDirectory; //!< Path to the test target binary directory.
AZStd::vector<AZStd::string> m_excludedTestTargets; //!< Test targets to always exclude from test run sequences.
AZStd::vector<ShardedTarget> m_shardedTestTargets; //!< Test target shard configurations (opt-in).
};
struct RuntimeConfig
{
ConfigMeta m_meta;
RepoConfig m_repo;
WorkspaceConfig m_workspace;
BuildTargetDescriptorConfig m_buildTargetDescriptor;
DependencyGraphDataConfig m_dependencyGraphData;
TestTargetMetaConfig m_testTargetMeta;
TestEngineConfig m_testEngine;
TargetConfig m_target;
};
} // namespace TestImpact
@@ -0,0 +1,26 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <TestImpactFramework/TestImpactException.h>
namespace TestImpact
{
//! Exception for configuration operations.
class ConfigurationException
: public Exception
{
public:
using Exception::Exception;
};
} // namespace TestImpact
@@ -0,0 +1,51 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <AzCore/std/string/string.h>
#include <stdexcept>
//! Evaluates the specified condition and throws the specified exception with the specified
// !message upon failure.
#define AZ_TestImpact_Eval(CONDITION, EXCEPTION_TYPE, MSG) \
do \
{ \
static_assert( \
AZStd::is_base_of_v<TestImpact::Exception, EXCEPTION_TYPE>, \
"TestImpact Eval macro must only be used with TestImpact exceptions"); \
if(!(CONDITION)) \
{ \
throw(EXCEPTION_TYPE(MSG)); \
} \
} \
while (0)
namespace TestImpact
{
//! Base class for test impact framework exceptions.
//! @note The message passed in to the constructor is copied and thus safe with dynamic strings.
class Exception
: public std::exception
{
public:
explicit Exception() = default;
explicit Exception(const AZStd::string& msg);
explicit Exception(const char* msg);
const char* what() const noexcept override;
private:
//! Error message detailing the reason for the exception.
AZStd::string m_msg;
};
} // namespace TestImpact
@@ -0,0 +1,85 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <TestImpactFramework/TestImpactException.h>
#include <TestImpactFramework/TestImpactRuntime.h>
#include <AzCore/IO/SystemFile.h>
#include <AzCore/std/containers/vector.h>
#include <AzCore/std/string/string.h>
#pragma once
namespace TestImpact
{
//! Attempts to read the contents of the specified file into a string.
//! @tparam ExceptionType The exception type to throw upon failure.
//! @param path The path to the file to read the contents of.
//! @returns The contents of the file.
template<typename ExceptionType>
AZStd::string ReadFileContents(const RepoPath& path)
{
const auto fileSize = AZ::IO::SystemFile::Length(path.c_str());
AZ_TestImpact_Eval(fileSize > 0, ExceptionType, AZStd::string::format("File %s does not exist", path.c_str()));
AZStd::vector<char> buffer(fileSize + 1);
buffer[fileSize] = '\0';
AZ_TestImpact_Eval(
AZ::IO::SystemFile::Read(path.c_str(), buffer.data()),
ExceptionType,
AZStd::string::format("Could not read contents of file %s", path.c_str()));
return AZStd::string(buffer.begin(), buffer.end());
}
//! Attempts to write the contents of the specified string to a file.
//! @tparam ExceptionType The exception type to throw upon failure.
//! @param contents The contents to write to the file.
//! @param path The path to the file to write the contents to.
template<typename ExceptionType>
void WriteFileContents(const AZStd::string& contents, const RepoPath& path)
{
AZ::IO::SystemFile file;
const AZStd::vector<char> bytes(contents.begin(), contents.end());
AZ_TestImpact_Eval(
file.Open(path.c_str(),
AZ::IO::SystemFile::SF_OPEN_CREATE | AZ::IO::SystemFile::SF_OPEN_CREATE_PATH | AZ::IO::SystemFile::SF_OPEN_WRITE_ONLY),
ExceptionType,
AZStd::string::format("Couldn't open file %s for writing", path.c_str()));
AZ_TestImpact_Eval(
file.Write(bytes.data(), bytes.size()), ExceptionType, AZStd::string::format("Couldn't write contents for file %s", path.c_str()));
}
//! Delete the files that match the pattern from the specified directory.
//! @param path The path to the directory to pattern match the files for deletion.
//! @param pattern The pattern to match files for deletion.
inline void DeleteFiles(const RepoPath& path, const AZStd::string& pattern)
{
AZ::IO::SystemFile::FindFiles(AZStd::string::format("%s/%s", path.c_str(), pattern.c_str()).c_str(),
[&path](const char* file, bool isFile)
{
if (isFile)
{
AZ::IO::SystemFile::Delete(AZStd::string::format("%s/%s", path.c_str(), file).c_str());
}
return true;
});
}
//! Deletes the specified file.
inline void DeleteFile(const RepoPath& file)
{
DeleteFiles(file.ParentPath(), file.Filename().Native());
}
} // namespace TestImpact
@@ -0,0 +1,98 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <AzCore/IO/Path/Path.h>
#include <AzCore/std/string/string.h>
namespace TestImpact
{
//! Wrapper class to ensure that all paths have the same path separator regardless of how they are sourced. This is critical
//! to the test impact analysis data as otherwise querying/retrieving test impact analysis data for the same source albeit
//! with different path separators will be considered different files entirely.
class RepoPath
{
public:
using string_type = AZ::IO::Path::string_type;
using string_view_type = AZ::IO::Path::string_view_type;
using value_type = AZ::IO::Path::value_type;
constexpr RepoPath() = default;
constexpr RepoPath(const RepoPath&) = default;
constexpr RepoPath(RepoPath&&) noexcept = default;
constexpr RepoPath::RepoPath(const string_type& path) noexcept;
constexpr RepoPath::RepoPath(const string_view_type& path) noexcept;
constexpr RepoPath::RepoPath(const value_type* path) noexcept;
constexpr RepoPath::RepoPath(const AZ::IO::PathView& path);
constexpr RepoPath::RepoPath(const AZ::IO::Path& path);
RepoPath& operator=(const RepoPath&) noexcept = default;
RepoPath& operator=(const string_type&) noexcept;
RepoPath& operator=(const value_type*) noexcept;
RepoPath& operator=(const AZ::IO::Path& str) noexcept;
const char* c_str() const { return m_path.c_str(); }
AZStd::string String() const { return m_path.String(); }
constexpr AZ::IO::PathView Stem() const { return m_path.Stem(); }
constexpr AZ::IO::PathView Extension() const { return m_path.Extension(); }
constexpr bool empty() const { return m_path.empty(); }
constexpr AZ::IO::PathView ParentPath() const { return m_path.ParentPath(); }
constexpr AZ::IO::PathView Filename() const { return m_path.Filename(); }
AZ::IO::Path LexicallyRelative(const RepoPath& base) const { return m_path.LexicallyRelative(base.m_path); }
[[nodiscard]] bool IsRelativeTo(const RepoPath& base) const { return m_path.IsRelativeTo(base.m_path); }
constexpr AZ::IO::PathView RootName() const { return m_path.RootName(); }
constexpr AZ::IO::PathView RelativePath() const { return m_path.RelativePath(); }
// Wrappers around the AZ::IO::Path concatenation operator
friend RepoPath operator/(const RepoPath& lhs, const AZ::IO::PathView& rhs);
friend RepoPath operator/(const RepoPath& lhs, AZStd::string_view rhs);
friend RepoPath operator/(const RepoPath& lhs, const typename value_type* rhs);
friend RepoPath operator/(const RepoPath& lhs, const RepoPath& rhs);
RepoPath& operator/=(const AZ::IO::PathView& rhs);
RepoPath& operator/=(AZStd::string_view rhs);
RepoPath& operator/=(const typename value_type* rhs);
RepoPath& operator/=(const RepoPath& rhs);
friend bool operator==(const RepoPath& lhs, const RepoPath& rhs) noexcept;
friend bool operator!=(const RepoPath& lhs, const RepoPath& rhs) noexcept;
friend bool operator<(const RepoPath& lhs, const RepoPath& rhs) noexcept;
private:
AZ::IO::Path m_path;
};
constexpr RepoPath::RepoPath(const string_type& path) noexcept
: m_path(AZ::IO::Path(path).MakePreferred())
{
}
constexpr RepoPath::RepoPath(const string_view_type& path) noexcept
: m_path(AZ::IO::Path(path).MakePreferred())
{
}
constexpr RepoPath::RepoPath(const value_type* path) noexcept
: m_path(AZ::IO::Path(path).MakePreferred())
{
}
constexpr RepoPath::RepoPath(const AZ::IO::PathView& path)
: m_path(AZ::IO::Path(path).MakePreferred())
{
}
constexpr RepoPath::RepoPath(const AZ::IO::Path& path)
: m_path(AZ::IO::Path(path).MakePreferred())
{
}
} // namespace TestImpact
@@ -0,0 +1,232 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <TestImpactFramework/TestImpactConfiguration.h>
#include <TestImpactFramework/TestImpactChangeList.h>
#include <TestImpactFramework/TestImpactClientTestSelection.h>
#include <TestImpactFramework/TestImpactClientTestRun.h>
#include <TestImpactFramework/TestImpactClientFailureReport.h>
#include <TestImpactFramework/TestImpactTestSequence.h>
#include <AzCore/std/string/string.h>
#include <AzCore/std/chrono/chrono.h>
#include <AzCore/std/optional.h>
#include <AzCore/std/functional.h>
#include <AzCore/std/containers/vector.h>
#include <AzCore/std/containers/unordered_set.h>
#include <AzCore/std/smart_ptr/unique_ptr.h>
namespace TestImpact
{
class ChangeDependencyList;
class DynamicDependencyMap;
class TestSelectorAndPrioritizer;
class TestEngine;
class TestTarget;
class SourceCoveringTestsList;
class TestEngineInstrumentedRun;
//! Callback for a test sequence that isn't using test impact analysis to determine selected tests.
//! @param tests The tests that will be run for this sequence.
using TestSequenceStartCallback = AZStd::function<void(Client::TestRunSelection&& tests)>;
//! Callback for a test sequence using test impact analysis.
//! @param selectedTests The tests that have been selected for this run by test impact analysis.
//! @param discardedTests The tests that have been rejected for this run by test impact analysis.
//! @param draftedTests The tests that have been drafted in for this run due to requirements outside of test impact analysis
//! (e.g. test targets that have been added to the repository since the last test impact analysis sequence or test that failed
//! to execute previously).
//! These tests will be run with coverage instrumentation.
//! @note discardedTests and draftedTests may contain overlapping tests.
using ImpactAnalysisTestSequenceStartCallback = AZStd::function<void(
Client::TestRunSelection&& selectedTests,
AZStd::vector<AZStd::string>&& discardedTests,
AZStd::vector<AZStd::string>&& draftedTests)>;
//! Callback for a test sequence using test impact analysis.
//! @param selectedTests The tests that have been selected for this run by test impact analysis.
//! @param discardedTests The tests that have been rejected for this run by test impact analysis.
//! These tests will not be run without coverage instrumentation unless there is an entry in the draftedTests list.
//! @param draftedTests The tests that have been drafted in for this run due to requirements outside of test impact analysis
//! (e.g. test targets that have been added to the repository since the last test impact analysis sequence or test that failed
//! to execute previously).
//! @note discardedTests and draftedTests may contain overlapping tests.
using SafeImpactAnalysisTestSequenceStartCallback = AZStd::function<void(
Client::TestRunSelection&& selectedTests,
Client::TestRunSelection&& discardedTests,
AZStd::vector<AZStd::string>&& draftedTests)>;
//! Callback for end of a test sequence.
//! @param failureReport The test runs that failed for any reason during this sequence.
//! @param duration The total duration of this test sequence.
using TestSequenceCompleteCallback = AZStd::function<void(
Client::SequenceFailure&& failureReport,
AZStd::chrono::milliseconds duration)>;
//! Callback for end of a test impact analysis test sequence.
//! @param selectedFailureReport The selected test runs that failed for any reason during this sequence.
//! @param discardedFailureReport The discarded test runs that failed for any reason during this sequence.
//! @param duration The total duration of this test sequence.
using SafeTestSequenceCompleteCallback = AZStd::function<void(
Client::SequenceFailure&& selectedFailureReport,
Client::SequenceFailure&& discardedFailureReport,
AZStd::chrono::milliseconds selectedDuration,
AZStd::chrono::milliseconds discardedDuration)>;
//! Callback for test runs that have completed for any reason.
//! @param selectedTests The test that has completed.
using TestRunCompleteCallback = AZStd::function<void(Client::TestRun&& selectedTests)>;
//! The API exposed to the client responsible for all test runs and persistent data management.
class Runtime
{
public:
//! Constructs a runtime with the specified configuration and policies.
//! @param config The configuration used for this runtime instance.
//! @param suiteFilter The test suite for which the coverage data and test selection will draw from.
//! @param executionFailurePolicy Determines how to handle test targets that fail to execute.
//! @param executionFailureDraftingPolicy Determines how test targets that previously failed to execute are drafted into subsequent test sequences.
//! @param testFailurePolicy Determines how to handle test targets that report test failures.
//! @param integrationFailurePolicy Determines how to handle instances where the build system model and/or test impact analysis data is compromised.
//! @param testShardingPolicy Determines how to handle test targets that have opted in to test sharding.
Runtime(
RuntimeConfig&& config,
SuiteType suiteFilter,
Policy::ExecutionFailure executionFailurePolicy,
Policy::FailedTestCoverage failedTestCoveragePolicy,
Policy::TestFailure testFailurePolicy,
Policy::IntegrityFailure integrationFailurePolicy,
Policy::TestSharding testShardingPolicy,
Policy::TargetOutputCapture targetOutputCapture,
AZStd::optional<size_t> maxConcurrency = AZStd::nullopt);
~Runtime();
//! Runs a test sequence where all tests with a matching suite in the suite filter and also not on the excluded list are selected.
//! @param testTargetTimeout The maximum duration individual test targets may be in flight for (infinite if empty).
//! @param globalTimeout The maximum duration the entire test sequence may run for (infinite if empty).
//! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests.
//! @param testSequenceCompleteCallback The client function to be called after the test sequence has completed.
//! @param testRunCompleteCallback The client function to be called after an individual test run has completed.
//! @returns
TestSequenceResult RegularTestSequence(
AZStd::optional<AZStd::chrono::milliseconds> testTargetTimeout,
AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
AZStd::optional<TestSequenceStartCallback> testSequenceStartCallback,
AZStd::optional<TestSequenceCompleteCallback> testSequenceCompleteCallback,
AZStd::optional<TestRunCompleteCallback> testRunCompleteCallback);
//! Runs a test sequence where tests are selected according to test impact analysis so long as they are not on the excluded list.
//! @param changeList The change list used to determine the tests to select.
//! @param testPrioritizationPolicy Determines how selected tests will be prioritized.
//! @param dynamicDependencyMapPolicy The policy to determine how the coverage data of produced by test sequences is used to update the dynamic dependency map.
//! @param testTargetTimeout The maximum duration individual test targets may be in flight for (infinite if empty).
//! @param globalTimeout The maximum duration the entire test sequence may run for (infinite if empty).
//! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests.
//! @param testSequenceCompleteCallback The client function to be called after the test sequence has completed.
//! @param testRunCompleteCallback The client function to be called after an individual test run has completed.
//! @returns
TestSequenceResult ImpactAnalysisTestSequence(
const ChangeList& changeList,
Policy::TestPrioritization testPrioritizationPolicy,
Policy::DynamicDependencyMap dynamicDependencyMapPolicy,
AZStd::optional<AZStd::chrono::milliseconds> testTargetTimeout,
AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
AZStd::optional<ImpactAnalysisTestSequenceStartCallback> testSequenceStartCallback,
AZStd::optional<TestSequenceCompleteCallback> testSequenceCompleteCallback,
AZStd::optional<TestRunCompleteCallback> testRunCompleteCallback);
//! Runs a test sequence as per the ImpactAnalysisTestSequence where the tests not selected are also run (albeit without instrumentation).
//! @param changeList The change list used to determine the tests to select.
//! @param testPrioritizationPolicy Determines how selected tests will be prioritized.
//! @param testTargetTimeout The maximum duration individual test targets may be in flight for (infinite if empty).
//! @param globalTimeout The maximum duration the entire test sequence may run for (infinite if empty).
//! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests.
//! @param testSequenceCompleteCallback The client function to be called after the test sequence has completed.
//! @param testRunCompleteCallback The client function to be called after an individual test run has completed.
//! @returns
AZStd::pair<TestSequenceResult, TestSequenceResult> SafeImpactAnalysisTestSequence(
const ChangeList& changeList,
Policy::TestPrioritization testPrioritizationPolicy,
AZStd::optional<AZStd::chrono::milliseconds> testTargetTimeout,
AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
AZStd::optional<SafeImpactAnalysisTestSequenceStartCallback> testSequenceStartCallback,
AZStd::optional<SafeTestSequenceCompleteCallback> testSequenceCompleteCallback,
AZStd::optional<TestRunCompleteCallback> testRunCompleteCallback);
//! Runs all tests not on the excluded list and uses their coverage data to seed the test impact analysis data (ant existing data will be overwritten).
//! @param testTargetTimeout The maximum duration individual test targets may be in flight for (infinite if empty).
//! @param globalTimeout The maximum duration the entire test sequence may run for (infinite if empty).
//! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests.
//! @param testSequenceCompleteCallback The client function to be called after the test sequence has completed.
//! @param testRunCompleteCallback The client function to be called after an individual test run has completed.
//!
TestSequenceResult SeededTestSequence(
AZStd::optional<AZStd::chrono::milliseconds> testTargetTimeout,
AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
AZStd::optional<TestSequenceStartCallback> testSequenceStartCallback,
AZStd::optional<TestSequenceCompleteCallback> testSequenceCompleteCallback,
AZStd::optional<TestRunCompleteCallback> testRunCompleteCallback);
//! Returns true if the runtime has test impact analysis data (either preexisting or generated).
bool HasImpactAnalysisData() const;
private:
//! Updates the test enumeration cache for test targets that had sources modified by a given change list.
//! @param changeDependencyList The resolved change dependency list generated for the change list.
void EnumerateMutatedTestTargets(const ChangeDependencyList& changeDependencyList);
//! Selects the test targets covering a given change list and updates the enumeration cache of the test targets with sources
//! modified in that change list.
//! @param changeList The change list for which the covering tests and enumeration cache updates will be generated for.
//! @param testPrioritizationPolicy The test prioritization strategy to use for the selected test targets.
//! @returns The pair of selected test targets and discarded test targets.
AZStd::pair<AZStd::vector<const TestTarget*>, AZStd::vector<const TestTarget*>> SelectCoveringTestTargetsAndUpdateEnumerationCache(
const ChangeList& changeList,
Policy::TestPrioritization testPrioritizationPolicy);
//! Selects the test targets from the specified list of test targets that are not on the test target exclusion list.
//! @param testTargets The list of test targets to select from.
//! @returns The subset of test targets in the specified list that are not on the target exclude list.
AZStd::pair<AZStd::vector<const TestTarget*>, AZStd::vector<const TestTarget*>> SelectTestTargetsByExcludeList(
AZStd::vector<const TestTarget*> testTargets) const;
//! Prunes the existing coverage for the specified jobs and creates the consolidated source covering tests list from the
//! test engine instrumented run jobs.
SourceCoveringTestsList CreateSourceCoveringTestFromTestCoverages(const AZStd::vector<TestEngineInstrumentedRun>& jobs);
//! Prepares the dynamic dependency map for a seed update by clearing all existing data and deleting the file that will be serialized.
void ClearDynamicDependencyMapAndRemoveExistingFile();
//! Updates the dynamic dependency map and serializes the entire map to disk.
void UpdateAndSerializeDynamicDependencyMap(const AZStd::vector<TestEngineInstrumentedRun>& jobs);
RuntimeConfig m_config;
SuiteType m_suiteFilter;
RepoPath m_sparTIAFile;
Policy::ExecutionFailure m_executionFailurePolicy;
Policy::FailedTestCoverage m_failedTestCoveragePolicy;
Policy::TestFailure m_testFailurePolicy;
Policy::IntegrityFailure m_integrationFailurePolicy;
Policy::TestSharding m_testShardingPolicy;
Policy::TargetOutputCapture m_targetOutputCapture;
size_t m_maxConcurrency = 0;
AZStd::unique_ptr<DynamicDependencyMap> m_dynamicDependencyMap;
AZStd::unique_ptr<TestSelectorAndPrioritizer> m_testSelectorAndPrioritizer;
AZStd::unique_ptr<TestEngine> m_testEngine;
AZStd::unordered_set<const TestTarget*> m_testTargetExcludeList;
AZStd::unordered_set<const TestTarget*> m_testTargetShardList;
bool m_hasImpactAnalysisData = false;
};
} // namespace TestImpact
@@ -0,0 +1,26 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <TestImpactFramework/TestImpactException.h>
namespace TestImpact
{
//! Exception for runtime related exceptions.
class RuntimeException
: public Exception
{
public:
using Exception::Exception;
};
} // namespace TestImpact
@@ -0,0 +1,127 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <TestImpactFramework/TestImpactRuntimeException.h>
#include <AzCore/std/containers/array.h>
namespace TestImpact
{
namespace Policy
{
//! Policy for handling of test targets that fail to execute (e.g. due to the binary not being found).
//! @note Test targets that fail to execute will be tagged such that their execution can be attempted at a later date. This is
//! important as otherwise it would be erroneously assumed that they cover no sources due to having no entries in the dynamic
//! dependency map.
enum class ExecutionFailure
{
Abort, //!< Abort the test sequence and report a failure.
Continue, //!< Continue the test sequence but treat the execution failures as test failures after the run.
Ignore //!< Continue the test sequence and ignore the execution failures.
};
//! Policy for handling the coverage data of failed tests targets (both test that failed to execute and tests that ran but failed).
enum class FailedTestCoverage
{
Discard, //!< Discard the coverage data produced by the failing tests, causing them to be drafted into future test runs.
Keep //!< Keep any existing coverage data and update the coverage data for failed test targetss that produce coverage.
};
//! Policy for prioritizing selected tests.
enum class TestPrioritization
{
None, //!< Do not attempt any test prioritization.
DependencyLocality //!< Prioritize test targets according to the locality of the production targets they cover in the build dependency graph.
};
//! Policy for handling test targets that report failing tests.
enum class TestFailure
{
Abort, //!< Abort the test sequence and report the test failure.
Continue //!< Continue the test sequence and report the test failures after the run.
};
//! Policy for handling integrity failures of the dynamic dependency map and the source to target mappings.
enum class IntegrityFailure
{
Abort, //!< Abort the test sequence and report the test failure.
Continue //!< Continue the test sequence and report the test failures after the run.
};
//! Policy for updating the dynamic dependency map with the coverage data of produced by test sequences.
enum class DynamicDependencyMap
{
Discard, //!< Discard the coverage data produced by test sequences.
Update //!< Update the dynamic dependency map with the coverage data produced by test sequences.
};
//! Policy for sharding test targets that have been marked for test sharding.
enum class TestSharding
{
Never, //!< Do not shard any test targets.
Always //!< Shard all test targets that have been marked for test sharding.
};
//! Standard output capture of test target runs.
enum class TargetOutputCapture
{
None, //!< Do not capture any output.
StdOut, //!< Send captured output to standard output
File, //!< Write captured output to file.
StdOutAndFile //!< Send captured output to standard output and write to file.
};
}
//! Configuration for test targets that opt in to test sharding.
enum class ShardConfiguration
{
Never, //!< Never shard this test target.
FixtureContiguous, //!< Each shard contains contiguous fixtures of tests (safest but least optimal).
TestContiguous, //!< Each shard contains contiguous tests agnostic of fixtures.
FixtureInterleaved, //!< Fixtures of tests are interleaved across shards.
TestInterleaved //!< Tests are interlaced across shards agnostic of fixtures (fastest but prone to inter-test dependency problems).
};
//! Test suite types to select from.
enum class SuiteType : AZ::u8
{
Main = 0,
Periodic,
Sandbox
};
//! User-friendly names for the test suite types.
inline AZStd::string GetSuiteTypeName(SuiteType suiteType)
{
switch (suiteType)
{
case SuiteType::Main:
return "main";
case SuiteType::Periodic:
return "periodic";
case SuiteType::Sandbox:
return "sandbox";
default:
throw(RuntimeException("Unexpected suite type"));
}
}
//! Result of a test sequence that was run.
enum class TestSequenceResult
{
Success, //!< All tests ran with no failures.
Failure, //!< One or more tests failed and/or timed out and/or failed to launch and/or an integrity failure was encountered.
Timeout //!< The global timeout for the sequence was exceeded.
};
} // namespace TestImpact
@@ -0,0 +1,42 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <TestImpactFramework/TestImpactRepoPath.h>
#include <AzCore/std/string/string.h>
#include <AzCore/std/containers/vector.h>
namespace TestImpact
{
//! Coverage information about a particular line.
struct LineCoverage
{
size_t m_lineNumber = 0; //!< The source line number this covers.
size_t m_hitCount = 0; //!< Number of times this line was covered (zero if not covered).
};
//! Coverage information about a particular source file.
struct SourceCoverage
{
RepoPath m_path; //!< Source file path.
AZStd::vector<LineCoverage> m_coverage; //!< Source file line coverage (empty if source level coverage only).
};
//! Coverage information about a particular module (executable, shared library).
struct ModuleCoverage
{
RepoPath m_path; //!< Module path.
AZStd::vector<SourceCoverage> m_sources; //!< Sources of this module that are covered.
};
} // namespace TestImpact
@@ -0,0 +1,21 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <Artifact/Dynamic/TestImpactTestSuite.h>
namespace TestImpact
{
using TestEnumerationCase = TestCase; //!< Test case for test enumeration artifacts.
using TestEnumerationSuite = TestSuite<TestEnumerationCase>; //!< Test suite for test enumeration artifacts.
} // namespace TestImpact
@@ -0,0 +1,51 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <Artifact/Dynamic/TestImpactTestSuite.h>
#include <AzCore/std/chrono/chrono.h>
#include <AzCore/std/optional.h>
namespace TestImpact
{
//! Result of a test that was ran.
enum class TestRunResult : bool
{
Failed, //! The given test failed.
Passed //! The given test passed.
};
//! Status of test as to whether or not it was ran.
enum class TestRunStatus : bool
{
NotRun, //!< The test was not run (typically because the test run was aborted by the client or runner before the test could run).
Run //!< The test was run (see TestRunResult for the result of this test).
};
//! Test case for test run artifacts.
struct TestRunCase
: public TestCase
{
AZStd::optional<TestRunResult> m_result;
AZStd::chrono::milliseconds m_duration = AZStd::chrono::milliseconds{0}; //! Duration this test took to run.
TestRunStatus m_status = TestRunStatus::NotRun;
};
//! Test suite for test run artifacts.
struct TestRunSuite
: public TestSuite<TestRunCase>
{
AZStd::chrono::milliseconds m_duration = AZStd::chrono::milliseconds{0}; //!< Duration this test suite took to run all of its tests.
};
} // namespace TestImpact
@@ -0,0 +1,35 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <AzCore/std/containers/vector.h>
#include <AzCore/std/string/string.h>
namespace TestImpact
{
//! Artifact describing basic information about a test case.
struct TestCase
{
AZStd::string m_name;
bool m_enabled = false;
};
//! Artifact describing basic information about a test suite.
template<typename Test>
struct TestSuite
{
AZStd::string m_name;
bool m_enabled = false;
AZStd::vector<Test> m_tests;
};
} // namespace TestImpact
@@ -0,0 +1,169 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <TestImpactFramework/TestImpactRuntime.h>
#include <Artifact/Factory/TestImpactBuildTargetDescriptorFactory.h>
#include <Artifact/TestImpactArtifactException.h>
#include <AzCore/JSON/document.h>
#include <AzCore/std/string/regex.h>
namespace TestImpact
{
AutogenSources PairAutogenSources(
const AZStd::vector<RepoPath>& inputSources,
const AZStd::vector<RepoPath>& outputSources,
const AZStd::string& autogenMatcher)
{
AutogenSources autogenSources;
const auto matcherPattern = AZStd::regex(autogenMatcher);
AZStd::smatch inputMatches, outputMatches;
// This has the potential to be optimized to O(n(n-1)/2) time complexity but to be perfectly honest it's not a serious
// bottleneck right now and easier gains would be achieved by constructing build target artifacts in parallel rather than
// trying to squeeze any more juice here as each build target is independent of one and other with no shared memory
for (const auto& input : inputSources)
{
AutogenPairs autogenPairs;
autogenPairs.m_input = input.String();
const AZStd::string inputString = input.Stem().Native();
if (AZStd::regex_search(inputString, inputMatches, matcherPattern))
{
for (const auto& output : outputSources)
{
const AZStd::string outputString = output.Stem().Native();
if (AZStd::regex_search(outputString, outputMatches, matcherPattern))
{
// Note: [0] contains the whole match, [1] contains the first capture group
const auto& inputMatch = inputMatches[1];
const auto& outputMatch = outputMatches[1];
if (inputMatch == outputMatch)
{
autogenPairs.m_outputs.emplace_back(output);
}
}
}
}
if (!autogenPairs.m_outputs.empty())
{
autogenSources.emplace_back(AZStd::move(autogenPairs));
}
}
return autogenSources;
}
BuildTargetDescriptor BuildTargetDescriptorFactory(
const AZStd::string& buildTargetData,
const AZStd::vector<AZStd::string>& staticSourceExtensionIncludes,
const AZStd::vector<AZStd::string>& autogenInputExtensionIncludes,
const AZStd::string& autogenMatcher)
{
// Keys for pertinent JSON node and attribute names
constexpr const char* Keys[] =
{
"target",
"name",
"output_name",
"path",
"sources",
"static",
"input",
"output"
};
enum
{
TargetKey,
NameKey,
OutputNameKey,
PathKey,
SourcesKey,
StaticKey,
InputKey,
OutputKey
};
AZ_TestImpact_Eval(!autogenMatcher.empty(), ArtifactException, "Autogen matcher cannot be empty");
BuildTargetDescriptor buildTargetDescriptor;
rapidjson::Document buildTarget;
if (buildTarget.Parse(buildTargetData.c_str()).HasParseError())
{
throw TestImpact::ArtifactException("Could not parse build target data");
}
const auto& target = buildTarget[Keys[TargetKey]];
buildTargetDescriptor.m_buildMetaData.m_name = target[Keys[NameKey]].GetString();
buildTargetDescriptor.m_buildMetaData.m_outputName = target[Keys[OutputNameKey]].GetString();
buildTargetDescriptor.m_buildMetaData.m_path = target["path"].GetString();
AZ_TestImpact_Eval(!buildTargetDescriptor.m_buildMetaData.m_name.empty(), ArtifactException, "Target name cannot be empty");
AZ_TestImpact_Eval(
!buildTargetDescriptor.m_buildMetaData.m_outputName.empty(), ArtifactException, "Target output name cannot be empty");
AZ_TestImpact_Eval(!buildTargetDescriptor.m_buildMetaData.m_path.empty(), ArtifactException, "Target path cannot be empty");
const auto& sources = buildTarget[Keys[SourcesKey]];
const auto& staticSources = sources[Keys[StaticKey]].GetArray();
if (!staticSources.Empty())
{
buildTargetDescriptor.m_sources.m_staticSources = AZStd::vector<RepoPath>();
for (const auto& source : staticSources)
{
const RepoPath sourcePath = RepoPath(source.GetString());
if (AZStd::find(
staticSourceExtensionIncludes.begin(), staticSourceExtensionIncludes.end(), sourcePath.Extension().Native()) !=
staticSourceExtensionIncludes.end())
{
buildTargetDescriptor.m_sources.m_staticSources.emplace_back(AZStd::move(sourcePath));
}
}
}
const auto& inputSources = buildTarget[Keys[SourcesKey]][Keys[InputKey]].GetArray();
const auto& outputSources = buildTarget[Keys[SourcesKey]][Keys[OutputKey]].GetArray();
if (!inputSources.Empty() || !outputSources.Empty())
{
AZ_TestImpact_Eval(
!inputSources.Empty() && !outputSources.Empty(), ArtifactException, "Autogen malformed, input or output sources are empty");
AZStd::vector<RepoPath> inputPaths;
AZStd::vector<RepoPath> outputPaths;
inputPaths.reserve(inputSources.Size());
outputPaths.reserve(outputSources.Size());
for (const auto& source : inputSources)
{
const RepoPath sourcePath = RepoPath(source.GetString());
if (AZStd::find(
autogenInputExtensionIncludes.begin(), autogenInputExtensionIncludes.end(), sourcePath.Extension().Native()) !=
autogenInputExtensionIncludes.end())
{
inputPaths.emplace_back(AZStd::move(sourcePath));
}
}
for (const auto& source : outputSources)
{
outputPaths.emplace_back(AZStd::move(RepoPath(source.GetString())));
}
buildTargetDescriptor.m_sources.m_autogenSources = PairAutogenSources(inputPaths, outputPaths, autogenMatcher);
}
return buildTargetDescriptor;
}
} // namespace TestImpact
@@ -0,0 +1,33 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <Artifact/Static/TestImpactBuildTargetDescriptor.h>
#include <AzCore/std/containers/vector.h>
#include <AzCore/std/string/string.h>
namespace TestImpact
{
//! Constructs a build target artifact from the specified build target data.
//! @param buildTargetData The raw build target data in JSON format.
//! @param staticSourceIncludes The list of file extensions to include for static sources.
//! @param autogenInputExtentsionIncludes The list of file extensions to include for autogen input sources.
//! @param autogenMatcher The regex pattern used to match autogen input filenames with output filenames.
//! @return The constructed build target artifact.
BuildTargetDescriptor BuildTargetDescriptorFactory(
const AZStd::string& buildTargetData,
const AZStd::vector<AZStd::string>& staticSourceExtentsionIncludes,
const AZStd::vector<AZStd::string>& autogenInputExtentsionIncludes,
const AZStd::string& autogenMatcher);
} // namespace TestImpact
@@ -0,0 +1,144 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <Artifact/Factory/TestImpactModuleCoverageFactory.h>
#include <Artifact/TestImpactArtifactException.h>
#include <AzCore/XML/rapidxml.h>
#include <AzCore/std/string/conversions.h>
namespace TestImpact
{
namespace Cobertura
{
// Note: OpenCppCoverage appears to have a very liberal interpretation of the Cobertura coverage file format so consider
// this implementation to be provisional and coupled to the Windows platform and OpenCppCoverage tool
AZStd::vector<ModuleCoverage> ModuleCoveragesFactory(const AZStd::string& coverageData)
{
// Keys for pertinent XML node and attribute names
constexpr const char* Keys[] =
{
"packages",
"name",
"filename",
"coverage",
"classes",
"lines",
"line",
"number",
"hits",
"sources",
"source"
};
enum
{
PackagesKey,
NameKey,
FileNameKey,
CoverageKey,
ClassesKey,
LinesKey,
LineKey,
NumberKey,
HitsKey,
SourcesKey,
SourceKey
};
AZ_TestImpact_Eval(!coverageData.empty(), ArtifactException, "Cannot parse coverage, string is empty");
AZStd::vector<ModuleCoverage> modules;
AZStd::vector<char> rawData(coverageData.begin(), coverageData.end());
try
{
AZ::rapidxml::xml_document<> doc;
// Parse the XML doc with default flags
doc.parse<0>(rawData.data());
// Coverage
const auto coverage_node = doc.first_node(Keys[CoverageKey]);
AZ_TestImpact_Eval(coverage_node, ArtifactException, "Could not parse coverage node");
// Sources
const auto sources_node = coverage_node->first_node(Keys[SourcesKey]);
if (!sources_node)
{
return {};
}
// Source
const auto source_node = sources_node->first_node(Keys[SourceKey]);
if (!source_node)
{
return {};
}
// Root drive (this seems to be an unconventional use of the sources section by OpenCppCoverage)
const AZStd::string pathRoot = AZStd::string(source_node->value(), source_node->value() + source_node->value_size()) + "\\";
const auto packages_node = coverage_node->first_node(Keys[PackagesKey]);
if (packages_node)
{
// Modules
for (auto package_node = packages_node->first_node(); package_node; package_node = package_node->next_sibling())
{
// Module
ModuleCoverage moduleCoverage;
moduleCoverage.m_path = package_node->first_attribute(Keys[NameKey])->value();
const auto classes_node = package_node->first_node(Keys[ClassesKey]);
if (classes_node)
{
// Sources
for (auto class_node = classes_node->first_node(); class_node; class_node = class_node->next_sibling())
{
// Source
SourceCoverage sourceCoverage;
sourceCoverage.m_path = pathRoot + class_node->first_attribute(Keys[FileNameKey])->value();
const auto lines_node = class_node->first_node(Keys[LinesKey]);
if (lines_node)
{
// Lines
for (auto line_node = lines_node->first_node(); line_node; line_node = line_node->next_sibling())
{
// Line
const size_t number =
AZStd::stol(AZStd::string(line_node->first_attribute(Keys[NumberKey])->value()));
const size_t hits = AZStd::stol(AZStd::string(line_node->first_attribute(Keys[HitsKey])->value()));
sourceCoverage.m_coverage.emplace_back(LineCoverage{number, hits});
}
}
moduleCoverage.m_sources.emplace_back(AZStd::move(sourceCoverage));
}
}
modules.emplace_back(AZStd::move(moduleCoverage));
}
}
}
catch (const std::exception& e)
{
AZ_Error("ModuleCoveragesFactory", false, e.what());
throw ArtifactException(e.what());
}
catch (...)
{
throw ArtifactException("An unknown error occurred parsing the XML data");
}
return modules;
}
} // namespace Cobertura
} // namespace TestImpact
@@ -0,0 +1,26 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <Artifact/Dynamic/TestImpactCoverage.h>
namespace TestImpact
{
namespace Cobertura
{
//! Constructs a list of module coverage artifacts from the specified coverage data.
//! @param coverageData The raw coverage data in XML format.
//! @return The constructed list of module coverage artifacts.
AZStd::vector<ModuleCoverage> ModuleCoveragesFactory(const AZStd::string& coverageData);
} // namespace Cobertura
} // namespace TestImpact
@@ -0,0 +1,91 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <Artifact/Factory/TestImpactTestEnumerationSuiteFactory.h>
#include <Artifact/TestImpactArtifactException.h>
#include <AzCore/XML/rapidxml.h>
#include <AzCore/std/containers/vector.h>
namespace TestImpact
{
namespace GTest
{
AZStd::vector<TestEnumerationSuite> TestEnumerationSuitesFactory(const AZStd::string& testEnumerationData)
{
// Keys for pertinent XML node and attribute names
constexpr const char* Keys[] =
{
"testsuites",
"testsuite",
"name",
"testcase"
};
enum
{
TestSuitesKey,
TestSuiteKey,
NameKey,
TestCaseKey
};
AZ_TestImpact_Eval(!testEnumerationData.empty(), ArtifactException, "Cannot parse enumeration, string is empty");
AZStd::vector<TestEnumerationSuite> testSuites;
AZStd::vector<char> rawData(testEnumerationData.begin(), testEnumerationData.end());
try
{
AZ::rapidxml::xml_document<> doc;
// Parse the XML doc with default flags
doc.parse<0>(rawData.data());
const auto testsuites_node = doc.first_node(Keys[TestSuitesKey]);
AZ_TestImpact_Eval(testsuites_node, ArtifactException, "Could not parse enumeration, XML is invalid");
for (auto testsuite_node = testsuites_node->first_node(Keys[TestSuiteKey]); testsuite_node;
testsuite_node = testsuite_node->next_sibling())
{
const auto isEnabled = [](const AZStd::string& name)
{
return !name.starts_with("DISABLED_") && name.find("/DISABLED_") == AZStd::string::npos;
};
TestEnumerationSuite testSuite;
testSuite.m_name = testsuite_node->first_attribute(Keys[NameKey])->value();
testSuite.m_enabled = isEnabled(testSuite.m_name);
for (auto testcase_node = testsuite_node->first_node(Keys[TestCaseKey]); testcase_node;
testcase_node = testcase_node->next_sibling())
{
TestEnumerationCase testCase;
testCase.m_name = testcase_node->first_attribute(Keys[NameKey])->value();
testCase.m_enabled = isEnabled(testCase.m_name);
testSuite.m_tests.emplace_back(AZStd::move(testCase));
}
testSuites.emplace_back(AZStd::move(testSuite));
}
}
catch (const std::exception& e)
{
AZ_Error("TestEnumerationSuitesFactory", false, e.what());
throw ArtifactException(e.what());
}
catch (...)
{
throw ArtifactException("An unknown error occured parsing the XML data");
}
return testSuites;
}
} // namespace GTest
} // namespace TestImpact
@@ -0,0 +1,26 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <Artifact/Dynamic/TestImpactTestEnumerationSuite.h>
namespace TestImpact
{
namespace GTest
{
//! Constructs a list of test enumeration suite artifacts from the specified test enumeraion data.
//! @param testEnumerationData The raw test enumeration data in XML format.
//! @return The constructed list of test enumeration suite artifacts.
AZStd::vector<TestEnumerationSuite> TestEnumerationSuitesFactory(const AZStd::string& testEnumerationData);
} // namespace GTest
} // namespace TestImpact
@@ -0,0 +1,140 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <Artifact/Factory/TestImpactTestRunSuiteFactory.h>
#include <Artifact/TestImpactArtifactException.h>
#include <AzCore/XML/rapidxml.h>
#include <AzCore/std/containers/vector.h>
#include <AzCore/std/string/conversions.h>
namespace TestImpact
{
namespace GTest
{
AZStd::vector<TestRunSuite> TestRunSuitesFactory(const AZStd::string& testEnumerationData)
{
// Keys for pertinent XML node and attribute names
constexpr const char* Keys[] =
{
"testsuites",
"testsuite",
"name",
"testcase",
"status",
"run",
"notrun",
"time"
};
enum
{
TestSuitesKey,
TestSuiteKey,
NameKey,
TestCaseKey,
StatusKey,
RunKey,
NotRunKey,
DurationKey
};
AZ_TestImpact_Eval(!testEnumerationData.empty(), ArtifactException, "Cannot parse test run, string is empty");
AZStd::vector<TestRunSuite> testSuites;
AZStd::vector<char> rawData(testEnumerationData.begin(), testEnumerationData.end());
try
{
AZ::rapidxml::xml_document<> doc;
// Parse the XML doc with default flags
doc.parse<0>(rawData.data());
const auto testsuites_node = doc.first_node(Keys[TestSuitesKey]);
AZ_TestImpact_Eval(testsuites_node, ArtifactException, "Could not parse enumeration, XML is invalid");
for (auto testsuite_node = testsuites_node->first_node(Keys[TestSuiteKey]); testsuite_node;
testsuite_node = testsuite_node->next_sibling())
{
const auto isEnabled = [](const AZStd::string& name)
{
return !name.starts_with("DISABLED_") && name.find("/DISABLED_") == AZStd::string::npos;
};
const auto getDuration = [&Keys](const AZ::rapidxml::xml_node<>* node)
{
const AZStd::string duration = node->first_attribute(Keys[DurationKey])->value();
return AZStd::chrono::milliseconds(AZStd::stof(duration) * 1000.f);
};
TestRunSuite testSuite;
testSuite.m_name = testsuite_node->first_attribute(Keys[NameKey])->value();
testSuite.m_enabled = isEnabled(testSuite.m_name);
testSuite.m_duration = getDuration(testsuite_node);
for (auto testcase_node = testsuite_node->first_node(Keys[TestCaseKey]); testcase_node;
testcase_node = testcase_node->next_sibling())
{
const auto getStatus = [&Keys](const AZ::rapidxml::xml_node<>* node)
{
const AZStd::string status = node->first_attribute(Keys[StatusKey])->value();
if (status == Keys[RunKey])
{
return TestRunStatus::Run;
}
else if (status == Keys[NotRunKey])
{
return TestRunStatus::NotRun;
}
throw ArtifactException(AZStd::string::format("Unexpected run status: %s", status.c_str()));
};
const auto getResult = [](const AZ::rapidxml::xml_node<>* node)
{
for (auto child_node = node->first_node("failure"); child_node; child_node = child_node->next_sibling())
{
return TestRunResult::Failed;
}
return TestRunResult::Passed;
};
TestRunCase testCase;
testCase.m_name = testcase_node->first_attribute(Keys[NameKey])->value();
testCase.m_enabled = isEnabled(testCase.m_name);
testCase.m_duration = getDuration(testcase_node);
testCase.m_status = getStatus(testcase_node);
if (testCase.m_status == TestRunStatus::Run)
{
testCase.m_result = getResult(testcase_node);
}
testSuite.m_tests.emplace_back(AZStd::move(testCase));
}
testSuites.emplace_back(AZStd::move(testSuite));
}
}
catch (const std::exception& e)
{
AZ_Error("TestRunSuitesFactory", false, e.what());
throw ArtifactException(e.what());
}
catch (...)
{
throw ArtifactException("An unknown error occurred parsing the XML data");
}
return testSuites;
}
} // namespace GTest
} // namespace TestImpact
@@ -0,0 +1,26 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <Artifact/Dynamic/TestImpactTestRunSuite.h>
namespace TestImpact
{
namespace GTest
{
//! Constructs a list of test run suite artifacts from the specified test run data.
//! @param testRunData The raw test run data in XML format.
//! @return The constructed list of test run suite artifacts.
AZStd::vector<TestRunSuite> TestRunSuitesFactory(const AZStd::string& testRunData);
} // namespace GTest
} // namespace TestImpact
@@ -0,0 +1,25 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <Artifact/Static/TestImpactTestTargetMetaArtifact.h>
#include <AzCore/std/containers/vector.h>
namespace TestImpact
{
//! Constructs a list of test target meta-data artifacts from the specified master test list data.
//! @param masterTestListData The raw master test list data in JSON format.
//! @return The constructed list of test target meta-data artifacts.
TestTargetMetas TestTargetMetaMapFactory(const AZStd::string& masterTestListData);
} // namespace TestImpact
@@ -0,0 +1,105 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <Artifact/Factory/TestImpactTestTargetMetaMapFactory.h>
#include <Artifact/TestImpactArtifactException.h>
#include <AzCore/JSON/document.h>
#include <cstring>
namespace TestImpact
{
TestTargetMetaMap TestTargetMetaMapFactory(const AZStd::string& masterTestListData, SuiteType suiteType)
{
// Keys for pertinent JSON node and attribute names
constexpr const char* Keys[] =
{
"google",
"test",
"tests",
"suites",
"suite",
"launch_method",
"test_runner",
"stand_alone",
"name",
"command",
"timeout"
};
enum
{
GoogleKey,
TestKey,
TestsKey,
TestSuitesKey,
SuiteKey,
LaunchMethodKey,
TestRunnerKey,
StandAloneKey,
NameKey,
CommandKey,
TimeoutKey
};
AZ_TestImpact_Eval(!masterTestListData.empty(), ArtifactException, "test meta-data cannot be empty");
TestTargetMetaMap testMetas;
rapidjson::Document masterTestList;
if (masterTestList.Parse(masterTestListData.c_str()).HasParseError())
{
throw TestImpact::ArtifactException("Could not parse test meta-data");
}
const auto tests = masterTestList[Keys[GoogleKey]][Keys[TestKey]][Keys[TestsKey]].GetArray();
for (const auto& test : tests)
{
TestTargetMeta testMeta;
const auto testSuites = test[Keys[TestSuitesKey]].GetArray();
for (const auto& suite : testSuites)
{
// Check to see if this test target has the suite we're looking for
if (const auto suiteName = suite[Keys[SuiteKey]].GetString();
strcmp(GetSuiteTypeName(suiteType).c_str(), suiteName) == 0)
{
testMeta.m_suite = suiteName;
testMeta.m_customArgs = suite[Keys[CommandKey]].GetString();
testMeta.m_timeout = AZStd::chrono::seconds{ suite[Keys[TimeoutKey]].GetUint() };
if (const auto buildTypeString = test[Keys[LaunchMethodKey]].GetString(); strcmp(buildTypeString, Keys[TestRunnerKey]) == 0)
{
testMeta.m_launchMethod = LaunchMethod::TestRunner;
}
else if (strcmp(buildTypeString, Keys[StandAloneKey]) == 0)
{
testMeta.m_launchMethod = LaunchMethod::StandAlone;
}
else
{
throw(ArtifactException("Unexpected test build type"));
}
AZStd::string name = test[Keys[NameKey]].GetString();
AZ_TestImpact_Eval(!name.empty(), ArtifactException, "Test name field cannot be empty");
testMetas.emplace(AZStd::move(name), AZStd::move(testMeta));
break;
}
}
}
// If there's no tests in the repo then something is seriously wrong
AZ_TestImpact_Eval(!testMetas.empty(), ArtifactException, "No tests were found in the repository");
return testMetas;
}
} // namespace TestImpact
@@ -0,0 +1,27 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <TestImpactFramework/TestImpactTestSequence.h>
#include <Artifact/Static/TestImpactTestTargetMeta.h>
#include <AzCore/std/containers/vector.h>
namespace TestImpact
{
//! Constructs a list of test target meta-data artifacts of the specified suite type from the specified master test list data.
//! @param masterTestListData The raw master test list data in JSON format.
//! @param suiteType The suite type to select the target meta-data artifacts from.
//! @return The constructed list of test target meta-data artifacts.
TestTargetMetaMap TestTargetMetaMapFactory(const AZStd::string& masterTestListData, SuiteType suiteType);
} // namespace TestImpact
@@ -0,0 +1,22 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <Artifact/Static/TestImpactBuildTargetDescriptor.h>
namespace TestImpact
{
BuildTargetDescriptor::BuildTargetDescriptor(BuildMetaData&& buildMetaData, TargetSources&& sources)
: m_buildMetaData(AZStd::move(buildMetaData))
, m_sources(AZStd::move(sources))
{
}
} // namespace TestImpact
@@ -0,0 +1,56 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <TestImpactFramework/TestImpactRepoPath.h>
#include <AzCore/std/containers/vector.h>
#include <AzCore/std/optional.h>
#include <AzCore/std/string/string.h>
namespace TestImpact
{
//! Pairing between a given autogen input source and the generated output source(s).
struct AutogenPairs
{
RepoPath m_input;
AZStd::vector<RepoPath> m_outputs;
};
using AutogenSources = AZStd::vector<AutogenPairs>;
//! Representation of a given built target's source list.
struct TargetSources
{
AZStd::vector<RepoPath> m_staticSources; //!< Source files used to build this target (if any).
AutogenSources m_autogenSources; //!< Autogen source files (if any).
};
//! Representation of a given build target's basic build infotmation.
struct BuildMetaData
{
AZStd::string m_name; //!< Build target name.
AZStd::string m_outputName; //!< Output name (sans extension) of build target binary.
RepoPath m_path; //!< Path to build target location in source tree (relative to repository root).
};
//! Artifact produced by the build system for each build target. Contains source and output information about said targets.
struct BuildTargetDescriptor
{
BuildTargetDescriptor() = default;
BuildTargetDescriptor(BuildMetaData&& buildMetaData, TargetSources&& sources);
BuildMetaData m_buildMetaData;
TargetSources m_sources;
};
} // namespace TestImpact
@@ -0,0 +1,27 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <AzCore/std/string/string.h>
#include <AzCore/std/containers/vector.h>
namespace TestImpact
{
//! Raw representation of the dependency graph for a given build target.
struct DependencyGraphData
{
AZStd::string m_root; //!< The build target this dependency graph is for.
AZStd::vector<AZStd::string> m_vertices; //!< The depender/depending built targets in this graph.
AZStd::vector<AZStd::pair<AZStd::string, AZStd::string>> m_edges; //!< The dependency connectivity of the build targets in this graph.
};
} // namespace TestImpact
@@ -0,0 +1,21 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <Artifact/Static/TestImpactProductionTargetDescriptor.h>
namespace TestImpact
{
ProductionTargetDescriptor::ProductionTargetDescriptor(BuildTargetDescriptor&& buildTargetDescriptor)
: BuildTargetDescriptor(AZStd::move(buildTargetDescriptor))
{
}
} // namespace TestImpact
@@ -0,0 +1,25 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <Artifact/Static/TestImpactBuildTargetDescriptor.h>
namespace TestImpact
{
//! Artifact produced by the target artifact compiler that represents a production build target in the repository.
struct ProductionTargetDescriptor
: public BuildTargetDescriptor
{
ProductionTargetDescriptor(BuildTargetDescriptor&& buildTarget);
};
} // namespace TestImpact
@@ -0,0 +1,45 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <Artifact/Static/TestImpactTargetDescriptorCompiler.h>
#include <Artifact/TestImpactArtifactException.h>
#include <AzCore/std/containers/unordered_map.h>
namespace TestImpact
{
AZStd::tuple<AZStd::vector<ProductionTargetDescriptor>, AZStd::vector<TestTargetDescriptor>> CompileTargetDescriptors(
AZStd::vector<BuildTargetDescriptor>&& buildTargets, TestTargetMetaMap&& testTargetMetaMap)
{
AZ_TestImpact_Eval(!buildTargets.empty(), ArtifactException, "Build target descriptor list cannot be null");
AZ_TestImpact_Eval(!testTargetMetaMap.empty(), ArtifactException, "Test target meta map cannot be null");
AZStd::tuple<AZStd::vector<ProductionTargetDescriptor>, AZStd::vector<TestTargetDescriptor>> outputTargets;
auto& [productionTargets, testTargets] = outputTargets;
for (auto&& buildTarget : buildTargets)
{
// If this build target has an associated test artifact then it is a test target, otherwise it is a production target
if (auto&& testTargetMeta = testTargetMetaMap.find(buildTarget.m_buildMetaData.m_name);
testTargetMeta != testTargetMetaMap.end())
{
testTargets.emplace_back(TestTargetDescriptor(AZStd::move(buildTarget), AZStd::move(testTargetMeta->second)));
}
else
{
productionTargets.emplace_back(ProductionTargetDescriptor(AZStd::move(buildTarget)));
}
}
return outputTargets;
}
} // namespace TestImpact
@@ -0,0 +1,31 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <Artifact/Static/TestImpactProductionTargetDescriptor.h>
#include <Artifact/Static/TestImpactTestTargetDescriptor.h>
#include <Artifact/Static/TestImpactTestTargetMeta.h>
#include <AzCore/std/containers/vector.h>
#include <AzCore/std/tuple.h>
namespace TestImpact
{
//! Compiles the production target artifacts and test target artifactss from the supplied build target artifacts and test target meta
//! map artifact.
//! @param buildTargets The list of build target artifacts to be sorted into production and test artifact types.
//! @param testTargetMetaMap The map of test target meta artifacts containing the additional meta-data about each test target.
//! @return A tuple containing the production artifacts and test artifacts.
AZStd::tuple<AZStd::vector<ProductionTargetDescriptor>, AZStd::vector<TestTargetDescriptor>> CompileTargetDescriptors(
AZStd::vector<BuildTargetDescriptor>&& buildTargets, TestTargetMetaMap&& testTargetMetaMap);
} // namespace TestImpact
@@ -0,0 +1,22 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <Artifact/Static/TestImpactTestTargetDescriptor.h>
namespace TestImpact
{
TestTargetDescriptor::TestTargetDescriptor(BuildTargetDescriptor&& buildTarget, TestTargetMeta&& testTargetMeta)
: BuildTargetDescriptor(AZStd::move(buildTarget))
, m_testMetaData(AZStd::move(testTargetMeta))
{
}
} // namespace TestImpact
@@ -0,0 +1,28 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <Artifact/Static/TestImpactBuildTargetDescriptor.h>
#include <Artifact/Static/TestImpactTestTargetMeta.h>
namespace TestImpact
{
//! Artifact produced by the target artifact compiler that represents a test build target in the repository.
struct TestTargetDescriptor
: public BuildTargetDescriptor
{
TestTargetDescriptor(BuildTargetDescriptor&& buildTarget, TestTargetMeta&& testTargetMeta);
TestTargetMeta m_testMetaData;
};
} // namespace TestImpact
@@ -0,0 +1,39 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <AzCore/std/containers/unordered_map.h>
#include <AzCore/std/string/string.h>
#include <AzCore/std/chrono/chrono.h>
namespace TestImpact
{
//! Method used to launch the test target.
enum class LaunchMethod : bool
{
TestRunner, //!< Target is launched through a separate test runner binary.
StandAlone //!< Target is launched directly by itself.
};
//! Artifact produced by the build system for each test target containing the additional meta-data about the test.
struct TestTargetMeta
{
AZStd::string m_suite;
AZStd::string m_customArgs;
AZStd::chrono::milliseconds m_timeout = AZStd::chrono::milliseconds{ 0 };
LaunchMethod m_launchMethod = LaunchMethod::TestRunner;
};
//! Map between test target name and test target meta-data.
using TestTargetMetaMap = AZStd::unordered_map<AZStd::string, TestTargetMeta>;
} // namespace TestImpact
@@ -0,0 +1,26 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <TestImpactFramework/TestImpactException.h>
namespace TestImpact
{
//! Exception for artifacts and artifact parsing operations.
class ArtifactException
: public Exception
{
public:
using Exception::Exception;
};
} // namespace TestImpact
@@ -0,0 +1,41 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <Dependency/TestImpactChangeDependencyList.h>
namespace TestImpact
{
ChangeDependencyList::ChangeDependencyList(
AZStd::vector<SourceDependency>&& createSourceDependencies,
AZStd::vector<SourceDependency>&& updateSourceDependencies,
AZStd::vector<SourceDependency>&& deleteSourceDependencies)
: m_createSourceDependencies(AZStd::move(createSourceDependencies))
, m_updateSourceDependencies(AZStd::move(updateSourceDependencies))
, m_deleteSourceDependencies(AZStd::move(deleteSourceDependencies))
{
}
const AZStd::vector<SourceDependency>& ChangeDependencyList::GetCreateSourceDependencies() const
{
return m_createSourceDependencies;
}
const AZStd::vector<SourceDependency>& ChangeDependencyList::GetUpdateSourceDependencies() const
{
return m_updateSourceDependencies;
}
const AZStd::vector<SourceDependency>& ChangeDependencyList::GetDeleteSourceDependencies() const
{
return m_deleteSourceDependencies;
}
} // namespace TestImpact
@@ -0,0 +1,41 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <Dependency/TestImpactSourceDependency.h>
namespace TestImpact
{
//! Representation of a change list where all CRUD sources have been resolved to source dependencies from the dynamic dependency map.
class ChangeDependencyList
{
public:
ChangeDependencyList(
AZStd::vector<SourceDependency>&& createSourceDependencies,
AZStd::vector<SourceDependency>&& updateSourceDependencies,
AZStd::vector<SourceDependency>&& deleteSourceDependencies);
//! Gets the sources dependencies of the created source files from the change list.
const AZStd::vector<SourceDependency>& GetCreateSourceDependencies() const;
//! Gets the sources dependencies of the updated source files from the change list.
const AZStd::vector<SourceDependency>& GetUpdateSourceDependencies() const;
//! Gets the sources dependencies of the deleted source files from the change list.
const AZStd::vector<SourceDependency>& GetDeleteSourceDependencies() const;
private:
AZStd::vector<SourceDependency> m_createSourceDependencies;
AZStd::vector<SourceDependency> m_updateSourceDependencies;
AZStd::vector<SourceDependency> m_deleteSourceDependencies;
};
} // namespace TestImpact
@@ -0,0 +1,26 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <TestImpactFramework/TestImpactException.h>
namespace TestImpact
{
//! Exception for dependency related operations.
class DependencyException
: public Exception
{
public:
using Exception::Exception;
};
} // namespace TestImpact
@@ -0,0 +1,507 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <Dependency/TestImpactDynamicDependencyMap.h>
#include <Dependency/TestImpactDependencyException.h>
namespace TestImpact
{
DynamicDependencyMap::DynamicDependencyMap(
AZStd::vector<ProductionTargetDescriptor>&& productionTargetDescriptors,
AZStd::vector<TestTargetDescriptor>&& testTargetDescriptors)
: m_productionTargets(AZStd::move(productionTargetDescriptors))
, m_testTargets(AZStd::move(testTargetDescriptors))
{
const auto mapBuildTargetSources = [this](const auto* target)
{
for (const auto& source : target->GetSources().m_staticSources)
{
if (auto mapping = m_sourceDependencyMap.find(source.String());
mapping != m_sourceDependencyMap.end())
{
// This is an existing entry in the dependency map so update the parent build targets with this target
mapping->second.m_parentTargets.insert(target);
}
else
{
// This is a new entry on the dependency map so create an entry with this parent target and no covering targets
m_sourceDependencyMap.emplace(source.String(), DependencyData{ {target}, {} });
}
}
// Populate the autogen input to output mapping with any autogen sources
for (const auto& autogen : target->GetSources().m_autogenSources)
{
for (const auto& output : autogen.m_outputs)
{
m_autogenInputToOutputMap[autogen.m_input.String()].push_back(output.String());
}
}
};
for (const auto& target : m_productionTargets.GetTargets())
{
mapBuildTargetSources(&target);
}
for (const auto& target : m_testTargets.GetTargets())
{
mapBuildTargetSources(&target);
m_testTargetSourceCoverage[&target] = {};
}
}
size_t DynamicDependencyMap::GetNumTargets() const
{
return m_productionTargets.GetNumTargets() + m_testTargets.GetNumTargets();
}
size_t DynamicDependencyMap::GetNumSources() const
{
return m_sourceDependencyMap.size();
}
const BuildTarget* DynamicDependencyMap::GetBuildTarget(const AZStd::string& name) const
{
const BuildTarget* buildTarget = nullptr;
AZStd::visit([&buildTarget](auto&& target)
{
if constexpr (IsProductionTarget<decltype(target)> || IsTestTarget<decltype(target)>)
{
buildTarget = target;
}
}, GetTarget(name));
return buildTarget;
}
const BuildTarget* DynamicDependencyMap::GetBuildTargetOrThrow(const AZStd::string& name) const
{
const BuildTarget* buildTarget = nullptr;
AZStd::visit([&buildTarget](auto&& target)
{
if constexpr (IsProductionTarget<decltype(target)> || IsTestTarget<decltype(target)>)
{
buildTarget = target;
}
}, GetTargetOrThrow(name));
return buildTarget;
}
OptionalTarget DynamicDependencyMap::GetTarget(const AZStd::string& name) const
{
if (const auto testTarget = m_testTargets.GetTarget(name);
testTarget != nullptr)
{
return testTarget;
}
else if (auto productionTarget = m_productionTargets.GetTarget(name);
productionTarget != nullptr)
{
return productionTarget;
}
return AZStd::monostate{};
}
Target DynamicDependencyMap::GetTargetOrThrow(const AZStd::string& name) const
{
Target buildTarget;
AZStd::visit([&buildTarget, &name](auto&& target)
{
if constexpr (IsProductionTarget<decltype(target)> || IsTestTarget<decltype(target)>)
{
buildTarget = target;
}
else
{
throw(TargetException(AZStd::string::format("Couldn't find target %s", name.c_str()).c_str()));
}
}, GetTarget(name));
return buildTarget;
}
void DynamicDependencyMap::ReplaceSourceCoverageInternal(const SourceCoveringTestsList& sourceCoverageDelta, bool pruneIfNoParentsOrCoverage)
{
AZStd::vector<AZStd::string> killList;
for (const auto& sourceCoverage : sourceCoverageDelta.GetCoverage())
{
// Autogen input files are not compiled sources and thus supplying coverage data for them makes no sense
AZ_TestImpact_Eval(
m_autogenInputToOutputMap.find(sourceCoverage.GetPath().String()) == m_autogenInputToOutputMap.end(),
DependencyException, AZStd::string::format("Couldn't replace source coverage for %s, source file is an autogen input file",
sourceCoverage.GetPath().c_str()).c_str());
auto [sourceDependencyIt, inserted] = m_sourceDependencyMap.insert(sourceCoverage.GetPath().String());
auto& [source, sourceDependency] = *sourceDependencyIt;
// Before we can replace the coverage for this source dependency, we must:
// 1. Remove the source from the test target covering sources map
// 2. Prune the covered targets for the parent test target(s) of this source dependency
// 3. Clear any existing coverage for the delta
// 1.
for (const auto& testTarget : sourceDependency.m_coveringTestTargets)
{
if (auto coveringTestTargetIt = m_testTargetSourceCoverage.find(testTarget);
coveringTestTargetIt != m_testTargetSourceCoverage.end())
{
coveringTestTargetIt->second.erase(source);
}
}
// 2.
// This step is prohibitively expensive as it requires iterating over all of the sources of the build targets covered by
// the parent test targets of this source dependency to ensure that this is in fact the last source being cleared and thus
// it can be determined that the parent test target is no longer covering the given build target
//
// The implications of this are that multiple calls to the test selector and priritizor's SelectTestTargets method will end
// up pulling in more test targets than needed for newly-created production sources until the next time the dynamic dependency
// map reconstructed, however until this use case materializes the implications described will not be addressed
// 3.
sourceDependency.m_coveringTestTargets.clear();
// Update the dependency with any new coverage data
for (const auto& unresolvedTestTarget : sourceCoverage.GetCoveringTestTargets())
{
if (const TestTarget* testTarget = m_testTargets.GetTarget(unresolvedTestTarget);
testTarget)
{
// Source to covering test target mapping
sourceDependency.m_coveringTestTargets.insert(testTarget);
// Add the source to the test target covering sources map
m_testTargetSourceCoverage[testTarget].insert(source);
// Build target to covering test target mapping
for (const auto& parentTarget : sourceDependency.m_parentTargets)
{
m_buildTargetCoverage[parentTarget.GetBuildTarget()].insert(testTarget);
}
}
else
{
AZ_Warning("ReplaceSourceCoverage", false, AZStd::string::format("Test target %s exists in the coverage data "
"but has since been removed from the build system", unresolvedTestTarget.c_str()).c_str());
}
}
// If the new coverage data results in a parentless and coverageless entry, consider it a dead entry and remove accordingly
if (sourceDependency.m_coveringTestTargets.empty() && sourceDependency.m_parentTargets.empty() && pruneIfNoParentsOrCoverage)
{
m_sourceDependencyMap.erase(sourceDependencyIt);
}
}
}
void DynamicDependencyMap::ReplaceSourceCoverage(const SourceCoveringTestsList& sourceCoverageDelta)
{
ReplaceSourceCoverageInternal(sourceCoverageDelta, true);
}
void DynamicDependencyMap::ClearSourceCoverage(const AZStd::vector<RepoPath>& paths)
{
for (const auto& path : paths)
{
if (const auto outputSources = m_autogenInputToOutputMap.find(path.String());
outputSources != m_autogenInputToOutputMap.end())
{
// Clearing the coverage data of an autogen input source instead clears the coverage data of its output sources
for (const auto& outputSource : outputSources->second)
{
ReplaceSourceCoverage(SourceCoveringTestsList(AZStd::vector<SourceCoveringTests>{ SourceCoveringTests(RepoPath(outputSource)) }));
}
}
else
{
ReplaceSourceCoverage(SourceCoveringTestsList(AZStd::vector<SourceCoveringTests>{ SourceCoveringTests(RepoPath(path)) }));
}
}
}
void DynamicDependencyMap::ClearAllSourceCoverage()
{
for (auto it = m_sourceDependencyMap.begin(); it != m_sourceDependencyMap.end(); ++it)
{
const auto& [path, coverage] = *it;
ReplaceSourceCoverageInternal(SourceCoveringTestsList(AZStd::vector<SourceCoveringTests>{ SourceCoveringTests(RepoPath(path)) }), false);
if (coverage.m_coveringTestTargets.empty() && coverage.m_parentTargets.empty())
{
it = m_sourceDependencyMap.erase(it);
}
}
}
const ProductionTargetList& DynamicDependencyMap::GetProductionTargetList() const
{
return m_productionTargets;
}
const TestTargetList& DynamicDependencyMap::GetTestTargetList() const
{
return m_testTargets;
}
AZStd::vector<const TestTarget*> DynamicDependencyMap::GetCoveringTestTargetsForProductionTarget(const ProductionTarget& productionTarget) const
{
AZStd::vector<const TestTarget*> coveringTestTargets;
if (const auto coverage = m_buildTargetCoverage.find(&productionTarget);
coverage != m_buildTargetCoverage.end())
{
coveringTestTargets.reserve(coverage->second.size());
AZStd::copy(coverage->second.begin(), coverage->second.end(), AZStd::back_inserter(coveringTestTargets));
}
return coveringTestTargets;
}
AZStd::optional<SourceDependency> DynamicDependencyMap::GetSourceDependency(const RepoPath& path) const
{
AZStd::unordered_set<ParentTarget> parentTargets;
AZStd::unordered_set<const TestTarget*> coveringTestTargets;
const auto getSourceDependency = [&parentTargets, &coveringTestTargets, this](const AZStd::string& path)
{
const auto sourceDependency = m_sourceDependencyMap.find(path);
if (sourceDependency != m_sourceDependencyMap.end())
{
for (const auto& parentTarget : sourceDependency->second.m_parentTargets)
{
parentTargets.insert(parentTarget);
}
for (const auto& testTarget : sourceDependency->second.m_coveringTestTargets)
{
coveringTestTargets.insert(testTarget);
}
}
};
if (const auto outputSources = m_autogenInputToOutputMap.find(path.String()); outputSources != m_autogenInputToOutputMap.end())
{
// Consolidate the parentage and coverage of each of the autogen input file's generated output files
for (const auto& outputSource : outputSources->second)
{
getSourceDependency(outputSource);
}
}
else
{
getSourceDependency(path.String());
}
if (!parentTargets.empty() || !coveringTestTargets.empty())
{
return SourceDependency(path, DependencyData{ AZStd::move(parentTargets), AZStd::move(coveringTestTargets) });
}
return AZStd::nullopt;
}
SourceDependency DynamicDependencyMap::GetSourceDependencyOrThrow(const RepoPath& path) const
{
auto sourceDependency = GetSourceDependency(path);
AZ_TestImpact_Eval(sourceDependency.has_value(), DependencyException, AZStd::string::format("Couldn't find source %s", path.c_str()).c_str());
return sourceDependency.value();
}
SourceCoveringTestsList DynamicDependencyMap::ExportSourceCoverage() const
{
AZStd::vector<SourceCoveringTests> coverage;
for (const auto& [path, dependency] : m_sourceDependencyMap)
{
AZStd::vector<AZStd::string> souceCoveringTests;
for (const auto& testTarget : dependency.m_coveringTestTargets)
{
souceCoveringTests.push_back(testTarget->GetName());
}
coverage.push_back(SourceCoveringTests(RepoPath(path), AZStd::move(souceCoveringTests)));
}
return SourceCoveringTestsList(AZStd::move(coverage));
}
AZStd::vector<AZStd::string> DynamicDependencyMap::GetOrphanSourceFiles() const
{
AZStd::vector<AZStd::string> orphans;
for (const auto& [source, dependency] : m_sourceDependencyMap)
{
if (dependency.m_parentTargets.empty())
{
orphans.push_back(source);
}
}
return orphans;
}
ChangeDependencyList DynamicDependencyMap::ApplyAndResoveChangeList(const ChangeList& changeList)
{
AZStd::vector<SourceDependency> createDependencies;
AZStd::vector<SourceDependency> updateDependencies;
AZStd::vector<SourceDependency> deleteDependencies;
// Keep track of the coverage to delete as a post step rather than deleting it in situ so that erroneous change lists
// do not corrupt the dynamic dependency map
AZStd::vector<RepoPath> coverageToDelete;
// Create operations
for (const auto& createdFile : changeList.m_createdFiles)
{
auto sourceDependency = GetSourceDependency(createdFile);
if (sourceDependency.has_value())
{
if (sourceDependency->GetNumCoveringTestTargets())
{
const AZStd::string msg = AZStd::string::format("The newly-created file %s belongs to a build target yet "
"still has coverage data in the source covering test list implying that a delete CRUD operation has been "
"missed, thus the integrity of the source covering test list has been compromised", createdFile.c_str());
AZ_Error("File Creation", false, msg.c_str());
throw DependencyException(msg);
}
if (sourceDependency->GetNumParentTargets())
{
createDependencies.emplace_back(AZStd::move(*sourceDependency));
}
}
}
// Update operations
for (const auto& updatedFile : changeList.m_updatedFiles)
{
auto sourceDependency = GetSourceDependency(updatedFile);
if (sourceDependency.has_value())
{
if (sourceDependency->GetNumParentTargets())
{
updateDependencies.emplace_back(AZStd::move(*sourceDependency));
}
else
{
if (sourceDependency->GetNumCoveringTestTargets())
{
AZ_Printf(
"File Update", AZStd::string::format("Source file '%s' is potentially an orphan (used by build targets "
"without explicitly being added to the build system, e.g. an include directive pulling in a header from the "
"repository). Running the covering tests for this file with instrumentation will confirm whether or nor this "
"is the case.\n", updatedFile.c_str()).c_str());
updateDependencies.emplace_back(AZStd::move(*sourceDependency));
coverageToDelete.push_back(updatedFile);
}
}
}
}
// Delete operations
for (const auto& deletedFile : changeList.m_deletedFiles)
{
auto sourceDependency = GetSourceDependency(deletedFile);
if (!sourceDependency.has_value())
{
continue;
}
if (sourceDependency->GetNumParentTargets())
{
if (sourceDependency->GetNumCoveringTestTargets())
{
const AZStd::string msg = AZStd::string::format("The deleted file %s still belongs to a build target and still "
"has coverage data in the source covering test list, implying that the integrity of both the source to target "
"mappings and the source covering test list has been compromised", deletedFile.c_str());
AZ_Error("File Delete", false, msg.c_str());
throw DependencyException(msg);
}
else
{
const AZStd::string msg = AZStd::string::format("The deleted file %s still belongs to a build target implying "
"that the integrity of the source to target mappings has been compromised", deletedFile.c_str());
AZ_Error("File Delete", false, msg.c_str());
throw DependencyException(msg);
}
}
else
{
if (sourceDependency->GetNumCoveringTestTargets())
{
deleteDependencies.emplace_back(AZStd::move(*sourceDependency));
coverageToDelete.push_back(deletedFile);
}
}
}
if (!coverageToDelete.empty())
{
ClearSourceCoverage(coverageToDelete);
}
return ChangeDependencyList(AZStd::move(createDependencies), AZStd::move(updateDependencies), AZStd::move(deleteDependencies));
}
void DynamicDependencyMap::RemoveTestTargetFromSourceCoverage(const TestTarget* testTarget)
{
if (const auto& it = m_testTargetSourceCoverage.find(testTarget);
it != m_testTargetSourceCoverage.end())
{
for (const auto& source : it->second)
{
const auto sourceDependency = m_sourceDependencyMap.find(source);
AZ_TestImpact_Eval(
sourceDependency != m_sourceDependencyMap.end(),
DependencyException,
AZStd::string::format("Test target '%s' has covering source '%s' yet cannot be found in the dependency map",
testTarget->GetName().c_str(), source.c_str()));
sourceDependency->second.m_coveringTestTargets.erase(testTarget);
}
m_testTargetSourceCoverage.erase(testTarget);
}
}
AZStd::vector<const TestTarget*> DynamicDependencyMap::GetCoveringTests() const
{
AZStd::vector<const TestTarget*> covering;
for (const auto& [testTarget, coveringSources] : m_testTargetSourceCoverage)
{
if (!coveringSources.empty())
{
covering.push_back(testTarget);
}
}
return covering;
}
AZStd::vector<const TestTarget*> DynamicDependencyMap::GetNotCoveringTests() const
{
AZStd::vector<const TestTarget*> notCovering;
for(const auto& [testTarget, coveringSources] : m_testTargetSourceCoverage)
{
if (coveringSources.empty())
{
notCovering.push_back(testTarget);
}
}
return notCovering;
}
} // namespace TestImpact
@@ -0,0 +1,142 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <TestImpactFramework/TestImpactChangeList.h>
#include <Artifact/Static/TestImpactProductionTargetDescriptor.h>
#include <Artifact/Static/TestImpactTestTargetDescriptor.h>
#include <Dependency/TestImpactSourceCoveringTestsList.h>
#include <Dependency/TestImpactSourceDependency.h>
#include <Dependency/TestImpactChangeDependencyList.h>
#include <Target/TestImpactProductionTargetList.h>
#include <Target/TestImpactTestTargetList.h>
#include <AzCore/std/containers/unordered_map.h>
#include <AzCore/std/containers/unordered_set.h>
#include <AzCore/std/containers/vector.h>
namespace TestImpact
{
//! Representation of the repository source tree and its relation to the build targets and coverage data.
class DynamicDependencyMap
{
public:
//! Constructs the dependency map with entries for each build target's source files with empty test coverage data.
DynamicDependencyMap(
AZStd::vector<ProductionTargetDescriptor>&& productionTargetDescriptors,
AZStd::vector<TestTargetDescriptor>&& testTargetDescriptors);
//! Gets the total number of production and test targets in the repository.
size_t GetNumTargets() const;
//! Gets the total number of unique source files in the repository.
//! @note This includes autogen output sources.
size_t GetNumSources() const;
//! Attempts to get the specified build target.
//! @param name The name of the build target to get.
//! @returns If found, the pointer to the specified build target, otherwise nullptr.
const BuildTarget* GetBuildTarget(const AZStd::string& name) const;
//! Attempts to get the specified build target or throw TargetException.
//! @param name The name of the build target to get.
const BuildTarget* GetBuildTargetOrThrow(const AZStd::string& name) const;
//! Attempts to get the specified target's specialized type.
//! @param name The name of the target to get.
//! @returns If found, the pointer to the specialized target, otherwise AZStd::monostate.
OptionalTarget GetTarget(const AZStd::string& name) const;
//! Attempts to get the specified target's specialized type or throw TargetException.
//! @param name The name of the target to get.
Target GetTargetOrThrow(const AZStd::string& name) const;
//! Get the list of production targets in the repository.
const ProductionTargetList& GetProductionTargetList() const;
//! Get the list of test targets in the repository.
const TestTargetList& GetTestTargetList() const;
//! Gets the test targets covering the specified production target.
//! @param productionTarget The production target to retrieve the covering tests for.
AZStd::vector<const TestTarget*> GetCoveringTestTargetsForProductionTarget(const ProductionTarget& productionTarget) const;
//! Gets the source dependency for the specified source file.
//! @note Autogen input source dependencies are the consolidated source dependencies of all of their generated output sources.
//! @returns If found, the source dependency information for the specified source file, otherwise empty.
AZStd::optional<SourceDependency> GetSourceDependency(const RepoPath& path) const;
//! Gets the source dependency for the specified source file or throw DependencyException.
SourceDependency GetSourceDependencyOrThrow(const RepoPath& path) const;
//! Replaces the source coverage of the specified sources with the specified source coverage.
//! @param sourceCoverageDelta The source coverage delta to replace in the dependency map.
void ReplaceSourceCoverage(const SourceCoveringTestsList& sourceCoverageDelta);
//! Clears all of the existing source coverage in the dependency map.
void ClearAllSourceCoverage();
//! Exports the coverage of all sources in the dependency map.
SourceCoveringTestsList ExportSourceCoverage() const;
//! Gets the list of orphaned source files in the dependency map that have coverage data but belong to no parent build targets.
AZStd::vector<AZStd::string> GetOrphanSourceFiles() const;
//! Applies the specified change list to the dependency map and resolves the change list to a change dependency list
//! containing the updated source dependencies for each source file in the change list.
//! @param changeList The change list to apply and resolve.
//! @returns The change list as resolved to the appropriate source dependencies.
[[nodiscard]] ChangeDependencyList ApplyAndResoveChangeList(const ChangeList& changeList);
//! Removes the specified test target from all source coverage.
void RemoveTestTargetFromSourceCoverage(const TestTarget* testTarget);
//! Returns the test targets that cover one or more sources in the repository.
AZStd::vector<const TestTarget*> GetCoveringTests() const;
//! Returns the test targets that do not cover any sources in the repository.
AZStd::vector<const TestTarget*> GetNotCoveringTests() const;
private:
//! Internal handler for ReplaceSourceCoverage where the pruning of parentless and coverageless source depenencies after the
//! source coverage has been replaced must be explicitly stated.
//! @note The covered targets for the source dependency's parent test target(s) will not be pruned if those covering targets are removed.
//! @param sourceCoverageDelta The source coverage delta to replace in the dependency map.
//! @param pruneIfNoParentsOrCoverage Flag to specify whether or not newly parentless and coverageless dependencies will be removed.
void ReplaceSourceCoverageInternal(const SourceCoveringTestsList& sourceCoverageDelta, bool pruneIfNoParentsOrCoverage);
//! Clears the source coverage of the specified sources.
//! @note The covering targets for the parent test target(s) will not be pruned if those covering targets are removed.
void ClearSourceCoverage(const AZStd::vector<RepoPath>& paths);
//! The sorted list of unique production targets in the repository.
ProductionTargetList m_productionTargets;
//! The sorted list of unique test targets in the repository.
TestTargetList m_testTargets;
//! The dependency map of sources to their parent build targets and covering test targets.
AZStd::unordered_map<AZStd::string, DependencyData> m_sourceDependencyMap;
//! Map of all test targets and the sources they cover.
AZStd::unordered_map<const TestTarget*, AZStd::unordered_set<AZStd::string>> m_testTargetSourceCoverage;
//! The map of build targets and their covering test targets.
//! @note As per the note for ReplaceSourceCoverageInternal, this map is currently not pruned when source coverage is replaced.
AZStd::unordered_map<const BuildTarget*, AZStd::unordered_set<const TestTarget*>> m_buildTargetCoverage;
//! Mapping of autogen input sources to their generated output sources.
AZStd::unordered_map<AZStd::string, AZStd::vector<AZStd::string>> m_autogenInputToOutputMap;
};
} // namespace TestImpact
@@ -0,0 +1,81 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <Dependency/TestImpactSourceCoveringTestsList.h>
#include <AzCore/std/sort.h>
namespace TestImpact
{
AZStd::vector<AZStd::string> ExtractTargetsFromSet(AZStd::unordered_set<AZStd::string>&& coveringTestTargets)
{
AZStd::vector<AZStd::string> testTargets;
testTargets.reserve(coveringTestTargets.size());
for (auto it = coveringTestTargets.begin(); it != coveringTestTargets.end(); )
{
testTargets.push_back(std::move(coveringTestTargets.extract(it++).value()));
}
return testTargets;
}
SourceCoveringTests::SourceCoveringTests(const RepoPath& path)
: m_path(path)
{
}
SourceCoveringTests::SourceCoveringTests(const RepoPath& path, AZStd::vector<AZStd::string>&& coveringTestTargets)
: m_path(path)
, m_coveringTestTargets(AZStd::move(coveringTestTargets))
{
}
SourceCoveringTests::SourceCoveringTests(const RepoPath& path, AZStd::unordered_set<AZStd::string>&& coveringTestTargets)
: m_path(path)
, m_coveringTestTargets(ExtractTargetsFromSet(AZStd::move(coveringTestTargets)))
{
}
const RepoPath& SourceCoveringTests::GetPath() const
{
return m_path;
}
size_t SourceCoveringTests::GetNumCoveringTestTargets() const
{
return m_coveringTestTargets.size();
}
const AZStd::vector<AZStd::string>& SourceCoveringTests::GetCoveringTestTargets() const
{
return m_coveringTestTargets;
}
SourceCoveringTestsList::SourceCoveringTestsList(AZStd::vector<SourceCoveringTests>&& sourceCoveringTests)
: m_coverage(AZStd::move(sourceCoveringTests))
{
AZStd::sort(m_coverage.begin(), m_coverage.end(), [](const SourceCoveringTests& lhs, const SourceCoveringTests& rhs)
{
return lhs.GetPath().String() < rhs.GetPath().String();
});
}
size_t SourceCoveringTestsList::GetNumSources() const
{
return m_coverage.size();
}
const AZStd::vector<SourceCoveringTests>& SourceCoveringTestsList::GetCoverage() const
{
return m_coverage;
}
} // namespace TestImpact
@@ -0,0 +1,59 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <TestImpactFramework/TestImpactRepoPath.h>
#include <AzCore/std/containers/vector.h>
#include <AzCore/std/containers/unordered_set.h>
#include <AzCore/std/string/string.h>
namespace TestImpact
{
//! Represents the unresolved test target coverage for a given source file.
class SourceCoveringTests
{
public:
//SourceCoveringTests(const SourceCoveringTests&);
explicit SourceCoveringTests(const RepoPath& path);
SourceCoveringTests(const RepoPath& path, AZStd::vector<AZStd::string>&& coveringTestTargets);
SourceCoveringTests(const RepoPath& path, AZStd::unordered_set<AZStd::string>&& coveringTestTargets);
//! Returns the path of this source file.
const RepoPath& GetPath() const;
//! Returns the number of unresolved test targets covering this source file.
size_t GetNumCoveringTestTargets() const;
//! Returns the unresolved test targets covering this source file.
const AZStd::vector<AZStd::string>& GetCoveringTestTargets() const;
private:
RepoPath m_path; //!< The path of this source file.
AZStd::vector<AZStd::string> m_coveringTestTargets; //!< The unresolved test targets that cover this source file.
};
//! Sorted collection of source file test coverage.
class SourceCoveringTestsList
{
public:
explicit SourceCoveringTestsList(AZStd::vector<SourceCoveringTests>&& sourceCoveringTests);
//! Returns the number of source files in the collection.
size_t GetNumSources() const;
//! Returns the source file coverages.
const AZStd::vector<SourceCoveringTests>& GetCoverage() const;
private:
AZStd::vector<SourceCoveringTests> m_coverage; //!< The collection of source file coverages.
};
} // namespace TestImpact
@@ -0,0 +1,93 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <Dependency/TestImpactDependencyException.h>
#include <Dependency/TestImpactSourceCoveringTestsSerializer.h>
#include <AzCore/JSON/document.h>
#include <AzCore/JSON/prettywriter.h>
#include <AzCore/JSON/rapidjson.h>
#include <AzCore/JSON/stringbuffer.h>
namespace TestImpact
{
// Tag used to indicate whether a given line is the name or a covering test target
constexpr char TargetTag = '-';
AZStd::string SerializeSourceCoveringTestsList(const SourceCoveringTestsList& sourceCoveringTestsList)
{
AZStd::string output;
output.reserve(1U << 24); // Reserve approx. 16Mib as the outputs can be quite large
for (const auto& source : sourceCoveringTestsList.GetCoverage())
{
// Source file
output += source.GetPath().String();
output += "\n";
// Covering test targets
for (const auto& testTarget : source.GetCoveringTestTargets())
{
output += AZStd::string::format("%c%s\n", TargetTag, testTarget.c_str());
}
}
// Add the newline so the deserializer can properly terminate on the last read line
output += "\n";
return output;
}
SourceCoveringTestsList DeserializeSourceCoveringTestsList(const AZStd::string& sourceCoveringTestsListString)
{
AZStd::vector<SourceCoveringTests> sourceCoveringTests;
AZStd::string source;
AZStd::vector<AZStd::string> coveringTests;
sourceCoveringTests.reserve(1U << 16); // Reserve for approx. 65k source files
const AZStd::string delim = "\n";
auto start = 0U;
auto end = sourceCoveringTestsListString.find(delim);
while (end != AZStd::string::npos)
{
const auto line = sourceCoveringTestsListString.substr(start, end - start);
if (line.starts_with(TargetTag))
{
// This is a test target covering the most recent source discovered
coveringTests.push_back(line.substr(1, line.length() - 1));
}
else
{
// This is a new source file so assign the accumulated test targets to the current source file before proceeding
if (!coveringTests.empty())
{
sourceCoveringTests.push_back(SourceCoveringTests(source, AZStd::move(coveringTests)));
coveringTests.clear();
}
source = line;
}
start = end + delim.length();
end = sourceCoveringTestsListString.find(delim, start);
}
// Ensure we properly assign the accumulated test targets to the most recent source discovered
if (!coveringTests.empty())
{
sourceCoveringTests.push_back(SourceCoveringTests(source, AZStd::move(coveringTests)));
coveringTests.clear();
}
return SourceCoveringTestsList(AZStd::move(sourceCoveringTests));
}
} // namespace TestImpact
@@ -0,0 +1,26 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <Dependency/TestImpactSourceCoveringTestsList.h>
#include <AzCore/std/string/string.h>
namespace TestImpact
{
//! Serializes the specified source covering tests list to plain text format.
AZStd::string SerializeSourceCoveringTestsList(const SourceCoveringTestsList& sourceCoveringTestsList);
//! Deserializes a source covering tests list from the specified source covering tests data in plain text format.
SourceCoveringTestsList DeserializeSourceCoveringTestsList(const AZStd::string& sourceCoveringTestsListString);
} // namespace TestImpact
@@ -0,0 +1,87 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <Dependency/TestImpactSourceDependency.h>
#include <Target/TestImpactBuildTarget.h>
#include <Target/TestImpactProductionTarget.h>
#include <Target/TestImpactTestTarget.h>
namespace TestImpact
{
ParentTarget::ParentTarget(const TestTarget* target)
: m_target(target)
{
}
ParentTarget::ParentTarget(const ProductionTarget* target)
: m_target(target)
{
}
bool ParentTarget::operator==(const ParentTarget& other) const
{
return GetBuildTarget() == other.GetBuildTarget();
}
const BuildTarget* ParentTarget::GetBuildTarget() const
{
const BuildTarget* buildTarget;
AZStd::visit([&buildTarget](auto&& target)
{
buildTarget = target;
}, m_target);
return buildTarget;
}
const Target& ParentTarget::GetTarget() const
{
return m_target;
}
SourceDependency::SourceDependency(
const RepoPath& path,
DependencyData&& dependencyData)
: m_path(path)
, m_dependencyData(AZStd::move(dependencyData))
{
}
const RepoPath& SourceDependency::GetPath() const
{
return m_path;
}
size_t SourceDependency::GetNumParentTargets() const
{
return m_dependencyData.m_parentTargets.size();
}
size_t SourceDependency::GetNumCoveringTestTargets() const
{
return m_dependencyData.m_coveringTestTargets.size();
}
const AZStd::unordered_set<ParentTarget>& SourceDependency::GetParentTargets() const
{
return m_dependencyData.m_parentTargets;
}
const AZStd::unordered_set<const TestTarget*>& SourceDependency::GetCoveringTestTargets() const
{
return m_dependencyData.m_coveringTestTargets;
}
} // namespace TestImpact
@@ -0,0 +1,95 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <Target/TestImpactBuildTarget.h>
#include <AzCore/std/string/string.h>
#include <AzCore/std/containers/unordered_map.h>
#include <AzCore/std/containers/unordered_set.h>
#include <AzCore/std/containers/vector.h>
namespace TestImpact
{
class ProductionTarget;
class TestTarget;
//! Representation of a source dependency's parent target.
class ParentTarget
{
public:
//! Constructor overload for test target types.
ParentTarget(const TestTarget* target);
//! Constructor overload for production target types.
ParentTarget(const ProductionTarget* target);
//! Returns the base build target pointer for this parent.
const BuildTarget* GetBuildTarget() const;
//! Returns the specialized target pointer for this parent.
const Target& GetTarget() const;
bool operator==(const ParentTarget& other) const;
private:
Target m_target; //! The specialized target pointer for this parent.
};
}
namespace AZStd
{
//! Hash function for ParentTarget types for use in maps and sets
template<> struct hash<TestImpact::ParentTarget>
{
size_t operator()(const TestImpact::ParentTarget& parentTarget) const noexcept
{
return reinterpret_cast<size_t>(parentTarget.GetBuildTarget());
}
};
}
namespace TestImpact
{
struct DependencyData
{
AZStd::unordered_set<ParentTarget> m_parentTargets;
AZStd::unordered_set<const TestTarget*> m_coveringTestTargets;
};
//! Test target coverage and build target dependency information for a given source file in the dynamic dependency map.
class SourceDependency
{
public:
SourceDependency(
const RepoPath& path,
DependencyData&& dependencyData);
//! Returns the path of this source file.
const RepoPath& GetPath() const;
//! Returns the number of parent build targets this source belongs to.
size_t GetNumParentTargets() const;
//! Returns the number of test targets covering this source file.
size_t GetNumCoveringTestTargets() const;
//! Returns the parent targets that this source file belongs to.
const AZStd::unordered_set<ParentTarget>& GetParentTargets() const;
//! Returns the test targets covering this source file.
const AZStd::unordered_set<const TestTarget*>& GetCoveringTestTargets() const;
private:
RepoPath m_path; //!< The path of this source file.
DependencyData m_dependencyData; //!< The dependency data for this source file.
};
} // namespace TestImpact
@@ -0,0 +1,226 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <Dependency/TestImpactDependencyException.h>
#include <Dependency/TestImpactDynamicDependencyMap.h>
#include <Dependency/TestImpactTestSelectorAndPrioritizer.h>
#include <Target/TestImpactTestTarget.h>
namespace TestImpact
{
TestSelectorAndPrioritizer::TestSelectorAndPrioritizer(
const DynamicDependencyMap* dynamicDependencyMap, DependencyGraphDataMap&& dependencyGraphDataMap)
: m_dynamicDependencyMap(dynamicDependencyMap)
, m_dependencyGraphDataMap(AZStd::move(dependencyGraphDataMap))
{
}
AZStd::vector<const TestTarget*> TestSelectorAndPrioritizer::SelectTestTargets(
const ChangeDependencyList& changeDependencyList, Policy::TestPrioritization testSelectionStrategy)
{
const auto selectedTestTargetAndDependerMap = SelectTestTargets(changeDependencyList);
const auto prioritizedSelectedTests = PrioritizeSelectedTestTargets(selectedTestTargetAndDependerMap, testSelectionStrategy);
return prioritizedSelectedTests;
}
TestSelectorAndPrioritizer::SelectedTestTargetAndDependerMap TestSelectorAndPrioritizer::SelectTestTargets(
const ChangeDependencyList& changeDependencyList)
{
SelectedTestTargetAndDependerMap selectedTestTargetMap;
// Create operations
for (const auto& sourceDependency : changeDependencyList.GetCreateSourceDependencies())
{
for (const auto& parentTarget : sourceDependency.GetParentTargets())
{
AZStd::visit([&selectedTestTargetMap, this](auto&& target)
{
if constexpr (IsProductionTarget<decltype(target)>)
{
// Parent Targets: Yes
// Coverage Data : No
// Source Type : Production
//
// Scenario
// 1. The file has been newly created
// 2. This file exists in one or more source to production target mapping artifacts
// 3. There exists no coverage data for this file in the source covering test list
//
// Action
// 1. Select all test targets covering the parent production targets
const auto coverage = m_dynamicDependencyMap->GetCoveringTestTargetsForProductionTarget(*target);
for (const auto* testTarget : coverage)
{
selectedTestTargetMap[testTarget].insert(target);
}
}
else
{
// Parent Targets: Yes
// Coverage Data : No
// Source Type : Test
//
// Scenario
// 1. The file has been newly created
// 2. This file exists in one or more source to test target mapping artifacts
// 3. There exists no coverage data for this file in the source covering test list
//
// Action
// 1. Select all parent test targets
selectedTestTargetMap.insert(target);
}
}, parentTarget.GetTarget());
}
}
// Update operations
for (const auto& sourceDependency : changeDependencyList.GetUpdateSourceDependencies())
{
if (sourceDependency.GetNumParentTargets())
{
if (sourceDependency.GetNumCoveringTestTargets())
{
for (const auto& parentTarget : sourceDependency.GetParentTargets())
{
AZStd::visit([&selectedTestTargetMap, &sourceDependency, this](auto&& target)
{
if constexpr (IsProductionTarget<decltype(target)>)
{
// Parent Targets: Yes
// Coverage Data : Yes
// Source Type : Production
//
// Scenario
// 1. The existing file has been modified
// 2. This file exists in one or more source to production target mapping artifacts
// 3. There exists coverage data for this file in the source covering test list
//
// Action
// 1. Select all test targets covering this file
for (const auto* testTarget : sourceDependency.GetCoveringTestTargets())
{
selectedTestTargetMap[testTarget].insert(target);
}
}
else
{
// Parent Targets: Yes
// Coverage Data : Yes
// Source Type : Test
//
// Scenario
// 1. The existing file has been modified
// 2. This file exists in one or more source to test target mapping artifacts
// 3. There exists coverage data for this file in the source covering test list
//
// Action
// 1. Select the parent test targets for this file
selectedTestTargetMap.insert(target);
}
}, parentTarget.GetTarget());
}
}
else
{
for (const auto& parentTarget : sourceDependency.GetParentTargets())
{
AZStd::visit([&selectedTestTargetMap, &sourceDependency, this](auto&& target)
{
if constexpr (IsTestTarget<decltype(target)>)
{
// Parent Targets: Yes
// Coverage Data : No
// Source Type : Test
//
// Scenario
// 1. The existing file has been modified
// 2. This file exists in one or more source to test target mapping artifacts
// 3. There exists no coverage data for this file in the source covering test list
//
// Action
// 1. Select the parent test targets for this file
selectedTestTargetMap.insert(target);
}
}, parentTarget.GetTarget());
}
}
}
else
{
// Parent Targets: No
// Coverage Data : Yes
// Source Type : Indeterminate
//
// Scenario
// 1. The existing file has been modified
// 2. Either:
// a) This file previously existed in one or more source to target mapping artifacts
// b) This file no longer exists in any source to target mapping artifacts
// c) The coverage data for this file was has yet to be deleted from the source covering test list
// 3. Or:
// a) The file is being used by build targets but has erroneously not been explicitly added to the build
// system (e.g. include directive pulling in a header from the repository that has not been added to
// any build targets due to an oversight)
//
// Action
// 1. Log potential orphaned source file warning
// 2. Select all test targets covering this file
// 3. Delete the existing coverage data from the source covering test list
for (const auto* testTarget : sourceDependency.GetCoveringTestTargets())
{
selectedTestTargetMap.insert(testTarget);
}
}
}
// Delete operations
for (const auto& sourceDependency : changeDependencyList.GetDeleteSourceDependencies())
{
// Parent Targets: No
// Coverage Data : Yes
// Source Type : Indeterminate
//
// Scenario
// 1. The existing file has been deleted
// 2. This file previously existed in one or more source to target mapping artifacts
// 2. This file does not exist in any source to target mapping artifacts
// 4. The coverage data for this file was has yet to be deleted from the source covering test list
//
// Action
// 1. Select all test targets covering this file
// 2. Delete the existing coverage data from the source covering test list
for (const auto* testTarget : sourceDependency.GetCoveringTestTargets())
{
selectedTestTargetMap.insert(testTarget);
}
}
return selectedTestTargetMap;
}
AZStd::vector<const TestTarget*> TestSelectorAndPrioritizer::PrioritizeSelectedTestTargets(
const SelectedTestTargetAndDependerMap& selectedTestTargetAndDependerMap,
[[maybe_unused]] Policy::TestPrioritization testSelectionStrategy)
{
AZStd::vector<const TestTarget*> selectedTestTargets;
// Prioritization disabled for now
// SPEC-6563
for (const auto& [testTarget, dependerTargets] : selectedTestTargetAndDependerMap)
{
selectedTestTargets.push_back(testTarget);
}
return selectedTestTargets;
}
} // namespace TestImpact
@@ -0,0 +1,72 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <TestImpactFramework/TestImpactTestSequence.h>
#include <Artifact/Static/TestImpactDependencyGraphData.h>
#include <Dependency/TestImpactChangeDependencyList.h>
#include <AzCore/std/containers/vector.h>
#include <AzCore/std/containers/unordered_map.h>
#include <AzCore/std/containers/unordered_set.h>
namespace TestImpact
{
class DynamicDependencyMap;
class BuildTarget;
class TestTarget;
//! Map of build targets and their dependency graph data.
//! For test targets, the dependency graph data is that of the build targets which the test target depends on.
//! For production targets, the dependency graph is that of the build targets that depend on it (dependers).
//! @note No dependency graph data is not an error, it simple means that the target cannot be prioritized.
using DependencyGraphDataMap = AZStd::unordered_map<const BuildTarget*, DependencyGraphData>;
//! Selects the test targets that cover a given set of changes based on the CRUD rules and optionally prioritizes the test
//! selection according to their locality of their covering production targets in the their dependency graphs.
//! @note the CRUD rules for how tests are selected can be found in the MicroRepo header file.
class TestSelectorAndPrioritizer
{
public:
//! Constructs the test selector and prioritizer for the given dynamic dependency map.
//! @param dynamicDependencyMap The dynamic dependency map representing the repository source tree.
//! @param dependencyGraphDataMap The map of build targets and their dependency graph data for use in test prioritization.
TestSelectorAndPrioritizer(const DynamicDependencyMap* dynamicDependencyMap, DependencyGraphDataMap&& dependencyGraphDataMap);
//! Select the covering test targets for the given set of source changes and optionally prioritizes said test selection.
//! @param changeDependencyList The resolved list of source dependencies for the CRUD source changes.
//! @param testSelectionStrategy The test selection and prioritization strategy to apply to the given CRUD source changes.
AZStd::vector<const TestTarget*> SelectTestTargets(const ChangeDependencyList& changeDependencyList, Policy::TestPrioritization testSelectionStrategy);
private:
//! Map of selected test targets and the production targets they cover for the given set of source changes.
using SelectedTestTargetAndDependerMap = AZStd::unordered_map<const TestTarget*, AZStd::unordered_set<const ProductionTarget*>>;
//! Selects the test targets covering the set of source changes in the change dependency list.
//! @param changeDependencyList The change dependency list containing the CRUD source changes to select tests for.
//! @returns The selected tests and their covering production targets for the given set of source changes.
SelectedTestTargetAndDependerMap SelectTestTargets(const ChangeDependencyList& changeDependencyList);
//! Prioritizes the selected tests according to the specified test selection strategy,
//! @note If no dependency graph data exists for a given test target then that test target still be selected albeit not prioritized.
//! @param selectedTestTargetAndDependerMap The selected tests to prioritize.
//! @param testSelectionStrategy The test selection strategy to prioritize the selected tests.
//! @returns The selected tests either in either arbitrary order or in prioritized with highest priority first.
AZStd::vector<const TestTarget*> PrioritizeSelectedTestTargets(
const SelectedTestTargetAndDependerMap& selectedTestTargetAndDependerMap, Policy::TestPrioritization testSelectionStrategy);
const DynamicDependencyMap* m_dynamicDependencyMap;
DependencyGraphDataMap m_dependencyGraphDataMap;
};
} // namespace TestImpact
@@ -1,11 +0,0 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
@@ -0,0 +1,12 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
set(LY_COMPILE_OPTIONS PUBLIC -fexceptions)
@@ -0,0 +1,12 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
set(LY_COMPILE_OPTIONS PUBLIC /EHsc)
@@ -1,11 +0,0 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
@@ -0,0 +1,92 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <AzCore/PlatformIncl.h>
namespace TestImpact
{
//! OS function to cleanup handle
using CleanupFunc = BOOL (*)(HANDLE);
//! RAII wrapper around OS handles.
template<CleanupFunc CleanupFuncT>
class Handle
{
public:
Handle() = default;
explicit Handle(HANDLE handle);
~Handle();
operator HANDLE&();
PHANDLE operator&();
HANDLE& operator=(HANDLE handle);
void Close();
private:
HANDLE m_handle = INVALID_HANDLE_VALUE;
};
template<CleanupFunc CleanupFuncT>
Handle<CleanupFuncT>::Handle(HANDLE handle)
: m_handle(handle)
{
}
template<CleanupFunc CleanupFuncT>
Handle<CleanupFuncT>::~Handle()
{
Close();
}
template<CleanupFunc CleanupFuncT>
Handle<CleanupFuncT>::operator HANDLE&()
{
return m_handle;
}
template<CleanupFunc CleanupFuncT>
PHANDLE Handle<CleanupFuncT>::operator&()
{
return &m_handle;
}
template<CleanupFunc CleanupFuncT>
HANDLE& Handle<CleanupFuncT>::operator=(HANDLE handle)
{
// Setting the handle to INVALID_HANDLE_VALUE will close the handle
if (handle == INVALID_HANDLE_VALUE && m_handle != INVALID_HANDLE_VALUE)
{
Close();
}
else
{
m_handle = handle;
}
return m_handle;
}
template<CleanupFunc CleanupFuncT>
void Handle<CleanupFuncT>::Close()
{
if (m_handle != INVALID_HANDLE_VALUE)
{
CleanupFuncT(m_handle);
m_handle = INVALID_HANDLE_VALUE;
}
}
using ObjectHandle = Handle<CloseHandle>;
using WaitHandle = Handle<UnregisterWait>;
} // namespace TestImpact
@@ -0,0 +1,67 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include "TestImpactWin32_Pipe.h"
#include <Process/TestImpactProcessException.h>
#include <AzCore/std/containers/vector.h>
#include <AzCore/std/optional.h>
namespace TestImpact
{
Pipe::Pipe(SECURITY_ATTRIBUTES& sa, HANDLE& stdChannel)
{
if (!CreatePipe(&m_parent, &m_child, &sa, 0))
{
throw ProcessException("Couldn't create pipe");
}
SetHandleInformation(m_parent, HANDLE_FLAG_INHERIT, 0);
stdChannel = m_child;
}
void Pipe::ReleaseChild()
{
m_child.Close();
}
void Pipe::EmptyPipe()
{
DWORD bytesAvailable = 0;
while (PeekNamedPipe(m_parent, NULL, 0, NULL, &bytesAvailable, NULL) && bytesAvailable > 0)
{
// Grow the buffer by the number of bytes available in the pipe and append the new data
DWORD bytesRead;
const size_t currentSize = m_buffer.size();
m_buffer.resize(m_buffer.size() + bytesAvailable);
if (!ReadFile(m_parent, m_buffer.data() + currentSize, bytesAvailable, &bytesRead, NULL) || bytesRead == 0)
{
throw ProcessException("Couldn't read child output from pipe");
}
}
}
AZStd::string Pipe::GetContentsAndClearInternalBuffer()
{
EmptyPipe();
AZStd::string contents;
if (m_buffer.size() > 0)
{
contents = AZStd::string(m_buffer.begin(), m_buffer.end());
m_buffer.clear();
}
return contents;
}
} // namespace TestImpact
@@ -0,0 +1,52 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include "TestImpactWin32_Handle.h"
#include <AzCore/PlatformIncl.h>
#include <AzCore/std/containers/vector.h>
#include <AzCore/std/string/string.h>
namespace TestImpact
{
//! RAII wrapper around OS pipes.
//! Used to connect the standard output and standard error of the child process to a sink accessible to the
//! parent process to allow the parent process to read the output(s) of the child process.
class Pipe
{
public:
Pipe(SECURITY_ATTRIBUTES& sa, HANDLE& stdChannel);
Pipe(Pipe&& other) = delete;
Pipe(Pipe& other) = delete;
Pipe& operator=(Pipe& other) = delete;
Pipe& operator=(Pipe&& other) = delete;
//! Releases the child end of the pipe (not needed once parent has their end).
void ReleaseChild();
//! Empties the contents of the pipe into the internal buffer.
void EmptyPipe();
//! Empties the contents of the pipe into a string, clearing the internal buffer.
AZStd::string GetContentsAndClearInternalBuffer();
private:
// Parent and child process ends of pipe
ObjectHandle m_parent;
ObjectHandle m_child;
// Buffer for emptying pipe upon child processes exit
AZStd::vector<char> m_buffer;
};
} // namespace TestImpact
@@ -0,0 +1,259 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include "TestImpactWin32_Process.h"
#include <Process/TestImpactProcessException.h>
#include <AzCore/std/parallel/lock.h>
namespace TestImpact
{
// Note: this is called from an OS thread
VOID ProcessWin32::ProcessExitCallback(PVOID processPtr, [[maybe_unused]] BOOLEAN EventSignalled)
{
// Lock the process destructor from being entered from the client thread
AZStd::lock_guard lifeLock(m_lifeCycleMutex);
ProcessId id = reinterpret_cast<ProcessId>(processPtr);
auto process = m_masterProcessList[id];
// Check that the process hasn't already been destructed from the client thread
if (process && process->m_isRunning)
{
// Lock state access and/or mutation from the client thread
AZStd::lock_guard stateLock(process->m_stateMutex);
process->RetrieveOSReturnCodeAndCleanUpProcess();
}
}
ProcessWin32::ProcessWin32(const ProcessInfo& processInfo)
: Process(processInfo)
{
AZStd::string args(m_processInfo.GetProcessPath().String());
if (m_processInfo.GetStartupArgs().length())
{
args = AZStd::string::format("%s %s", args.c_str(), m_processInfo.GetStartupArgs().c_str());
}
SECURITY_ATTRIBUTES sa;
sa.nLength = sizeof(sa);
sa.lpSecurityDescriptor = nullptr;
sa.bInheritHandle = IsPiping();
STARTUPINFO si;
ZeroMemory(&si, sizeof(STARTUPINFO));
si.cb = sizeof(STARTUPINFO);
PROCESS_INFORMATION pi;
ZeroMemory(&pi, sizeof(PROCESS_INFORMATION));
CreatePipes(sa, si);
if (!CreateProcess(
NULL,
&args[0],
NULL,
NULL,
IsPiping(),
CREATE_NEW_PROCESS_GROUP | CREATE_NO_WINDOW,
NULL, NULL,
&si, &pi))
{
throw ProcessException(AZStd::string::format("Couldn't create process with args: %s", args.c_str()));
}
ReleaseChildPipes();
m_process = pi.hProcess;
m_thread = pi.hThread;
m_isRunning = true;
{
// Lock reading of the master process list from the OS thread
AZStd::lock_guard lock(m_lifeCycleMutex);
// Register this process with a unique id in the master process list
m_uniqueId = m_uniqueIdCounter++;
m_masterProcessList[m_uniqueId] = this;
}
// Register the process exit signal callback
if (!RegisterWaitForSingleObject(
&m_waitCallback,
pi.hProcess,
ProcessExitCallback,
reinterpret_cast<PVOID>(m_uniqueId),
INFINITE,
WT_EXECUTEONLYONCE))
{
throw ProcessException("Couldn't register wait object for process exit event");
}
}
bool ProcessWin32::IsPiping() const
{
return m_processInfo.ParentHasStdOutput() || m_processInfo.ParentHasStdError();
}
void ProcessWin32::CreatePipes(SECURITY_ATTRIBUTES& sa, STARTUPINFO& si)
{
if (IsPiping())
{
si.dwFlags = STARTF_USESTDHANDLES;
if (m_processInfo.ParentHasStdOutput())
{
m_stdOutPipe.emplace(sa, si.hStdOutput);
}
if (m_processInfo.ParentHasStdError())
{
m_stdErrPipe.emplace(sa, si.hStdError);
}
}
}
void ProcessWin32::ReleaseChildPipes()
{
if (m_stdOutPipe)
{
m_stdOutPipe->ReleaseChild();
}
if (m_stdErrPipe)
{
m_stdErrPipe->ReleaseChild();
}
}
void ProcessWin32::EmptyPipes()
{
if (m_stdOutPipe)
{
m_stdOutPipe->EmptyPipe();
}
if (m_stdErrPipe)
{
m_stdErrPipe->EmptyPipe();
}
}
AZStd::optional<AZStd::string> ProcessWin32::ConsumeStdOut()
{
if (m_stdOutPipe)
{
AZStd::string contents = m_stdOutPipe->GetContentsAndClearInternalBuffer();
if (!contents.empty())
{
return contents;
}
}
return AZStd::nullopt;
}
AZStd::optional<AZStd::string> ProcessWin32::ConsumeStdErr()
{
if (m_stdErrPipe)
{
AZStd::string contents = m_stdErrPipe->GetContentsAndClearInternalBuffer();
if (!contents.empty())
{
return contents;
}
}
return AZStd::nullopt;
}
void ProcessWin32::Terminate(ReturnCode returnCode)
{
// Lock process cleanup from the OS thread
AZStd::lock_guard lock(m_stateMutex);
if (m_isRunning)
{
// Cancel the callback so we can wait for the signal ourselves
// Note: we keep the state mutex locked as closing the callback is not guaranteed to be instantaneous
m_waitCallback.Close();
// Terminate the process and set the error code to the terminate code
TerminateProcess(m_process, returnCode);
SetReturnCodeAndCleanUpProcesses(returnCode);
}
}
bool ProcessWin32::IsRunning() const
{
return m_isRunning;
}
void ProcessWin32::BlockUntilExit()
{
// Lock process cleanup from the OS thread
AZStd::lock_guard lock(m_stateMutex);
if (m_isRunning)
{
// Cancel the callback so we can wait for the signal ourselves
// Note: we keep the state mutex locked as closing the callback is not guaranteed to be instantaneous
m_waitCallback.Close();
if (IsPiping())
{
// This process will be blocked from exiting if pipe not emptied so will deadlock if we wait
// indefintely whilst there is still output in the pipes so instead keep waiting and checking
// if the pipes need emptying until the process exits
while (WAIT_OBJECT_0 != WaitForSingleObject(m_process, 1))
{
EmptyPipes();
}
}
else
{
// No possibility of pipe deadlocking, safe to wait indefinitely for process exit
WaitForSingleObject(m_process, INFINITE);
}
// Now that the this process has definately exited we are safe to clean up
RetrieveOSReturnCodeAndCleanUpProcess();
}
}
void ProcessWin32::RetrieveOSReturnCodeAndCleanUpProcess()
{
DWORD returnCode;
GetExitCodeProcess(m_process, &returnCode);
SetReturnCodeAndCleanUpProcesses(returnCode);
}
void ProcessWin32::SetReturnCodeAndCleanUpProcesses(ReturnCode returnCode)
{
m_returnCode = returnCode;
m_process.Close();
m_thread.Close();
m_waitCallback.Close();
m_isRunning = false;
}
ProcessWin32::~ProcessWin32()
{
// Lock the process exit signal callback from being entered OS thread
AZStd::lock_guard lock(m_lifeCycleMutex);
// Remove this process from the master list so the process exit signal doesn't attempt to cleanup
// this process if it is deleted client side
m_masterProcessList[m_uniqueId] = nullptr;
}
} // namespace TestImpact
@@ -0,0 +1,101 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include "TestImpactWin32_Handle.h"
#include "TestImpactWin32_Pipe.h"
#include <Process/TestImpactProcess.h>
#include <AzCore/PlatformIncl.h>
#include <AzCore/std/containers/vector.h>
#include <AzCore/std/optional.h>
#include <AzCore/std/parallel/atomic.h>
#include <AzCore/std/parallel/mutex.h>
#include <AzCore/std/smart_ptr/unique_ptr.h>
#include <AzCore/std/string/string.h>
namespace TestImpact
{
//! Platform-specific implementation of Process.
class ProcessWin32
: public Process
{
public:
explicit ProcessWin32(const ProcessInfo& processInfo);
ProcessWin32(ProcessWin32&& other) = delete;
ProcessWin32(ProcessWin32& other) = delete;
ProcessWin32& operator=(ProcessWin32& other) = delete;
ProcessWin32& operator=(ProcessWin32&& other) = delete;
~ProcessWin32();
// Process overrides...
void Terminate(ReturnCode returnCode) override;
void BlockUntilExit() override;
bool IsRunning() const override;
AZStd::optional<AZStd::string> ConsumeStdOut() override;
AZStd::optional<AZStd::string> ConsumeStdErr() override;
private:
//! Callback for process exit signal.
static VOID ProcessExitCallback(PVOID processPtr, BOOLEAN EventSignalled);
//! Retrieves the return code and cleans up the OS handles.
void RetrieveOSReturnCodeAndCleanUpProcess();
//! Sets the return code and cleans up the OS handles
void SetReturnCodeAndCleanUpProcesses(ReturnCode returnCode);
//! Returns true if either stdout or stderr is beign redirected.
bool IsPiping() const;
//! Creates the parent and child pipes for stdout and/or stderr.
void CreatePipes(SECURITY_ATTRIBUTES& sa, STARTUPINFO& si);
//! Empties all pipes so the process can exit without deadlocking.
void EmptyPipes();
//! Releases the child end of the stdout and/or stderr pipes/
void ReleaseChildPipes();
// Flag to determine whether or not the process is in flight
AZStd::atomic_bool m_isRunning = false;
// Unique id assigned to this process (not the same as the id assigned by the client in the ProcessInfo class)
// as used in the master process list
size_t m_uniqueId = 0;
// Handles to OS process
ObjectHandle m_process;
ObjectHandle m_thread;
// Handle to process exit signal callback
WaitHandle m_waitCallback;
// Process to parent standard output piping
AZStd::optional<Pipe> m_stdOutPipe;
AZStd::optional<Pipe> m_stdErrPipe;
// Mutex protecting process state access/mutation from the OS thread and client thread
mutable AZStd::mutex m_stateMutex;
// Mutex keeping the process life cycles in sync between the OS thread and client thread
inline static AZStd::mutex m_lifeCycleMutex;
// Unique counter to give each launched process a unique id
inline static size_t m_uniqueIdCounter = 1;
// Master process list used to ensure consistency of process lifecycles between OS thread and client thread
inline static std::unordered_map<ProcessId, ProcessWin32*> m_masterProcessList;
};
} // namespace TestImpact
@@ -0,0 +1,24 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include "TestImpactWin32_Process.h"
#include <Process/TestImpactProcess.h>
#include <Process/TestImpactProcessLauncher.h>
namespace TestImpact
{
AZStd::unique_ptr<Process> LaunchProcess(const ProcessInfo& processInfo)
{
return AZStd::make_unique<ProcessWin32>(processInfo);
}
} // namespace TestImpact
@@ -0,0 +1,44 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <Target/TestImpactTestTarget.h>
#include <TestEngine/TestImpactTestEngineException.h>
#include <TestEngine/JobRunner/TestImpactTestTargetExtension.h>
namespace TestImpact
{
AZStd::string GetTestTargetExtension(const TestTarget* testTarget)
{
static constexpr char* const standAloneExtension = ".exe";
static constexpr char* const testRunnerExtension = ".dll";
switch (const auto launchMethod = testTarget->GetLaunchMethod(); launchMethod)
{
case LaunchMethod::StandAlone:
{
return standAloneExtension;
}
case LaunchMethod::TestRunner:
{
return testRunnerExtension;
}
default:
{
throw TestEngineException(
AZStd::string::format(
"Unexpected launch method for target %s: %u",
testTarget->GetName().c_str(),
aznumeric_cast<unsigned int>(launchMethod)));
}
}
}
} // namespace TestImpact
@@ -0,0 +1,35 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <TestEngine/TestImpactTestEngineJobFailure.h>
namespace TestImpact
{
// Known error codes for test instrumentation
namespace ErrorCodes
{
namespace OpenCppCoverage
{
static constexpr ReturnCode InvalidArgs = 0x9F8C8E5C;
}
}
AZStd::optional<Client::TestRunResult> CheckForKnownTestInstrumentErrorCode(ReturnCode returnCode)
{
if (returnCode == ErrorCodes::OpenCppCoverage::InvalidArgs)
{
return Client::TestRunResult::FailedToExecute;
}
return AZStd::nullopt;
}
} // namespace TestImpact
@@ -10,5 +10,12 @@
#
set(FILES
Dummy_Windows.cpp
Process/TestImpactWin32_ProcessLauncher.cpp
Process/TestImpactWin32_Process.cpp
Process/TestImpactWin32_Process.h
Process/TestImpactWin32_Handle.h
Process/TestImpactWin32_Pipe.cpp
Process/TestImpactWin32_Pipe.h
TestEngine/JobRunner/TestImpactWin32_TestTargetExtension.cpp
TestEngine/TestImpactWin32_TestEngineJobFailure.cpp
)
@@ -0,0 +1,74 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <Process/JobRunner/TestImpactProcessJobInfo.h>
#include <Process/JobRunner/TestImpactProcessJobMeta.h>
namespace TestImpact
{
//! Representation of a unit of work to be performed by a process.
//! @tparam JobInfoT The JobInfo structure containing the information required to run this job.
//! @tparam JobPayloadT The resulting output of the processed artifact produced by this job.
template<typename JobInfoT, typename JobPayloadT>
class Job
: public JobMetaWrapper
{
public:
using Info = JobInfoT;
using Payload = JobPayloadT;
//! Constructor with r-values for the specific use case of the job runner.
Job(const Info& jobInfo, JobMeta&& jobMeta, AZStd::optional<Payload>&& payload);
//! Returns the job info associated with this job.
const Info& GetJobInfo() const;
//! Returns the payload produced by this job.
const AZStd::optional<Payload>& GetPayload() const;
//! Facilitates the client consuming the payload.
//! @note It is valid for a job life cycle to continue after having released its payload.
AZStd::optional<Payload> ReleasePayload();
private:
Info m_jobInfo;
AZStd::optional<Payload> m_payload;
};
template<typename JobInfoT, typename JobPayloadT>
Job<JobInfoT, JobPayloadT>::Job(const Info& jobInfo, JobMeta&& jobMeta, AZStd::optional<Payload>&& payload)
: JobMetaWrapper(AZStd::move(jobMeta))
, m_jobInfo(jobInfo)
, m_payload(AZStd::move(payload))
{
}
template<typename JobInfoT, typename JobPayloadT>
const JobInfoT& Job<JobInfoT, JobPayloadT>::GetJobInfo() const
{
return m_jobInfo;
}
template<typename JobInfoT, typename JobPayloadT>
const AZStd::optional<JobPayloadT>& Job<JobInfoT, JobPayloadT>::GetPayload() const
{
return m_payload;
}
template<typename JobInfoT, typename JobPayloadT>
AZStd::optional<JobPayloadT> Job<JobInfoT, JobPayloadT>::ReleasePayload()
{
return AZStd::exchange(m_payload, AZStd::nullopt);
}
} // namespace TestImpact
@@ -0,0 +1,81 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <AzCore/std/string/string.h>
namespace TestImpact
{
//! Per-job information to configure and run jobs and process the resulting artifacts.
//! @tparam AdditionalInfo Additional information to be provided to each job to be consumed by client.
template<typename AdditionalInfo>
class JobInfo
: public AdditionalInfo
{
public:
using IdType = size_t;
using CommandType = AZStd::string;
//! Client-provided identifier to distinguish between different jobs.
//! @note Ids of different job types are not interchangeable.
struct Id
{
IdType m_value;
};
//! Command used my ProcessScheduler to execute this job.
//! @note Commands of different job types are not interchangeable.
struct Command
{
CommandType m_args;
};
//! Constructs the job information with any additional information required by the job.
//! @param jobId The client-provided unique identifier for the job.
//! @param command The command used to launch the process running the job.
//! @param additionalInfo The arguments to be provided to the additional information data structure.
template<typename... AdditionalInfoArgs>
JobInfo(Id jobId, const Command& command, AdditionalInfoArgs&&... additionalInfo);
//! Returns the id of this job.
Id GetId() const;
//! Returns the command arguments used to execute this job.
const Command& GetCommand() const;
private:
Id m_id;
Command m_command;
};
template<typename AdditionalInfo>
template<typename... AdditionalInfoArgs>
JobInfo<AdditionalInfo>::JobInfo(Id jobId, const Command& command, AdditionalInfoArgs&&... additionalInfo)
: AdditionalInfo{std::forward<AdditionalInfoArgs>(additionalInfo)...}
, m_id(jobId)
, m_command(command)
{
}
template<typename AdditionalInfo>
typename JobInfo<AdditionalInfo>::Id JobInfo<AdditionalInfo>::GetId() const
{
return m_id;
}
template<typename AdditionalInfo>
const typename JobInfo<AdditionalInfo>::Command& JobInfo<AdditionalInfo>::GetCommand() const
{
return m_command;
}
} // namespace TestImpact
@@ -0,0 +1,61 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <Process/JobRunner/TestImpactProcessJobMeta.h>
namespace TestImpact
{
JobMetaWrapper::JobMetaWrapper(const JobMeta& jobMeta)
: m_meta(jobMeta)
{
}
JobMetaWrapper::JobMetaWrapper(JobMeta&& jobMeta)
: m_meta(AZStd::move(jobMeta))
{
}
JobResult JobMetaWrapper::GetJobResult() const
{
return m_meta.m_result;
}
AZStd::optional<ReturnCode> JobMetaWrapper::GetReturnCode() const
{
return m_meta.m_returnCode;
}
AZStd::chrono::high_resolution_clock::time_point JobMetaWrapper::GetStartTime() const
{
return m_meta.m_startTime.value_or(AZStd::chrono::high_resolution_clock::time_point());
}
AZStd::chrono::high_resolution_clock::time_point JobMetaWrapper::GetEndTime() const
{
if (m_meta.m_startTime.has_value() && m_meta.m_duration.has_value())
{
return m_meta.m_startTime.value() + m_meta.m_duration.value();
}
else
{
return AZStd::chrono::high_resolution_clock::time_point();
}
}
AZStd::chrono::milliseconds JobMetaWrapper::GetDuration() const
{
return m_meta.m_duration.value_or(AZStd::chrono::milliseconds{ 0 });
}
} // namespace TestImpact
@@ -0,0 +1,68 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <Process/TestImpactProcessInfo.h>
#include <AzCore/std/chrono/chrono.h>
#include <AzCore/std/optional.h>
namespace TestImpact
{
//! Result of a job that was run.
enum class JobResult
{
NotExecuted, //!< The job was not executed (e.g. the job runner terminated before the job could be executed).
FailedToExecute, //!< The job failed to execute (e.g. due to the arguments used to execute the job being invalid).
Timeout, //!< The job was terminated by the job runner (e.g. job timeout exceeded while job was in-flight).
Terminated, //!< The job was terminated by the job runner (e.g. global timeout exceeded while job was in-flight).
ExecutedWithFailure, //!< The job was executed but exited in an erroneous state (the underlying process returned non-zero).
ExecutedWithSuccess //!< The job was executed and exited in a successful state (the underlying processes returned zero).
};
//! The meta-data for a given job.
struct JobMeta
{
JobResult m_result = JobResult::NotExecuted;
AZStd::optional<AZStd::chrono::high_resolution_clock::time_point> m_startTime; //!< The time, relative to the job runner start, that this job started.
AZStd::optional<AZStd::chrono::milliseconds> m_duration; //!< The duration that this job took to complete.
AZStd::optional<ReturnCode> m_returnCode; //!< The return code of the underlying processes of this job.
};
//! Wrapper for job meta structure to inheritance/aggregation without being coupled to the JobInfo or Job classes.
class JobMetaWrapper
{
public:
JobMetaWrapper(const JobMeta& jobMeta);
JobMetaWrapper(JobMeta&& jobMeta);
//! Returns the result of this job.
JobResult GetJobResult() const;
//! Returns the start time, relative to the job runner start, that this job started.
AZStd::chrono::high_resolution_clock::time_point GetStartTime() const;
//! Returns the end time, relative to the job runner start, that this job ended.
AZStd::chrono::high_resolution_clock::time_point GetEndTime() const;
//! Returns the duration that this job took to complete.
AZStd::chrono::milliseconds GetDuration() const;
//! Returns the return code of the underlying processes of this job.
AZStd::optional<ReturnCode> GetReturnCode() const;
private:
JobMeta m_meta;
};
} // namespace TestImpact
@@ -0,0 +1,183 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <Process/Scheduler/TestImpactProcessScheduler.h>
#include <AzCore/std/containers/unordered_map.h>
#include <AzCore/std/containers/vector.h>
#include <AzCore/std/functional.h>
#include <AzCore/std/string/string.h>
namespace TestImpact
{
//! Callback for job completion/failure.
//! @param jobInfo The job information associated with this job.
//! @param meta The meta-data about the job run.
//! @param std The standard output and standard error of the process running the job.
template<typename Job>
using JobCallback = AZStd::function<ProcessCallbackResult(const typename Job::Info& jobInfo, const JobMeta& meta, StdContent&& std)>;
//! The payloads produced by the job-specific payload producer in the form of a map associating each job id with the job's payload.
template<typename Job>
using PayloadMap = AZStd::unordered_map<typename Job::Info::IdType, AZStd::optional<typename Job::Payload>>;
//! The map used by the client to associate the job information and meta-data with the job ids.
template<typename Job>
using JobDataMap = AZStd::unordered_map<typename Job::Info::IdType, AZStd::pair<JobMeta, const typename Job::Info*>>;
//! The callback for producing the payloads for the jobs after all jobs have finished executing.
//! @param jobInfos The information for each job run.
//! @param jobDataMap The job data (in the form of job info and meta-data) for each job run.
template<typename Job>
using PayloadMapProducer = AZStd::function<PayloadMap<Job>(const JobDataMap<Job>& jobDataMap)>;
//! Generic job runner that launches a process for each job, records metrics about each job run and hands the payload artifacts
//! produced by each job to the client before compositing the metrics and payload artifacts for each job into a single interface
//! to be consumed by the client.
template<typename JobT>
class JobRunner
{
public:
//! Constructs the job runner with the specified parameters to constrain job runs.
//! @param maxConcurrentProcesses he maximum number of concurrent jobs in-flight.
explicit JobRunner(size_t maxConcurrentProcesses);
//! Executes the specified jobs and returns the products of their labor.
//! @param jobs The arguments (and other pertinent information) required for each job to be run.
//! @param stdOutRouting The standard output routing to be specified for all jobs.
//! @param stdErrRouting The standard error routing to be specified for all jobs.
//! @param jobTimeout The maximum duration a job may be in-flight before being forcefully terminated (nullopt if no timeout).
//! @param runnerTimeout The maximum duration the scheduler may run before forcefully terminating all in-flight jobs (nullopt if no timeout).
//! @param payloadMapProducer The client callback to be called when all jobs have finished to transform the work produced by each job into the desired output.
//! @param jobCallback The client callback to be called when each job changes state.
//! @return The result of the run sequence and the jobs with their associated payloads.
AZStd::pair<ProcessSchedulerResult, AZStd::vector<typename JobT>> Execute(
const AZStd::vector<typename JobT::Info>& jobs,
PayloadMapProducer<JobT> payloadMapProducer,
StdOutputRouting stdOutRouting,
StdErrorRouting stdErrRouting,
AZStd::optional<AZStd::chrono::milliseconds> jobTimeout,
AZStd::optional<AZStd::chrono::milliseconds> runnerTimeout,
JobCallback<typename JobT> jobCallback);
private:
ProcessScheduler m_processScheduler;
StdOutputRouting m_stdOutRouting; //!< Standard output routing from each job process to job runner.
StdErrorRouting m_stdErrRouting; //!< Standard error routing from each job process to job runner
AZStd::optional<AZStd::chrono::milliseconds> m_jobTimeout; //!< Maximum time a job can run for before being forcefully terminated.
AZStd::optional<AZStd::chrono::milliseconds> m_runnerTimeout; //!< Maximum time the job runner can run before forcefully terminating all in-flight jobs and shutting down.
};
template<typename JobT>
JobRunner<JobT>::JobRunner(size_t maxConcurrentProcesses)
: m_processScheduler(maxConcurrentProcesses)
{
}
template<typename JobT>
AZStd::pair<ProcessSchedulerResult, AZStd::vector<typename JobT>> JobRunner<JobT>::Execute(
const AZStd::vector<typename JobT::Info>& jobInfos,
PayloadMapProducer<JobT> payloadMapProducer,
StdOutputRouting stdOutRouting,
StdErrorRouting stdErrRouting,
AZStd::optional<AZStd::chrono::milliseconds> jobTimeout,
AZStd::optional<AZStd::chrono::milliseconds> runnerTimeout,
JobCallback<typename JobT> jobCallback)
{
AZStd::vector<ProcessInfo> processes;
AZStd::unordered_map<JobT::Info::IdType, AZStd::pair<JobMeta, const typename JobT::Info*>> metas;
AZStd::vector<JobT> jobs;
jobs.reserve(jobInfos.size());
processes.reserve(jobInfos.size());
// Transform the job infos into the underlying process infos required for each job
for (size_t jobIndex = 0; jobIndex < jobInfos.size(); jobIndex++)
{
const auto* jobInfo = &jobInfos[jobIndex];
const auto jobId = jobInfo->GetId().m_value;
metas.emplace(jobId, AZStd::pair<JobMeta, const typename JobT::Info*>{JobMeta{}, jobInfo});
processes.emplace_back(jobId, stdOutRouting, stdErrRouting, jobInfo->GetCommand().m_args);
}
// Wrapper around low-level process launch callback to gather job meta-data and present a simplified callback interface to the client
const ProcessLaunchCallback processLaunchCallback = [&jobCallback, &jobInfos, &metas](
TestImpact::ProcessId pid,
TestImpact::LaunchResult launchResult,
AZStd::chrono::high_resolution_clock::time_point createTime)
{
auto& [meta, jobInfo] = metas.at(pid);
if (launchResult == LaunchResult::Failure)
{
meta.m_result = JobResult::FailedToExecute;
return jobCallback(*jobInfo, meta, {});
}
else
{
meta.m_startTime = createTime;
return ProcessCallbackResult::Continue;
}
};
// Wrapper around low-level process exit callback to gather job meta-data and present a simplified callback interface to the client
const ProcessExitCallback processExitCallback = [&jobCallback, &jobInfos, &metas](
TestImpact::ProcessId pid,
TestImpact::ExitCondition exitCondition,
TestImpact::ReturnCode returnCode,
TestImpact::StdContent&& std,
AZStd::chrono::high_resolution_clock::time_point exitTime)
{
auto& [meta, jobInfo] = metas.at(pid);
meta.m_returnCode = returnCode;
meta.m_duration = AZStd::chrono::duration_cast<AZStd::chrono::milliseconds>(exitTime - *meta.m_startTime);
if (exitCondition == ExitCondition::Gracefull && returnCode == 0)
{
meta.m_result = JobResult::ExecutedWithSuccess;
}
else if (exitCondition == ExitCondition::Terminated)
{
meta.m_result = JobResult::Terminated;
}
else if (exitCondition == ExitCondition::Timeout)
{
meta.m_result = JobResult::Timeout;
}
else
{
meta.m_result = JobResult::ExecutedWithFailure;
}
return jobCallback(*jobInfo, meta, AZStd::move(std));
};
// Schedule all jobs for execution
const auto result = m_processScheduler.Execute(
processes,
jobTimeout,
runnerTimeout,
processLaunchCallback,
processExitCallback);
// Hand off the jobs to the client for payload generation
auto payloadMap = payloadMapProducer(metas);
// Unpack the payload map produced by the client into a vector of jobs containing the job data and payload for each job
for (const auto& jobInfo : jobInfos)
{
const auto jobId = jobInfo.GetId().m_value;
jobs.emplace_back(JobT(jobInfo, AZStd::move(metas.at(jobId).first), AZStd::move(payloadMap[jobId])));
}
return { result, jobs };
}
} // namespace TestImpact
@@ -0,0 +1,321 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <Process/Scheduler/TestImpactProcessScheduler.h>
#include <Process/TestImpactProcess.h>
#include <Process/TestImpactProcessException.h>
#include <Process/TestImpactProcessInfo.h>
#include <Process/TestImpactProcessLauncher.h>
namespace TestImpact
{
struct ProcessInFlight
{
AZStd::unique_ptr<Process> m_process;
AZStd::optional<AZStd::chrono::high_resolution_clock::time_point> m_startTime;
AZStd::string m_stdOutput;
AZStd::string m_stdError;
};
class ProcessScheduler::ExecutionState
{
public:
ExecutionState(
size_t maxConcurrentProcesses,
AZStd::optional<AZStd::chrono::milliseconds> processTimeout,
AZStd::optional<AZStd::chrono::milliseconds> scheduleTimeout,
ProcessLaunchCallback& processLaunchCallback,
ProcessExitCallback& processExitCallback);
~ExecutionState();
ProcessSchedulerResult MonitorProcesses(const AZStd::vector<ProcessInfo>& processes);
void TerminateAllProcesses(ExitCondition exitStatus);
private:
ProcessCallbackResult PopAndLaunch(ProcessInFlight& processInFlight);
StdContent ConsumeProcessStdContent(ProcessInFlight& processInFlight);
void AccumulateProcessStdContent(ProcessInFlight& processInFlight);
size_t m_maxConcurrentProcesses = 0;
ProcessLaunchCallback m_processLaunchCallback;
ProcessExitCallback m_processExitCallback;
AZStd::optional<AZStd::chrono::milliseconds> m_processTimeout;
AZStd::optional<AZStd::chrono::milliseconds> m_scheduleTimeout;
AZStd::chrono::high_resolution_clock::time_point m_startTime;
AZStd::vector<ProcessInFlight> m_processPool;
AZStd::queue<ProcessInfo> m_processQueue;
};
ProcessScheduler::ExecutionState::ExecutionState(
size_t maxConcurrentProcesses,
AZStd::optional<AZStd::chrono::milliseconds> processTimeout,
AZStd::optional<AZStd::chrono::milliseconds> scheduleTimeout,
ProcessLaunchCallback& processLaunchCallback,
ProcessExitCallback& processExitCallback)
: m_maxConcurrentProcesses(maxConcurrentProcesses)
, m_processLaunchCallback(processLaunchCallback)
, m_processExitCallback(processExitCallback)
, m_processTimeout(processTimeout)
, m_scheduleTimeout(scheduleTimeout)
{
AZ_TestImpact_Eval(
!m_processTimeout.has_value() || m_processTimeout->count() > 0, ProcessException,
"Process timeout must be empty or non-zero value");
AZ_TestImpact_Eval(
!m_scheduleTimeout.has_value() || m_scheduleTimeout->count() > 0, ProcessException,
"Scheduler timeout must be empty or non-zero value");
}
ProcessScheduler::ExecutionState::~ExecutionState()
{
TerminateAllProcesses(ExitCondition::Terminated);
}
ProcessSchedulerResult ProcessScheduler::ExecutionState::MonitorProcesses(const AZStd::vector<ProcessInfo>& processes)
{
AZ_TestImpact_Eval(!processes.empty(), ProcessException, "Number of processes to launch cannot be 0");
m_startTime = AZStd::chrono::high_resolution_clock::now();
const size_t numConcurrentProcesses = AZStd::min(processes.size(), m_maxConcurrentProcesses);
m_processPool.resize(numConcurrentProcesses);
for (const auto& process : processes)
{
m_processQueue.emplace(process);
}
for (auto& process : m_processPool)
{
if (PopAndLaunch(process) == ProcessCallbackResult::Abort)
{
// Client chose to abort the scheduler
TerminateAllProcesses(ExitCondition::Terminated);
return ProcessSchedulerResult::UserAborted;
}
}
while (true)
{
// Check to see whether or not the scheduling has exceeded its specified runtime
if (m_scheduleTimeout.has_value())
{
const auto shedulerRunTime = AZStd::chrono::milliseconds(AZStd::chrono::high_resolution_clock::now() - m_startTime);
if (shedulerRunTime > m_scheduleTimeout)
{
// Runtime exceeded, terminate all proccesses and schedule no further
TerminateAllProcesses(ExitCondition::Timeout);
return ProcessSchedulerResult::Timeout;
}
}
// Flag to determine whether or not there are currently any processes in-flight
bool processesInFlight = false;
// Loop round the process pool and visit round robin queued up processes for launch
for (auto& processInFlight : m_processPool)
{
if (processInFlight.m_process)
{
// Process is alive (note: not necessarily currently running)
AccumulateProcessStdContent(processInFlight);
const ProcessId processId = processInFlight.m_process->GetProcessInfo().GetId();
if (!processInFlight.m_process->IsRunning())
{
// Process has exited of its own accord
const ReturnCode returnCode = processInFlight.m_process->GetReturnCode().value();
processInFlight.m_process.reset();
const auto exitTime = AZStd::chrono::high_resolution_clock::now();
// Inform the client that the processes has exited
if (ProcessCallbackResult::Abort == m_processExitCallback(
processId,
ExitCondition::Gracefull,
returnCode,
ConsumeProcessStdContent(processInFlight),
exitTime))
{
// Client chose to abort the scheduler
TerminateAllProcesses(ExitCondition::Terminated);
return ProcessSchedulerResult::UserAborted;
}
else if (!m_processQueue.empty())
{
// This slot in the pool is free so launch one of the processes waiting in the queue
if (PopAndLaunch(processInFlight) == ProcessCallbackResult::Abort)
{
// Client chose to abort the scheduler
TerminateAllProcesses(ExitCondition::Terminated);
return ProcessSchedulerResult::UserAborted;
}
else
{
// We know from the above PopAndLaunch there is at least one process in-flight this iteration
processesInFlight = true;
}
}
}
else
{
// Process is still in-flight
const auto exitTime = AZStd::chrono::high_resolution_clock::now();
const auto runTime = AZStd::chrono::milliseconds(exitTime - processInFlight.m_startTime.value());
// Check to see whether or not the processes has exceeded its specified flight time
if (m_processTimeout.has_value() && runTime > m_processTimeout)
{
processInFlight.m_process->Terminate(ProcessTimeoutErrorCode);
const ReturnCode returnCode = processInFlight.m_process->GetReturnCode().value();
processInFlight.m_process.reset();
if (ProcessCallbackResult::Abort == m_processExitCallback(
processId,
ExitCondition::Timeout,
returnCode,
ConsumeProcessStdContent(processInFlight),
exitTime))
{
// Client chose to abort the scheduler
TerminateAllProcesses(ExitCondition::Terminated);
return ProcessSchedulerResult::UserAborted;
}
}
// We know that at least this process is in-flight this iteration
processesInFlight = true;
}
}
else
{
// Queue is empty, no more processes to launch
if (!m_processQueue.empty())
{
if (PopAndLaunch(processInFlight) == ProcessCallbackResult::Abort)
{
// Client chose to abort the scheduler
TerminateAllProcesses(ExitCondition::Terminated);
return ProcessSchedulerResult::UserAborted;
}
else
{
// We know from the above PopAndLaunch there is at least one process in-flight this iteration
processesInFlight = true;
}
}
}
}
if (!processesInFlight)
{
break;
}
}
return ProcessSchedulerResult::Graceful;
}
ProcessCallbackResult ProcessScheduler::ExecutionState::PopAndLaunch(ProcessInFlight& processInFlight)
{
auto processInfo = m_processQueue.front();
m_processQueue.pop();
const auto createTime = AZStd::chrono::high_resolution_clock::now();
LaunchResult createResult = LaunchResult::Success;
try
{
processInFlight.m_process = LaunchProcess(AZStd::move(processInfo));
processInFlight.m_startTime = createTime;
}
catch (ProcessException& e)
{
AZ_Warning("ProcessScheduler", false, e.what());
createResult = LaunchResult::Failure;
}
return m_processLaunchCallback(processInfo.GetId(), createResult, createTime);
}
void ProcessScheduler::ExecutionState::AccumulateProcessStdContent(ProcessInFlight& processInFlight)
{
// Accumulate the stdout/stderr so we don't deadlock with the process waiting for the pipe to empty before finishing
processInFlight.m_stdOutput += processInFlight.m_process->ConsumeStdOut().value_or("");
processInFlight.m_stdError += processInFlight.m_process->ConsumeStdErr().value_or("");
}
StdContent ProcessScheduler::ExecutionState::ConsumeProcessStdContent(ProcessInFlight& processInFlight)
{
return
{
!processInFlight.m_stdOutput.empty()
? AZStd::optional<AZStd::string>{AZStd::move(processInFlight.m_stdOutput)}
: AZStd::nullopt,
!processInFlight.m_stdError.empty()
? AZStd::optional<AZStd::string>{AZStd::move(processInFlight.m_stdError)}
: AZStd::nullopt
};
}
void ProcessScheduler::ExecutionState::TerminateAllProcesses(ExitCondition exitStatus)
{
bool isCallingBackToClient = true;
const ReturnCode returnCode = static_cast<ReturnCode>(exitStatus);
for (auto& processInFlight : m_processPool)
{
if (processInFlight.m_process)
{
processInFlight.m_process->Terminate(ProcessTerminateErrorCode);
AccumulateProcessStdContent(processInFlight);
const ProcessId processId = processInFlight.m_process->GetProcessInfo().GetId();
if (isCallingBackToClient)
{
const auto exitTime = AZStd::chrono::high_resolution_clock::now();
if (ProcessCallbackResult::Abort == m_processExitCallback(
processInFlight.m_process->GetProcessInfo().GetId(),
exitStatus,
returnCode,
ConsumeProcessStdContent(processInFlight),
exitTime))
{
// Client chose to abort the scheduler, do not make any further callbacks
isCallingBackToClient = false;
}
}
processInFlight.m_process.reset();
}
}
}
ProcessScheduler::ProcessScheduler(size_t maxConcurrentProcesses)
: m_maxConcurrentProcesses(maxConcurrentProcesses)
{
AZ_TestImpact_Eval(maxConcurrentProcesses != 0, ProcessException, "Max Number of concurrent processes in flight cannot be 0");
}
ProcessScheduler::~ProcessScheduler() = default;
ProcessSchedulerResult ProcessScheduler::Execute(
const AZStd::vector<ProcessInfo>& processes,
AZStd::optional<AZStd::chrono::milliseconds> processTimeout,
AZStd::optional<AZStd::chrono::milliseconds> scheduleTimeout,
ProcessLaunchCallback processLaunchCallback,
ProcessExitCallback processExitCallback)
{
AZ_TestImpact_Eval(!m_executionState, ProcessException, "Couldn't execute schedule, schedule already in progress");
m_executionState = AZStd::make_unique<ExecutionState>(
m_maxConcurrentProcesses, processTimeout, scheduleTimeout, processLaunchCallback, processExitCallback);
const auto result = m_executionState->MonitorProcesses(processes);
m_executionState.reset();
return result;
}
} // namespace TestImpact
@@ -0,0 +1,115 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <TestImpactFramework/TestImpactRuntime.h>
#include <Process/TestImpactProcessInfo.h>
#include <AzCore/std/chrono/chrono.h>
#include <AzCore/std/containers/queue.h>
#include <AzCore/std/containers/vector.h>
#include <AzCore/std/functional.h>
#include <AzCore/std/optional.h>
#include <AzCore/std/string/string.h>
#include <AzCore/std/smart_ptr/unique_ptr.h>
namespace TestImpact
{
//! Result of the attempt to launch a process.
enum class LaunchResult : bool
{
Failure,
Success
};
//! The condition under which the processes exited.
//! @note For convinience, the terminate and timeout condition values are set to the corresponding return value sent to the
//! process.
enum class ExitCondition : ReturnCode
{
Gracefull, //!< Process has exited of its own accord.
Terminated = ProcessTerminateErrorCode, //!< The process was terminated by the client/scheduler.
Timeout = ProcessTimeoutErrorCode //!< The process was terminated by the scheduler due to exceeding runtime limit.
};
//! Client result for process scheduler callbacks.
enum class ProcessCallbackResult : bool
{
Continue, //!< Continune scheduling.
Abort //!< Abort scheduling immediately.
};
//! Result of the process scheduling sequence.
enum class ProcessSchedulerResult : AZ::u8
{
Graceful, //!< The scheduler completed its run without incident or was terminated gracefully in response to a client callback result.
UserAborted, //!< The scheduler aborted prematurely due to the user returning an abort value from thier callback handler.
Timeout //!< The scheduler aborted its run prematurely due to its runtime exceeding the scheduler timeout value.
};
//! Callback for process launch attempt.
//! @param processId The id of the process that attempted to launch.
//! @param launchResult The result of the process launch attempt.
//! @param createTime The timestamp of the process launch attempt.
using ProcessLaunchCallback =
AZStd::function<ProcessCallbackResult(
ProcessId processId,
LaunchResult launchResult,
AZStd::chrono::high_resolution_clock::time_point createTime)>;
//! Callback for process exit of successfully launched process.
//! @param processId The id of the process that attempted to launch.
//! @param exitStatus The circumstances upon which the processes exited.
//! @param returnCode The return code of the exited process.
//! @param std The standard output and standard error of the process.
//! @param createTime The timestamp of the process exit.
using ProcessExitCallback =
AZStd::function<ProcessCallbackResult(
ProcessId processId,
ExitCondition exitStatus,
ReturnCode returnCode,
StdContent&& std,
AZStd::chrono::high_resolution_clock::time_point exitTime)>;
//! Schedules a batch of processes for launch using a round robin approach to distribute the in-flight processes over
//! the specified number of concurrent process slots.
class ProcessScheduler
{
public:
//! Constructs the scheduler with the specified batch of processes.
//! @param maxConcurrentProcesses The maximum number of concurrent processes in-flight.
explicit ProcessScheduler(size_t maxConcurrentProcesses);
~ProcessScheduler();
//! Executes the specified processes and calls the client callbacks (if any) as each process progresses in its life cycle.
//! @note Multiple subsequent calls to Execute are permitted.
//! @param processes The batch of processes to schedule.
//! @param processTimeout The maximum duration a process may be in-flight for before being forcefully terminated.
//! @param scheduleTimeout The maximum duration the scheduler may run before forcefully terminating all in-flight processes.
//! @param processLaunchCallback The process launch callback function.
//! @param processExitCallback The process exit callback function.
//! @returns The state that triggered the end of the schedule sequence.
ProcessSchedulerResult Execute(
const AZStd::vector<ProcessInfo>& processes,
AZStd::optional<AZStd::chrono::milliseconds> processTimeout,
AZStd::optional<AZStd::chrono::milliseconds> scheduleTimeout,
ProcessLaunchCallback processLaunchCallback,
ProcessExitCallback processExitCallback);
private:
class ExecutionState;
AZStd::unique_ptr<ExecutionState> m_executionState;
size_t m_maxConcurrentProcesses = 0;
};
} // namespace TestImpact
@@ -0,0 +1,32 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <Process/TestImpactProcess.h>
#include <Process/TestImpactProcessInfo.h>
namespace TestImpact
{
Process::Process(const ProcessInfo& processInfo)
: m_processInfo(processInfo)
{
}
const ProcessInfo& Process::GetProcessInfo() const
{
return m_processInfo;
}
AZStd::optional<ReturnCode> Process::GetReturnCode() const
{
return m_returnCode;
}
} // namespace TestImpact
@@ -0,0 +1,61 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <Process/TestImpactProcessInfo.h>
#include <AzCore/std/optional.h>
namespace TestImpact
{
//! Abstraction of platform-specific process.
class Process
{
public:
explicit Process(const ProcessInfo& processInfo);
virtual ~Process() = default;
//! Terminates the process with the specified return code.
virtual void Terminate(ReturnCode returnCode) = 0;
//! Block the calling thread until the process exits.
virtual void BlockUntilExit() = 0;
//! Returns whether or not the process is still running.
virtual bool IsRunning() const = 0;
//! Returns the process info associated with this process.
const ProcessInfo& GetProcessInfo() const;
//! Returns the return code of the exited process.
//! Will be empty if the process is still running or was not successfully launched.
AZStd::optional<ReturnCode> GetReturnCode() const;
//! Flushes the internal buffer and returns the process's buffered standard output.
//! Subsequent calls will keep returning data so long as the process is producing output.
//! Will return nullopt if no output routing or no output produced.
virtual AZStd::optional<AZStd::string> ConsumeStdOut() = 0;
//! Flushes the internal buffer and returns the process's buffered standard error.
//! Subsequent calls will keep returning data so long as the process is producing errors.
//! Will return nullopt if no error routing or no errors produced.
virtual AZStd::optional<AZStd::string> ConsumeStdErr() = 0;
protected:
//! The information used to launch the process.
ProcessInfo m_processInfo;
//! The return code of a successfully launched process (otherwise is empty)
AZStd::optional<ReturnCode> m_returnCode;
};
} // namespace TestImpact
@@ -0,0 +1,26 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <TestImpactFramework/TestImpactException.h>
namespace TestImpact
{
//! Exception for processes and process-related operations.
class ProcessException
: public Exception
{
public:
using Exception::Exception;
};
}
@@ -0,0 +1,67 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <Process/TestImpactProcessException.h>
#include <Process/TestImpactProcessInfo.h>
namespace TestImpact
{
ProcessInfo::ProcessInfo(ProcessId id, const RepoPath& processPath, const AZStd::string& startupArgs)
: m_id(id)
, m_parentHasStdOutput(false)
, m_parentHasStdErr(false)
, m_processPath(processPath)
, m_startupArgs(startupArgs)
{
AZ_TestImpact_Eval(processPath.String().length() > 0, ProcessException, "Process path cannot be empty");
}
ProcessInfo::ProcessInfo(
ProcessId id,
StdOutputRouting stdOut,
StdErrorRouting stdErr,
const RepoPath& processPath,
const AZStd::string& startupArgs)
: m_id(id)
, m_processPath(processPath)
, m_startupArgs(startupArgs)
, m_parentHasStdOutput(stdOut == StdOutputRouting::ToParent ? true : false)
, m_parentHasStdErr(stdErr == StdErrorRouting::ToParent ? true : false)
{
AZ_TestImpact_Eval(processPath.String().length() > 0, ProcessException, "Process path cannot be empty");
}
ProcessId ProcessInfo::GetId() const
{
return m_id;
}
const RepoPath& ProcessInfo::GetProcessPath() const
{
return m_processPath;
}
const AZStd::string& ProcessInfo::GetStartupArgs() const
{
return m_startupArgs;
}
bool ProcessInfo::ParentHasStdOutput() const
{
return m_parentHasStdOutput;
}
bool ProcessInfo::ParentHasStdError() const
{
return m_parentHasStdErr;
}
} // namespace TestImpact
@@ -0,0 +1,94 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <TestImpactFramework/TestImpactRuntime.h>
#include <AzCore/std/optional.h>
#include <AzCore/std/string/string.h>
namespace TestImpact
{
//! Identifier to distinguish between processes.
using ProcessId = size_t;
//! Return code of successfully launched process.
using ReturnCode = int;
//! Error code for processes that are forcefully terminated whilst in-flight by the client.
inline constexpr const ReturnCode ProcessTerminateErrorCode = 0xF10BAD;
//! Error code for processes that are forcefully terminated whilst in-flight by the scheduler due to timing out.
inline constexpr const ReturnCode ProcessTimeoutErrorCode = 0xBADF10;
//! Specifier for how the process's standard out willt be routed
enum class StdOutputRouting
{
ToParent,
None
};
enum class StdErrorRouting
{
ToParent,
None
};
//! Container for process standard output and standard error.
struct StdContent
{
AZStd::optional<AZStd::string> m_out;
AZStd::optional<AZStd::string> m_err;
};
//! Information about a process the arguments used to launch it.
class ProcessInfo
{
public:
//! Provides the information required to launch a process.
//! @param processId Client-supplied id to diffrentiate between processes.
//! @param stdOut Routing of process standard output.
//! @param stdErr Routing of process standard error.
//! @param processPath Path to executable binary to launch.
//! @param startupArgs Arguments to launch the process with.
ProcessInfo(
ProcessId processId,
StdOutputRouting stdOut,
StdErrorRouting stdErr,
const RepoPath& processPath,
const AZStd::string& startupArgs = "");
ProcessInfo(ProcessId processId, const RepoPath& processPath, const AZStd::string& startupArgs = "");
//! Returns the identifier of this process.
ProcessId GetId() const;
//! Returns whether or not stdoutput is routed to the parent process.
bool ParentHasStdOutput() const;
//! Returns whether or not stderror is routed to the parent process.
bool ParentHasStdError() const;
// Returns the path to the process binary.
const RepoPath& GetProcessPath() const;
//! Returns the command line arguments used to launch the process.
const AZStd::string& GetStartupArgs() const;
private:
const ProcessId m_id;
const bool m_parentHasStdOutput;
const bool m_parentHasStdErr;
const RepoPath m_processPath;
const AZStd::string m_startupArgs;
};
} // namespace TestImpact

Some files were not shown because too many files have changed in this diff Show More