From 684ef8046fad906a0952d2832f6473696fb17629 Mon Sep 17 00:00:00 2001 From: jonawals Date: Fri, 28 May 2021 10:46:25 +0100 Subject: [PATCH 1/3] Add console command line options --- .../Frontend/Console/CMakeLists.txt | 12 - .../Frontend/Console/Code/CMakeLists.txt | 23 - ...pactframework_frontend_console_files.cmake | 14 - .../Source/TestImpactCommandLineOptions.cpp | 463 ++++++++++++++++++ .../Source/TestImpactCommandLineOptions.h | 111 +++++ .../TestImpactCommandLineOptionsException.h} | 17 +- .../TestImpactCommandLineOptionsUtils.cpp | 77 +++ .../TestImpactCommandLineOptionsUtils.h | 119 +++++ 8 files changed, 783 insertions(+), 53 deletions(-) delete mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/CMakeLists.txt delete mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/Code/CMakeLists.txt delete mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/Code/testimpactframework_frontend_console_files.cmake create mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp create mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.h rename Code/Tools/TestImpactFramework/Frontend/Console/{Code/Source/TestImpactConsole.cpp => Static/Code/Source/TestImpactCommandLineOptionsException.h} (65%) create mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.cpp create mode 100644 Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.h diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/CMakeLists.txt b/Code/Tools/TestImpactFramework/Frontend/Console/CMakeLists.txt deleted file mode 100644 index 20a680bce9..0000000000 --- a/Code/Tools/TestImpactFramework/Frontend/Console/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -# -# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or -# its licensors. -# -# For complete copyright and license terms please see the LICENSE at the root of this -# distribution (the "License"). All use of this software is governed by the License, -# or, if provided, by the license below or the license accompanying this file. Do not -# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# - -add_subdirectory(Code) diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Code/CMakeLists.txt b/Code/Tools/TestImpactFramework/Frontend/Console/Code/CMakeLists.txt deleted file mode 100644 index da2c707cb8..0000000000 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Code/CMakeLists.txt +++ /dev/null @@ -1,23 +0,0 @@ -# -# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or -# its licensors. -# -# For complete copyright and license terms please see the LICENSE at the root of this -# distribution (the "License"). All use of this software is governed by the License, -# or, if provided, by the license below or the license accompanying this file. Do not -# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# - -ly_add_target( - NAME TestImpact.Frontend.Console EXECUTABLE - NAMESPACE AZ - FILES_CMAKE - testimpactframework_frontend_console_files.cmake - INCLUDE_DIRECTORIES - PRIVATE - Source - BUILD_DEPENDENCIES - PRIVATE - AZ::TestImpact.Runtime.Static -) \ No newline at end of file diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Code/testimpactframework_frontend_console_files.cmake b/Code/Tools/TestImpactFramework/Frontend/Console/Code/testimpactframework_frontend_console_files.cmake deleted file mode 100644 index 19e99649a7..0000000000 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Code/testimpactframework_frontend_console_files.cmake +++ /dev/null @@ -1,14 +0,0 @@ -# -# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or -# its licensors. -# -# For complete copyright and license terms please see the LICENSE at the root of this -# distribution (the "License"). All use of this software is governed by the License, -# or, if provided, by the license below or the license accompanying this file. Do not -# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# - -set(FILES - Source/TestImpactConsole.cpp -) diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp new file mode 100644 index 0000000000..e4083641e5 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp @@ -0,0 +1,463 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include + +#include + +namespace TestImpact +{ + namespace + { + enum + { + // Options + Config, + ChangeList, + OutputChangeList, + Sequence, + TestPrioritizationPolicy, + ExecutionFailurePolicy, + ExecutionFailureDraftingPolicy, + TestFailurePolicy, + IntegrityFailurePolicy, + TestShardingPolicy, + TargetOutputCapture, + MaxConcurrency, + TestTargetTimeout, + GlobalTimeout, + SuitesFilter, + SafeMode, + // Values + None, + Seed, + Regular, + ImpactAnalysis, + ImpactAnalysisOrSeed, + Locality, + Abort, + Continue, + Ignore, + StdOut, + File, + AllSuites + }; + + constexpr const char* OptionKeys[] = + { + // Options + "config", + "changelist", + "ochangelist", + "sequence", + "ppolicy", + "epolicy", + "rexecfailures", + "fpolicy", + "ipolicy", + "shard", + "targetout", + "maxconcurrency", + "ttimeout", + "gtimeout", + "suites", + "safemode", + // Values + "none", + "seed", + "regular", + "tia", + "tiaorseed", + "locality", + "abort", + "continue", + "ignore", + "stdout", + "file", + "*" + }; + + RepoPath ParseConfigurationFile(const AZ::CommandLine& cmd) + { + return ParsePathOption(OptionKeys[Config], cmd).value_or(LY_TEST_IMPACT_DEFAULT_CONFIG_FILE); + } + + AZStd::optional ParseChangeListFile(const AZ::CommandLine& cmd) + { + return ParsePathOption(OptionKeys[ChangeList], cmd); + } + + bool ParseOutputChangeList(const AZ::CommandLine& cmd) + { + return ParseOnOffOption(OptionKeys[OutputChangeList], BinaryStateValue{ false, true }, cmd).value_or(false); + } + + TestSequenceType ParseTestSequenceType(const AZ::CommandLine& cmd) + { + const AZStd::vector> states = + { + {OptionKeys[None], TestSequenceType::None}, + {OptionKeys[Seed], TestSequenceType::Seed}, + {OptionKeys[Regular], TestSequenceType::Regular}, + {OptionKeys[ImpactAnalysis], TestSequenceType::ImpactAnalysis}, + {OptionKeys[ImpactAnalysisOrSeed], TestSequenceType::ImpactAnalysisOrSeed} + }; + + return ParseMultiStateOption(OptionKeys[Sequence], states, cmd).value_or(TestSequenceType::None); + } + + Policy::TestPrioritization ParseTestPrioritizationPolicy(const AZ::CommandLine& cmd) + { + const BinaryStateOption states = + { + {OptionKeys[None], Policy::TestPrioritization::None}, + {OptionKeys[Locality], Policy::TestPrioritization::DependencyLocality} + }; + + return ParseBinaryStateOption(OptionKeys[TestPrioritizationPolicy], states, cmd).value_or(Policy::TestPrioritization::None); + } + + Policy::ExecutionFailure ParseExecutionFailurePolicy(const AZ::CommandLine& cmd) + { + const AZStd::vector> states = + { + {OptionKeys[Abort], Policy::ExecutionFailure::Abort}, + {OptionKeys[Continue], Policy::ExecutionFailure::Continue}, + {OptionKeys[Ignore], Policy::ExecutionFailure::Ignore} + }; + return ParseMultiStateOption(OptionKeys[ExecutionFailurePolicy], states, cmd).value_or(Policy::ExecutionFailure::Continue); + } + + Policy::ExecutionFailureDrafting ParseExecutionFailureDraftingPolicy(const AZ::CommandLine& cmd) + { + const BinaryStateValue states = + { + Policy::ExecutionFailureDrafting::Never, + Policy::ExecutionFailureDrafting::Always + }; + + return ParseOnOffOption(OptionKeys[ExecutionFailureDraftingPolicy], states, cmd).value_or(Policy::ExecutionFailureDrafting::Always); + } + + Policy::TestFailure ParseTestFailurePolicy(const AZ::CommandLine& cmd) + { + const BinaryStateValue states = + { + Policy::TestFailure::Abort, + Policy::TestFailure::Continue + }; + + return ParseAbortContinueOption(OptionKeys[TestFailurePolicy], states, cmd).value_or(Policy::TestFailure::Abort); + } + + Policy::IntegrityFailure ParseIntegrityFailurePolicy(const AZ::CommandLine& cmd) + { + const BinaryStateValue states = + { + Policy::IntegrityFailure::Abort, + Policy::IntegrityFailure::Continue + }; + + return ParseAbortContinueOption(OptionKeys[IntegrityFailurePolicy], states, cmd).value_or(Policy::IntegrityFailure::Abort); + } + + Policy::TestSharding ParseTestShardingPolicy(const AZ::CommandLine& cmd) + { + const BinaryStateValue states = + { + Policy::TestSharding::Never, + Policy::TestSharding::Always + }; + + return ParseOnOffOption("shard", states, cmd).value_or(Policy::TestSharding::Never); + } + + Policy::TargetOutputCapture ParseTargetOutputCapture(const AZ::CommandLine& cmd) + { + const auto numSwitchValues = cmd.GetNumSwitchValues(OptionKeys[TargetOutputCapture]); + if (numSwitchValues) + { + AZ_TestImpact_Eval( + numSwitchValues <= 2, CommandLineOptionsException, "Unexpected parameters for target output capture option"); + + Policy::TargetOutputCapture targetOutputCapture = Policy::TargetOutputCapture::None; + for (auto i = 0; i < numSwitchValues; i++) + { + const auto option = cmd.GetSwitchValue(OptionKeys[TargetOutputCapture], i); + if (option == OptionKeys[StdOut]) + { + if (targetOutputCapture == Policy::TargetOutputCapture::File) + { + targetOutputCapture = Policy::TargetOutputCapture::StdOutAndFile; + } + else + { + targetOutputCapture = Policy::TargetOutputCapture::StdOut; + } + } + else if (option == OptionKeys[File]) + { + if (targetOutputCapture == Policy::TargetOutputCapture::StdOut) + { + targetOutputCapture = Policy::TargetOutputCapture::StdOutAndFile; + } + else + { + targetOutputCapture = Policy::TargetOutputCapture::File; + } + } + else + { + throw CommandLineOptionsException( + AZStd::string::format("Unexpected value for target output capture option: %s", option.c_str())); + } + } + + return targetOutputCapture; + } + + return Policy::TargetOutputCapture::None; + } + + AZStd::optional ParseMaxConcurrency(const AZ::CommandLine& cmd) + { + return ParseUnsignedIntegerOption(OptionKeys[MaxConcurrency], cmd); + } + + AZStd::optional ParseTestTargetTimeout(const AZ::CommandLine& cmd) + { + return ParseSecondsOption(OptionKeys[TestTargetTimeout], cmd); + } + + AZStd::optional ParseGlobalTimeout(const AZ::CommandLine& cmd) + { + return ParseSecondsOption(OptionKeys[GlobalTimeout], cmd); + } + + bool ParseSafeMode(const AZ::CommandLine& cmd) + { + const BinaryStateValue states = { false, true }; + return ParseOnOffOption(OptionKeys[SafeMode], states, cmd).value_or(false); + } + + AZStd::unordered_set ParseSuitesFilter(const AZ::CommandLine& cmd) + { + AZStd::unordered_set suitesFilter; + const auto numSwitchValues = cmd.GetNumSwitchValues(OptionKeys[SuitesFilter]); + if (numSwitchValues) + { + for (auto i = 0; i < numSwitchValues; i++) + { + const auto value = cmd.GetSwitchValue(OptionKeys[SuitesFilter], i); + AZ_TestImpact_Eval(!value.empty(), CommandLineOptionsException, "Suites option value is empty"); + if (value == OptionKeys[AllSuites]) + { + AZ_TestImpact_Eval( + suitesFilter.empty(), CommandLineOptionsException, "The * suite cannot be used with other suites"); + } + + suitesFilter.insert(value); + } + } + + if (suitesFilter.find(OptionKeys[AllSuites]) != suitesFilter.end()) + { + return {}; + } + + return suitesFilter; + } + } + + CommandLineOptions::CommandLineOptions(int argc, char** argv) + { + AZ::CommandLine cmd; + cmd.Parse(argc, argv); + + m_configurationFile = ParseConfigurationFile(cmd); + m_changeListFile = ParseChangeListFile(cmd); + m_outputChangeList = ParseOutputChangeList(cmd); + m_testSequenceType = ParseTestSequenceType(cmd); + m_testPrioritizationPolicy = ParseTestPrioritizationPolicy(cmd); + m_executionFailurePolicy = ParseExecutionFailurePolicy(cmd); + m_executionFailureDraftingPolicy = ParseExecutionFailureDraftingPolicy(cmd); + m_testFailurePolicy = ParseTestFailurePolicy(cmd); + m_integrityFailurePolicy = ParseIntegrityFailurePolicy(cmd); + m_testShardingPolicy = ParseTestShardingPolicy(cmd); + m_targetOutputCapture = ParseTargetOutputCapture(cmd); + m_maxConcurrency = ParseMaxConcurrency(cmd); + m_testTargetTimeout = ParseTestTargetTimeout(cmd); + m_globalTimeout = ParseGlobalTimeout(cmd); + m_safeMode = ParseSafeMode(cmd); + m_suitesFilter = ParseSuitesFilter(cmd); + } + + bool CommandLineOptions::HasChangeListFile() const + { + return m_changeListFile.has_value(); + } + + bool CommandLineOptions::HasSafeMode() const + { + return m_safeMode; + } + + const AZStd::optional& CommandLineOptions::GetChangeListFile() const + { + return m_changeListFile; + } + + bool CommandLineOptions::HasOutputChangeList() const + { + return m_outputChangeList; + } + + const RepoPath& CommandLineOptions::GetConfigurationFile() const + { + return m_configurationFile; + } + + TestSequenceType CommandLineOptions::GetTestSequenceType() const + { + return m_testSequenceType; + } + + Policy::TestPrioritization CommandLineOptions::GetTestPrioritizationPolicy() const + { + return m_testPrioritizationPolicy; + } + + Policy::ExecutionFailure CommandLineOptions::GetExecutionFailurePolicy() const + { + return m_executionFailurePolicy; + } + + Policy::ExecutionFailureDrafting CommandLineOptions::GetExecutionFailureDraftingPolicy() const + { + return m_executionFailureDraftingPolicy; + } + + Policy::TestFailure CommandLineOptions::GetTestFailurePolicy() const + { + return m_testFailurePolicy; + } + + Policy::IntegrityFailure CommandLineOptions::GetIntegrityFailurePolicy() const + { + return m_integrityFailurePolicy; + } + + Policy::TestSharding CommandLineOptions::GetTestShardingPolicy() const + { + return m_testShardingPolicy; + } + + Policy::TargetOutputCapture CommandLineOptions::GetTargetOutputCapture() const + { + return m_targetOutputCapture; + } + + const AZStd::optional& CommandLineOptions::GetMaxConcurrency() const + { + return m_maxConcurrency; + } + + const AZStd::optional& CommandLineOptions::GetTestTargetTimeout() const + { + return m_testTargetTimeout; + } + + const AZStd::optional& CommandLineOptions::GetGlobalTimeout() const + { + return m_globalTimeout; + } + + const AZStd::unordered_set& CommandLineOptions::GetSuitesFilter() const + { + return m_suitesFilter; + } + + AZStd::string CommandLineOptions::GetCommandLineUsageString() + { + AZStd::string help = + "usage: tiaf [options]\n" + " options:\n" + " -config= Path to the configuration file for the TIAF runtime (default: \n" + " ..json).\n" + " -changelist= Path to the JSON of source file changes to perform test impact \n" + " analysis on.\n" + " -gtimeout= Global timeout value to terminate the entire test sequence should it \n" + " be exceeded.\n" + " -ttimeout= Timeout value to terminate individual test targets should it be \n" + " exceeded.\n" + " -sequence= The type of test sequence to perform, where none runs no tests and\n" + " will report a all tests successful, seed removes any prior coverage \n" + " data and runs all test targets with instrumentation to reseed the \n" + " data from scratch, regular runs all of the test targets without any \n" + " instrumentation to generate coverage data(any prior coverage data is \n" + " left intact), tia uses any prior coverage data to run the instrumented \n" + " subset of selected tests(if no prior coverage data a regular run is \n" + " performed instead) and tiaorseed uses any prior coverage data to run \n" + " the instrumented subset of selected tests(if no prior coverage data a \n" + " seed run is performed instead).\n" + " -safemode= Flag to specify a safe mode sequence where the set of unselected \n" + " tests is run without instrumentation after the set of selected \n" + " instrumented tests is run (this has the effect of ensuring all \n" + " tests are run regardless).\n" + " -shard= Break any test targets with a sharding policy into the number of \n" + " shards according to the maximum concurrency value.\n" + " -rexecfailures= Attempt to execute test targets that previously failed to execute.\n" + " -targetout= Capture of individual test run stdout, where stdout will capture \n" + " each individual test target's stdout and output each one to stdout \n" + " and file will capture each individual test target's stdout and output \n" + " each one individually to a file (multiple values are accepted).\n" + " -epolicy= Policy for handling test execution failure (test targets could not be \n" + " launched due to the binary not being built, incorrect paths, etc.), \n" + " where abort will abort the entire test sequence upon the first test\n" + " target execution failureand report a failure(along with the return \n" + " code of the test target that failed to launch), continue will continue \n" + " with the test sequence in the event of test target execution failures\n" + " and treat the test targets that failed to launch as as test failures\n" + " (along with the return codes of the test targets that failed to \n" + " launch), ignore will continue with the test sequence in the event of \n" + " test target execution failuresand treat the test targets that failed\n" + " to launch as as test passes(along with the return codes of the test \n" + " targets that failed to launch).\n" + " -fpolicy Policy for handling test failures (test targets report failing tests), \n" + " where abort will abort the entire test sequenceupon the first test \n" + " failureand report a failure and continue will continue with the test\n" + " sequence in the event of test failuresand report the test failures.\n" + " -ipolicy= Policy for handling coverage data integrity failures, where abort will \n" + " abort the test sequenceand report a failure, seed will attempt another \n" + " sequence using the seed sequence type, otherwise will abort and report \n" + " a failure (this option has no effect for regularand seed sequence \n" + " types) and rerun will attempt another sequence using the regular \n" + " sequence type, otherwise will abortand report a failure(this option has \n" + " no effect for regular sequence type).\n" + " -ppolicy= Policy for prioritizing selected test targets, where none will not \n" + " attempt any test target prioritization and locality will attempt to \n" + " prioritize test targets according to the locality of their covering \n" + " production targets in the dependency graph(if no dependency graph data \n" + " available, no prioritization will occur).\n" + " -maxconcurrency= The maximum number of concurrent test targets/shards to be in flight at \n" + " any given moment.\n" + " -ochangelist= Outputs the change list used for test selection.\n" + " -suites= The test suites to select from for this test sequence (multiple values are \n" + " allowed). The suite all has special significance and will allow tests from \n" + " any suite to be selected, however this particular suite is mutually exclusive\n" + " with other suite. Note: this option is only applicable to the regular sequence\n" + " and, if safe mode is enables, the tia and tiaorseed sequences."; + + return help; + } +} diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.h b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.h new file mode 100644 index 0000000000..6958f03b80 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.h @@ -0,0 +1,111 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include +#include + +#include +#include +#include +#include + +namespace TestImpact +{ + //! The type of test sequence to run. + enum class TestSequenceType + { + None, //!< Runs no tests and will report a all tests successful. + Seed, //!< Removes any prior coverage data and runs all test targets with instrumentation to reseed the data from scratch. + Regular, //!< Runs all of the test targets without any instrumentation to generate coverage data (any prior coverage data is left intact). + ImpactAnalysis, //!< Uses any prior coverage data to run the instrumented subset of selected tests (if no prior coverage data a regular run is performed instead). + ImpactAnalysisOrSeed //!< Uses any prior coverage data to run the instrumented subset of selected tests (if no prior coverage data a seed run is performed instead). + }; + + //! Representation of the command line options supplied to the console frontend application. + class CommandLineOptions + { + public: + CommandLineOptions(int argc, char** argv); + static AZStd::string GetCommandLineUsageString(); + + //! Returns true if a change list file path has been supplied, otherwise false. + bool HasChangeListFile() const; + + //! Returns true if the safe mode option has been enabled, otherwise false. + bool HasSafeMode() const; + + //! Returns true if the output change list option has been enabled, otherwise false. + bool HasOutputChangeList() const; + + //! Returns the path to the runtime configuration file. + const RepoPath& GetConfigurationFile() const; + + //! Returns the path to the change list file (if any). + const AZStd::optional& GetChangeListFile() const; + + //! Returns the test sequence type to run. + TestSequenceType GetTestSequenceType() const; + + //! Returns the test prioritization policy to use. + Policy::TestPrioritization GetTestPrioritizationPolicy() const; + + //! Returns the test execution failure policy to use. + Policy::ExecutionFailure GetExecutionFailurePolicy() const; + + //! Returns the test historic test execution failure drafting policy to use. + Policy::ExecutionFailureDrafting GetExecutionFailureDraftingPolicy() const; + + //! Returns the test failure policy to use. + Policy::TestFailure GetTestFailurePolicy() const; + + //! Returns the integration failure policy to use. + Policy::IntegrityFailure GetIntegrityFailurePolicy() const; + + //! Returns the test sharding policy to use. + Policy::TestSharding GetTestShardingPolicy() const; + + //! Returns the test target standard output capture policy to use. + Policy::TargetOutputCapture GetTargetOutputCapture() const; + + //! Returns the maximum number of test targets to be in flight at any given time. + const AZStd::optional& GetMaxConcurrency() const; + + //! Returns the individual test target timeout to use (if any). + const AZStd::optional& GetTestTargetTimeout() const; + + //! Returns the global test sequence timeout to use (if any). + const AZStd::optional& GetGlobalTimeout() const; + + //! Returns the filter for test suites that will be allowed to be run. + const AZStd::unordered_set& GetSuitesFilter() const; + + private: + RepoPath m_configurationFile; + AZStd::optional m_changeListFile; + bool m_outputChangeList = false; + TestSequenceType m_testSequenceType; + Policy::TestPrioritization m_testPrioritizationPolicy = Policy::TestPrioritization::None; + Policy::ExecutionFailure m_executionFailurePolicy = Policy::ExecutionFailure::Continue; + Policy::ExecutionFailureDrafting m_executionFailureDraftingPolicy = Policy::ExecutionFailureDrafting::Always; + Policy::TestFailure m_testFailurePolicy = Policy::TestFailure::Abort; + Policy::IntegrityFailure m_integrityFailurePolicy = Policy::IntegrityFailure::Abort; + Policy::TestSharding m_testShardingPolicy = Policy::TestSharding::Never; + Policy::TargetOutputCapture m_targetOutputCapture = Policy::TargetOutputCapture::None; + AZStd::optional m_maxConcurrency; + AZStd::optional m_testTargetTimeout; + AZStd::optional m_globalTimeout; + AZStd::unordered_set m_suitesFilter; + bool m_safeMode = false; + }; +} diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactConsole.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsException.h similarity index 65% rename from Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactConsole.cpp rename to Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsException.h index 12743f4b38..fd0a58b0f5 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactConsole.cpp +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsException.h @@ -10,8 +10,17 @@ * */ -int main([[maybe_unused]] int argc, [[maybe_unused]] char** argv) -{ - return 0; -} +#pragma once +#include + +namespace TestImpact +{ + //! Exception for command line options. + class CommandLineOptionsException + : public Exception + { + public: + using Exception::Exception; + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.cpp new file mode 100644 index 0000000000..3315f68ae5 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.cpp @@ -0,0 +1,77 @@ +/* +* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +* its licensors. +* +* For complete copyright and license terms please see the LICENSE at the root of this +* distribution (the "License"). All use of this software is governed by the License, +* or, if provided, by the license below or the license accompanying this file. Do not +* remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* +*/ + +#include + +namespace TestImpact +{ + //! Attempts to parse a path option value. + AZStd::optional ParsePathOption(const AZStd::string& optionName, const AZ::CommandLine& cmd) + { + const auto numSwitchValues = cmd.GetNumSwitchValues(optionName); + if (numSwitchValues) + { + AZ_TestImpact_Eval( + numSwitchValues == 1, + CommandLineOptionsException, + AZStd::string::format("Unexpected number of parameters for %s option", optionName.c_str())); + + const auto value = cmd.GetSwitchValue(optionName, 0); + AZ_TestImpact_Eval( + !value.empty(), + CommandLineOptionsException, + AZStd::string::format("%s file option value is empty", optionName.c_str())); + + return value; + } + + return AZStd::nullopt; + } + + //! Attempts to pass an unsigned integer option value. + AZStd::optional ParseUnsignedIntegerOption(const AZStd::string& optionName, const AZ::CommandLine& cmd) + { + const auto numSwitchValues = cmd.GetNumSwitchValues(optionName); + if (numSwitchValues) + { + AZ_TestImpact_Eval( + numSwitchValues == 1, + CommandLineOptionsException, + AZStd::string::format("Unexpected number of parameters for %s option", optionName.c_str())); + + const auto str = cmd.GetSwitchValue(optionName, 0); + char* end = nullptr; + auto value = strtoul(str.c_str(), &end, 0); + + AZ_TestImpact_Eval( + end != str, + CommandLineOptionsException, + AZStd::string::format("Couldn't parse unsigned integer option value: %s", str.c_str())); + + return aznumeric_caster(value); + } + + return AZStd::nullopt; + } + + //! Attempts to parse an option value in seconds. + AZStd::optional ParseSecondsOption(const AZStd::string& optionName, const AZ::CommandLine& cmd) + { + if (const auto option = ParseUnsignedIntegerOption(optionName, cmd); + option.has_value()) + { + return AZStd::chrono::seconds(option.value()); + } + + return AZStd::nullopt; + } +} diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.h b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.h new file mode 100644 index 0000000000..0da7f1f16e --- /dev/null +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.h @@ -0,0 +1,119 @@ +/* +* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +* its licensors. +* +* For complete copyright and license terms please see the LICENSE at the root of this +* distribution (the "License"). All use of this software is governed by the License, +* or, if provided, by the license below or the license accompanying this file. Do not +* remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* +*/ + +#pragma once + +#include +#include + +#include + +namespace TestImpact +{ + //! Representation of a command line option value name and its typed value. + template + using OptionValue = AZStd::pair; + + //! Representation of a binary state command line option with its two values. + template + using BinaryStateOption = AZStd::pair, OptionValue>; + + //! Representation of the values for a binary state option. + template + using BinaryStateValue = AZStd::pair; + + //! Attempts to parse the specified binary state option. + template + AZStd::optional ParseBinaryStateOption( + const AZStd::string& optionName, + const AZStd::pair, + OptionValue>& state, const AZ::CommandLine& cmd) + { + const auto numSwitchValues = cmd.GetNumSwitchValues(optionName); + if (numSwitchValues) + { + AZ_TestImpact_Eval( + numSwitchValues == 1, + CommandLineOptionsException, + AZStd::string::format("Unexpected number of parameters for %s option", optionName.c_str())); + + const auto option = cmd.GetSwitchValue(optionName, 0); + if (option == state.first.first) + { + return state.first.second; + } + else if (option == state.second.first) + { + return state.second.second; + } + + throw CommandLineOptionsException( + AZStd::string::format("Unexpected value for %s option: %s", optionName.c_str(), option.c_str())); + } + + return AZStd::nullopt; + } + + //! Attempts to pass an arbitrarily sized state option. + template + AZStd::optional ParseMultiStateOption( + const AZStd::string& optionName, + const AZStd::vector>& states, + const AZ::CommandLine& cmd) + { + const auto numSwitchValues = cmd.GetNumSwitchValues(optionName); + if (numSwitchValues) + { + AZ_TestImpact_Eval( + numSwitchValues == 1, + CommandLineOptionsException, + AZStd::string::format("Unexpected number of parameters for %s option", optionName.c_str())); + + const auto option = cmd.GetSwitchValue(optionName, 0); + for (const auto& state : states) + { + if (option == state.first) + { + return state.second; + } + } + + throw CommandLineOptionsException( + AZStd::string::format("Unexpected value for %s option: %s", optionName.c_str(), option.c_str())); + } + + return AZStd::nullopt; + } + + //! Attempts to pass a specialization of the binary state option where the command line values are "on" and "off". + template + AZStd::optional ParseOnOffOption(const AZStd::string& optionName, const AZStd::pair& states, const AZ::CommandLine& cmd) + { + return ParseBinaryStateOption(optionName, BinaryStateOption{ {"off", states.first}, { "on", states.second } }, cmd); + } + + //! Attempts to pass a specialization of the binary state option where the command line values are "abort" and "continue". + template + AZStd::optional ParseAbortContinueOption(const AZStd::string& optionName, const AZStd::pair& states, const AZ::CommandLine& cmd) + { + return ParseBinaryStateOption(optionName, BinaryStateOption{ {"abort", states.first}, { "continue", states.second } }, cmd); + } + + //! Attempts to parse a path option value. + AZStd::optional ParsePathOption(const AZStd::string& optionName, const AZ::CommandLine& cmd); + + //! Attempts to pass an unsigned integer option value. + AZStd::optional ParseUnsignedIntegerOption(const AZStd::string& optionName, const AZ::CommandLine& cmd); + + //! Attempts to parse an option value in seconds. + AZStd::optional ParseSecondsOption(const AZStd::string& optionName, const AZ::CommandLine& cmd); +} From 728d3b4b980d29637d9b528548284e5772b45dd5 Mon Sep 17 00:00:00 2001 From: jonawals Date: Fri, 28 May 2021 12:33:27 +0100 Subject: [PATCH 2/3] Address PR comments --- .../Source/TestImpactCommandLineOptions.cpp | 10 +++++----- .../Source/TestImpactCommandLineOptions.h | 4 ++-- .../TestImpactCommandLineOptionsUtils.cpp | 20 ++++++++++--------- .../TestImpactCommandLineOptionsUtils.h | 10 +++++----- 4 files changed, 23 insertions(+), 21 deletions(-) diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp index e4083641e5..a6eb2e886e 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp @@ -184,8 +184,8 @@ namespace TestImpact Policy::TargetOutputCapture ParseTargetOutputCapture(const AZ::CommandLine& cmd) { - const auto numSwitchValues = cmd.GetNumSwitchValues(OptionKeys[TargetOutputCapture]); - if (numSwitchValues) + if (const auto numSwitchValues = cmd.GetNumSwitchValues(OptionKeys[TargetOutputCapture]); + numSwitchValues) { AZ_TestImpact_Eval( numSwitchValues <= 2, CommandLineOptionsException, "Unexpected parameters for target output capture option"); @@ -253,8 +253,8 @@ namespace TestImpact AZStd::unordered_set ParseSuitesFilter(const AZ::CommandLine& cmd) { AZStd::unordered_set suitesFilter; - const auto numSwitchValues = cmd.GetNumSwitchValues(OptionKeys[SuitesFilter]); - if (numSwitchValues) + if (const auto numSwitchValues = cmd.GetNumSwitchValues(OptionKeys[SuitesFilter]); + numSwitchValues) { for (auto i = 0; i < numSwitchValues; i++) { @@ -460,4 +460,4 @@ namespace TestImpact return help; } -} +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.h b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.h index 6958f03b80..73961b5fcc 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.h +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.h @@ -25,7 +25,7 @@ namespace TestImpact //! The type of test sequence to run. enum class TestSequenceType { - None, //!< Runs no tests and will report a all tests successful. + None, //!< Runs no tests and will report all tests successful. Seed, //!< Removes any prior coverage data and runs all test targets with instrumentation to reseed the data from scratch. Regular, //!< Runs all of the test targets without any instrumentation to generate coverage data (any prior coverage data is left intact). ImpactAnalysis, //!< Uses any prior coverage data to run the instrumented subset of selected tests (if no prior coverage data a regular run is performed instead). @@ -108,4 +108,4 @@ namespace TestImpact AZStd::unordered_set m_suitesFilter; bool m_safeMode = false; }; -} +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.cpp index 3315f68ae5..c03ba98303 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.cpp +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.cpp @@ -12,13 +12,15 @@ #include +#include + namespace TestImpact { //! Attempts to parse a path option value. AZStd::optional ParsePathOption(const AZStd::string& optionName, const AZ::CommandLine& cmd) - { - const auto numSwitchValues = cmd.GetNumSwitchValues(optionName); - if (numSwitchValues) + { + if (const auto numSwitchValues = cmd.GetNumSwitchValues(optionName); + numSwitchValues) { AZ_TestImpact_Eval( numSwitchValues == 1, @@ -40,8 +42,8 @@ namespace TestImpact //! Attempts to pass an unsigned integer option value. AZStd::optional ParseUnsignedIntegerOption(const AZStd::string& optionName, const AZ::CommandLine& cmd) { - const auto numSwitchValues = cmd.GetNumSwitchValues(optionName); - if (numSwitchValues) + if (const auto numSwitchValues = cmd.GetNumSwitchValues(optionName); + numSwitchValues) { AZ_TestImpact_Eval( numSwitchValues == 1, @@ -49,11 +51,11 @@ namespace TestImpact AZStd::string::format("Unexpected number of parameters for %s option", optionName.c_str())); const auto str = cmd.GetSwitchValue(optionName, 0); - char* end = nullptr; - auto value = strtoul(str.c_str(), &end, 0); + size_t end = 0; + auto value = AZStd::stoul(str, &end, 0); AZ_TestImpact_Eval( - end != str, + end, CommandLineOptionsException, AZStd::string::format("Couldn't parse unsigned integer option value: %s", str.c_str())); @@ -74,4 +76,4 @@ namespace TestImpact return AZStd::nullopt; } -} +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.h b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.h index 0da7f1f16e..e635d05710 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.h +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.h @@ -38,8 +38,8 @@ namespace TestImpact const AZStd::pair, OptionValue>& state, const AZ::CommandLine& cmd) { - const auto numSwitchValues = cmd.GetNumSwitchValues(optionName); - if (numSwitchValues) + if (const auto numSwitchValues = cmd.GetNumSwitchValues(optionName); + numSwitchValues) { AZ_TestImpact_Eval( numSwitchValues == 1, @@ -70,8 +70,8 @@ namespace TestImpact const AZStd::vector>& states, const AZ::CommandLine& cmd) { - const auto numSwitchValues = cmd.GetNumSwitchValues(optionName); - if (numSwitchValues) + if (const auto numSwitchValues = cmd.GetNumSwitchValues(optionName); + numSwitchValues) { AZ_TestImpact_Eval( numSwitchValues == 1, @@ -116,4 +116,4 @@ namespace TestImpact //! Attempts to parse an option value in seconds. AZStd::optional ParseSecondsOption(const AZStd::string& optionName, const AZ::CommandLine& cmd); -} +} // namespace TestImpact From c63de35b688aa4479133b8f456339148cd4476b3 Mon Sep 17 00:00:00 2001 From: jonawals Date: Fri, 28 May 2021 15:39:28 +0100 Subject: [PATCH 3/3] Address PR feedback --- .../Source/TestImpactCommandLineOptionsUtils.cpp | 10 +++++----- .../Source/TestImpactCommandLineOptionsUtils.h | 15 +++++++++------ 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.cpp index c03ba98303..b96716059f 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.cpp +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.cpp @@ -50,14 +50,14 @@ namespace TestImpact CommandLineOptionsException, AZStd::string::format("Unexpected number of parameters for %s option", optionName.c_str())); - const auto str = cmd.GetSwitchValue(optionName, 0); - size_t end = 0; - auto value = AZStd::stoul(str, &end, 0); + const auto strValue = cmd.GetSwitchValue(optionName, 0); + size_t successfulParse = 0; // Will be non-zero if the parse was successful + auto value = AZStd::stoul(strValue, &successfulParse, 0); AZ_TestImpact_Eval( - end, + successfulParse, CommandLineOptionsException, - AZStd::string::format("Couldn't parse unsigned integer option value: %s", str.c_str())); + AZStd::string::format("Couldn't parse unsigned integer option value: %s", strValue.c_str())); return aznumeric_caster(value); } diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.h b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.h index e635d05710..1708efadd9 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.h +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptionsUtils.h @@ -47,13 +47,15 @@ namespace TestImpact AZStd::string::format("Unexpected number of parameters for %s option", optionName.c_str())); const auto option = cmd.GetSwitchValue(optionName, 0); - if (option == state.first.first) + if (const auto& [optionValueText, optionValue] = state.first; + option == optionValueText) { - return state.first.second; + return optionValue; } - else if (option == state.second.first) + if (const auto& [optionValueText, optionValue] = state.second; + option == optionValueText) { - return state.second.second; + return optionValue; } throw CommandLineOptionsException( @@ -81,9 +83,10 @@ namespace TestImpact const auto option = cmd.GetSwitchValue(optionName, 0); for (const auto& state : states) { - if (option == state.first) + if (const auto& [optionValueText, optionValue] = state; + option == optionValueText) { - return state.second; + return optionValue; } }