diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Executable/Code/Source/TestImpactConsole.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Executable/Code/Source/TestImpactConsole.cpp new file mode 100644 index 0000000000..631eb17b32 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Executable/Code/Source/TestImpactConsole.cpp @@ -0,0 +1,29 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include + +#include +#include + +int main(int argc, char** argv) +{ + AZ::AllocatorInstance::Create(); + AZ::AllocatorInstance::Create(); + + TestImpact::Console::ReturnCode returnCode = TestImpact::Console::Main(argc, argv); + + AZ::AllocatorInstance::Destroy(); + AZ::AllocatorInstance::Destroy(); + + return static_cast(returnCode); +} diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Include/TestImpactFramework/TestImpactConsoleMain.h b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Include/TestImpactFramework/TestImpactConsoleMain.h new file mode 100644 index 0000000000..c6ad90a36f --- /dev/null +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Include/TestImpactFramework/TestImpactConsoleMain.h @@ -0,0 +1,35 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +namespace TestImpact +{ + namespace Console + { + enum class ReturnCode : int + { + Success = 0, //!< The instigation operation(s) returned without error. + InvalidArgs, //!< The specified command line arguments were incorrect. + InvalidUnifiedDiff, //!< The specified unified diff could not be transformed into a valid change list. + InvalidConfiguration, //!< The runtime configuration is malformed. + RuntimeError, //!< The runtime encountered an error that it could not recover from. + UnhandledError, //!< The framework encountered an error that it anticipated but did not handle and could not recover from. + UnknownError, //!< An error of unknown origin was encountered that the console or runtime could not recover from. + TestFailure, //!< The test sequence had one or more test failures. + Timeout //!< The test sequence runtime exceeded the global timeout value. + }; + + //! Entry point for the console front end application. + [[nodiscard]] ReturnCode Main(int argc, char** argv); + } // namespace Console +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleMain.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleMain.cpp new file mode 100644 index 0000000000..0d4f08406e --- /dev/null +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleMain.cpp @@ -0,0 +1,279 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include + +#include + +namespace TestImpact +{ + namespace Console + { + //! Generates a string to be used for printing to the console for the specified change list. + AZStd::string GenerateChangeListString(const ChangeList& changeList) + { + AZStd::string output; + + const auto& outputFiles = [&output](const AZStd::vector& files) + { + for (const auto& file : files) + { + output += AZStd::string::format("\t%s\n", file.c_str()); + } + }; + + output += AZStd::string::format("Created files (%u):\n", changeList.m_createdFiles.size()); + outputFiles(changeList.m_createdFiles); + + output += AZStd::string::format("Updated files (%u):\n", changeList.m_updatedFiles.size()); + outputFiles(changeList.m_updatedFiles); + + output += AZStd::string::format("Deleted files (%u):\n", changeList.m_deletedFiles.size()); + outputFiles(changeList.m_deletedFiles); + + return output; + } + + //! Gets the appropriate console return code for the specified test sequence result. + ReturnCode GetReturnCodeForTestSequenceResult(TestSequenceResult result) + { + switch (result) + { + case TestSequenceResult::Success: + return ReturnCode::Success; + case TestSequenceResult::Failure: + return ReturnCode::TestFailure; + case TestSequenceResult::Timeout: + return ReturnCode::Timeout; + default: + std::cout << "Unexpected TestSequenceResult value: " << aznumeric_cast(result) << std::endl; + return ReturnCode::UnknownError; + } + } + + //! Wrapper around impact analysis sequences to handle the case where the safe mode option is active. + ReturnCode WrappedImpactAnalysisTestSequence( + TestSequenceEventHandler& sequenceEventHandler, + const CommandLineOptions& options, + Runtime& runtime, + const AZStd::optional& changeList) + { + AZ_TestImpact_Eval( + changeList.has_value(), + CommandLineOptionsException, + "Expected a change list for impact analysis but none was provided"); + + TestSequenceResult result = TestSequenceResult::Failure; + if (options.HasSafeMode()) + { + auto [selectedResult, discardedResult] = runtime.SafeImpactAnalysisTestSequence( + changeList.value(), + options.GetSuitesFilter(), + options.GetTestPrioritizationPolicy(), + options.GetTestTargetTimeout(), + options.GetGlobalTimeout(), + AZStd::ref(sequenceEventHandler), + AZStd::ref(sequenceEventHandler), + AZStd::ref(sequenceEventHandler)); + + // Handling the possible timeout and failure permutations of the selected and discarded test results is splitting hairs + // so apply the following, admittedly arbitrary, rules to determine what the composite test sequence result should be + if (selectedResult == TestSequenceResult::Success && discardedResult == TestSequenceResult::Success) + { + // Trivial case: both sequences succeeded + result = TestSequenceResult::Success; + } + else if (selectedResult == TestSequenceResult::Failure || discardedResult == TestSequenceResult::Failure) + { + // One sequence failed whilst the other sequence either succeeded or timed out + result = TestSequenceResult::Failure; + } + else + { + // One sequence timed out whilst the other sequence succeeded or both sequences timed out + result = TestSequenceResult::Timeout; + } + } + else + { + result = runtime.ImpactAnalysisTestSequence( + changeList.value(), + options.GetTestPrioritizationPolicy(), + options.GetTestTargetTimeout(), + options.GetGlobalTimeout(), + AZStd::ref(sequenceEventHandler), + AZStd::ref(sequenceEventHandler), + AZStd::ref(sequenceEventHandler)); + } + + return GetReturnCodeForTestSequenceResult(result); + }; + + //! Entry point for the test impact analysis framework console front end application. + ReturnCode Main(int argc, char** argv) + { + try + { + CommandLineOptions options(argc, argv); + AZStd::optional changeList; + + // If we have a change list, check to see whether or not the client has requested the printing of said change list + if (options.HasChangeListFile()) + { + changeList = DeserializeChangeList(ReadFileContents(*options.GetChangeListFile())); + if (options.HasOutputChangeList()) + { + std::cout << "Change List:\n"; + std::cout << GenerateChangeListString(*changeList).c_str(); + + if (options.GetTestSequenceType() == TestSequenceType::None) + { + return ReturnCode::Success; + } + } + } + + // As of now, there are no other non-test operations other than printing a change list so getting this far is considered an error + AZ_TestImpact_Eval(options.GetTestSequenceType() != TestSequenceType::None, CommandLineOptionsException, "No action specified"); + + std::cout << "Constructing in-memory model of source tree and test coverage, this may take a moment...\n"; + Runtime runtime( + ConfigurationFactory(ReadFileContents(options.GetConfigurationFile())), + options.GetExecutionFailurePolicy(), + options.GetExecutionFailureDraftingPolicy(), + options.GetTestFailurePolicy(), + options.GetIntegrityFailurePolicy(), + options.GetTestShardingPolicy(), + options.GetTargetOutputCapture(), + options.GetMaxConcurrency()); + + if (runtime.HasImpactAnalysisData()) + { + std::cout << "Test impact analysis data for this repository was found.\n"; + } + else + { + std::cout << "Test impact analysis data for this repository was not found, seed or regular sequence fallbacks will be used.\n"; + } + + TestSequenceEventHandler sequenceEventHandler(&options.GetSuitesFilter()); + + switch (const auto type = options.GetTestSequenceType()) + { + case TestSequenceType::Regular: + { + const auto result = runtime.RegularTestSequence( + options.GetSuitesFilter(), + options.GetTestTargetTimeout(), + options.GetGlobalTimeout(), + AZStd::ref(sequenceEventHandler), + AZStd::ref(sequenceEventHandler), + AZStd::ref(sequenceEventHandler)); + + return GetReturnCodeForTestSequenceResult(result); + } + case TestSequenceType::Seed: + { + const auto result = runtime.SeededTestSequence( + options.GetTestTargetTimeout(), + options.GetGlobalTimeout(), + AZStd::ref(sequenceEventHandler), + AZStd::ref(sequenceEventHandler), + AZStd::ref(sequenceEventHandler)); + + return GetReturnCodeForTestSequenceResult(result); + } + case TestSequenceType::ImpactAnalysis: + { + return WrappedImpactAnalysisTestSequence(sequenceEventHandler, options, runtime, changeList); + } + case TestSequenceType::ImpactAnalysisOrSeed: + { + if (runtime.HasImpactAnalysisData()) + { + return WrappedImpactAnalysisTestSequence(sequenceEventHandler, options, runtime, changeList); + } + else + { + const auto result = runtime.SeededTestSequence( + options.GetTestTargetTimeout(), + options.GetGlobalTimeout(), + AZStd::ref(sequenceEventHandler), + AZStd::ref(sequenceEventHandler), + AZStd::ref(sequenceEventHandler)); + + return GetReturnCodeForTestSequenceResult(result); + } + } + default: + std::cout << "Unexpected TestSequenceType value: " << static_cast(type) << std::endl; + return ReturnCode::UnknownError; + } + } + catch (const CommandLineOptionsException& e) + { + std::cout << e.what() << std::endl; + std::cout << CommandLineOptions::GetCommandLineUsageString().c_str() << std::endl; + return ReturnCode::InvalidArgs; + } + catch (const ChangeListException& e) + { + std::cout << e.what() << std::endl; + return ReturnCode::InvalidUnifiedDiff; + } + catch (const ConfigurationException& e) + { + std::cout << e.what() << std::endl; + return ReturnCode::InvalidConfiguration; + } + catch (const RuntimeException& e) + { + std::cout << e.what() << std::endl; + return ReturnCode::RuntimeError; + } + catch (const Exception& e) + { + std::cout << e.what() << std::endl; + return ReturnCode::UnhandledError; + } + catch (const std::exception& e) + { + std::cout << e.what() << std::endl; + return ReturnCode::UnknownError; + } + catch (...) + { + std::cout << "An unknown error occurred" << std::endl; + return ReturnCode::UnknownError; + } + } + } // namespace Console +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/platform_windows_files.cmake b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/platform_windows_files.cmake index 5f5e8fc3d1..e994bcdbbd 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/platform_windows_files.cmake +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/platform_windows_files.cmake @@ -17,4 +17,5 @@ set(FILES Process/TestImpactWin32_Pipe.cpp Process/TestImpactWin32_Pipe.h TestEngine/JobRunner/TestImpactWin32_TestTargetExtension.cpp + TestEngine/TestImpactWin32_TestEngineJobFailure.cpp ) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRepoPath.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRepoPath.cpp index b2ffed2caf..86b9cbbed5 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRepoPath.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRepoPath.cpp @@ -32,70 +32,70 @@ namespace TestImpact return *this; } - inline RepoPath operator/(const RepoPath& lhs, const AZ::IO::PathView& rhs) + RepoPath operator/(const RepoPath& lhs, const AZ::IO::PathView& rhs) { RepoPath result(lhs); result.m_path /= RepoPath(rhs).m_path; return result; } - inline RepoPath operator/(const RepoPath& lhs, AZStd::string_view rhs) + RepoPath operator/(const RepoPath& lhs, AZStd::string_view rhs) { RepoPath result(lhs); result.m_path /= RepoPath(rhs).m_path; return result; } - inline RepoPath operator/(const RepoPath& lhs, const RepoPath::value_type* rhs) + RepoPath operator/(const RepoPath& lhs, const RepoPath::value_type* rhs) { RepoPath result(lhs); result.m_path /= RepoPath(rhs).m_path; return result; } - inline RepoPath operator/(const RepoPath& lhs, const RepoPath& rhs) + RepoPath operator/(const RepoPath& lhs, const RepoPath& rhs) { RepoPath result(lhs); result.m_path /= rhs.m_path; return result; } - inline RepoPath& RepoPath::operator/=(const AZ::IO::PathView& rhs) + RepoPath& RepoPath::operator/=(const AZ::IO::PathView& rhs) { m_path /= RepoPath(rhs).m_path; return *this; } - inline RepoPath& RepoPath::operator/=(AZStd::string_view rhs) + RepoPath& RepoPath::operator/=(AZStd::string_view rhs) { m_path /= RepoPath(rhs).m_path; return *this; } - inline RepoPath& RepoPath::operator/=(const RepoPath::value_type* rhs) + RepoPath& RepoPath::operator/=(const RepoPath::value_type* rhs) { m_path /= RepoPath(rhs).m_path; return *this; } - inline RepoPath& RepoPath::operator/=(const RepoPath& rhs) + RepoPath& RepoPath::operator/=(const RepoPath& rhs) { m_path /= rhs.m_path; return *this; } - inline bool operator==(const RepoPath& lhs, const RepoPath& rhs) noexcept + bool operator==(const RepoPath& lhs, const RepoPath& rhs) noexcept { return lhs.m_path.Compare(rhs.m_path) == 0; } - inline bool operator!=(const RepoPath& lhs, const RepoPath& rhs) noexcept + bool operator!=(const RepoPath& lhs, const RepoPath& rhs) noexcept { return lhs.m_path.Compare(rhs.m_path) != 0; } - inline bool operator<([[maybe_unused]] const RepoPath& lhs, [[maybe_unused]] const RepoPath& rhs) noexcept + bool operator<([[maybe_unused]] const RepoPath& lhs, [[maybe_unused]] const RepoPath& rhs) noexcept { return lhs.m_path.String() < rhs.m_path.String(); } diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp index 9c9351244f..e02bee03c6 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp @@ -309,7 +309,7 @@ namespace TestImpact if (testSequenceEndCallback.has_value()) { - (*testSequenceEndCallback)(CreateSequenceFailureReport(testJobs), timer.Elapsed()); + (*testSequenceEndCallback)(GenerateSequenceFailureReport(testJobs), timer.Elapsed()); } return result; @@ -353,7 +353,7 @@ namespace TestImpact if (testSequenceEndCallback.has_value()) { - (*testSequenceEndCallback)(CreateSequenceFailureReport(testJobs), timer.Elapsed()); + (*testSequenceEndCallback)(GenerateSequenceFailureReport(testJobs), timer.Elapsed()); } return result; @@ -419,8 +419,8 @@ namespace TestImpact if (testSequenceEndCallback.has_value()) { (*testSequenceEndCallback)( - CreateSequenceFailureReport(selectedTestJobs), - CreateSequenceFailureReport(discardedTestJobs), + GenerateSequenceFailureReport(selectedTestJobs), + GenerateSequenceFailureReport(discardedTestJobs), timer.Elapsed()); } @@ -471,7 +471,7 @@ namespace TestImpact if (testSequenceEndCallback.has_value()) { - (*testSequenceEndCallback)(CreateSequenceFailureReport(testJobs), timer.Elapsed()); + (*testSequenceEndCallback)(GenerateSequenceFailureReport(testJobs), timer.Elapsed()); } return result; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.h index e9d1c5425b..56775b1c58 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.h @@ -114,7 +114,7 @@ namespace TestImpact } case Client::TestRunResult::TestFailures: { - testRunFailures.push_back(ExtractTestRunFailure(testJob)); + testRunFailures.push_back(GenerateTestRunFailure(testJob)); break; } default: