Merge branch 'TIF/Jenkins' of https://github.com/aws-lumberyard-dev/o3de into TIF/Jenkins
This commit is contained in:
jonawals
+28
@@ -0,0 +1,28 @@
|
||||
/*
|
||||
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
* its licensors.
|
||||
*
|
||||
* For complete copyright and license terms please see the LICENSE at the root of this
|
||||
* distribution (the "License"). All use of this software is governed by the License,
|
||||
* or, if provided, by the license below or the license accompanying this file. Do not
|
||||
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <TestImpactFramework/TestImpactRepoPath.h>
|
||||
|
||||
#include <AzCore/std/containers/vector.h>
|
||||
|
||||
namespace TestImpact
|
||||
{
|
||||
//! Representation of the file CRUD operations of a given set of source changes.
|
||||
struct ChangeList
|
||||
{
|
||||
AZStd::vector<RepoPath> m_createdFiles; //!< Files that were newly created.
|
||||
AZStd::vector<RepoPath> m_updatedFiles; //!< Files that were updated.
|
||||
AZStd::vector<RepoPath> m_deletedFiles; //!< Files that were deleted.
|
||||
};
|
||||
} // namespace TestImpact
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
/*
|
||||
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
* its licensors.
|
||||
*
|
||||
* For complete copyright and license terms please see the LICENSE at the root of this
|
||||
* distribution (the "License"). All use of this software is governed by the License,
|
||||
* or, if provided, by the license below or the license accompanying this file. Do not
|
||||
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <TestImpactFramework/TestImpactException.h>
|
||||
|
||||
namespace TestImpact
|
||||
{
|
||||
//! Exception for change list operations.
|
||||
class ChangeListException
|
||||
: public Exception
|
||||
{
|
||||
public:
|
||||
using Exception::Exception;
|
||||
};
|
||||
} // namespace TestImpact
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
/*
|
||||
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
* its licensors.
|
||||
*
|
||||
* For complete copyright and license terms please see the LICENSE at the root of this
|
||||
* distribution (the "License"). All use of this software is governed by the License,
|
||||
* or, if provided, by the license below or the license accompanying this file. Do not
|
||||
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <TestImpactFramework/TestImpactChangeList.h>
|
||||
|
||||
#include <AzCore/std/string/string.h>
|
||||
|
||||
namespace TestImpact
|
||||
{
|
||||
//! Serializes the specified change list to JSON format.
|
||||
AZStd::string SerializeChangeList(const ChangeList& changeList);
|
||||
|
||||
//! Deserializes a change list from the specified test run data in JSON format.
|
||||
ChangeList DeserializeChangeList(const AZStd::string& changeListString);
|
||||
} // namespace TestImpact
|
||||
+177
@@ -0,0 +1,177 @@
|
||||
/*
|
||||
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
* its licensors.
|
||||
*
|
||||
* For complete copyright and license terms please see the LICENSE at the root of this
|
||||
* distribution (the "License"). All use of this software is governed by the License,
|
||||
* or, if provided, by the license below or the license accompanying this file. Do not
|
||||
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <AzCore/std/string/string.h>
|
||||
#include <AzCore/std/containers/vector.h>
|
||||
|
||||
namespace TestImpact
|
||||
{
|
||||
namespace Client
|
||||
{
|
||||
//! Represents a test target that failed, either due to failing to execute, completing in an abnormal state or completing with failing tests.
|
||||
class TargetFailure
|
||||
{
|
||||
public:
|
||||
TargetFailure(const AZStd::string& targetName);
|
||||
|
||||
//! Returns the name of the test target this failure pertains to.
|
||||
const AZStd::string& GetTargetName() const;
|
||||
private:
|
||||
AZStd::string m_targetName;
|
||||
};
|
||||
|
||||
//! Represents a test target that failed to execute.
|
||||
class ExecutionFailure
|
||||
: public TargetFailure
|
||||
{
|
||||
public:
|
||||
ExecutionFailure(const AZStd::string& targetName, const AZStd::string& command);
|
||||
|
||||
//! Returns the command string used to execute this test target.
|
||||
const AZStd::string& GetCommandString() const;
|
||||
private:
|
||||
AZStd::string m_commandString;
|
||||
};
|
||||
|
||||
//! Represents a test target that terminated abnormally.
|
||||
class LauncherFailure
|
||||
: public ExecutionFailure
|
||||
{
|
||||
public:
|
||||
LauncherFailure(const AZStd::string& targetName, const AZStd::string& command, int returnCode);
|
||||
|
||||
//! The return code of the test target that terminated abnormally.
|
||||
int GetReturnCode() const;
|
||||
private:
|
||||
int m_returnCode;
|
||||
};
|
||||
|
||||
//! Represents an individual test of a test target that failed.
|
||||
class TestFailure
|
||||
{
|
||||
public:
|
||||
TestFailure(const AZStd::string& testName, const AZStd::string& errorMessage);
|
||||
|
||||
//! Returns the name of the test that failed.
|
||||
const AZStd::string& GetName() const;
|
||||
|
||||
//! Returns the error message of the test that failed.
|
||||
const AZStd::string& GetErrorMessage() const;
|
||||
|
||||
private:
|
||||
AZStd::string m_name;
|
||||
AZStd::string m_errorMessage;
|
||||
};
|
||||
|
||||
//! Represents a collection of tests that failed.
|
||||
//! @note Only the failing tests are included in the collection.
|
||||
class TestCaseFailure
|
||||
{
|
||||
public:
|
||||
TestCaseFailure(const AZStd::string& testCaseName, AZStd::vector<TestFailure>&& testFailures);
|
||||
|
||||
//! Returns the name of the test case containing the failing tests.
|
||||
const AZStd::string& GetName() const;
|
||||
|
||||
//! Returns the collection of tests in this test case that failed.
|
||||
const AZStd::vector<TestFailure>& GetTestFailures() const;
|
||||
|
||||
private:
|
||||
AZStd::string m_name;
|
||||
AZStd::vector<TestFailure> m_testFailures;
|
||||
};
|
||||
|
||||
//! Represents a test target that launched successfully but contains failing tests.
|
||||
class TestRunFailure
|
||||
: public TargetFailure
|
||||
{
|
||||
public:
|
||||
TestRunFailure(const AZStd::string& targetName, AZStd::vector<TestCaseFailure>&& testFailures);
|
||||
|
||||
//! Returns the total number of failing tests in this run.
|
||||
size_t GetNumTestFailures() const;
|
||||
|
||||
//! Returns the test cases in this run containing failing tests.
|
||||
const AZStd::vector<TestCaseFailure>& GetTestCaseFailures() const;
|
||||
|
||||
private:
|
||||
AZStd::vector<TestCaseFailure> m_testCaseFailures;
|
||||
};
|
||||
|
||||
//! Base class for reporting failing test sequences.
|
||||
class SequenceFailure
|
||||
{
|
||||
public:
|
||||
SequenceFailure(
|
||||
AZStd::vector<ExecutionFailure>&& executionFailures,
|
||||
AZStd::vector<LauncherFailure>&& launcherFailures,
|
||||
AZStd::vector<TargetFailure>&& unexecutedTests);
|
||||
|
||||
//! Returns the test targets in this sequence that failed to execute.
|
||||
const AZStd::vector<ExecutionFailure>& GetExecutionFailures() const;
|
||||
|
||||
//! Returns the test targets in this sequence that terminated abnormally.
|
||||
const AZStd::vector<LauncherFailure>& GetLauncherFailures() const;
|
||||
|
||||
//! Returns the test targets in this sequence that were not executed due to the sequence terminating prematurely.
|
||||
const AZStd::vector<TargetFailure>& GetUnexecutedTest() const;
|
||||
|
||||
private:
|
||||
AZStd::vector<ExecutionFailure> m_executionFailures;
|
||||
AZStd::vector<LauncherFailure> m_launcherFailures;
|
||||
AZStd::vector<TargetFailure> m_unexecutedTestsTests;
|
||||
};
|
||||
|
||||
//! Represents the report for a failed regular test sequence run without test impact analysis.
|
||||
class RegularSequenceFailure
|
||||
: public SequenceFailure
|
||||
{
|
||||
public:
|
||||
RegularSequenceFailure(
|
||||
AZStd::vector<ExecutionFailure>&& executionFailures,
|
||||
AZStd::vector<LauncherFailure>&& launcherFailures,
|
||||
AZStd::vector<TestRunFailure>&& testRunFailures,
|
||||
AZStd::vector<TargetFailure>&& unexecutedTests);
|
||||
|
||||
//! Returns the test targets that contain failing tests.
|
||||
const AZStd::vector<TestRunFailure>& GetTestRunFailures() const;
|
||||
|
||||
private:
|
||||
AZStd::vector<TestRunFailure> m_testRunFailures;
|
||||
};
|
||||
|
||||
//! Represents the report for a failed test sequence run with test impact analysis.
|
||||
class ImpactAnalysisSequenceFailure
|
||||
: public SequenceFailure
|
||||
{
|
||||
public:
|
||||
ImpactAnalysisSequenceFailure(
|
||||
AZStd::vector<ExecutionFailure>&& executionFailures,
|
||||
AZStd::vector<LauncherFailure>&& launcherFailures,
|
||||
AZStd::vector<TestRunFailure>&& selectedTestRunFailures,
|
||||
AZStd::vector<TestRunFailure>&& discardedTestRunFailures,
|
||||
AZStd::vector<TargetFailure>&& unexecutedTests);
|
||||
|
||||
//! Returns the test targets that were selected to run but contain failing tests.
|
||||
const AZStd::vector<TestRunFailure> GetSelectedTestRunFailures() const;
|
||||
|
||||
//! Returns the test targets that were not selected but still run but contain failing tests.
|
||||
const AZStd::vector<TestRunFailure> GetDiscardedTestRunFailures() const;
|
||||
|
||||
private:
|
||||
AZStd::vector<TestRunFailure> m_selectedTestRunFailures;
|
||||
AZStd::vector<TestRunFailure> m_discardedTestRunFailures;
|
||||
};
|
||||
} // namespace Client
|
||||
} // namespace TestImpact
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
/*
|
||||
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
* its licensors.
|
||||
*
|
||||
* For complete copyright and license terms please see the LICENSE at the root of this
|
||||
* distribution (the "License"). All use of this software is governed by the License,
|
||||
* or, if provided, by the license below or the license accompanying this file. Do not
|
||||
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
*
|
||||
*/
|
||||
|
||||
#include <AzCore/std/string/string.h>
|
||||
#include <AzCore/std/chrono/chrono.h>
|
||||
|
||||
#pragma once
|
||||
|
||||
namespace TestImpact
|
||||
{
|
||||
namespace Client
|
||||
{
|
||||
//! Result of a test run.
|
||||
enum class TestRunResult
|
||||
{
|
||||
NotRun, //!< The test run was not executed due to the test sequence terminating prematurely.
|
||||
FailedToExecute, //!< The test run failed to execute either due to the target binary missing or incorrect arguments.
|
||||
Timeout, //!< The test run timed out whilst in flight before being able to complete its run.
|
||||
TestFailures, //!< The test run completed its run but there were failing tests.
|
||||
AllTestsPass //!< The test run completed its run and all tests passed.
|
||||
};
|
||||
|
||||
class TestRun
|
||||
{
|
||||
public:
|
||||
TestRun(const AZStd::string& name, TestRunResult result, AZStd::chrono::milliseconds duration);
|
||||
const AZStd::string& GetTargetName() const;
|
||||
TestRunResult GetResult() const;
|
||||
AZStd::chrono::milliseconds GetDuration() const;
|
||||
|
||||
private:
|
||||
AZStd::string m_targetName;
|
||||
TestRunResult m_result;
|
||||
AZStd::chrono::milliseconds m_duration;
|
||||
};
|
||||
} // namespace Client
|
||||
} // namespace TestImpact
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
/*
|
||||
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
* its licensors.
|
||||
*
|
||||
* For complete copyright and license terms please see the LICENSE at the root of this
|
||||
* distribution (the "License"). All use of this software is governed by the License,
|
||||
* or, if provided, by the license below or the license accompanying this file. Do not
|
||||
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
*
|
||||
*/
|
||||
|
||||
#include <AzCore/std/string/string.h>
|
||||
#include <AzCore/std/containers/vector.h>
|
||||
|
||||
#pragma once
|
||||
|
||||
namespace TestImpact
|
||||
{
|
||||
namespace Client
|
||||
{
|
||||
//! The set of test targets selected to run regardless of whether or not the test targets are to be excluded either for being on the primary exclude
|
||||
//! list and/or being part of a test suite excluded from this run.
|
||||
//! @note Only the included test targets will be run. The excluded test targets, although selected, will not be run.
|
||||
class TestRunSelection
|
||||
{
|
||||
public:
|
||||
TestRunSelection(AZStd::vector<AZStd::string>&& includedTests, AZStd::vector<AZStd::string>&& excludedTests);
|
||||
|
||||
//! Returns the test runs that were selected to be run and will actually be run.
|
||||
const AZStd::vector<AZStd::string>& GetIncludededTestRuns() const;
|
||||
|
||||
//! Returns the test runs that were selected to be run but will not actually be run.
|
||||
const AZStd::vector<AZStd::string>& GetExcludedTestRuns() const;
|
||||
|
||||
//! Returns the number of selected test runs that will be run.
|
||||
size_t GetNumIncludedTestRuns() const;
|
||||
|
||||
//! Returns the number of selected test runs that will not be run.
|
||||
size_t GetNumNumExcludedTestRuns() const;
|
||||
|
||||
private:
|
||||
AZStd::vector<AZStd::string> m_includedTestRuns;
|
||||
AZStd::vector<AZStd::string> m_excludedTestRuns;
|
||||
};
|
||||
} // namespace Client
|
||||
} // namespace TestImpact
|
||||
+137
@@ -0,0 +1,137 @@
|
||||
/*
|
||||
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
* its licensors.
|
||||
*
|
||||
* For complete copyright and license terms please see the LICENSE at the root of this
|
||||
* distribution (the "License"). All use of this software is governed by the License,
|
||||
* or, if provided, by the license below or the license accompanying this file. Do not
|
||||
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <TestImpactFramework/TestImpactTestSequence.h>
|
||||
#include <TestImpactFramework/TestImpactRepoPath.h>
|
||||
|
||||
#include <AzCore/std/containers/vector.h>
|
||||
#include <AzCore/std/string/string.h>
|
||||
|
||||
namespace TestImpact
|
||||
{
|
||||
//! Meta-data about the configuration.
|
||||
struct ConfigMeta
|
||||
{
|
||||
AZStd::string m_platform; //!< The platform for which the configuration pertains to.
|
||||
};
|
||||
|
||||
//! Repository configuration.
|
||||
struct RepoConfig
|
||||
{
|
||||
RepoPath m_root; //!< The absolute path to the repository root.
|
||||
};
|
||||
|
||||
//! Test impact analysis framework workspace configuration.
|
||||
struct WorkspaceConfig
|
||||
{
|
||||
//! Temporary workspace configuration.
|
||||
struct Temp
|
||||
{
|
||||
//! Paths relative to root.
|
||||
struct RelativePaths
|
||||
{
|
||||
RepoPath m_artifactDirectory; //!< Path to read and write runtime artifacts to and from.
|
||||
};
|
||||
|
||||
RepoPath m_root; //!< Path to the temporary workspace (cleaned prior to use).
|
||||
RelativePaths m_relativePaths;
|
||||
};
|
||||
|
||||
//! Active persistent data workspace configuration.
|
||||
struct Active
|
||||
{
|
||||
//! Paths relative to root.
|
||||
struct RelativePaths
|
||||
{
|
||||
RepoPath m_sparTIAFile; //!< Path to the test impact analysis data.
|
||||
RepoPath m_enumerationCacheDirectory; //!< Path to the test enumerations cache.
|
||||
};
|
||||
|
||||
RepoPath m_root; //!< Path to the persistent workspace tracked by the repository.
|
||||
RelativePaths m_relativePaths;
|
||||
};
|
||||
|
||||
Temp m_temp;
|
||||
Active m_active;
|
||||
};
|
||||
|
||||
//! Build target descriptor configuration.
|
||||
struct BuildTargetDescriptorConfig
|
||||
{
|
||||
RepoPath m_mappingDirectory; //!< Path to the source to target mapping files.
|
||||
AZStd::vector<AZStd::string> m_staticInclusionFilters; //!< File extensions to include for static files.
|
||||
AZStd::string m_inputOutputPairer; //!< Regex for matching autogen input files with autogen outputs files.
|
||||
AZStd::vector<AZStd::string> m_inputInclusionFilters; //!< File extensions fo include for autogen input files.
|
||||
};
|
||||
|
||||
//! Dependency graph configuration.
|
||||
struct DependencyGraphDataConfig
|
||||
{
|
||||
RepoPath m_graphDirectory; //!< Path to the dependency graph files.
|
||||
AZStd::string m_targetDependencyFileMatcher; //!< Regex for matching dependency graph files to build targets.
|
||||
AZStd::string m_targetVertexMatcher; //!< Regex form matching dependency graph vertices to build targets.
|
||||
};
|
||||
|
||||
//! Test target meta configuration.
|
||||
struct TestTargetMetaConfig
|
||||
{
|
||||
RepoPath m_metaFile; //!< Path to the test target meta file.
|
||||
};
|
||||
|
||||
//! Test engine configuration.
|
||||
struct TestEngineConfig
|
||||
{
|
||||
//! Test runner configuration.
|
||||
struct TestRunner
|
||||
{
|
||||
RepoPath m_binary; //!< Path to the test runner binary.
|
||||
};
|
||||
|
||||
//! Test instrumentation configuration.
|
||||
struct Instrumentation
|
||||
{
|
||||
RepoPath m_binary; //!< Path to the test instrumentation binary.
|
||||
};
|
||||
|
||||
TestRunner m_testRunner;
|
||||
Instrumentation m_instrumentation;
|
||||
};
|
||||
|
||||
//! Build target configuration.
|
||||
struct TargetConfig
|
||||
{
|
||||
//! Test target sharding configuration.
|
||||
struct ShardedTarget
|
||||
{
|
||||
AZStd::string m_name; //!< Name of test target this sharding configuration applies to.
|
||||
ShardConfiguration m_configuration; //!< The shard configuration to use.
|
||||
};
|
||||
|
||||
RepoPath m_outputDirectory; //!< Path to the test target binary directory.
|
||||
AZStd::vector<AZStd::string> m_excludedTestTargets; //!< Test targets to always exclude from test run sequences.
|
||||
AZStd::vector<ShardedTarget> m_shardedTestTargets; //!< Test target shard configurations (opt-in).
|
||||
};
|
||||
|
||||
struct RuntimeConfig
|
||||
{
|
||||
ConfigMeta m_meta;
|
||||
RepoConfig m_repo;
|
||||
WorkspaceConfig m_workspace;
|
||||
BuildTargetDescriptorConfig m_buildTargetDescriptor;
|
||||
DependencyGraphDataConfig m_dependencyGraphData;
|
||||
TestTargetMetaConfig m_testTargetMeta;
|
||||
TestEngineConfig m_testEngine;
|
||||
TargetConfig m_target;
|
||||
};
|
||||
} // namespace TestImpact
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
/*
|
||||
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
* its licensors.
|
||||
*
|
||||
* For complete copyright and license terms please see the LICENSE at the root of this
|
||||
* distribution (the "License"). All use of this software is governed by the License,
|
||||
* or, if provided, by the license below or the license accompanying this file. Do not
|
||||
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <TestImpactFramework/TestImpactException.h>
|
||||
|
||||
namespace TestImpact
|
||||
{
|
||||
//! Exception for configuration operations.
|
||||
class ConfigurationException
|
||||
: public Exception
|
||||
{
|
||||
public:
|
||||
using Exception::Exception;
|
||||
};
|
||||
} // namespace TestImpact
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
/*
|
||||
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
* its licensors.
|
||||
*
|
||||
* For complete copyright and license terms please see the LICENSE at the root of this
|
||||
* distribution (the "License"). All use of this software is governed by the License,
|
||||
* or, if provided, by the license below or the license accompanying this file. Do not
|
||||
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <AzCore/IO/Path/Path.h>
|
||||
#include <AzCore/std/string/string.h>
|
||||
|
||||
namespace TestImpact
|
||||
{
|
||||
//! Wrapper class to ensure that all paths have the same path separator regardless of how they are sourced. This is critical
|
||||
//! to the test impact analysis data as otherwise querying/retrieving test impact analysis data for the same source albeit
|
||||
//! with different path separators will be considered different files entirely.
|
||||
class RepoPath
|
||||
: public AZ::IO::Path
|
||||
{
|
||||
public:
|
||||
constexpr RepoPath() = default;
|
||||
constexpr RepoPath(const RepoPath&) = default;
|
||||
constexpr RepoPath(RepoPath&&) noexcept = default;
|
||||
constexpr RepoPath(const string_type&) noexcept;
|
||||
constexpr RepoPath(const value_type*) noexcept;
|
||||
constexpr RepoPath(const AZ::IO::PathView&);
|
||||
constexpr RepoPath(const AZ::IO::Path&);
|
||||
|
||||
RepoPath& operator=(const RepoPath&) noexcept = default;
|
||||
RepoPath& operator=(const string_type&) noexcept;
|
||||
RepoPath& operator=(const value_type*) noexcept;
|
||||
RepoPath& operator=(const AZ::IO::Path& str) noexcept;
|
||||
|
||||
using AZ::IO::Path::operator AZ::IO::PathView;
|
||||
};
|
||||
} // namespace TestImpact
|
||||
+183
@@ -0,0 +1,183 @@
|
||||
/*
|
||||
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
* its licensors.
|
||||
*
|
||||
* For complete copyright and license terms please see the LICENSE at the root of this
|
||||
* distribution (the "License"). All use of this software is governed by the License,
|
||||
* or, if provided, by the license below or the license accompanying this file. Do not
|
||||
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <TestImpactFramework/TestImpactConfiguration.h>
|
||||
#include <TestImpactFramework/TestImpactChangeList.h>
|
||||
#include <TestImpactFramework/TestImpactClientTestSelection.h>
|
||||
#include <TestImpactFramework/TestImpactClientTestRun.h>
|
||||
#include <TestImpactFramework/TestImpactClientFailureReport.h>
|
||||
#include <TestImpactFramework/TestImpactTestSequence.h>
|
||||
|
||||
#include <AzCore/std/string/string.h>
|
||||
#include <AzCore/std/chrono/chrono.h>
|
||||
#include <AzCore/std/optional.h>
|
||||
#include <AzCore/std/functional.h>
|
||||
#include <AzCore/std/containers/vector.h>
|
||||
#include <AzCore/std/containers/unordered_set.h>
|
||||
#include <AzCore/std/smart_ptr/unique_ptr.h>
|
||||
|
||||
namespace TestImpact
|
||||
{
|
||||
class DynamicDependencyMap;
|
||||
class TestEngine;
|
||||
class TestTarget;
|
||||
|
||||
//! Callback for a test sequence that isn't using test impact analysis to determine selected tests.
|
||||
//! @param tests The tests that will be run for this sequence.
|
||||
using TestSequenceStartCallback = AZStd::function<void(Client::TestRunSelection&& tests)>;
|
||||
|
||||
//! Callback for a test sequence using test impact analysis.
|
||||
//! @param selectedTests The tests that have been selected for this run by test impact analysis.
|
||||
//! @param discardedTests The tests that have been rejected for this run by test impact analysis.
|
||||
//! @param draftedTests The tests that have been drafted in for this run due to requirements outside of test impact analysis
|
||||
//! (e.g. test targets that have been added to the repository since the last test impact analysis sequence or test that failed
|
||||
//! to execute previously).
|
||||
//! These tests will be run with coverage instrumentation.
|
||||
//! @note discardedTests and draftedTests may contain overlapping tests.
|
||||
using ImpactAnalysisTestSequenceStartCallback = AZStd::function<void(
|
||||
Client::TestRunSelection&& selectedTests,
|
||||
AZStd::vector<AZStd::string>&& discardedTests,
|
||||
AZStd::vector<AZStd::string>&& draftedTests)>;
|
||||
|
||||
//! Callback for a test sequence using test impact analysis.
|
||||
//! @param selectedTests The tests that have been selected for this run by test impact analysis.
|
||||
//! @param discardedTests The tests that have been rejected for this run by test impact analysis.
|
||||
//! These tests will not be run without coverage instrumentation unless there is an entry in the draftedTests list.
|
||||
//! @param draftedTests The tests that have been drafted in for this run due to requirements outside of test impact analysis
|
||||
//! (e.g. test targets that have been added to the repository since the last test impact analysis sequence or test that failed
|
||||
//! to execute previously).
|
||||
//! @note discardedTests and draftedTests may contain overlapping tests.
|
||||
using SafeImpactAnalysisTestSequenceStartCallback = AZStd::function<void(
|
||||
Client::TestRunSelection&& selectedTests,
|
||||
Client::TestRunSelection&& discardedTests,
|
||||
AZStd::vector<AZStd::string>&& draftedTests)>;
|
||||
|
||||
//! Callback for end of a test sequence.
|
||||
//! @param failureReport The test runs that failed for any reason during this sequence.
|
||||
//! @param duration The total duration of this test sequence.
|
||||
using TestSequenceCompleteCallback = AZStd::function<void(Client::RegularSequenceFailure&& failureReport, AZStd::chrono::milliseconds duration)>;
|
||||
|
||||
//! Callback for end of a test impact analysis test sequence.
|
||||
//! @param failureReport The test runs that failed for any reason during this sequence.
|
||||
//! @param duration The total duration of this test sequence.
|
||||
using ImpactAnalysisTestSequenceCompleteCallback = AZStd::function<void(Client::ImpactAnalysisSequenceFailure&& failureReport, AZStd::chrono::milliseconds duration)>;
|
||||
|
||||
//! Callback for test runs that have completed for any reason.
|
||||
//! test The test that has completed.
|
||||
using TestCompleteCallback = AZStd::function<void(Client::TestRun&& test)>;
|
||||
|
||||
//! The API exposed to the client responsible for all test runs and persistent data management.
|
||||
class Runtime
|
||||
{
|
||||
public:
|
||||
//! Constructs a runtime with the specified configuration and policies.
|
||||
//! @param config The configuration used for this runtime instance.
|
||||
//! @param executionFailurePolicy Determines how to handle test targets that fail to execute.
|
||||
//! @param executionFailureDraftingPolicy Determines how test targets that previously failed to execute are drafted into subsequent test sequences.
|
||||
//! @param testFailurePolicy Determines how to handle test targets that report test failures.
|
||||
//! @param integrationFailurePolicy Determines how to handle instances where the build system model and/or test impact analysis data is compromised.
|
||||
//! @param testShardingPolicy Determines how to handle test targets that have opted in to test sharding.
|
||||
Runtime(
|
||||
RuntimeConfig&& config,
|
||||
Policy::ExecutionFailure executionFailurePolicy,
|
||||
Policy::ExecutionFailureDrafting executionFailureDraftingPolicy,
|
||||
Policy::TestFailure testFailurePolicy,
|
||||
Policy::IntegrityFailure integrationFailurePolicy,
|
||||
Policy::TestSharding testShardingPolicy,
|
||||
TargetOutputCapture targetOutputCapture,
|
||||
AZStd::optional<size_t> maxConcurrency = AZStd::nullopt);
|
||||
|
||||
~Runtime();
|
||||
|
||||
//! Runs a test sequence where all tests with a matching suite in the suite filter and also not on the excluded list are selected.
|
||||
//! @param suitesFilter The test suites that will be included in the test selection.
|
||||
//! @param testTargetTimeout The maximum duration individual test targets may be in flight for (infinite if empty).
|
||||
//! @param testTargetTimeout The maximum duration the entire test sequence may run for (infinite if empty).
|
||||
//! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests.
|
||||
//! @param testSequenceCompleteCallback The client function to be called after the test sequence has completed.
|
||||
//! @param testRunCompleteCallback The client function to be called after an individual test run has completed.
|
||||
TestSequenceResult RegularTestSequence(
|
||||
const AZStd::unordered_set<AZStd::string> suitesFilter,
|
||||
AZStd::optional<AZStd::chrono::milliseconds> testTargetTimeout,
|
||||
AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
|
||||
AZStd::optional<TestSequenceStartCallback> testSequenceStartCallback,
|
||||
AZStd::optional<TestSequenceCompleteCallback> testSequenceCompleteCallback,
|
||||
AZStd::optional<TestCompleteCallback> testRunCompleteCallback);
|
||||
|
||||
//! Runs a test sequence where tests are selected according to test impact analysis so long as they are not on the excluded list.
|
||||
//! @param changeList The change list used to determine the tests to select.
|
||||
//! @param testPrioritizationPolicy Determines how selected tests will be prioritized.
|
||||
//! @param testTargetTimeout The maximum duration individual test targets may be in flight for (infinite if empty).
|
||||
//! @param testTargetTimeout The maximum duration the entire test sequence may run for (infinite if empty).
|
||||
//! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests.
|
||||
//! @param testSequenceCompleteCallback The client function to be called after the test sequence has completed.
|
||||
//! @param testRunCompleteCallback The client function to be called after an individual test run has completed.
|
||||
TestSequenceResult ImpactAnalysisTestSequence(
|
||||
const ChangeList& changeList,
|
||||
Policy::TestPrioritization testPrioritizationPolicy,
|
||||
AZStd::optional<AZStd::chrono::milliseconds> testTargetTimeout,
|
||||
AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
|
||||
AZStd::optional<ImpactAnalysisTestSequenceStartCallback> testSequenceStartCallback,
|
||||
AZStd::optional<ImpactAnalysisTestSequenceCompleteCallback> testSequenceCompleteCallback,
|
||||
AZStd::optional<TestCompleteCallback> testRunCompleteCallback);
|
||||
|
||||
//! Runs a test sequence as per the ImpactAnalysisTestSequence where the tests not selected are also run (albeit without instrumentation).
|
||||
//! @param changeList The change list used to determine the tests to select.
|
||||
//! @param suitesFilter The test suites that will be included in the test selection.
|
||||
//! @param testPrioritizationPolicy Determines how selected tests will be prioritized.
|
||||
//! @param testTargetTimeout The maximum duration individual test targets may be in flight for (infinite if empty).
|
||||
//! @param testTargetTimeout The maximum duration the entire test sequence may run for (infinite if empty).
|
||||
//! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests.
|
||||
//! @param testSequenceCompleteCallback The client function to be called after the test sequence has completed.
|
||||
//! @param testRunCompleteCallback The client function to be called after an individual test run has completed.
|
||||
TestSequenceResult SafeImpactAnalysisTestSequence(
|
||||
const ChangeList& changeList,
|
||||
const AZStd::unordered_set<AZStd::string> suitesFilter,
|
||||
Policy::TestPrioritization testPrioritizationPolicy,
|
||||
AZStd::optional<AZStd::chrono::milliseconds> testTargetTimeout,
|
||||
AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
|
||||
AZStd::optional<SafeImpactAnalysisTestSequenceStartCallback> testSequenceStartCallback,
|
||||
AZStd::optional<ImpactAnalysisTestSequenceCompleteCallback> testSequenceCompleteCallback,
|
||||
AZStd::optional<TestCompleteCallback> testRunCompleteCallback);
|
||||
|
||||
//! Runs all tests not on the excluded list and uses their coverage data to seed the test impact analysis data (ant existing data will be overwritten).
|
||||
//! @param testTargetTimeout The maximum duration the entire test sequence may run for (infinite if empty).
|
||||
//! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests.
|
||||
//! @param testSequenceCompleteCallback The client function to be called after the test sequence has completed.
|
||||
//! @param testRunCompleteCallback The client function to be called after an individual test run has completed.
|
||||
TestSequenceResult SeededTestSequence(
|
||||
AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
|
||||
AZStd::optional<TestSequenceStartCallback> testSequenceStartCallback,
|
||||
AZStd::optional<TestSequenceCompleteCallback> testSequenceCompleteCallback,
|
||||
AZStd::optional<TestCompleteCallback> testRunCompleteCallback);
|
||||
|
||||
//! Returns true if the runtime has test impact analysis data (either preexisting or generated).
|
||||
bool HasImpactAnalysisData() const;
|
||||
|
||||
private:
|
||||
RuntimeConfig m_config;
|
||||
Policy::ExecutionFailure m_executionFailurePolicy;
|
||||
Policy::ExecutionFailureDrafting m_executionFailureDraftingPolicy;
|
||||
Policy::TestFailure m_testFailurePolicy;
|
||||
Policy::IntegrityFailure m_integrationFailurePolicy;
|
||||
Policy::TestSharding m_testShardingPolicy;
|
||||
TargetOutputCapture m_targetOutputCapture;
|
||||
size_t m_maxConcurrency = 0;
|
||||
AZStd::unique_ptr<DynamicDependencyMap> m_dynamicDependencyMap;
|
||||
AZStd::unique_ptr<TestEngine> m_testEngine;
|
||||
AZStd::unordered_set<const TestTarget*> m_testTargetExcludeList;
|
||||
AZStd::unordered_set<const TestTarget*> m_testTargetShardList;
|
||||
bool m_hasImpactAnalysisData = false;
|
||||
};
|
||||
} // namespace TestImpact
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
/*
|
||||
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
* its licensors.
|
||||
*
|
||||
* For complete copyright and license terms please see the LICENSE at the root of this
|
||||
* distribution (the "License"). All use of this software is governed by the License,
|
||||
* or, if provided, by the license below or the license accompanying this file. Do not
|
||||
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <TestImpactFramework/TestImpactException.h>
|
||||
|
||||
namespace TestImpact
|
||||
{
|
||||
//! Exception for runtime related exceptions.
|
||||
class RuntimeException
|
||||
: public Exception
|
||||
{
|
||||
public:
|
||||
using Exception::Exception;
|
||||
};
|
||||
} // namespace TestImpact
|
||||
+92
@@ -0,0 +1,92 @@
|
||||
/*
|
||||
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
* its licensors.
|
||||
*
|
||||
* For complete copyright and license terms please see the LICENSE at the root of this
|
||||
* distribution (the "License"). All use of this software is governed by the License,
|
||||
* or, if provided, by the license below or the license accompanying this file. Do not
|
||||
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
namespace TestImpact
|
||||
{
|
||||
namespace Policy
|
||||
{
|
||||
//! Policy for handling of test targets that fail to execute (e.g. due to the binary not being found).
|
||||
//! @note Test targets that fail to execute will be tagged such that their execution can be attempted at a later date. This is
|
||||
//! important as otherwise it would be erroneously assumed that they cover no sources due to having no entries in the dynamic
|
||||
//! dependency map.
|
||||
enum class ExecutionFailure
|
||||
{
|
||||
Abort, //!< Abort the test sequence and report a failure.
|
||||
Continue, //!< Continue the test sequence but treat the execution failures as test failures after the run.
|
||||
Ignore //!< Continue the test sequence and ignore the execution failures.
|
||||
};
|
||||
|
||||
//! Policy for reattempting the execution of test targets that failed to execute in previous runs.
|
||||
enum class ExecutionFailureDrafting
|
||||
{
|
||||
Never, //!< Do not attempt to execute historic execution failures.
|
||||
Always //!< Reattempt the exectution of historic execution failures.
|
||||
};
|
||||
|
||||
//! Policy for prioritizing selected tests.
|
||||
enum class TestPrioritization
|
||||
{
|
||||
None, //!< Do not attempt any test prioritization.
|
||||
DependencyLocality //!< Prioritize test targets according to the locality of the production targets they cover in the build dependency graph.
|
||||
};
|
||||
|
||||
//! Policy for handling test targets that report failing tests.
|
||||
enum class TestFailure
|
||||
{
|
||||
Abort, //!< Abort the test sequence and report the test failure.
|
||||
Continue //!< Continue the test sequence and report the test failures after the run.
|
||||
};
|
||||
|
||||
//! Policy for handling integrity failures of the dynamic dependency map and the source to target mappings.
|
||||
enum class IntegrityFailure
|
||||
{
|
||||
Abort, //!< Abort the test sequence and report the test failure.
|
||||
Continue //!< Continue the test sequence and report the test failures after the run.
|
||||
};
|
||||
|
||||
//! Policy for sharding test targets that have been marked for test sharding.
|
||||
enum class TestSharding
|
||||
{
|
||||
Never, //!< Do not shard any test targets.
|
||||
Always //!< Shard all test targets that have been marked for test sharding.
|
||||
};
|
||||
}
|
||||
|
||||
//! Standard output capture of test target runs.
|
||||
enum class TargetOutputCapture
|
||||
{
|
||||
None, //!< Do not capture any output.
|
||||
StdOut, //!< Send captured output to standard output
|
||||
File, //!< Write captured output to file.
|
||||
StdOutAndFile //!< Send captured output to standard output and write to file.
|
||||
};
|
||||
|
||||
//! Configuration for test targets that opt in to test sharding.
|
||||
enum class ShardConfiguration
|
||||
{
|
||||
Never, //!< Never shard this test target.
|
||||
FixtureContiguous, //!< Each shard contains contiguous fixtures of tests (safest but least optimal).
|
||||
TestContiguous, //!< Each shard contains contiguous tests agnostic of fixtures.
|
||||
FixtureInterleaved, //!< Fixtures of tests are interleaved across shards.
|
||||
TestInterleaved //!< Tests are interlaced across shards agnostic of fixtures (fastest but prone to inter-test dependency problems).
|
||||
};
|
||||
|
||||
//! Result of a test sequence that was run.
|
||||
enum class TestSequenceResult
|
||||
{
|
||||
Success, //!< All tests ran with no failures.
|
||||
Failure, //!< One or more tests failed and/or timed out and/or failed to launch and/or an integrity failure was encountered.
|
||||
Timeout //!< The global timeout for the sequence was exceeded.
|
||||
};
|
||||
} // namespace TestImpact
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
/*
|
||||
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
* its licensors.
|
||||
*
|
||||
* For complete copyright and license terms please see the LICENSE at the root of this
|
||||
* distribution (the "License"). All use of this software is governed by the License,
|
||||
* or, if provided, by the license below or the license accompanying this file. Do not
|
||||
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
*
|
||||
*/
|
||||
|
||||
#include <TestImpactFramework/TestImpactException.h>
|
||||
#include <TestImpactFramework/TestImpactRuntime.h>
|
||||
|
||||
#include <AzCore/IO/SystemFile.h>
|
||||
#include <AzCore/std/containers/vector.h>
|
||||
#include <AzCore/std/string/string.h>
|
||||
|
||||
#pragma once
|
||||
|
||||
namespace TestImpact
|
||||
{
|
||||
//! Attempts to read the contents of the specified file into a string.
|
||||
//! @tparam ExceptionType The exception type to throw upon failure.
|
||||
//! @param path The path to the file to read the contents of.
|
||||
//! @returns The contents of the file.
|
||||
template<typename ExceptionType>
|
||||
AZStd::string ReadFileContents(const RepoPath& path)
|
||||
{
|
||||
const auto fileSize = AZ::IO::SystemFile::Length(path.c_str());
|
||||
AZ_TestImpact_Eval(fileSize > 0, ExceptionType, AZStd::string::format("File %s does not exist", path.c_str()));
|
||||
|
||||
AZStd::vector<char> buffer(fileSize + 1);
|
||||
buffer[fileSize] = '\0';
|
||||
AZ_TestImpact_Eval(
|
||||
AZ::IO::SystemFile::Read(path.c_str(), buffer.data()),
|
||||
ExceptionType,
|
||||
AZStd::string::format("Could not read contents of file %s", path.c_str()));
|
||||
|
||||
return AZStd::string(buffer.begin(), buffer.end());
|
||||
}
|
||||
|
||||
//! Attempts to write the contents of the specified string to a file.
|
||||
//! @tparam ExceptionType The exception type to throw upon failure.
|
||||
//! @param contents The contents to write to the file.
|
||||
//! @param path The path to the file to write the contents to.
|
||||
template<typename ExceptionType>
|
||||
void WriteFileContents(const AZStd::string& contents, const RepoPath& path)
|
||||
{
|
||||
AZ::IO::SystemFile file;
|
||||
const AZStd::vector<char> bytes(contents.begin(), contents.end());
|
||||
AZ_TestImpact_Eval(
|
||||
file.Open(path.c_str(),
|
||||
AZ::IO::SystemFile::SF_OPEN_CREATE | AZ::IO::SystemFile::SF_OPEN_CREATE_PATH | AZ::IO::SystemFile::SF_OPEN_WRITE_ONLY),
|
||||
ExceptionType,
|
||||
AZStd::string::format("Couldn't open file %s for writing", path.c_str()));
|
||||
|
||||
AZ_TestImpact_Eval(
|
||||
file.Write(bytes.data(), bytes.size()), ExceptionType, AZStd::string::format("Couldn't write contents for file %s", path.c_str()));
|
||||
}
|
||||
} // namespace TestImpact
|
||||
Reference in New Issue
Block a user