Add AWSI automation tests to a new test suite and pipe (#3913)
Signed-off-by: junbo <junbo@amazon.com>
This commit is contained in:
Junbo Liang
GitHub
@@ -17,15 +17,15 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS)
|
||||
return()
|
||||
endif()
|
||||
|
||||
# Enable after installing NodeJS and CDK on jenkins Windows AMI.
|
||||
ly_add_pytest(
|
||||
NAME AutomatedTesting::AWSTests
|
||||
TEST_SUITE periodic
|
||||
TEST_SUITE awsi
|
||||
TEST_SERIAL
|
||||
PATH ${CMAKE_CURRENT_LIST_DIR}/${PAL_PLATFORM_NAME}/
|
||||
RUNTIME_DEPENDENCIES
|
||||
Legacy::Editor
|
||||
AZ::AssetProcessor
|
||||
AutomatedTesting.GameLauncher
|
||||
AutomatedTesting.Assets
|
||||
COMPONENT
|
||||
AWS
|
||||
|
||||
+1
-1
@@ -133,7 +133,7 @@ def update_kinesis_analytics_application_status(aws_metrics_utils: pytest.fixtur
|
||||
aws_metrics_utils.stop_kinesis_data_analytics_application(
|
||||
resource_mappings.get_resource_name_id('AWSMetrics.AnalyticsApplicationName'))
|
||||
|
||||
@pytest.mark.SUITE_periodic
|
||||
@pytest.mark.SUITE_awsi
|
||||
@pytest.mark.usefixtures('automatic_process_killer')
|
||||
@pytest.mark.usefixtures('aws_credentials')
|
||||
@pytest.mark.usefixtures('resource_mappings')
|
||||
|
||||
+1
-1
@@ -21,7 +21,7 @@ AWS_CLIENT_AUTH_FEATURE_NAME = 'AWSClientAuth'
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@pytest.mark.SUITE_periodic
|
||||
@pytest.mark.SUITE_awsi
|
||||
@pytest.mark.usefixtures('asset_processor')
|
||||
@pytest.mark.usefixtures('automatic_process_killer')
|
||||
@pytest.mark.usefixtures('aws_utils')
|
||||
|
||||
@@ -74,7 +74,7 @@ def write_test_data_to_dynamodb_table(resource_mappings: pytest.fixture, aws_uti
|
||||
raise
|
||||
|
||||
|
||||
@pytest.mark.SUITE_periodic
|
||||
@pytest.mark.SUITE_awsi
|
||||
@pytest.mark.usefixtures('automatic_process_killer')
|
||||
@pytest.mark.usefixtures('asset_processor')
|
||||
@pytest.mark.parametrize('feature_name', [AWS_CORE_FEATURE_NAME])
|
||||
|
||||
+3
-2
@@ -266,7 +266,8 @@ namespace TestImpact
|
||||
{
|
||||
{ SuiteTypeAsString(SuiteType::Main), SuiteType::Main },
|
||||
{ SuiteTypeAsString(SuiteType::Periodic), SuiteType::Periodic },
|
||||
{ SuiteTypeAsString(SuiteType::Sandbox), SuiteType::Sandbox }
|
||||
{ SuiteTypeAsString(SuiteType::Sandbox), SuiteType::Sandbox },
|
||||
{ SuiteTypeAsString(SuiteType::AWSI), SuiteType::AWSI }
|
||||
};
|
||||
|
||||
return ParseMultiStateOption(OptionKeys[SuiteFilterKey], states, cmd).value_or(SuiteType::Main);
|
||||
@@ -472,7 +473,7 @@ namespace TestImpact
|
||||
" available, no prioritization will occur).\n"
|
||||
" -maxconcurrency=<number> The maximum number of concurrent test targets/shards to be in flight at \n"
|
||||
" any given moment.\n"
|
||||
" -suite=<main, periodic, sandbox> The test suite to select from for this test sequence.";
|
||||
" -suite=<main, periodic, sandbox, awsi> The test suite to select from for this test sequence.";
|
||||
|
||||
return help;
|
||||
}
|
||||
|
||||
+2
-1
@@ -30,7 +30,8 @@ namespace TestImpact
|
||||
{
|
||||
Main = 0,
|
||||
Periodic,
|
||||
Sandbox
|
||||
Sandbox,
|
||||
AWSI
|
||||
};
|
||||
|
||||
//! Result of a test sequence that was run.
|
||||
|
||||
@@ -53,6 +53,8 @@ namespace TestImpact
|
||||
return "periodic";
|
||||
case SuiteType::Sandbox:
|
||||
return "sandbox";
|
||||
case SuiteType::AWSI:
|
||||
return "awsi";
|
||||
default:
|
||||
throw(Exception("Unexpected suite type"));
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@ set(LY_GOOGLETEST_EXTRA_PARAMS CACHE STRING "Allows injection of additional opti
|
||||
find_package(Python REQUIRED MODULE)
|
||||
|
||||
ly_set(LY_PYTEST_EXECUTABLE ${LY_PYTHON_CMD} -B -m pytest -v --tb=short --show-capture=log -c ${LY_ROOT_FOLDER}/ctest_pytest.ini --build-directory "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/$<CONFIG>")
|
||||
ly_set(LY_TEST_GLOBAL_KNOWN_SUITE_NAMES "smoke" "main" "periodic" "benchmark" "sandbox")
|
||||
ly_set(LY_TEST_GLOBAL_KNOWN_SUITE_NAMES "smoke" "main" "periodic" "benchmark" "sandbox" "awsi")
|
||||
ly_set(LY_TEST_GLOBAL_KNOWN_REQUIREMENTS "gpu")
|
||||
|
||||
# Set default test aborts to 25 minutes, avoids hitting the CI pipeline inactivity timeout usually set to 30 minutes
|
||||
@@ -71,11 +71,14 @@ endfunction()
|
||||
# \arg:PARENT_NAME(optional) - Name of the parent test run target (if this is a subsequent call to specify a suite)
|
||||
# \arg:TEST_REQUIRES(optional) - List of system resources that are required to run this test.
|
||||
# Only available option is "gpu"
|
||||
# \arg:TEST_SUITE(optional) - "smoke" or "periodic" or "sandbox" - prevents the test from running normally
|
||||
# \arg:TEST_SUITE(optional) - "smoke" or "periodic" or "benchmark" or "sandbox" or "awsi" - prevents the test from running normally
|
||||
# and instead places it a special suite of tests that only run when requested.
|
||||
# Otherwise, do not specify a TEST_SUITE value and the default ("main") will apply.
|
||||
# "smoke" is tiny, quick tests of fundamental operation (tests with no suite marker will also execute here in CI)
|
||||
# "periodic" is low-priority verification, which should not block code submission
|
||||
# "benchmark" is currently reserved for Google Benchmarks
|
||||
# "sandbox" should be only be used for the workflow of flaky tests
|
||||
# "awsi" Time consuming AWS integration end-to-end tests
|
||||
# \arg:TIMEOUT (optional) The timeout in seconds for the module. Defaults to LY_TEST_DEFAULT_TIMEOUT.
|
||||
# \arg:TEST_COMMAND - Command which runs the tests. It is a required argument
|
||||
# \arg:NON_IDE_PARAMS - extra params that will be run in ctest, but will not be used in the IDE.
|
||||
@@ -279,7 +282,7 @@ endfunction()
|
||||
# \arg:RUNTIME_DEPENDENCIES (optional) - List of additional runtime dependencies required by this test.
|
||||
# \arg:COMPONENT (optional) - Scope of the feature area that the test belongs to (eg. physics, graphics, etc.).
|
||||
# \arg:EXCLUDE_TEST_RUN_TARGET_FROM_IDE(bool) - If set the test run target will be not be shown in the IDE
|
||||
# \arg:TEST_SUITE(optional) - "smoke" or "periodic" or "sandbox" - prevents the test from running normally
|
||||
# \arg:TEST_SUITE(optional) - "smoke" or "periodic" or "sandbox" or "awsi" - prevents the test from running normally
|
||||
# and instead places it a special suite of tests that only run when requested.
|
||||
# \arg:TIMEOUT (optional) The timeout in seconds for the module. If not set defaults to LY_TEST_DEFAULT_TIMEOUT
|
||||
#
|
||||
@@ -331,7 +334,7 @@ endfunction()
|
||||
# \arg:TARGET Name of the target module that is being run for tests. If not provided, will default to 'NAME'
|
||||
# \arg:TEST_REQUIRES(optional) List of system resources that are required to run this test.
|
||||
# Only available option is "gpu"
|
||||
# \arg:TEST_SUITE(optional) - "smoke" or "periodic" or "sandbox" - prevents the test from running normally
|
||||
# \arg:TEST_SUITE(optional) - "smoke" or "periodic" or "sandbox" or "awsi" - prevents the test from running normally
|
||||
# and instead places it a special suite of tests that only run when requested.
|
||||
# \arg:TEST_COMMAND(optional) - Command which runs the tests.
|
||||
# If not supplied, a default of "AzTestRunner $<TARGET_FILE:${NAME}> AzRunUnitTests" will be used
|
||||
@@ -372,7 +375,7 @@ function(ly_add_googletest)
|
||||
# will actually run everything in main OR everything tagged as requiring a GPU
|
||||
# instead of only tests tagged with BOTH main and gpu...
|
||||
# so we have to do it this way (negating all others)
|
||||
set(non_ide_params "--gtest_filter=-*SUITE_smoke*:*SUITE_periodic*:*SUITE_benchmark*:*SUITE_sandbox*")
|
||||
set(non_ide_params "--gtest_filter=-*SUITE_smoke*:*SUITE_periodic*:*SUITE_benchmark*:*SUITE_sandbox*:*SUITE_awsi*")
|
||||
endif()
|
||||
|
||||
if(NOT ly_add_googletest_TEST_COMMAND)
|
||||
|
||||
@@ -18,6 +18,7 @@ markers = SUITE_smoke: Tiny, quick tests of fundamental operation (tests with no
|
||||
SUITE_periodic: low-priority verification, which should not block code submission
|
||||
SUITE_benchmark: Benchmarks which do not pass or fail but instead output statistics
|
||||
SUITE_sandbox: Temporarly contains flaky/unstable tests, this should not block code submission. This test suite should be ideally empty
|
||||
SUITE_awsi: Time consuming AWS integration end-to-end tests
|
||||
# secondary markers which may appear alongisde a suite marker:
|
||||
REQUIRES_gpu: Tests which require a physical GPU
|
||||
# custom markers not listed above will cause pytest to emit a typo warning
|
||||
|
||||
@@ -83,7 +83,7 @@
|
||||
"CMAKE_OPTIONS": "-G 'Ninja Multi-Config' -DCMAKE_C_COMPILER=clang-6.0 -DCMAKE_CXX_COMPILER=clang++-6.0 -DLY_UNITY_BUILD=TRUE -DLY_PARALLEL_LINK_JOBS=4",
|
||||
"CMAKE_LY_PROJECTS": "AutomatedTesting",
|
||||
"CMAKE_TARGET": "all",
|
||||
"CTEST_OPTIONS": "-E Gem::EMotionFX.Editor.Tests -LE SUITE_sandbox -L FRAMEWORK_googletest"
|
||||
"CTEST_OPTIONS": "-E Gem::EMotionFX.Editor.Tests -LE (SUITE_sandbox|SUITE_awsi) -L FRAMEWORK_googletest"
|
||||
}
|
||||
},
|
||||
"test_profile_nounity": {
|
||||
@@ -95,7 +95,7 @@
|
||||
"CMAKE_OPTIONS": "-G 'Ninja Multi-Config' -DCMAKE_C_COMPILER=clang-6.0 -DCMAKE_CXX_COMPILER=clang++-6.0 -DLY_UNITY_BUILD=FALSE -DLY_PARALLEL_LINK_JOBS=4",
|
||||
"CMAKE_LY_PROJECTS": "AutomatedTesting",
|
||||
"CMAKE_TARGET": "all",
|
||||
"CTEST_OPTIONS": "-E Gem::EMotionFX.Editor.Tests -LE SUITE_sandbox -L FRAMEWORK_googletest"
|
||||
"CTEST_OPTIONS": "-E Gem::EMotionFX.Editor.Tests -LE (SUITE_sandbox|SUITE_awsi) -L FRAMEWORK_googletest"
|
||||
}
|
||||
},
|
||||
"asset_profile": {
|
||||
|
||||
@@ -213,19 +213,38 @@
|
||||
"ASSET_PROCESSOR_PLATFORMS": "pc,server"
|
||||
}
|
||||
},
|
||||
"periodic_test_profile_vs2019_pipe": {
|
||||
"awsi_test_profile_vs2019_pipe": {
|
||||
"TAGS": [
|
||||
"nightly-incremental",
|
||||
"nightly-clean"
|
||||
],
|
||||
"steps": [
|
||||
"awsi_deployment",
|
||||
"periodic_test_profile_vs2019"
|
||||
"awsi_test_profile_vs2019"
|
||||
]
|
||||
},
|
||||
"awsi_test_profile_vs2019": {
|
||||
"TAGS": [
|
||||
"weekly-build-metrics"
|
||||
],
|
||||
"COMMAND": "build_test_windows.cmd",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION": "profile",
|
||||
"OUTPUT_DIRECTORY": "build\\windows_vs2019",
|
||||
"CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE",
|
||||
"CMAKE_LY_PROJECTS": "AutomatedTesting",
|
||||
"CMAKE_TARGET": "TEST_SUITE_awsi",
|
||||
"CMAKE_NATIVE_BUILD_ARGS": "/m /nologo",
|
||||
"CTEST_OPTIONS": "-L \"(SUITE_awsi)\" -T Test",
|
||||
"TEST_METRICS": "True",
|
||||
"TEST_RESULTS": "True"
|
||||
}
|
||||
},
|
||||
"periodic_test_profile_vs2019": {
|
||||
"TAGS": [
|
||||
"weekly-build-metrics"
|
||||
"nightly-incremental",
|
||||
"nightly-clean",
|
||||
"weekly-build-metrics"
|
||||
],
|
||||
"COMMAND": "build_test_windows.cmd",
|
||||
"PARAMETERS": {
|
||||
|
||||
@@ -19,7 +19,8 @@ SUITES_AND_DESCRIPTIONS = {
|
||||
"main": "The default set of tests, covers most of all testing.",
|
||||
"periodic": "Tests which can take a long time and should be done periodially instead of every commit - these should not block code submission",
|
||||
"benchmark": "Benchmarks - instead of pass/fail, these collect data for comparison against historic data",
|
||||
"sandbox": "Flaky/Intermittent failing tests, this is used as a temporary spot to hold flaky tests, this will not block code submission. Ideally, this suite should always be empty"
|
||||
"sandbox": "Flaky/Intermittent failing tests, this is used as a temporary spot to hold flaky tests, this will not block code submission. Ideally, this suite should always be empty",
|
||||
"awsi": "Time consuming AWS integration end-to-end tests"
|
||||
}
|
||||
|
||||
BUILD_CONFIGURATIONS = [
|
||||
|
||||
@@ -32,6 +32,10 @@ def test_Sanity_Benchmark_Pass():
|
||||
def test_Sanity_Sandbox_Pass():
|
||||
pass
|
||||
|
||||
@pytest.mark.SUITE_awsi
|
||||
def test_Sanity_AWSI_Pass():
|
||||
pass
|
||||
|
||||
@pytest.mark.REQUIRES_gpu
|
||||
def test_Sanity_RequireGpu_Pass():
|
||||
pass
|
||||
|
||||
Reference in New Issue
Block a user