diff --git a/AutomatedTesting/Gem/PythonTests/AWS/CMakeLists.txt b/AutomatedTesting/Gem/PythonTests/AWS/CMakeLists.txt index a589a395c1..2a5e1d7cab 100644 --- a/AutomatedTesting/Gem/PythonTests/AWS/CMakeLists.txt +++ b/AutomatedTesting/Gem/PythonTests/AWS/CMakeLists.txt @@ -17,15 +17,15 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS) return() endif() - # Enable after installing NodeJS and CDK on jenkins Windows AMI. ly_add_pytest( NAME AutomatedTesting::AWSTests - TEST_SUITE periodic + TEST_SUITE awsi TEST_SERIAL PATH ${CMAKE_CURRENT_LIST_DIR}/${PAL_PLATFORM_NAME}/ RUNTIME_DEPENDENCIES Legacy::Editor AZ::AssetProcessor + AutomatedTesting.GameLauncher AutomatedTesting.Assets COMPONENT AWS diff --git a/AutomatedTesting/Gem/PythonTests/AWS/Windows/aws_metrics/aws_metrics_automation_test.py b/AutomatedTesting/Gem/PythonTests/AWS/Windows/aws_metrics/aws_metrics_automation_test.py index 061db991cf..1fb35cb9e8 100644 --- a/AutomatedTesting/Gem/PythonTests/AWS/Windows/aws_metrics/aws_metrics_automation_test.py +++ b/AutomatedTesting/Gem/PythonTests/AWS/Windows/aws_metrics/aws_metrics_automation_test.py @@ -133,7 +133,7 @@ def update_kinesis_analytics_application_status(aws_metrics_utils: pytest.fixtur aws_metrics_utils.stop_kinesis_data_analytics_application( resource_mappings.get_resource_name_id('AWSMetrics.AnalyticsApplicationName')) -@pytest.mark.SUITE_periodic +@pytest.mark.SUITE_awsi @pytest.mark.usefixtures('automatic_process_killer') @pytest.mark.usefixtures('aws_credentials') @pytest.mark.usefixtures('resource_mappings') diff --git a/AutomatedTesting/Gem/PythonTests/AWS/Windows/client_auth/aws_client_auth_automation_test.py b/AutomatedTesting/Gem/PythonTests/AWS/Windows/client_auth/aws_client_auth_automation_test.py index b56d3f88f5..198fb934d9 100644 --- a/AutomatedTesting/Gem/PythonTests/AWS/Windows/client_auth/aws_client_auth_automation_test.py +++ b/AutomatedTesting/Gem/PythonTests/AWS/Windows/client_auth/aws_client_auth_automation_test.py @@ -21,7 +21,7 @@ AWS_CLIENT_AUTH_FEATURE_NAME = 'AWSClientAuth' logger = logging.getLogger(__name__) -@pytest.mark.SUITE_periodic +@pytest.mark.SUITE_awsi @pytest.mark.usefixtures('asset_processor') @pytest.mark.usefixtures('automatic_process_killer') @pytest.mark.usefixtures('aws_utils') diff --git a/AutomatedTesting/Gem/PythonTests/AWS/Windows/core/test_aws_resource_interaction.py b/AutomatedTesting/Gem/PythonTests/AWS/Windows/core/test_aws_resource_interaction.py index 529151f3f6..949186ad50 100644 --- a/AutomatedTesting/Gem/PythonTests/AWS/Windows/core/test_aws_resource_interaction.py +++ b/AutomatedTesting/Gem/PythonTests/AWS/Windows/core/test_aws_resource_interaction.py @@ -74,7 +74,7 @@ def write_test_data_to_dynamodb_table(resource_mappings: pytest.fixture, aws_uti raise -@pytest.mark.SUITE_periodic +@pytest.mark.SUITE_awsi @pytest.mark.usefixtures('automatic_process_killer') @pytest.mark.usefixtures('asset_processor') @pytest.mark.parametrize('feature_name', [AWS_CORE_FEATURE_NAME]) diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactCommandLineOptions.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactCommandLineOptions.cpp index 0d1f352f21..1bd57e49aa 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactCommandLineOptions.cpp +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactCommandLineOptions.cpp @@ -266,7 +266,8 @@ namespace TestImpact { { SuiteTypeAsString(SuiteType::Main), SuiteType::Main }, { SuiteTypeAsString(SuiteType::Periodic), SuiteType::Periodic }, - { SuiteTypeAsString(SuiteType::Sandbox), SuiteType::Sandbox } + { SuiteTypeAsString(SuiteType::Sandbox), SuiteType::Sandbox }, + { SuiteTypeAsString(SuiteType::AWSI), SuiteType::AWSI } }; return ParseMultiStateOption(OptionKeys[SuiteFilterKey], states, cmd).value_or(SuiteType::Main); @@ -472,7 +473,7 @@ namespace TestImpact " available, no prioritization will occur).\n" " -maxconcurrency= The maximum number of concurrent test targets/shards to be in flight at \n" " any given moment.\n" - " -suite= The test suite to select from for this test sequence."; + " -suite= The test suite to select from for this test sequence."; return help; } diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactTestSequence.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactTestSequence.h index 38b716398c..0a62b7a004 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactTestSequence.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactTestSequence.h @@ -30,7 +30,8 @@ namespace TestImpact { Main = 0, Periodic, - Sandbox + Sandbox, + AWSI }; //! Result of a test sequence that was run. diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactUtils.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactUtils.cpp index 993aee9af5..3f6c9dafd4 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactUtils.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactUtils.cpp @@ -53,6 +53,8 @@ namespace TestImpact return "periodic"; case SuiteType::Sandbox: return "sandbox"; + case SuiteType::AWSI: + return "awsi"; default: throw(Exception("Unexpected suite type")); } diff --git a/cmake/LYTestWrappers.cmake b/cmake/LYTestWrappers.cmake index 8e71cb3db1..a305f0be38 100644 --- a/cmake/LYTestWrappers.cmake +++ b/cmake/LYTestWrappers.cmake @@ -18,7 +18,7 @@ set(LY_GOOGLETEST_EXTRA_PARAMS CACHE STRING "Allows injection of additional opti find_package(Python REQUIRED MODULE) ly_set(LY_PYTEST_EXECUTABLE ${LY_PYTHON_CMD} -B -m pytest -v --tb=short --show-capture=log -c ${LY_ROOT_FOLDER}/ctest_pytest.ini --build-directory "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/$") -ly_set(LY_TEST_GLOBAL_KNOWN_SUITE_NAMES "smoke" "main" "periodic" "benchmark" "sandbox") +ly_set(LY_TEST_GLOBAL_KNOWN_SUITE_NAMES "smoke" "main" "periodic" "benchmark" "sandbox" "awsi") ly_set(LY_TEST_GLOBAL_KNOWN_REQUIREMENTS "gpu") # Set default test aborts to 25 minutes, avoids hitting the CI pipeline inactivity timeout usually set to 30 minutes @@ -71,11 +71,14 @@ endfunction() # \arg:PARENT_NAME(optional) - Name of the parent test run target (if this is a subsequent call to specify a suite) # \arg:TEST_REQUIRES(optional) - List of system resources that are required to run this test. # Only available option is "gpu" -# \arg:TEST_SUITE(optional) - "smoke" or "periodic" or "sandbox" - prevents the test from running normally +# \arg:TEST_SUITE(optional) - "smoke" or "periodic" or "benchmark" or "sandbox" or "awsi" - prevents the test from running normally # and instead places it a special suite of tests that only run when requested. # Otherwise, do not specify a TEST_SUITE value and the default ("main") will apply. +# "smoke" is tiny, quick tests of fundamental operation (tests with no suite marker will also execute here in CI) +# "periodic" is low-priority verification, which should not block code submission # "benchmark" is currently reserved for Google Benchmarks # "sandbox" should be only be used for the workflow of flaky tests +# "awsi" Time consuming AWS integration end-to-end tests # \arg:TIMEOUT (optional) The timeout in seconds for the module. Defaults to LY_TEST_DEFAULT_TIMEOUT. # \arg:TEST_COMMAND - Command which runs the tests. It is a required argument # \arg:NON_IDE_PARAMS - extra params that will be run in ctest, but will not be used in the IDE. @@ -279,7 +282,7 @@ endfunction() # \arg:RUNTIME_DEPENDENCIES (optional) - List of additional runtime dependencies required by this test. # \arg:COMPONENT (optional) - Scope of the feature area that the test belongs to (eg. physics, graphics, etc.). # \arg:EXCLUDE_TEST_RUN_TARGET_FROM_IDE(bool) - If set the test run target will be not be shown in the IDE -# \arg:TEST_SUITE(optional) - "smoke" or "periodic" or "sandbox" - prevents the test from running normally +# \arg:TEST_SUITE(optional) - "smoke" or "periodic" or "sandbox" or "awsi" - prevents the test from running normally # and instead places it a special suite of tests that only run when requested. # \arg:TIMEOUT (optional) The timeout in seconds for the module. If not set defaults to LY_TEST_DEFAULT_TIMEOUT # @@ -331,7 +334,7 @@ endfunction() # \arg:TARGET Name of the target module that is being run for tests. If not provided, will default to 'NAME' # \arg:TEST_REQUIRES(optional) List of system resources that are required to run this test. # Only available option is "gpu" -# \arg:TEST_SUITE(optional) - "smoke" or "periodic" or "sandbox" - prevents the test from running normally +# \arg:TEST_SUITE(optional) - "smoke" or "periodic" or "sandbox" or "awsi" - prevents the test from running normally # and instead places it a special suite of tests that only run when requested. # \arg:TEST_COMMAND(optional) - Command which runs the tests. # If not supplied, a default of "AzTestRunner $ AzRunUnitTests" will be used @@ -372,7 +375,7 @@ function(ly_add_googletest) # will actually run everything in main OR everything tagged as requiring a GPU # instead of only tests tagged with BOTH main and gpu... # so we have to do it this way (negating all others) - set(non_ide_params "--gtest_filter=-*SUITE_smoke*:*SUITE_periodic*:*SUITE_benchmark*:*SUITE_sandbox*") + set(non_ide_params "--gtest_filter=-*SUITE_smoke*:*SUITE_periodic*:*SUITE_benchmark*:*SUITE_sandbox*:*SUITE_awsi*") endif() if(NOT ly_add_googletest_TEST_COMMAND) diff --git a/ctest_pytest.ini b/ctest_pytest.ini index 8365e07988..93310a522d 100644 --- a/ctest_pytest.ini +++ b/ctest_pytest.ini @@ -18,6 +18,7 @@ markers = SUITE_smoke: Tiny, quick tests of fundamental operation (tests with no SUITE_periodic: low-priority verification, which should not block code submission SUITE_benchmark: Benchmarks which do not pass or fail but instead output statistics SUITE_sandbox: Temporarly contains flaky/unstable tests, this should not block code submission. This test suite should be ideally empty + SUITE_awsi: Time consuming AWS integration end-to-end tests # secondary markers which may appear alongisde a suite marker: REQUIRES_gpu: Tests which require a physical GPU # custom markers not listed above will cause pytest to emit a typo warning diff --git a/scripts/build/Platform/Linux/build_config.json b/scripts/build/Platform/Linux/build_config.json index e60b5deee9..e70fa32878 100644 --- a/scripts/build/Platform/Linux/build_config.json +++ b/scripts/build/Platform/Linux/build_config.json @@ -83,7 +83,7 @@ "CMAKE_OPTIONS": "-G 'Ninja Multi-Config' -DCMAKE_C_COMPILER=clang-6.0 -DCMAKE_CXX_COMPILER=clang++-6.0 -DLY_UNITY_BUILD=TRUE -DLY_PARALLEL_LINK_JOBS=4", "CMAKE_LY_PROJECTS": "AutomatedTesting", "CMAKE_TARGET": "all", - "CTEST_OPTIONS": "-E Gem::EMotionFX.Editor.Tests -LE SUITE_sandbox -L FRAMEWORK_googletest" + "CTEST_OPTIONS": "-E Gem::EMotionFX.Editor.Tests -LE (SUITE_sandbox|SUITE_awsi) -L FRAMEWORK_googletest" } }, "test_profile_nounity": { @@ -95,7 +95,7 @@ "CMAKE_OPTIONS": "-G 'Ninja Multi-Config' -DCMAKE_C_COMPILER=clang-6.0 -DCMAKE_CXX_COMPILER=clang++-6.0 -DLY_UNITY_BUILD=FALSE -DLY_PARALLEL_LINK_JOBS=4", "CMAKE_LY_PROJECTS": "AutomatedTesting", "CMAKE_TARGET": "all", - "CTEST_OPTIONS": "-E Gem::EMotionFX.Editor.Tests -LE SUITE_sandbox -L FRAMEWORK_googletest" + "CTEST_OPTIONS": "-E Gem::EMotionFX.Editor.Tests -LE (SUITE_sandbox|SUITE_awsi) -L FRAMEWORK_googletest" } }, "asset_profile": { diff --git a/scripts/build/Platform/Windows/build_config.json b/scripts/build/Platform/Windows/build_config.json index 583a73b58b..689ba6935d 100644 --- a/scripts/build/Platform/Windows/build_config.json +++ b/scripts/build/Platform/Windows/build_config.json @@ -213,19 +213,38 @@ "ASSET_PROCESSOR_PLATFORMS": "pc,server" } }, - "periodic_test_profile_vs2019_pipe": { + "awsi_test_profile_vs2019_pipe": { "TAGS": [ "nightly-incremental", "nightly-clean" ], "steps": [ "awsi_deployment", - "periodic_test_profile_vs2019" + "awsi_test_profile_vs2019" ] }, + "awsi_test_profile_vs2019": { + "TAGS": [ + "weekly-build-metrics" + ], + "COMMAND": "build_test_windows.cmd", + "PARAMETERS": { + "CONFIGURATION": "profile", + "OUTPUT_DIRECTORY": "build\\windows_vs2019", + "CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE", + "CMAKE_LY_PROJECTS": "AutomatedTesting", + "CMAKE_TARGET": "TEST_SUITE_awsi", + "CMAKE_NATIVE_BUILD_ARGS": "/m /nologo", + "CTEST_OPTIONS": "-L \"(SUITE_awsi)\" -T Test", + "TEST_METRICS": "True", + "TEST_RESULTS": "True" + } + }, "periodic_test_profile_vs2019": { "TAGS": [ - "weekly-build-metrics" + "nightly-incremental", + "nightly-clean", + "weekly-build-metrics" ], "COMMAND": "build_test_windows.cmd", "PARAMETERS": { diff --git a/scripts/ctest/ctest_driver.py b/scripts/ctest/ctest_driver.py index 3fc07921fd..aaad523186 100755 --- a/scripts/ctest/ctest_driver.py +++ b/scripts/ctest/ctest_driver.py @@ -19,7 +19,8 @@ SUITES_AND_DESCRIPTIONS = { "main": "The default set of tests, covers most of all testing.", "periodic": "Tests which can take a long time and should be done periodially instead of every commit - these should not block code submission", "benchmark": "Benchmarks - instead of pass/fail, these collect data for comparison against historic data", - "sandbox": "Flaky/Intermittent failing tests, this is used as a temporary spot to hold flaky tests, this will not block code submission. Ideally, this suite should always be empty" + "sandbox": "Flaky/Intermittent failing tests, this is used as a temporary spot to hold flaky tests, this will not block code submission. Ideally, this suite should always be empty", + "awsi": "Time consuming AWS integration end-to-end tests" } BUILD_CONFIGURATIONS = [ diff --git a/scripts/ctest/sanity_test.py b/scripts/ctest/sanity_test.py index dcbab281bb..2972546c65 100755 --- a/scripts/ctest/sanity_test.py +++ b/scripts/ctest/sanity_test.py @@ -32,6 +32,10 @@ def test_Sanity_Benchmark_Pass(): def test_Sanity_Sandbox_Pass(): pass +@pytest.mark.SUITE_awsi +def test_Sanity_AWSI_Pass(): + pass + @pytest.mark.REQUIRES_gpu def test_Sanity_RequireGpu_Pass(): pass