Add AWSI automation tests to a new test suite and pipe (#3913)

Signed-off-by: junbo <junbo@amazon.com>

AutomatedTesting/Gem/PythonTests/AWS/CMakeLists.txt

@@ -17,15 +17,15 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS)
         return()
     endif()
 
-    # Enable after installing NodeJS and CDK on jenkins Windows AMI.
     ly_add_pytest(
         NAME AutomatedTesting::AWSTests
-        TEST_SUITE periodic
+        TEST_SUITE awsi
         TEST_SERIAL
         PATH ${CMAKE_CURRENT_LIST_DIR}/${PAL_PLATFORM_NAME}/
         RUNTIME_DEPENDENCIES
             Legacy::Editor
             AZ::AssetProcessor
+            AutomatedTesting.GameLauncher
             AutomatedTesting.Assets
         COMPONENT
             AWS

AutomatedTesting/Gem/PythonTests/AWS/Windows/aws_metrics/aws_metrics_automation_test.py

@@ -133,7 +133,7 @@ def update_kinesis_analytics_application_status(aws_metrics_utils: pytest.fixtur
         aws_metrics_utils.stop_kinesis_data_analytics_application(
             resource_mappings.get_resource_name_id('AWSMetrics.AnalyticsApplicationName'))
 
-@pytest.mark.SUITE_periodic
+@pytest.mark.SUITE_awsi
 @pytest.mark.usefixtures('automatic_process_killer')
 @pytest.mark.usefixtures('aws_credentials')
 @pytest.mark.usefixtures('resource_mappings')

AutomatedTesting/Gem/PythonTests/AWS/Windows/client_auth/aws_client_auth_automation_test.py

@@ -21,7 +21,7 @@ AWS_CLIENT_AUTH_FEATURE_NAME = 'AWSClientAuth'
 logger = logging.getLogger(__name__)
 
 
-@pytest.mark.SUITE_periodic
+@pytest.mark.SUITE_awsi
 @pytest.mark.usefixtures('asset_processor')
 @pytest.mark.usefixtures('automatic_process_killer')
 @pytest.mark.usefixtures('aws_utils')

AutomatedTesting/Gem/PythonTests/AWS/Windows/core/test_aws_resource_interaction.py

@@ -74,7 +74,7 @@ def write_test_data_to_dynamodb_table(resource_mappings: pytest.fixture, aws_uti
         raise
 
 
-@pytest.mark.SUITE_periodic
+@pytest.mark.SUITE_awsi
 @pytest.mark.usefixtures('automatic_process_killer')
 @pytest.mark.usefixtures('asset_processor')
 @pytest.mark.parametrize('feature_name', [AWS_CORE_FEATURE_NAME])

Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactCommandLineOptions.cpp

@@ -266,7 +266,8 @@ namespace TestImpact
             {
                 { SuiteTypeAsString(SuiteType::Main), SuiteType::Main },
                 { SuiteTypeAsString(SuiteType::Periodic), SuiteType::Periodic },
-                { SuiteTypeAsString(SuiteType::Sandbox), SuiteType::Sandbox }
+                { SuiteTypeAsString(SuiteType::Sandbox), SuiteType::Sandbox },
+                { SuiteTypeAsString(SuiteType::AWSI), SuiteType::AWSI }
             };
 
             return ParseMultiStateOption(OptionKeys[SuiteFilterKey], states, cmd).value_or(SuiteType::Main);
@@ -472,7 +473,7 @@ namespace TestImpact
             "                                                                available, no prioritization will occur).\n"
             "    -maxconcurrency=<number>                                    The maximum number of concurrent test targets/shards to be in flight at \n"
             "                                                                any given moment.\n"
-            "    -suite=<main, periodic, sandbox>                            The test suite to select from for this test sequence.";
+            "    -suite=<main, periodic, sandbox, awsi>                      The test suite to select from for this test sequence.";
 
         return help;
     }

Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactTestSequence.h

@@ -30,7 +30,8 @@ namespace TestImpact
     {
         Main = 0,
         Periodic,
-        Sandbox
+        Sandbox,
+        AWSI
     };
 
     //! Result of a test sequence that was run.

Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactUtils.cpp

@@ -53,6 +53,8 @@ namespace TestImpact
             return "periodic";
         case SuiteType::Sandbox:
             return "sandbox";
+        case SuiteType::AWSI:
+            return "awsi";
         default:
             throw(Exception("Unexpected suite type"));
         }

cmake/LYTestWrappers.cmake

@@ -18,7 +18,7 @@ set(LY_GOOGLETEST_EXTRA_PARAMS CACHE STRING "Allows injection of additional opti
 find_package(Python REQUIRED MODULE)
 
 ly_set(LY_PYTEST_EXECUTABLE ${LY_PYTHON_CMD} -B -m pytest -v --tb=short --show-capture=log -c ${LY_ROOT_FOLDER}/ctest_pytest.ini --build-directory "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/$<CONFIG>")
-ly_set(LY_TEST_GLOBAL_KNOWN_SUITE_NAMES "smoke" "main" "periodic" "benchmark" "sandbox")
+ly_set(LY_TEST_GLOBAL_KNOWN_SUITE_NAMES "smoke" "main" "periodic" "benchmark" "sandbox" "awsi")
 ly_set(LY_TEST_GLOBAL_KNOWN_REQUIREMENTS "gpu")
 
 # Set default test aborts to 25 minutes, avoids hitting the CI pipeline inactivity timeout usually set to 30 minutes
@@ -71,11 +71,14 @@ endfunction()
 # \arg:PARENT_NAME(optional) - Name of the parent test run target (if this is a subsequent call to specify a suite)
 # \arg:TEST_REQUIRES(optional) - List of system resources that are required to run this test.
 #      Only available option is "gpu"
-# \arg:TEST_SUITE(optional) - "smoke" or "periodic" or "sandbox" - prevents the test from running normally
+# \arg:TEST_SUITE(optional) - "smoke" or "periodic" or "benchmark" or "sandbox" or "awsi" - prevents the test from running normally
 #      and instead places it a special suite of tests that only run when requested.
 #      Otherwise, do not specify a TEST_SUITE value and the default ("main") will apply.
+#      "smoke" is tiny, quick tests of fundamental operation (tests with no suite marker will also execute here in CI)
+#      "periodic" is low-priority verification, which should not block code submission
 #      "benchmark" is currently reserved for Google Benchmarks
 #      "sandbox" should be only be used for the workflow of flaky tests
+#      "awsi" Time consuming AWS integration end-to-end tests
 # \arg:TIMEOUT (optional) The timeout in seconds for the module. Defaults to LY_TEST_DEFAULT_TIMEOUT.
 # \arg:TEST_COMMAND - Command which runs the tests. It is a required argument
 # \arg:NON_IDE_PARAMS - extra params that will be run in ctest, but will not be used in the IDE.
@@ -279,7 +282,7 @@ endfunction()
 # \arg:RUNTIME_DEPENDENCIES (optional) - List of additional runtime dependencies required by this test.
 # \arg:COMPONENT (optional) - Scope of the feature area that the test belongs to (eg. physics, graphics, etc.).
 # \arg:EXCLUDE_TEST_RUN_TARGET_FROM_IDE(bool) - If set the test run target will be not be shown in the IDE
-# \arg:TEST_SUITE(optional) - "smoke" or "periodic" or "sandbox" - prevents the test from running normally
+# \arg:TEST_SUITE(optional) - "smoke" or "periodic" or "sandbox" or "awsi" - prevents the test from running normally
 #      and instead places it a special suite of tests that only run when requested.
 # \arg:TIMEOUT (optional) The timeout in seconds for the module. If not set defaults to LY_TEST_DEFAULT_TIMEOUT
 #
@@ -331,7 +334,7 @@ endfunction()
 # \arg:TARGET Name of the target module that is being run for tests. If not provided, will default to 'NAME'
 # \arg:TEST_REQUIRES(optional) List of system resources that are required to run this test.
 #      Only available option is "gpu"
-# \arg:TEST_SUITE(optional) - "smoke" or "periodic" or "sandbox" - prevents the test from running normally
+# \arg:TEST_SUITE(optional) - "smoke" or "periodic" or "sandbox" or "awsi" - prevents the test from running normally
 #      and instead places it a special suite of tests that only run when requested.
 # \arg:TEST_COMMAND(optional) - Command which runs the tests.
 #      If not supplied, a default of "AzTestRunner $<TARGET_FILE:${NAME}> AzRunUnitTests" will be used
@@ -372,7 +375,7 @@ function(ly_add_googletest)
         # will actually run everything in main OR everything tagged as requiring a GPU
         # instead of only tests tagged with BOTH main and gpu...
         # so we have to do it this way (negating all others)
-        set(non_ide_params "--gtest_filter=-*SUITE_smoke*:*SUITE_periodic*:*SUITE_benchmark*:*SUITE_sandbox*")
+        set(non_ide_params "--gtest_filter=-*SUITE_smoke*:*SUITE_periodic*:*SUITE_benchmark*:*SUITE_sandbox*:*SUITE_awsi*")
     endif()
 
     if(NOT ly_add_googletest_TEST_COMMAND)

ctest_pytest.ini

@@ -18,6 +18,7 @@ markers = SUITE_smoke: Tiny, quick tests of fundamental operation (tests with no
           SUITE_periodic: low-priority verification, which should not block code submission
           SUITE_benchmark: Benchmarks which do not pass or fail but instead output statistics
           SUITE_sandbox: Temporarly contains flaky/unstable tests, this should not block code submission. This test suite should be ideally empty
+          SUITE_awsi: Time consuming AWS integration end-to-end tests
 # secondary markers which may appear alongisde a suite marker:
           REQUIRES_gpu: Tests which require a physical GPU
 # custom markers not listed above will cause pytest to emit a typo warning

scripts/build/Platform/Linux/build_config.json

@@ -83,7 +83,7 @@
       "CMAKE_OPTIONS": "-G 'Ninja Multi-Config' -DCMAKE_C_COMPILER=clang-6.0 -DCMAKE_CXX_COMPILER=clang++-6.0 -DLY_UNITY_BUILD=TRUE -DLY_PARALLEL_LINK_JOBS=4",
       "CMAKE_LY_PROJECTS": "AutomatedTesting",
       "CMAKE_TARGET": "all",
-      "CTEST_OPTIONS": "-E Gem::EMotionFX.Editor.Tests -LE SUITE_sandbox -L FRAMEWORK_googletest"
+      "CTEST_OPTIONS": "-E Gem::EMotionFX.Editor.Tests -LE (SUITE_sandbox|SUITE_awsi) -L FRAMEWORK_googletest"
     }
   },
   "test_profile_nounity": {
@@ -95,7 +95,7 @@
       "CMAKE_OPTIONS": "-G 'Ninja Multi-Config' -DCMAKE_C_COMPILER=clang-6.0 -DCMAKE_CXX_COMPILER=clang++-6.0 -DLY_UNITY_BUILD=FALSE -DLY_PARALLEL_LINK_JOBS=4",
       "CMAKE_LY_PROJECTS": "AutomatedTesting",
       "CMAKE_TARGET": "all",
-      "CTEST_OPTIONS": "-E Gem::EMotionFX.Editor.Tests -LE SUITE_sandbox -L FRAMEWORK_googletest"
+      "CTEST_OPTIONS": "-E Gem::EMotionFX.Editor.Tests -LE (SUITE_sandbox|SUITE_awsi) -L FRAMEWORK_googletest"
     }
   },
   "asset_profile": {

scripts/build/Platform/Windows/build_config.json

@@ -213,19 +213,38 @@
       "ASSET_PROCESSOR_PLATFORMS": "pc,server"
     }
   },
-  "periodic_test_profile_vs2019_pipe": {
+  "awsi_test_profile_vs2019_pipe": {
     "TAGS": [
       "nightly-incremental",
       "nightly-clean"
     ],
     "steps": [
       "awsi_deployment",
-      "periodic_test_profile_vs2019"
+      "awsi_test_profile_vs2019"
     ]
   },
+  "awsi_test_profile_vs2019": {
+    "TAGS": [
+        "weekly-build-metrics"
+    ],
+    "COMMAND": "build_test_windows.cmd",
+    "PARAMETERS": {
+      "CONFIGURATION": "profile",
+      "OUTPUT_DIRECTORY": "build\\windows_vs2019",
+      "CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE",
+      "CMAKE_LY_PROJECTS": "AutomatedTesting",
+      "CMAKE_TARGET": "TEST_SUITE_awsi",
+      "CMAKE_NATIVE_BUILD_ARGS": "/m /nologo",
+      "CTEST_OPTIONS": "-L \"(SUITE_awsi)\" -T Test",
+      "TEST_METRICS": "True",
+      "TEST_RESULTS": "True"
+    }
+  },
   "periodic_test_profile_vs2019": {
     "TAGS": [
-      "weekly-build-metrics"
+        "nightly-incremental",
+        "nightly-clean",
+        "weekly-build-metrics"
     ],
     "COMMAND": "build_test_windows.cmd",
     "PARAMETERS": {

scripts/ctest/ctest_driver.py

@@ -19,7 +19,8 @@ SUITES_AND_DESCRIPTIONS = {
     "main": "The default set of tests, covers most of all testing.",
     "periodic": "Tests which can take a long time and should be done periodially instead of every commit - these should not block code submission",
     "benchmark": "Benchmarks - instead of pass/fail, these collect data for comparison against historic data",
-    "sandbox": "Flaky/Intermittent failing tests, this is used as a temporary spot to hold flaky tests, this will not block code submission. Ideally, this suite should always be empty"
+    "sandbox": "Flaky/Intermittent failing tests, this is used as a temporary spot to hold flaky tests, this will not block code submission. Ideally, this suite should always be empty",
+    "awsi": "Time consuming AWS integration end-to-end tests"
 }
 
 BUILD_CONFIGURATIONS = [

scripts/ctest/sanity_test.py

@@ -32,6 +32,10 @@ def test_Sanity_Benchmark_Pass():
 def test_Sanity_Sandbox_Pass():
     pass
 
+@pytest.mark.SUITE_awsi
+def test_Sanity_AWSI_Pass():
+    pass
+
 @pytest.mark.REQUIRES_gpu
 def test_Sanity_RequireGpu_Pass():
     pass