Add AWSI automation tests to a new test suite and pipe (#3913)

Signed-off-by: junbo <junbo@amazon.com>

scripts/build/Platform/Linux/build_config.json

@@ -83,7 +83,7 @@
       "CMAKE_OPTIONS": "-G 'Ninja Multi-Config' -DCMAKE_C_COMPILER=clang-6.0 -DCMAKE_CXX_COMPILER=clang++-6.0 -DLY_UNITY_BUILD=TRUE -DLY_PARALLEL_LINK_JOBS=4",
       "CMAKE_LY_PROJECTS": "AutomatedTesting",
       "CMAKE_TARGET": "all",
-      "CTEST_OPTIONS": "-E Gem::EMotionFX.Editor.Tests -LE SUITE_sandbox -L FRAMEWORK_googletest"
+      "CTEST_OPTIONS": "-E Gem::EMotionFX.Editor.Tests -LE (SUITE_sandbox|SUITE_awsi) -L FRAMEWORK_googletest"
     }
   },
   "test_profile_nounity": {
@@ -95,7 +95,7 @@
       "CMAKE_OPTIONS": "-G 'Ninja Multi-Config' -DCMAKE_C_COMPILER=clang-6.0 -DCMAKE_CXX_COMPILER=clang++-6.0 -DLY_UNITY_BUILD=FALSE -DLY_PARALLEL_LINK_JOBS=4",
       "CMAKE_LY_PROJECTS": "AutomatedTesting",
       "CMAKE_TARGET": "all",
-      "CTEST_OPTIONS": "-E Gem::EMotionFX.Editor.Tests -LE SUITE_sandbox -L FRAMEWORK_googletest"
+      "CTEST_OPTIONS": "-E Gem::EMotionFX.Editor.Tests -LE (SUITE_sandbox|SUITE_awsi) -L FRAMEWORK_googletest"
     }
   },
   "asset_profile": {

scripts/build/Platform/Windows/build_config.json

@@ -213,19 +213,38 @@
       "ASSET_PROCESSOR_PLATFORMS": "pc,server"
     }
   },
-  "periodic_test_profile_vs2019_pipe": {
+  "awsi_test_profile_vs2019_pipe": {
     "TAGS": [
       "nightly-incremental",
       "nightly-clean"
     ],
     "steps": [
       "awsi_deployment",
-      "periodic_test_profile_vs2019"
+      "awsi_test_profile_vs2019"
     ]
   },
+  "awsi_test_profile_vs2019": {
+    "TAGS": [
+        "weekly-build-metrics"
+    ],
+    "COMMAND": "build_test_windows.cmd",
+    "PARAMETERS": {
+      "CONFIGURATION": "profile",
+      "OUTPUT_DIRECTORY": "build\\windows_vs2019",
+      "CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE",
+      "CMAKE_LY_PROJECTS": "AutomatedTesting",
+      "CMAKE_TARGET": "TEST_SUITE_awsi",
+      "CMAKE_NATIVE_BUILD_ARGS": "/m /nologo",
+      "CTEST_OPTIONS": "-L \"(SUITE_awsi)\" -T Test",
+      "TEST_METRICS": "True",
+      "TEST_RESULTS": "True"
+    }
+  },
   "periodic_test_profile_vs2019": {
     "TAGS": [
-      "weekly-build-metrics"
+        "nightly-incremental",
+        "nightly-clean",
+        "weekly-build-metrics"
     ],
     "COMMAND": "build_test_windows.cmd",
     "PARAMETERS": {

scripts/ctest/ctest_driver.py

@@ -19,7 +19,8 @@ SUITES_AND_DESCRIPTIONS = {
     "main": "The default set of tests, covers most of all testing.",
     "periodic": "Tests which can take a long time and should be done periodially instead of every commit - these should not block code submission",
     "benchmark": "Benchmarks - instead of pass/fail, these collect data for comparison against historic data",
-    "sandbox": "Flaky/Intermittent failing tests, this is used as a temporary spot to hold flaky tests, this will not block code submission. Ideally, this suite should always be empty"
+    "sandbox": "Flaky/Intermittent failing tests, this is used as a temporary spot to hold flaky tests, this will not block code submission. Ideally, this suite should always be empty",
+    "awsi": "Time consuming AWS integration end-to-end tests"
 }
 
 BUILD_CONFIGURATIONS = [

scripts/ctest/sanity_test.py

@@ -32,6 +32,10 @@ def test_Sanity_Benchmark_Pass():
 def test_Sanity_Sandbox_Pass():
     pass
 
+@pytest.mark.SUITE_awsi
+def test_Sanity_AWSI_Pass():
+    pass
+
 @pytest.mark.REQUIRES_gpu
 def test_Sanity_RequireGpu_Pass():
     pass