[AWS Automation] Stack tear down mechanism for the deployment on Jenkins (#3656)

Make awsi_test_profile_vs2019 a nonblocking step to make sure that resources are cleaned up even though automation tests failed

Signed-off-by: Junbo Liang <junbo@amazon.com>

AutomatedTesting/Gem/PythonTests/AWS/README.md

@@ -8,19 +8,21 @@
 ## Deploy CDK Applications
 1. Go to the AWS IAM console and create an IAM role called o3de-automation-tests which adds your own account as as a trusted entity and uses the "AdministratorAccess" permissions policy.
 2. Copy {engine_root}\scripts\build\Platform\Windows\deploy_cdk_applications.cmd to your engine root folder.
-3. Open a Command Prompt window at the engine root and set the following environment variables:  
-   Set O3DE_AWS_PROJECT_NAME=AWSAUTO  
-   Set O3DE_AWS_DEPLOY_REGION=us-east-1  
-   Set ASSUME_ROLE_ARN="arn:aws:iam::{your_aws_account_id}:role/o3de-automation-tests"  
-   Set COMMIT_ID=HEAD  
-4. Deploy the CDK applications for AWS gems by running deploy_cdk_applications.cmd in the same Command Prompt window.
-5. Edit AWS\common\constants.py to replace the assume role ARN with your own:  
-   arn:aws:iam::{your_aws_account_id}:role/o3de-automation-tests
+3. Open a new Command Prompt window at the engine root and set the following environment variables:  
+   Set O3DE_AWS_PROJECT_NAME=AWSAUTO
+   Set O3DE_AWS_DEPLOY_REGION=us-east-1
+   Set ASSUME_ROLE_ARN="arn:aws:iam::{your_aws_account_id}:role/o3de-automation-tests"
+   Set COMMIT_ID=HEAD
+4. In the same Command Prompt window, Deploy the CDK applications for AWS gems by running deploy_cdk_applications.cmd.
    
 ## Run Automation Tests
 ### CLI
-Open a Command Prompt window at the engine root and run the following CLI command:  
+In the same Command Prompt window, run the following CLI command:  
 python\python.cmd -m pytest {path_to_the_test_file} --build-directory {directory_to_the_profile_build}
 
 ### Pycharm
-You can also run any specific automation test directly from Pycharm by providing the "--build-directory" argument in the Run Configuration.
+You can also run any specific automation test directly from Pycharm by providing the "--build-directory" argument in the Run Configuration.
+
+## Destroy CDK Applications
+1. Copy {engine_root}\scripts\build\Platform\Windows\destroy_cdk_applications.cmd to your engine root folder.
+2. In the same Command Prompt window, destroy the CDK applications for AWS gems by running destroy_cdk_applications.cmd.

AutomatedTesting/Gem/PythonTests/AWS/Windows/aws_metrics/aws_metrics_automation_test.py

@@ -167,10 +167,6 @@ class TestAWSMetricsWindows(object):
                                                                 args=(aws_metrics_utils, resource_mappings, True))
         kinesis_analytics_application_thread.start()
 
-        # Clear the analytics bucket objects before sending new metrics.
-        aws_metrics_utils.empty_bucket(
-            resource_mappings.get_resource_name_id('AWSMetrics.AnalyticsBucketName'))
-
         log_monitor = setup(launcher, asset_processor)
 
         # Kinesis analytics application needs to be in the running state before we start the game launcher.
@@ -205,6 +201,10 @@ class TestAWSMetricsWindows(object):
         for thread in operational_threads:
             thread.join()
 
+        # Clear the analytics bucket objects so that the S3 bucket can be destroyed during tear down.
+        aws_metrics_utils.empty_bucket(
+            resource_mappings.get_resource_name_id('AWSMetrics.AnalyticsBucketName'))
+
     def test_unauthorized_user_request_rejected(self,
                                                 level: str,
                                                 launcher: pytest.fixture,

scripts/build/Platform/Windows/build_config.json

@@ -220,13 +220,17 @@
     ],
     "steps": [
       "awsi_deployment",
-      "awsi_test_profile_vs2019"
+      "awsi_test_profile_vs2019",
+      "awsi_destruction"
     ]
   },
   "awsi_test_profile_vs2019": {
     "TAGS": [
         "weekly-build-metrics"
     ],
+    "PIPELINE_ENV": {
+      "NONBLOCKING_STEP": "True"
+    },
     "COMMAND": "build_test_windows.cmd",
     "PARAMETERS": {
       "CONFIGURATION": "profile",
@@ -411,5 +415,13 @@
     },
     "COMMAND": "deploy_cdk_applications.cmd",
     "PARAMETERS": {}
+  },
+  "awsi_destruction": {
+    "TAGS": [],
+    "PIPELINE_ENV": {
+      "NONBLOCKING_STEP": "True"
+    },
+    "COMMAND": "destroy_cdk_applications.cmd",
+    "PARAMETERS": {}
   }
 }

scripts/build/Platform/Windows/destroy_cdk_applications.cmd

@@ -0,0 +1,92 @@
+@ECHO OFF
+REM
+REM Copyright (c) Contributors to the Open 3D Engine Project.
+REM For complete copyright and license terms please see the LICENSE at the root of this distribution.
+REM
+REM SPDX-License-Identifier: Apache-2.0 OR MIT
+REM
+REM
+
+REM Destroy the CDK applcations for AWS gems (Windows only)
+REM Prerequisites:
+REM 1) Node.js is installed
+REM 2) Node.js version >= 10.13.0, except for versions 13.0.0 - 13.6.0. A version in active long-term support is recommended.
+SETLOCAL EnableDelayedExpansion
+
+SET SOURCE_DIRECTORY=%CD%
+SET PATH=%SOURCE_DIRECTORY%\python;%PATH%
+SET GEM_DIRECTORY=%SOURCE_DIRECTORY%\Gems
+
+REM Create and activate a virtualenv for the CDK destruction
+CALL python -m venv .env
+IF ERRORLEVEL 1 (
+    ECHO [cdk_bootstrap] Failed to create a virtualenv for the CDK destruction
+    exit /b 1
+)
+CALL .env\Scripts\activate.bat
+IF ERRORLEVEL 1 (
+    ECHO [cdk_bootstrap] Failed to activate the virtualenv for the CDK destruction
+    exit /b 1
+)
+
+ECHO [cdk_installation] Install the latest version of CDK
+CALL npm uninstall -g aws-cdk
+IF ERRORLEVEL 1 (
+    ECHO [cdk_bootstrap] Failed to uninstall the current version of CDK
+    exit /b 1
+)
+CALL npm install -g aws-cdk@latest
+IF ERRORLEVEL 1 (
+    ECHO [cdk_bootstrap] Failed to install the latest version of CDK
+    exit /b 1
+)
+
+REM Set temporary AWS credentials from the assume role
+FOR /f "tokens=1,2,3" %%a IN ('CALL aws sts assume-role --query Credentials.[SecretAccessKey^,SessionToken^,AccessKeyId] --output text --role-arn %ASSUME_ROLE_ARN% --role-session-name o3de-Automation-session') DO (
+    SET AWS_SECRET_ACCESS_KEY=%%a
+    SET AWS_SESSION_TOKEN=%%b
+    SET AWS_ACCESS_KEY_ID=%%c
+)
+FOR /F "tokens=4 delims=:" %%a IN ("%ASSUME_ROLE_ARN%") DO SET O3DE_AWS_DEPLOY_ACCOUNT=%%a
+
+SET ERROR_EXISTS=0
+CALL :DestroyCDKApplication AWSCore,ERROR_EXISTS
+CALL :DestroyCDKApplication AWSClientAuth,ERROR_EXISTS
+CALL :DestroyCDKApplication AWSMetrics,ERROR_EXISTS
+
+IF %ERROR_EXISTS% EQU 1 (
+    EXIT /b 1
+)
+
+EXIT /b 0
+
+:DestroyCDKApplication
+REM Destroy the CDK application for a specific AWS gem
+SET GEM_NAME=%~1
+ECHO [cdk_destruction] Destroy the CDK application for the %GEM_NAME% gem
+PUSHD %GEM_DIRECTORY%\%GEM_NAME%\cdk
+
+REM Revert the CDK application code to a stable state using the provided commit ID
+CALL git checkout %COMMIT_ID% -- .
+IF ERRORLEVEL 1 (
+    ECHO [git_checkout] Failed to checkout the CDK application for the %GEM_NAME% gem using commit ID %COMMIT_ID%
+    POPD
+    SET %~2=1
+)
+
+REM Install required packages for the CDK application
+CALL python -m pip install -r requirements.txt
+IF ERRORLEVEL 1 (
+    ECHO [cdk_destruction] Failed to install required packages for the %GEM_NAME% gem
+    POPD
+    SET %~2=1
+)
+
+REM Destroy the CDK application
+CALL cdk destroy --all -f
+IF ERRORLEVEL 1 (
+    ECHO [cdk_destruction] Failed to destroy the CDK application for the %GEM_NAME% gem
+    POPD
+    SET %~2=1
+)
+POPD