birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Enable AWSI Automation Tests (#1330)

Browse Source 
* Add AWS AutomatedTesting levels

* Update AWSCoreConfiguration to use default profile

* Update lambda name

* Add cdk update command before tests run

* Update global cdk version before runnign tests.

* Add npm update command

* More cdk changes

* More cdk changes

* Shortening project names for cdk

* increase timeout for AWSTests

* Add comments

* Set AWSTests to periodic test suite

* Update logic to re install cdk and deploy bootstrap

* change version to list to catch version mismatch

* Move AWS fixtures to module directory scope

* Fixing issues with cdk utils

* Add cdk setup to be called on cdk fixture function
This commit is contained in:
amzn-hdoke
2021-06-18 12:39:02 -07:00
committed by GitHub
parent acd23698ea
commit 04f6c7b90f
31 changed files with 13813 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AutomatedTesting/Gem/PythonTests/AWS/CMakeLists.txt
+1
View File
@@ -21,6 +21,7 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS)
TEST_SUITE periodic
TEST_SERIAL
PATH ${CMAKE_CURRENT_LIST_DIR}/${PAL_PLATFORM_NAME}/
TIMEOUT 3000
RUNTIME_DEPENDENCIES
Legacy::Editor
AZ::AssetProcessor
AutomatedTesting/Gem/PythonTests/AWS/Windows/aws_metrics/aws_metrics_automation_test.py
+31 -31
View File
@@ -18,11 +18,12 @@ import typing
from datetime import datetime
import ly_test_tools.log.log_monitor
from assetpipeline.ap_fixtures.asset_processor_fixture import asset_processor as asset_processor
from AWS.common.aws_utils import aws_utils
from AWS.common.aws_credentials import aws_credentials
# fixture imports
from AWS.Windows.resource_mappings.resource_mappings import resource_mappings
from AWS.Windows.cdk.cdk import cdk
from AWS.Windows.cdk.cdk_utils import Cdk
from AWS.common.aws_utils import AwsUtils
from assetpipeline.ap_fixtures.asset_processor_fixture import asset_processor as asset_processor
from AWS.common.aws_credentials import aws_credentials
from .aws_metrics_utils import aws_metrics_utils
AWS_METRICS_FEATURE_NAME = 'AWSMetrics'
@@ -32,7 +33,7 @@ logger = logging.getLogger(__name__)
def setup(launcher: ly_test_tools.launchers.Launcher,
cdk: cdk,
cdk: Cdk,
asset_processor: asset_processor,
resource_mappings: resource_mappings,
context_variable: str = '') -> typing.Tuple[ly_test_tools.log.log_monitor.LogMonitor, str, str]:
@@ -116,18 +117,18 @@ def remove_file(file_path: str) -> None:
@pytest.mark.parametrize('region_name', ['us-west-2'])
@pytest.mark.parametrize('assume_role_arn', ['arn:aws:iam::645075835648:role/o3de-automation-tests'])
@pytest.mark.parametrize('session_name', ['o3de-Automation-session'])
class TestAWSMetrics_Windows(object):
def test_AWSMetrics_RealTimeAnalytics_MetricsSentToCloudWatch(self,
level: str,
launcher: ly_test_tools.launchers.Launcher,
asset_processor: pytest.fixture,
workspace: pytest.fixture,
aws_utils: aws_utils,
aws_credentials: aws_credentials,
resource_mappings: resource_mappings,
cdk: cdk,
aws_metrics_utils: aws_metrics_utils,
):
class TestAWSMetricsWindows(object):
def test_realtime_analytics_metrics_sent_to_cloudwatch(self,
level: str,
launcher: ly_test_tools.launchers.Launcher,
asset_processor: pytest.fixture,
workspace: pytest.fixture,
aws_utils: pytest.fixture,
aws_credentials: aws_credentials,
resource_mappings: pytest.fixture,
cdk: pytest.fixture,
aws_metrics_utils: aws_metrics_utils,
):
"""
Tests that the submitted metrics are sent to CloudWatch for real-time analytics.
"""
@@ -148,7 +149,7 @@ class TestAWSMetrics_Windows(object):
'AWS/Lambda',
'Invocations',
[{'Name': 'FunctionName',
'Value': f'{stack_name}-AnalyticsProcessingLambda'}],
'Value': f'{stack_name}-AnalyticsProcessingLambdaName'}],
start_time)
logger.info('Operational health metrics sent to CloudWatch.')
@@ -162,14 +163,14 @@ class TestAWSMetrics_Windows(object):
# Stop the Kinesis Data Analytics application.
aws_metrics_utils.stop_kinesis_data_analytics_application(analytics_application_name)
def test_AWSMetrics_UnauthorizedUser_RequestRejected(self,
level: str,
launcher: ly_test_tools.launchers.Launcher,
cdk: cdk,
aws_credentials: aws_credentials,
asset_processor: pytest.fixture,
resource_mappings: resource_mappings,
workspace: pytest.fixture):
def test_unauthorized_user_request_rejected(self,
level: str,
launcher: ly_test_tools.launchers.Launcher,
cdk: pytest.fixture,
aws_credentials: aws_credentials,
asset_processor: pytest.fixture,
resource_mappings: pytest.fixture,
workspace: pytest.fixture):
"""
Tests that unauthorized users cannot send metrics events to the AWS backed backend.
"""
@@ -187,14 +188,14 @@ class TestAWSMetrics_Windows(object):
assert result, 'Metrics events are sent successfully by unauthorized user'
logger.info('Unauthorized user is rejected to send metrics.')
def test_AWSMetrics_BatchAnalytics_MetricsDeliveredToS3(self,
def test_batch_analytics_metrics_delivered_to_s3(self,
level: str,
launcher: ly_test_tools.launchers.Launcher,
cdk: cdk,
cdk: pytest.fixture,
aws_credentials: aws_credentials,
asset_processor: pytest.fixture,
resource_mappings: resource_mappings,
aws_utils: aws_utils,
resource_mappings: pytest.fixture,
aws_utils: pytest.fixture,
aws_metrics_utils: aws_metrics_utils,
workspace: pytest.fixture):
"""
@@ -234,4 +235,3 @@ class TestAWSMetrics_Windows(object):
time.sleep(60)
# Empty the S3 bucket. S3 buckets can only be deleted successfully when it doesn't contain any object.
aws_metrics_utils.empty_s3_bucket(analytics_bucket_name)
AutomatedTesting/Gem/PythonTests/AWS/Windows/cdk/cdk.py → AutomatedTesting/Gem/PythonTests/AWS/Windows/cdk/cdk_utils.py
+97 -66
View File
@@ -12,6 +12,10 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
import os
import pytest
import boto3
import uuid
import logging
import subprocess
import botocore
import ly_test_tools.environment.process_utils as process_utils
from typing import List
@@ -19,22 +23,71 @@ from typing import List
BOOTSTRAP_STACK_NAME = 'CDKToolkit'
BOOTSTRAP_STAGING_BUCKET_LOGIC_ID = 'StagingBucket'
logger = logging.getLogger(__name__)
class Cdk:
"""
Cdk class that provides methods to run cdk application commands.
Expects system to have NodeJS, AWS CLI and CDK installed globally and have their paths setup as env variables.
"""
def __init__(self, cdk_path: str, project: str, account_id: str,
workspace: pytest.fixture, session: boto3.session.Session):
def __init__(self):
self._cdk_env = ''
self._stacks = []
self._cdk_path = os.path.dirname(os.path.realpath(__file__))
self._session = ''
cdk_npm_latest_version_cmd = ['npm', 'view', 'aws-cdk', 'version']
output = process_utils.check_output(
cdk_npm_latest_version_cmd,
cwd=self._cdk_path,
shell=True)
cdk_npm_latest_version = output.split()[0]
cdk_version_cmd = ['cdk', 'version']
output = process_utils.check_output(
cdk_version_cmd,
cwd=self._cdk_path,
shell=True)
cdk_version = output.split()[0]
logger.info(f'Current CDK version {cdk_version}')
if cdk_version != cdk_npm_latest_version:
try:
logger.info(f'Updating CDK to latest')
# uninstall and reinstall cdk in case npm has been updated.
output = process_utils.check_output(
'npm uninstall -g aws-cdk',
cwd=self._cdk_path,
shell=True)
logger.info(f'Uninstall CDK output: {output}')
output = process_utils.check_output(
'npm install -g aws-cdk@latest',
cwd=self._cdk_path,
shell=True)
logger.info(f'Install CDK output: {output}')
except subprocess.CalledProcessError as error:
logger.warning(f'Failed reinstalling latest CDK on npm'
f'\nError:{error.stderr}')
def setup(self, cdk_path: str, project: str, account_id: str,
workspace: pytest.fixture, session: boto3.session.Session, bootstrap_required: bool):
"""
:param cdk_path: Path where cdk app.py is stored.
:param project: Project name used for cdk project name env variable.
:param account_id: AWS account id to use with cdk application.
:param workspace: ly_test_tools workspace fixture.
:param workspace: bootstrap_required deploys bootstrap stack.
"""
self._cdk_env = os.environ.copy()
self._cdk_env['O3DE_AWS_PROJECT_NAME'] = project
unique_id = uuid.uuid4().hex[-4:]
self._cdk_env['O3DE_AWS_PROJECT_NAME'] = project[:4] + unique_id if len(project) > 4 else project + unique_id
self._cdk_env['O3DE_AWS_DEPLOY_REGION'] = session.region_name
self._cdk_env['O3DE_AWS_DEPLOY_ACCOUNT'] = account_id
self._cdk_env['PATH'] = f'{workspace.paths.engine_root()}\\python;' + self._cdk_env['PATH']
@@ -43,27 +96,37 @@ class Cdk:
self._cdk_env['AWS_ACCESS_KEY_ID'] = credentials.access_key
self._cdk_env['AWS_SECRET_ACCESS_KEY'] = credentials.secret_key
self._cdk_env['AWS_SESSION_TOKEN'] = credentials.token
self._stacks = []
self._cdk_path = cdk_path
self._session = session
output = process_utils.check_output(
'python -m pip install -r requirements.txt',
cwd=self._cdk_path,
env=self._cdk_env,
shell=True)
logger.info(f'Installing cdk python dependencies: {output}')
if bootstrap_required:
self.bootstrap()
def bootstrap(self) -> None:
"""
Deploy the bootstrap stack.
"""
bootstrap_cmd = ['cdk', 'bootstrap',
f'aws://{self._cdk_env["O3DE_AWS_DEPLOY_ACCOUNT"]}/{self._cdk_env["O3DE_AWS_DEPLOY_REGION"]}']
try:
bootstrap_cmd = ['cdk', 'bootstrap',
f'aws://{self._cdk_env["O3DE_AWS_DEPLOY_ACCOUNT"]}/{self._cdk_env["O3DE_AWS_DEPLOY_REGION"]}']
process_utils.check_call(
bootstrap_cmd,
cwd=self._cdk_path,
env=self._cdk_env,
shell=True)
process_utils.check_call(
bootstrap_cmd,
cwd=self._cdk_path,
env=self._cdk_env,
shell=True)
except botocore.exceptions.ClientError as clientError:
logger.warning(f'Failed creating Bootstrap stack {BOOTSTRAP_STACK_NAME} not found. '
f'\nError:{clientError["Error"]["Message"]}')
def list(self) -> List[str]:
"""
@@ -131,83 +194,51 @@ class Cdk:
"""
Destroys the cdk application.
"""
destroy_cdk_application_cmd = ['cdk', 'destroy', '-f']
process_utils.check_output(
destroy_cdk_application_cmd,
cwd=self._cdk_path,
env=self._cdk_env,
shell=True)
logger.info(f'CDK Path {self._cdk_path}')
destroy_cdk_application_cmd = ['cdk', 'destroy', '--all', '-f']
try:
process_utils.check_output(
destroy_cdk_application_cmd,
cwd=self._cdk_path,
env=self._cdk_env,
shell=True)
except subprocess.CalledProcessError as e:
logger.error(e.output)
raise e
self._stacks = []
self._cdk_path = ''
@staticmethod
def remove_bootstrap_stack(aws_utils: pytest.fixture) -> None:
def remove_bootstrap_stack(self) -> None:
"""
Remove the CDK bootstrap stack.
:param aws_utils: aws_utils fixture.
"""
# Check if the bootstrap stack exists.
response = aws_utils.client('cloudformation').describe_stacks(
response = self._session.client('cloudformation').describe_stacks(
StackName=BOOTSTRAP_STACK_NAME
)
stacks = response.get('Stacks', [])
if not stacks:
if not stacks or len(stacks) is 0:
return
# Clear the bootstrap staging bucket before deleting the bootstrap stack.
response = aws_utils.client('cloudformation').describe_stack_resource(
response = self._session.client('cloudformation').describe_stack_resource(
StackName=BOOTSTRAP_STACK_NAME,
LogicalResourceId=BOOTSTRAP_STAGING_BUCKET_LOGIC_ID
)
staging_bucket_name = response.get('StackResourceDetail', {}).get('PhysicalResourceId', '')
if staging_bucket_name:
s3 = aws_utils.resource('s3')
s3 = self._session.resource('s3')
bucket = s3.Bucket(staging_bucket_name)
for key in bucket.objects.all():
key.delete()
# Delete the bootstrap stack.
aws_utils.client('cloudformation').delete_stack(
StackName=BOOTSTRAP_STACK_NAME
)
@pytest.fixture(scope='function')
def cdk(
request: pytest.fixture,
project: str,
feature_name: str,
workspace: pytest.fixture,
aws_utils: pytest.fixture,
bootstrap_required: bool = True,
destroy_stacks_on_teardown: bool = True) -> Cdk:
"""
Fixture for setting up a Cdk
:param request: _pytest.fixtures.SubRequest class that handles getting
a pytest fixture from a pytest function/fixture.
:param project: Project name used for cdk project name env variable.
:param feature_name: Feature gem name to expect cdk folder in.
:param workspace: ly_test_tools workspace fixture.
:param aws_utils: aws_utils fixture.
:param bootstrap_required: Whether the bootstrap stack needs to be created to
provision resources the AWS CDK needs to perform the deployment.
:param destroy_stacks_on_teardown: option to control calling destroy ot the end of test.
:return Cdk class object.
"""
cdk_path = f'{workspace.paths.engine_root()}/Gems/{feature_name}/cdk'
cdk_obj = Cdk(cdk_path, project, aws_utils.assume_account_id(), workspace, aws_utils.assume_session())
if bootstrap_required:
cdk_obj.bootstrap()
def teardown():
if destroy_stacks_on_teardown:
cdk_obj.destroy()
cdk_obj.remove_bootstrap_stack(aws_utils)
request.addfinalizer(teardown)
return cdk_obj
# Should not need to delete the stack if S3 bucket can be cleaned.
# self._session.client('cloudformation').delete_stack(
# StackName=BOOTSTRAP_STACK_NAME
# )
AutomatedTesting/Gem/PythonTests/AWS/Windows/client_auth/test_anonymous_credentials.py
+4 -3
View File
@@ -12,9 +12,10 @@ import os
import logging
import ly_test_tools.log.log_monitor
# fixture imports
from AWS.Windows.resource_mappings.resource_mappings import resource_mappings
from AWS.Windows.cdk.cdk import cdk
from AWS.common.aws_utils import aws_utils
from AWS.Windows.cdk.cdk_utils import Cdk
from AWS.common.aws_utils import AwsUtils
from assetpipeline.ap_fixtures.asset_processor_fixture import asset_processor as asset_processor
AWS_PROJECT_NAME = 'AWS-AutomationTest'
@@ -75,5 +76,5 @@ class TestAWSClientAuthAnonymousCredentials(object):
expected_lines=['(Script) - Success anonymous credentials'],
unexpected_lines=['(Script) - Fail anonymous credentials'],
halt_on_unexpected=True,
)
)
assert result, 'Anonymous credentials fetched successfully.'
AutomatedTesting/Gem/PythonTests/AWS/Windows/client_auth/test_password_signin.py
+3 -2
View File
@@ -12,9 +12,10 @@ import os
import logging
import ly_test_tools.log.log_monitor
# fixture imports
from AWS.Windows.resource_mappings.resource_mappings import resource_mappings
from AWS.Windows.cdk.cdk import cdk
from AWS.common.aws_utils import aws_utils
from AWS.Windows.cdk.cdk_utils import Cdk
from AWS.common.aws_utils import AwsUtils
from assetpipeline.ap_fixtures.asset_processor_fixture import asset_processor as asset_processor
AWS_PROJECT_NAME = 'AWS-AutomationTest'
AutomatedTesting/Gem/PythonTests/AWS/common/aws_utils.py
+3 -27
View File
@@ -8,11 +8,12 @@ remove or modify any license notices. This file is distributed on an "AS IS" BAS
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
"""
import boto3
import pytest
import logging
logger = logging.getLogger(__name__)
logging.getLogger('boto').setLevel(logging.CRITICAL)
logging.getLogger('boto3').setLevel(logging.WARNING)
logging.getLogger('botocore').setLevel(logging.WARNING)
logging.getLogger('nose').setLevel(logging.WARNING)
class AwsUtils:
@@ -63,28 +64,3 @@ class AwsUtils:
clears stored session
"""
self._assume_session = None
@pytest.fixture(scope='function')
def aws_utils(
request: pytest.fixture,
assume_role_arn: str,
session_name: str,
region_name: str):
"""
Fixture for AWS util functions
:param request: _pytest.fixtures.SubRequest class that handles getting
a pytest fixture from a pytest function/fixture.
:param assume_role_arn: Role used to fetch temporary aws credentials, configure service clients with obtained credentials.
:param session_name: Session name to set.
:param region_name: AWS account region to set for session.
:return AWSUtils class object.
"""
aws_utils_obj = AwsUtils(assume_role_arn, session_name, region_name)
def teardown():
aws_utils_obj.destroy()
request.addfinalizer(teardown)
return aws_utils_obj
AutomatedTesting/Gem/PythonTests/AWS/conftest.py
+87
View File
@@ -0,0 +1,87 @@
"""
All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
its licensors.
For complete copyright and license terms please see the LICENSE at the root of this
distribution (the "License"). All use of this software is governed by the License,
or, if provided, by the license below or the license accompanying this file. Do not
remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
"""
import pytest
import logging
from AWS.common.aws_utils import AwsUtils
from AWS.Windows.cdk.cdk_utils import Cdk
logger = logging.getLogger(__name__)
@pytest.fixture(scope='function')
def aws_utils(
request: pytest.fixture,
assume_role_arn: str,
session_name: str,
region_name: str):
"""
Fixture for AWS util functions
:param request: _pytest.fixtures.SubRequest class that handles getting
a pytest fixture from a pytest function/fixture.
:param assume_role_arn: Role used to fetch temporary aws credentials, configure service clients with obtained credentials.
:param session_name: Session name to set.
:param region_name: AWS account region to set for session.
:return AWSUtils class object.
"""
aws_utils_obj = AwsUtils(assume_role_arn, session_name, region_name)
def teardown():
aws_utils_obj.destroy()
request.addfinalizer(teardown)
return aws_utils_obj
# Set global pytest variable for cdk to avoid recreating instance
pytest.cdk_obj = None
@pytest.fixture(scope='function')
def cdk(
request: pytest.fixture,
project: str,
feature_name: str,
workspace: pytest.fixture,
aws_utils: pytest.fixture,
bootstrap_required: bool = True,
destroy_stacks_on_teardown: bool = True) -> Cdk:
"""
Fixture for setting up a Cdk
:param request: _pytest.fixtures.SubRequest class that handles getting
a pytest fixture from a pytest function/fixture.
:param project: Project name used for cdk project name env variable.
:param feature_name: Feature gem name to expect cdk folder in.
:param workspace: ly_test_tools workspace fixture.
:param aws_utils: aws_utils fixture.
:param bootstrap_required: Whether the bootstrap stack needs to be created to
provision resources the AWS CDK needs to perform the deployment.
:param destroy_stacks_on_teardown: option to control calling destroy ot the end of test.
:return Cdk class object.
"""
cdk_path = f'{workspace.paths.engine_root()}/Gems/{feature_name}/cdk'
logger.info(f'CDK Path {cdk_path}')
if pytest.cdk_obj is None:
pytest.cdk_obj = Cdk()
pytest.cdk_obj.setup(cdk_path, project, aws_utils.assume_account_id(), workspace, aws_utils.assume_session(),
bootstrap_required)
def teardown():
if destroy_stacks_on_teardown:
pytest.cdk_obj.destroy()
# Enable after https://github.com/aws/aws-cdk/issues/986 is fixed.
# Until then clean the bootstrap bucket manually.
# cdk_obj.remove_bootstrap_stack()
request.addfinalizer(teardown)
return pytest.cdk_obj
AutomatedTesting/Gem/PythonTests/CMakeLists.txt
+1 -2
View File
@@ -60,5 +60,4 @@ add_subdirectory(streaming)
add_subdirectory(smoke)
## AWS ##
# Enable when AWS Gems work on Linux and Android.
# add_subdirectory(AWS)
add_subdirectory(AWS)