Merge branch 'development' of https://github.com/o3de/o3de into jckand/DynVegXfailFixes

This commit is contained in:
jckand-amzn
2021-07-20 17:19:15 -05:00
106 changed files with 13836 additions and 13958 deletions
@@ -8,58 +8,43 @@ SPDX-License-Identifier: Apache-2.0 OR MIT
import logging
import os
import pytest
import time
import typing
from datetime import datetime
import ly_test_tools.log.log_monitor
# fixture imports
from AWS.Windows.resource_mappings.resource_mappings import resource_mappings
from assetpipeline.ap_fixtures.asset_processor_fixture import asset_processor
from .aws_metrics_utils import aws_metrics_utils
from .aws_metrics_custom_thread import AWSMetricsThread
AWS_METRICS_FEATURE_NAME = 'AWSMetrics'
GAME_LOG_NAME = 'Game.log'
CONTEXT_VARIABLE = ['-c', 'batch_processing=true']
logger = logging.getLogger(__name__)
def setup(launcher: ly_test_tools.launchers.Launcher,
cdk: pytest.fixture,
asset_processor: asset_processor,
resource_mappings: resource_mappings,
context_variable: str = '') -> typing.Tuple[ly_test_tools.log.log_monitor.LogMonitor, str, str]:
def setup(launcher: pytest.fixture,
asset_processor: pytest.fixture) -> pytest.fixture:
"""
Set up the CDK application and start the log monitor.
Set up the resource mapping configuration and start the log monitor.
:param launcher: Client launcher for running the test level.
:param cdk: CDK application for deploying the AWS resources.
:param asset_processor: asset_processor fixture.
:param resource_mappings: resource_mappings fixture.
:param context_variable: context_variable for enable optional CDK feature.
:return log monitor object, metrics file path and the metrics stack name.
"""
logger.info(f'Cdk stack names:\n{cdk.list()}')
stacks = cdk.deploy(context_variable=context_variable)
resource_mappings.populate_output_keys(stacks)
asset_processor.start()
asset_processor.wait_for_idle()
metrics_file_path = os.path.join(launcher.workspace.paths.project(), 'user',
AWS_METRICS_FEATURE_NAME, 'metrics.json')
remove_file(metrics_file_path)
file_to_monitor = os.path.join(launcher.workspace.paths.project_log(), GAME_LOG_NAME)
remove_file(file_to_monitor)
# Initialize the log monitor.
log_monitor = ly_test_tools.log.log_monitor.LogMonitor(launcher=launcher, log_file_path=file_to_monitor)
return log_monitor, metrics_file_path, stacks[0]
return log_monitor
def monitor_metrics_submission(log_monitor: ly_test_tools.log.log_monitor.LogMonitor) -> None:
def monitor_metrics_submission(log_monitor: pytest.fixture) -> None:
"""
Monitor the messages and notifications for submitting metrics.
:param log_monitor: Log monitor to check the log messages.
@@ -67,6 +52,7 @@ def monitor_metrics_submission(log_monitor: ly_test_tools.log.log_monitor.LogMon
expected_lines = [
'(Script) - Submitted metrics without buffer.',
'(Script) - Submitted metrics with buffer.',
'(Script) - Flushed the buffered metrics.',
'(Script) - Metrics is sent successfully.'
]
@@ -87,87 +73,132 @@ def monitor_metrics_submission(log_monitor: ly_test_tools.log.log_monitor.LogMon
f'unexpected_lines values: {unexpected_lines}')
def remove_file(file_path: str) -> None:
def query_metrics_from_s3(aws_metrics_utils: pytest.fixture, stack_name: str) -> None:
"""
Remove a local file and its directory.
:param file_path: Path to the local file.
Verify that the metrics events are delivered to the S3 bucket and can be queried.
aws_metrics_utils: aws_metrics_utils fixture.
stack_name: name of the CloudFormation stack.
"""
if os.path.exists(file_path):
os.remove(file_path)
analytics_bucket_name = aws_metrics_utils.get_analytics_bucket_name(stack_name)
aws_metrics_utils.verify_s3_delivery(analytics_bucket_name)
logger.info('Metrics are sent to S3.')
file_dir = os.path.dirname(file_path)
if os.path.exists(file_dir) and len(os.listdir(file_dir)) == 0:
os.rmdir(file_dir)
aws_metrics_utils.run_glue_crawler(f'{stack_name}-EventsCrawler')
aws_metrics_utils.run_named_queries(f'{stack_name}-AthenaWorkGroup')
logger.info('Query metrics from S3 successfully.')
# Empty the S3 bucket. S3 buckets can only be deleted successfully when it doesn't contain any object.
aws_metrics_utils.empty_batch_analytics_bucket(analytics_bucket_name)
def verify_operational_metrics(aws_metrics_utils: pytest.fixture, stack_name: str, start_time: datetime) -> None:
"""
Verify that operational health metrics are delivered to CloudWatch.
aws_metrics_utils: aws_metrics_utils fixture.
stack_name: name of the CloudFormation stack.
start_time: Time when the game launcher starts.
"""
aws_metrics_utils.verify_cloud_watch_delivery(
'AWS/Lambda',
'Invocations',
[{'Name': 'FunctionName',
'Value': f'{stack_name}-AnalyticsProcessingLambdaName'}],
start_time)
logger.info('AnalyticsProcessingLambda metrics are sent to CloudWatch.')
aws_metrics_utils.verify_cloud_watch_delivery(
'AWS/Lambda',
'Invocations',
[{'Name': 'FunctionName',
'Value': f'{stack_name}-EventsProcessingLambda'}],
start_time)
logger.info('EventsProcessingLambda metrics are sent to CloudWatch.')
def start_kinesis_analytics_application(aws_metrics_utils: pytest.fixture, stack_name: str) -> None:
"""
Start the Kinesis analytics application for real-time analytics.
aws_metrics_utils: aws_metrics_utils fixture.
stack_name: name of the CloudFormation stack.
"""
analytics_application_name = f'{stack_name}-AnalyticsApplication'
aws_metrics_utils.start_kinesis_data_analytics_application(analytics_application_name)
@pytest.mark.SUITE_periodic
@pytest.mark.usefixtures('automatic_process_killer')
@pytest.mark.parametrize('project', ['AutomatedTesting'])
@pytest.mark.parametrize('level', ['AWS/Metrics'])
@pytest.mark.parametrize('feature_name', [AWS_METRICS_FEATURE_NAME])
@pytest.mark.usefixtures('resource_mappings')
@pytest.mark.parametrize('resource_mappings_filename', ['default_aws_resource_mappings.json'])
@pytest.mark.usefixtures('aws_credentials')
@pytest.mark.parametrize('profile_name', ['AWSAutomationTest'])
@pytest.mark.parametrize('region_name', ['us-west-2'])
@pytest.mark.parametrize('assume_role_arn', ['arn:aws:iam::645075835648:role/o3de-automation-tests'])
@pytest.mark.usefixtures('cdk')
@pytest.mark.parametrize('session_name', ['o3de-Automation-session'])
@pytest.mark.parametrize('deployment_params', [CONTEXT_VARIABLE])
class TestAWSMetricsWindows(object):
def test_realtime_analytics_metrics_sent_to_cloudwatch(self,
level: str,
launcher: ly_test_tools.launchers.Launcher,
asset_processor: pytest.fixture,
workspace: pytest.fixture,
aws_utils: pytest.fixture,
aws_credentials: pytest.fixture,
resource_mappings: pytest.fixture,
cdk: pytest.fixture,
aws_metrics_utils: aws_metrics_utils,
):
"""
Tests that the submitted metrics are sent to CloudWatch for real-time analytics.
"""
log_monitor, metrics_file_path, stack_name = setup(launcher, cdk, asset_processor, resource_mappings)
"""
Test class to verify the real-time and batch analytics for metrics.
"""
# Start the Kinesis Data Analytics application for real-time analytics.
analytics_application_name = f'{stack_name}-AnalyticsApplication'
aws_metrics_utils.start_kinesis_data_analytics_application(analytics_application_name)
@pytest.mark.parametrize('destroy_stacks_on_teardown', [False])
def test_realtime_and_batch_analytics(self,
level: str,
launcher: pytest.fixture,
asset_processor: pytest.fixture,
workspace: pytest.fixture,
aws_utils: pytest.fixture,
cdk: pytest.fixture,
aws_metrics_utils: pytest.fixture):
"""
Verify that the metrics events are sent to CloudWatch and S3 for analytics.
"""
# Start Kinesis analytics application on a separate thread to avoid blocking the test.
kinesis_analytics_application_thread = AWSMetricsThread(target=start_kinesis_analytics_application,
args=(aws_metrics_utils, cdk.stacks[0]))
kinesis_analytics_application_thread.start()
log_monitor = setup(launcher, asset_processor)
# Kinesis analytics application needs to be in the running state before we start the game launcher.
kinesis_analytics_application_thread.join()
launcher.args = ['+LoadLevel', level]
launcher.args.extend(['-rhi=null'])
start_time = datetime.utcnow()
with launcher.start(launch_ap=False):
start_time = datetime.utcnow()
monitor_metrics_submission(log_monitor)
# Verify that operational health metrics are delivered to CloudWatch.
aws_metrics_utils.verify_cloud_watch_delivery(
'AWS/Lambda',
'Invocations',
[{'Name': 'FunctionName',
'Value': f'{stack_name}-AnalyticsProcessingLambdaName'}],
start_time)
logger.info('Operational health metrics sent to CloudWatch.')
# Verify that real-time analytics metrics are delivered to CloudWatch.
aws_metrics_utils.verify_cloud_watch_delivery(
AWS_METRICS_FEATURE_NAME,
'TotalLogins',
[],
start_time)
logger.info('Real-time metrics sent to CloudWatch.')
logger.info('Real-time metrics are sent to CloudWatch.')
# Stop the Kinesis Data Analytics application.
aws_metrics_utils.stop_kinesis_data_analytics_application(analytics_application_name)
# Run time-consuming verifications on separate threads to avoid blocking the test.
verification_threads = list()
verification_threads.append(
AWSMetricsThread(target=query_metrics_from_s3, args=(aws_metrics_utils, cdk.stacks[0])))
verification_threads.append(
AWSMetricsThread(target=verify_operational_metrics, args=(aws_metrics_utils, cdk.stacks[0], start_time)))
for thread in verification_threads:
thread.start()
for thread in verification_threads:
thread.join()
@pytest.mark.parametrize('destroy_stacks_on_teardown', [True])
def test_unauthorized_user_request_rejected(self,
level: str,
launcher: ly_test_tools.launchers.Launcher,
cdk: pytest.fixture,
aws_credentials: pytest.fixture,
launcher: pytest.fixture,
asset_processor: pytest.fixture,
resource_mappings: pytest.fixture,
workspace: pytest.fixture):
"""
Tests that unauthorized users cannot send metrics events to the AWS backed backend.
Verify that unauthorized users cannot send metrics events to the AWS backed backend.
"""
log_monitor, metrics_file_path, stack_name = setup(launcher, cdk, asset_processor, resource_mappings)
log_monitor = setup(launcher, asset_processor)
# Set invalid AWS credentials.
launcher.args = ['+LoadLevel', level, '+cl_awsAccessKey', 'AKIAIOSFODNN7EXAMPLE',
'+cl_awsSecretKey', 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY']
@@ -180,51 +211,3 @@ class TestAWSMetricsWindows(object):
halt_on_unexpected=True)
assert result, 'Metrics events are sent successfully by unauthorized user'
logger.info('Unauthorized user is rejected to send metrics.')
def test_batch_analytics_metrics_delivered_to_s3(self,
level: str,
launcher: ly_test_tools.launchers.Launcher,
cdk: pytest.fixture,
aws_credentials: pytest.fixture,
asset_processor: pytest.fixture,
resource_mappings: pytest.fixture,
aws_utils: pytest.fixture,
aws_metrics_utils: aws_metrics_utils,
workspace: pytest.fixture):
"""
Tests that the submitted metrics are sent to the data lake for batch analytics.
"""
log_monitor, metrics_file_path, stack_name = setup(launcher, cdk, asset_processor, resource_mappings,
context_variable='batch_processing=true')
analytics_bucket_name = aws_metrics_utils.get_analytics_bucket_name(stack_name)
launcher.args = ['+LoadLevel', level]
launcher.args.extend(['-rhi=null'])
with launcher.start(launch_ap=False):
start_time = datetime.utcnow()
monitor_metrics_submission(log_monitor)
# Verify that operational health metrics are delivered to CloudWatch.
aws_metrics_utils.verify_cloud_watch_delivery(
'AWS/Lambda',
'Invocations',
[{'Name': 'FunctionName',
'Value': f'{stack_name}-EventsProcessingLambda'}],
start_time)
logger.info('Operational health metrics sent to CloudWatch.')
aws_metrics_utils.verify_s3_delivery(analytics_bucket_name)
logger.info('Metrics sent to S3.')
# Run the glue crawler to populate the AWS Glue Data Catalog with tables.
aws_metrics_utils.run_glue_crawler(f'{stack_name}-EventsCrawler')
# Run named queries on the table to verify the batch analytics.
aws_metrics_utils.run_named_queries(f'{stack_name}-AthenaWorkGroup')
logger.info('Query metrics from S3 successfully.')
# Kinesis Data Firehose buffers incoming data before it delivers it to Amazon S3. Sleep for the
# default interval (60s) to make sure that all the metrics are sent to the bucket before cleanup.
time.sleep(60)
# Empty the S3 bucket. S3 buckets can only be deleted successfully when it doesn't contain any object.
aws_metrics_utils.empty_s3_bucket(analytics_bucket_name)
@@ -0,0 +1,29 @@
"""
Copyright (c) Contributors to the Open 3D Engine Project.
For complete copyright and license terms please see the LICENSE at the root of this distribution.
SPDX-License-Identifier: Apache-2.0 OR MIT
"""
from threading import Thread
class AWSMetricsThread(Thread):
"""
Custom thread for raising assertion errors on the main thread.
"""
def __init__(self, **kwargs):
super().__init__(**kwargs)
self._error = None
def run(self) -> None:
try:
super().run()
except AssertionError as e:
self._error = e
def join(self, **kwargs) -> None:
super().join(**kwargs)
if self._error:
raise AssertionError(self._error)
@@ -13,7 +13,6 @@ import typing
from datetime import datetime
from botocore.exceptions import WaiterError
from AWS.common.aws_utils import AwsUtils
from .aws_metrics_waiters import KinesisAnalyticsApplicationUpdatedWaiter, \
CloudWatchMetricsDeliveredWaiter, DataLakeMetricsDeliveredWaiter, GlueCrawlerReadyWaiter
@@ -29,7 +28,7 @@ class AWSMetricsUtils:
Provide utils functions for the AWSMetrics gem to interact with the deployed resources.
"""
def __init__(self, aws_utils: AwsUtils):
def __init__(self, aws_utils: pytest.fixture):
self._aws_util = aws_utils
def start_kinesis_data_analytics_application(self, application_name: str) -> None:
@@ -199,14 +198,13 @@ class AWSMetricsUtils:
assert state == 'SUCCEEDED', f'Failed to run the named query {named_query.get("Name", {})}'
def empty_s3_bucket(self, bucket_name: str) -> None:
def empty_batch_analytics_bucket(self, bucket_name: str) -> None:
"""
Empty the S3 bucket following:
https://boto3.amazonaws.com/v1/documentation/api/latest/guide/migrations3.html
:param bucket_name: Name of the S3 bucket.
"""
s3 = self._aws_util.resource('s3')
bucket = s3.Bucket(bucket_name)
@@ -72,15 +72,14 @@ class Cdk:
f'\nError:{error.stderr}')
def setup(self, cdk_path: str, project: str, account_id: str,
workspace: pytest.fixture, session: boto3.session.Session, bootstrap_required: bool):
workspace: pytest.fixture, session: boto3.session.Session):
"""
:param cdk_path: Path where cdk app.py is stored.
:param project: Project name used for cdk project name env variable.
:param account_id: AWS account id to use with cdk application.
:param workspace: ly_test_tools workspace fixture.
:param workspace: bootstrap_required deploys bootstrap stack.
:param session: Current boto3 session, provides credentials and region.
"""
self._cdk_env = os.environ.copy()
unique_id = uuid.uuid4().hex[-4:]
self._cdk_env['O3DE_AWS_PROJECT_NAME'] = project[:4] + unique_id if len(project) > 4 else project + unique_id
@@ -104,8 +103,7 @@ class Cdk:
logger.info(f'Installing cdk python dependencies: {output}')
if bootstrap_required:
self.bootstrap()
self.bootstrap()
def bootstrap(self) -> None:
"""
@@ -124,15 +122,19 @@ class Cdk:
logger.warning(f'Failed creating Bootstrap stack {BOOTSTRAP_STACK_NAME} not found. '
f'\nError:{clientError["Error"]["Message"]}')
def list(self) -> List[str]:
def list(self, deployment_params: List[str] = None) -> List[str]:
"""
lists cdk stack names
:return List of cdk stack names
lists cdk stack names.
:param deployment_params: Deployment parameters like --all can be passed in this way.
:return List of cdk stack names.
"""
if not self._cdk_path:
return []
list_cdk_application_cmd = ['cdk', 'list']
if deployment_params:
list_cdk_application_cmd.extend(deployment_params)
output = process_utils.check_output(
list_cdk_application_cmd,
cwd=self._cdk_path,
@@ -141,36 +143,36 @@ class Cdk:
return output.splitlines()
def synthesize(self) -> None:
def synthesize(self, deployment_params: List[str] = None) -> None:
"""
Synthesizes all cdk stacks
Synthesizes all cdk stacks.
:param deployment_params: Deployment parameters like --all can be passed in this way.
"""
if not self._cdk_path:
return
list_cdk_application_cmd = ['cdk', 'synth']
synth_cdk_application_cmd = ['cdk', 'synth']
if deployment_params:
synth_cdk_application_cmd.extend(deployment_params)
process_utils.check_output(
list_cdk_application_cmd,
synth_cdk_application_cmd,
cwd=self._cdk_path,
env=self._cdk_env,
shell=True)
def deploy(self, context_variable: str = '', additonal_params: List[str] = None) -> List[str]:
def deploy(self, deployment_params: List[str] = None) -> List[str]:
"""
Deploys all the CDK stacks.
:param context_variable: Context variable for enabling optional features.
:param additonal_params: Additonal parameters like --all can be passed in this way.
:param deployment_params: Deployment parameters like --all can be passed in this way.
:return List of deployed stack arns.
"""
if not self._cdk_path:
return []
deploy_cdk_application_cmd = ['cdk', 'deploy', '--require-approval', 'never']
if additonal_params:
deploy_cdk_application_cmd.extend(additonal_params)
if context_variable:
deploy_cdk_application_cmd.extend(['-c', f'{context_variable}'])
if deployment_params:
deploy_cdk_application_cmd.extend(deployment_params)
output = process_utils.check_output(
deploy_cdk_application_cmd,
@@ -178,21 +180,23 @@ class Cdk:
env=self._cdk_env,
shell=True)
stacks = []
for line in output.splitlines():
line_sections = line.split('/')
assert len(line_sections), 3
stacks.append(line.split('/')[-2])
self._stacks.append(line.split('/')[-2])
return stacks
return self._stacks
def destroy(self) -> None:
def destroy(self, deployment_params: List[str] = None) -> None:
"""
Destroys the cdk application.
:param deployment_params: Deployment parameters like --all can be passed in this way.
"""
logger.info(f'CDK Path {self._cdk_path}')
destroy_cdk_application_cmd = ['cdk', 'destroy', '--all', '-f']
destroy_cdk_application_cmd = ['cdk', 'destroy', '-f']
if deployment_params:
destroy_cdk_application_cmd.extend(deployment_params)
try:
process_utils.check_output(
@@ -238,3 +242,7 @@ class Cdk:
# self._session.client('cloudformation').delete_stack(
# StackName=BOOTSTRAP_STACK_NAME
# )
@property
def stacks(self):
return self._stacks
@@ -10,10 +10,7 @@ import logging
import ly_test_tools.log.log_monitor
# fixture imports
from AWS.Windows.resource_mappings.resource_mappings import resource_mappings
from AWS.Windows.cdk.cdk_utils import Cdk
from AWS.common.aws_utils import AwsUtils
from assetpipeline.ap_fixtures.asset_processor_fixture import asset_processor as asset_processor
from assetpipeline.ap_fixtures.asset_processor_fixture import asset_processor
AWS_PROJECT_NAME = 'AWS-AutomationTest'
AWS_CLIENT_AUTH_FEATURE_NAME = 'AWSClientAuth'
@@ -37,11 +34,47 @@ logger = logging.getLogger(__name__)
@pytest.mark.parametrize('region_name', ['us-west-2'])
@pytest.mark.parametrize('assume_role_arn', ['arn:aws:iam::645075835648:role/o3de-automation-tests'])
@pytest.mark.parametrize('session_name', ['o3de-Automation-session'])
class TestAWSClientAuthPasswordSignIn(object):
@pytest.mark.usefixtures('cdk')
@pytest.mark.parametrize('deployment_params', [[]])
class TestAWSClientAuthWindows(object):
"""
Test class to verify AWS Cognito IDP Password sign in and Cognito Identity pool authenticated authorization.
Test class to verify AWS Client Auth gem features on Windows.
"""
@pytest.mark.parametrize('level', ['AWS/ClientAuth'])
@pytest.mark.parametrize('destroy_stacks_on_teardown', [False])
def test_anonymous_credentials(self,
level: str,
launcher: pytest.fixture,
resource_mappings: pytest.fixture,
workspace: pytest.fixture,
asset_processor: pytest.fixture
):
"""
Test to verify AWS Cognito Identity pool anonymous authorization.
Setup: Deploys cdk and updates resource mapping file.
Tests: Getting credentials when no credentials are configured
Verification: Log monitor looks for success credentials log.
"""
asset_processor.start()
asset_processor.wait_for_idle()
file_to_monitor = os.path.join(launcher.workspace.paths.project_log(), GAME_LOG_NAME)
log_monitor = ly_test_tools.log.log_monitor.LogMonitor(launcher=launcher, log_file_path=file_to_monitor)
launcher.args = ['+LoadLevel', level]
launcher.args.extend(['-rhi=null'])
with launcher.start(launch_ap=False):
result = log_monitor.monitor_log_for_lines(
expected_lines=['(Script) - Success anonymous credentials'],
unexpected_lines=['(Script) - Fail anonymous credentials'],
halt_on_unexpected=True,
)
assert result, 'Anonymous credentials fetched successfully.'
@pytest.mark.parametrize('destroy_stacks_on_teardown', [True])
def test_password_signin_credentials(self,
launcher: pytest.fixture,
cdk: pytest.fixture,
@@ -51,13 +84,12 @@ class TestAWSClientAuthPasswordSignIn(object):
aws_utils: pytest.fixture
):
"""
Test to verify AWS Cognito IDP Password sign in and Cognito Identity pool authenticated authorization.
Setup: Deploys cdk and updates resource mapping file.
Tests: Sign up new test user, admin confirm the user, sign in and get aws credentials.
Verification: Log monitor looks for success credentials log.
"""
logger.info(f'Cdk stack names:\n{cdk.list()}')
stacks = cdk.deploy()
resource_mappings.populate_output_keys(stacks)
asset_processor.start()
asset_processor.wait_for_idle()
@@ -1,77 +0,0 @@
"""
Copyright (c) Contributors to the Open 3D Engine Project.
For complete copyright and license terms please see the LICENSE at the root of this distribution.
SPDX-License-Identifier: Apache-2.0 OR MIT
"""
import pytest
import os
import logging
import ly_test_tools.log.log_monitor
# fixture imports
from AWS.Windows.resource_mappings.resource_mappings import resource_mappings
from AWS.Windows.cdk.cdk_utils import Cdk
from AWS.common.aws_utils import AwsUtils
from assetpipeline.ap_fixtures.asset_processor_fixture import asset_processor as asset_processor
AWS_PROJECT_NAME = 'AWS-AutomationTest'
AWS_CLIENT_AUTH_FEATURE_NAME = 'AWSClientAuth'
AWS_CLIENT_AUTH_DEFAULT_PROFILE_NAME = 'default'
GAME_LOG_NAME = 'Game.log'
logger = logging.getLogger(__name__)
@pytest.mark.SUITE_periodic
@pytest.mark.usefixtures('automatic_process_killer')
@pytest.mark.usefixtures('asset_processor')
@pytest.mark.usefixtures('workspace')
@pytest.mark.parametrize('project', ['AutomatedTesting'])
@pytest.mark.parametrize('level', ['AWS/ClientAuth'])
@pytest.mark.usefixtures('cdk')
@pytest.mark.parametrize('feature_name', [AWS_CLIENT_AUTH_FEATURE_NAME])
@pytest.mark.usefixtures('resource_mappings')
@pytest.mark.parametrize('resource_mappings_filename', ['default_aws_resource_mappings.json'])
@pytest.mark.usefixtures('aws_utils')
@pytest.mark.parametrize('region_name', ['us-west-2'])
@pytest.mark.parametrize('assume_role_arn', ['arn:aws:iam::645075835648:role/o3de-automation-tests'])
@pytest.mark.parametrize('session_name', ['o3de-Automation-session'])
class TestAWSClientAuthAnonymousCredentials(object):
"""
Test class to verify AWS Cognito Identity pool anonymous authorization.
"""
def test_anonymous_credentials(self,
level: str,
launcher: pytest.fixture,
cdk: pytest.fixture,
resource_mappings: pytest.fixture,
workspace: pytest.fixture,
asset_processor: pytest.fixture
):
"""
Setup: Deploys cdk and updates resource mapping file.
Tests: Getting AWS credentials for no signed in user.
Verification: Log monitor looks for success credentials log.
"""
logger.info(f'Cdk stack names:\n{cdk.list()}')
stacks = cdk.deploy()
resource_mappings.populate_output_keys(stacks)
asset_processor.start()
asset_processor.wait_for_idle()
file_to_monitor = os.path.join(launcher.workspace.paths.project_log(), GAME_LOG_NAME)
log_monitor = ly_test_tools.log.log_monitor.LogMonitor(launcher=launcher, log_file_path=file_to_monitor)
launcher.args = ['+LoadLevel', level]
launcher.args.extend(['-rhi=null'])
with launcher.start(launch_ap=False):
result = log_monitor.monitor_log_for_lines(
expected_lines=['(Script) - Success anonymous credentials'],
unexpected_lines=['(Script) - Fail anonymous credentials'],
halt_on_unexpected=True,
)
assert result, 'Anonymous credentials fetched successfully.'
@@ -7,7 +7,8 @@ SPDX-License-Identifier: Apache-2.0 OR MIT
import os
import logging
import time
import typing
import shutil
import pytest
import ly_test_tools
@@ -16,7 +17,6 @@ import ly_test_tools.environment.process_utils as process_utils
import ly_test_tools.o3de.asset_processor_utils as asset_processor_utils
from botocore.exceptions import ClientError
from AWS.Windows.resource_mappings.resource_mappings import resource_mappings
from assetpipeline.ap_fixtures.asset_processor_fixture import asset_processor
AWS_CORE_FEATURE_NAME = 'AWSCore'
@@ -29,18 +29,49 @@ GAME_LOG_NAME = 'Game.log'
logger = logging.getLogger(__name__)
def setup(launcher: pytest.fixture, cdk: pytest.fixture, resource_mappings: pytest.fixture, asset_processor: pytest.fixture):
def setup(launcher: pytest.fixture, asset_processor: pytest.fixture) -> typing.Tuple[pytest.fixture, str]:
"""
Set up the resource mapping configuration and start the log monitor.
:param launcher: Client launcher for running the test level.
:param asset_processor: asset_processor fixture.
:return log monitor object, metrics file path and the metrics stack name.
"""
# Create the temporary directory for downloading test file from S3.
user_dir = os.path.join(launcher.workspace.paths.project(), 'user')
s3_download_dir = os.path.join(user_dir, 's3_download')
if not os.path.exists(s3_download_dir):
os.makedirs(s3_download_dir)
asset_processor_utils.kill_asset_processor()
logger.info(f'Cdk stack names:\n{cdk.list()}')
stacks = cdk.deploy(additonal_params=['--all'])
resource_mappings.populate_output_keys(stacks)
asset_processor.start()
asset_processor.wait_for_idle()
file_to_monitor = os.path.join(launcher.workspace.paths.project_log(), GAME_LOG_NAME)
log_monitor = ly_test_tools.log.log_monitor.LogMonitor(launcher=launcher, log_file_path=file_to_monitor)
return log_monitor
return log_monitor, s3_download_dir
def write_test_data_to_dynamodb_table(resource_mappings: pytest.fixture, aws_utils: pytest.fixture) -> None:
"""
Write test data to the DynamoDB table created by the CDK application.
:param resource_mappings: resource_mappings fixture.
:param aws_utils: aws_utils fixture.
"""
table_name = resource_mappings.get_resource_name_id("AWSCore.ExampleDynamoTableOutput")
try:
aws_utils.client('dynamodb').put_item(
TableName=table_name,
Item={
'id': {
'S': 'Item1'
}
}
)
logger.info(f'Loaded data into table {table_name}')
except ClientError:
logger.exception(f'Failed to load data into table {table_name}')
raise
@pytest.mark.SUITE_periodic
@@ -58,130 +89,57 @@ def setup(launcher: pytest.fixture, cdk: pytest.fixture, resource_mappings: pyte
@pytest.mark.parametrize('resource_mappings_filename', [AWS_RESOURCE_MAPPING_FILE_NAME])
@pytest.mark.usefixtures('aws_credentials')
@pytest.mark.parametrize('profile_name', ['AWSAutomationTest'])
@pytest.mark.usefixtures('cdk')
@pytest.mark.parametrize('deployment_params', [['--all']])
@pytest.mark.parametrize('destroy_stacks_on_teardown', [True])
class TestAWSCoreAWSResourceInteraction(object):
"""
Test class to verify AWSCore can downloading a file from S3.
Test class to verify the scripting behavior for the AWSCore gem.
"""
def test_download_from_s3(self,
level: str,
launcher: pytest.fixture,
cdk: pytest.fixture,
workspace: pytest.fixture,
asset_processor: pytest.fixture,
resource_mappings: pytest.fixture
):
@pytest.mark.parametrize('expected_lines', [
['(Script) - [S3] Head object request is done',
'(Script) - [S3] Head object success: Object example.txt is found.',
'(Script) - [S3] Get object success: Object example.txt is downloaded.',
'(Script) - [Lambda] Completed Invoke',
'(Script) - [Lambda] Invoke success: {"statusCode": 200, "body": {}}',
'(Script) - [DynamoDB] Results finished']])
@pytest.mark.parametrize('unexpected_lines', [
['(Script) - [S3] Head object error: No response body.',
'(Script) - [S3] Get object error: Request validation failed, output file directory doesn\'t exist.',
'(Script) - Request validation failed, output file miss full path.',
'(Script) - ']])
def test_scripting_behavior(self,
level: str,
launcher: pytest.fixture,
workspace: pytest.fixture,
asset_processor: pytest.fixture,
resource_mappings: pytest.fixture,
aws_utils: pytest.fixture,
expected_lines: typing.List[str],
unexpected_lines: typing.List[str]):
"""
Setup: Deploys cdk and updates resource mapping file.
Tests: Getting AWS credentials for no signed in user.
Verification: Log monitor looks for success download. The existence and contents of the file are also verified.
Tests: Interact with AWS S3, DynamoDB and Lambda services.
Verification: Script canvas nodes can communicate with AWS services successfully.
"""
log_monitor = setup(launcher, cdk, resource_mappings, asset_processor)
log_monitor, s3_download_dir = setup(launcher, asset_processor)
write_test_data_to_dynamodb_table(resource_mappings, aws_utils)
launcher.args = ['+LoadLevel', level]
launcher.args.extend(['-rhi=null'])
user_dir = os.path.join(workspace.paths.project(), 'user')
download_dir = os.path.join(user_dir, 's3_download')
if not os.path.exists(download_dir):
os.makedirs(download_dir)
with launcher.start(launch_ap=False):
result = log_monitor.monitor_log_for_lines(
expected_lines=['(Script) - [S3] Head object request is done',
'(Script) - [S3] Head object success: Object example.txt is found.',
'(Script) - [S3] Get object success: Object example.txt is downloaded.'],
unexpected_lines=['(Script) - [S3] Head object error: No response body.',
'(Script) - [S3] Get object error: Request validation failed, output file directory doesn\'t exist.'],
expected_lines=expected_lines,
unexpected_lines=unexpected_lines,
halt_on_unexpected=True
)
assert result, "Expected lines weren't found."
download_path = os.path.join(download_dir, 'output.txt')
file_was_downloaded = os.path.exists(download_path)
assert os.path.exists(os.path.join(s3_download_dir, 'output.txt')), \
'The expected file wasn\'t successfully downloaded.'
# clean up the file directories.
if file_was_downloaded:
os.remove(download_path)
os.rmdir(download_dir)
assert file_was_downloaded, 'The expected file wasn\'t successfully downloaded'
def test_invoke_lambda(self,
level: str,
launcher: pytest.fixture,
cdk: pytest.fixture,
resource_mappings: pytest.fixture,
workspace: pytest.fixture,
asset_processor: pytest.fixture
):
"""
Setup: Deploys the CDK.
Tests: Runs the test level.
Verification: Searches the logs for the expected output from the example lambda.
"""
log_monitor = setup(launcher, cdk, resource_mappings, asset_processor)
launcher.args = ['+LoadLevel', level]
launcher.args.extend(['-rhi=null'])
with launcher.start(launch_ap=False):
result = log_monitor.monitor_log_for_lines(
expected_lines=['(Script) - [Lambda] Completed Invoke',
'(Script) - [Lambda] Invoke success: {"statusCode": 200, "body": {}}'],
unexpected_lines=['(Script) - Request validation failed, output file miss full path.',
'(Script) - '],
halt_on_unexpected=True
)
assert result
def test_get_dynamodb_value(self,
level: str,
launcher: pytest.fixture,
cdk: pytest.fixture,
resource_mappings: pytest.fixture,
workspace: pytest.fixture,
asset_processor: pytest.fixture,
aws_utils: pytest.fixture,
):
"""
Setup: Deploys the CDK application
Test: Runs a launcher with a level that loads a scriptcanvas that pulls a DynamoDB table value.
Verification: The value is output in the logs and verified by the test.
"""
def write_test_table_data():
client = aws_utils.client('dynamodb')
table_name = resource_mappings.get_resource_name_id("AWSCore.ExampleDynamoTableOutput")
try:
client.put_item(
TableName=table_name,
Item={
'id': {
'S': 'Item1'
}
}
)
logger.info(f'Loaded data into table {table_name}')
except ClientError:
logger.exception(f'Failed to load data into table {table_name}')
raise
log_monitor = setup(launcher, cdk, resource_mappings, asset_processor)
write_test_table_data()
launcher.args = ['+LoadLevel', level]
launcher.args.extend(['-rhi=null'])
with launcher.start(launch_ap=False):
result = log_monitor.monitor_log_for_lines(
expected_lines=['(Script) - [DynamoDB] Results finished'],
unexpected_lines=['(Script) - Request validation failed, output file miss full path.',
'(Script) - '],
halt_on_unexpected=True
)
assert result
shutil.rmtree(s3_download_dir)
@@ -6,7 +6,6 @@ SPDX-License-Identifier: Apache-2.0 OR MIT
"""
import os
from os.path import abspath
import pytest
import json
import logging
@@ -107,38 +106,3 @@ class ResourceMappings:
def get_resource_name_id(self, resource_key: str):
return self._resource_mappings[AWS_RESOURCE_MAPPINGS_KEY][resource_key]['Name/ID']
@pytest.fixture(scope='function')
def resource_mappings(
request: pytest.fixture,
project: str,
feature_name: str,
resource_mappings_filename: str,
workspace: pytest.fixture,
aws_utils: pytest.fixture) -> ResourceMappings:
"""
Fixture for setting up resource mappings file.
:param request: _pytest.fixtures.SubRequest class that handles getting
a pytest fixture from a pytest function/fixture.
:param project: Project to find resource mapping file.
:param feature_name: AWS Gem name that is prepended to resource mapping keys.
:param resource_mappings_filename: Name of resource mapping file.
:param workspace: ly_test_tools workspace fixture.
:param aws_utils: AWS utils fixture.
:return: ResourceMappings class object.
"""
path = f'{workspace.paths.engine_root()}/{project}/Config/{resource_mappings_filename}'
logger.info(f'Resource mapping path : {path}')
logger.info(f'Resource mapping resolved path : {abspath(path)}')
resource_mappings_obj = ResourceMappings(abspath(path), aws_utils.assume_session().region_name, feature_name,
aws_utils.assume_account_id(), workspace,
aws_utils.client('cloudformation'))
def teardown():
resource_mappings_obj.clear_output_keys()
request.addfinalizer(teardown)
return resource_mappings_obj
@@ -4,11 +4,15 @@ For complete copyright and license terms please see the LICENSE at the root of t
SPDX-License-Identifier: Apache-2.0 OR MIT
"""
import pytest
import logging
from os.path import abspath
import pytest
import typing
from AWS.common.aws_utils import AwsUtils
from AWS.common.aws_credentials import AwsCredentials
from AWS.Windows.cdk.cdk_utils import Cdk
from AWS.Windows.resource_mappings.resource_mappings import ResourceMappings
logger = logging.getLogger(__name__)
@@ -42,6 +46,41 @@ def aws_utils(
pytest.cdk_obj = None
@pytest.fixture(scope='function')
def resource_mappings(
request: pytest.fixture,
project: str,
feature_name: str,
resource_mappings_filename: str,
workspace: pytest.fixture,
aws_utils: pytest.fixture) -> ResourceMappings:
"""
Fixture for setting up resource mappings file.
:param request: _pytest.fixtures.SubRequest class that handles getting
a pytest fixture from a pytest function/fixture.
:param project: Project to find resource mapping file.
:param feature_name: AWS Gem name that is prepended to resource mapping keys.
:param resource_mappings_filename: Name of resource mapping file.
:param workspace: ly_test_tools workspace fixture.
:param aws_utils: AWS utils fixture.
:return: ResourceMappings class object.
"""
path = f'{workspace.paths.engine_root()}/{project}/Config/{resource_mappings_filename}'
logger.info(f'Resource mapping path : {path}')
logger.info(f'Resource mapping resolved path : {abspath(path)}')
resource_mappings_obj = ResourceMappings(abspath(path), aws_utils.assume_session().region_name, feature_name,
aws_utils.assume_account_id(), workspace,
aws_utils.client('cloudformation'))
def teardown():
resource_mappings_obj.clear_output_keys()
request.addfinalizer(teardown)
return resource_mappings_obj
@pytest.fixture(scope='function')
def cdk(
request: pytest.fixture,
@@ -49,8 +88,9 @@ def cdk(
feature_name: str,
workspace: pytest.fixture,
aws_utils: pytest.fixture,
bootstrap_required: bool = True,
destroy_stacks_on_teardown: bool = True) -> Cdk:
resource_mappings: pytest.fixture,
deployment_params: typing.List[str],
destroy_stacks_on_teardown: bool) -> Cdk:
"""
Fixture for setting up a Cdk
:param request: _pytest.fixtures.SubRequest class that handles getting
@@ -59,8 +99,8 @@ def cdk(
:param feature_name: Feature gem name to expect cdk folder in.
:param workspace: ly_test_tools workspace fixture.
:param aws_utils: aws_utils fixture.
:param bootstrap_required: Whether the bootstrap stack needs to be created to
provision resources the AWS CDK needs to perform the deployment.
:param resource_mappings: resource_mappings fixture.
:param deployment_params: Parameters for the CDK application deployment.
:param destroy_stacks_on_teardown: option to control calling destroy ot the end of test.
:return Cdk class object.
"""
@@ -70,22 +110,27 @@ def cdk(
if pytest.cdk_obj is None:
pytest.cdk_obj = Cdk()
pytest.cdk_obj.setup(cdk_path, project, aws_utils.assume_account_id(), workspace, aws_utils.assume_session())
stacks = pytest.cdk_obj.deploy(deployment_params=deployment_params)
logger.info(f'Cdk stack names:\n{stacks}')
resource_mappings.populate_output_keys(stacks)
pytest.cdk_obj.setup(cdk_path, project, aws_utils.assume_account_id(), workspace, aws_utils.assume_session(),
bootstrap_required)
def teardown():
if destroy_stacks_on_teardown:
pytest.cdk_obj.destroy()
pytest.cdk_obj.destroy(deployment_params=deployment_params)
# Enable after https://github.com/aws/aws-cdk/issues/986 is fixed.
# Until then clean the bootstrap bucket manually.
# cdk_obj.remove_bootstrap_stack()
# pytest.cdk_obj.remove_bootstrap_stack()
pytest.cdk_obj = None
request.addfinalizer(teardown)
return pytest.cdk_obj
@pytest.fixture(scope='function')
def aws_credentials(request: pytest.fixture, aws_utils: pytest.fixture, profile_name: str):
"""
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:f0f4d4e0155feaa76c80a14128000a0fd9570ab76e79f4847eaef9006324a4d2
size 9084
oid sha256:19f2c4454bb395cdc0a36d1e45e6a384bbd23037af1a2fb93e088ecfa0f10e5b
size 9343
File diff suppressed because it is too large Load Diff
@@ -1 +0,0 @@
<Environment />
@@ -1 +0,0 @@
<TimeOfDay Time="12"/>
@@ -1,6 +1,6 @@
<download name="ClientAuth" type="Map">
<index src="filelist.xml" dest="filelist.xml"/>
<files>
<file src="level.pak" dest="level.pak" size="ED0" md5="dbf5115226e4b0ea38ebdc3967ba3aa9"/>
<file src="level.pak" dest="level.pak" size="E1A" md5="fecbc160ebc2186184504482e3e2eba3"/>
</files>
</download>
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:da041115014f11696d5878d5c21247c17b8d694fa9674e30692259261a7223a2
size 3792
oid sha256:8a674e05824e5ceec13a0487b318923568710bc8269e5be84adad59c495a7ceb
size 3610
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:43b1a23b62fe2ffa05545ac99524f40b6fff49d6e35925b9d6138c00d8082e86
size 9073
oid sha256:a1c0b621525b8e88c3775ea4c60c2197d1e1b060ace9bad9d6efcb0532817e44
size 9356
File diff suppressed because it is too large Load Diff
@@ -1,6 +1,6 @@
<download name="ClientAuthPasswordSignIn" type="Map">
<index src="filelist.xml" dest="filelist.xml"/>
<files>
<file src="level.pak" dest="level.pak" size="DDF" md5="ebe91ae5f1ea1ec735b6650b14f3f95b"/>
<file src="level.pak" dest="level.pak" size="E53" md5="12728d49c7efe0c8e83e3651a1d13ca6"/>
</files>
</download>
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:a58292341785cb260dc0ccf346259e35e2817ee48fc401a21ab528f6afb97b52
size 3551
oid sha256:f318a1787069385de291660f79e350cea2ca2c3ef3b5e0576686066bd9c49395
size 3667
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:f3d5121b26608b02747e245071ccff29ac57358cb6349ec9495a7a003ac12467
size 8942
oid sha256:afc5d665128738e6bea09e78a16ee38acc923a8ecefff90d987858ce72c395fa
size 9360
File diff suppressed because it is too large Load Diff
@@ -1,6 +1,6 @@
<download name="ClientAuthPasswordSignUp" type="Map">
<index src="filelist.xml" dest="filelist.xml"/>
<files>
<file src="level.pak" dest="level.pak" size="DDA" md5="aa6df891d505d6d9175beee4b55626db"/>
<file src="level.pak" dest="level.pak" size="E44" md5="acfaa325178533f135cd683d3782cc8e"/>
</files>
</download>
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:605391d415b828b100bada11d108099520c0b6a020f17588887b610475805d90
size 3546
oid sha256:87882b64688a77815d93c6973929fa21b89dc6c13d4866c710124ce2cd0f411e
size 3652
@@ -30,9 +30,6 @@ function metrics:OnSendMetricsFailure(requestId, errorMessage)
end
function metrics:OnDeactivate()
AWSMetricsRequestBus.Broadcast.FlushMetrics()
Debug.Log("Stop generating new test events and flushed the buffered metrics.")
self.tickBusHandler:Disconnect()
self.metricsNotificationHandler:Disconnect()
end
@@ -40,7 +37,7 @@ end
function metrics:OnTick(deltaTime, timePoint)
self.tickTime = self.tickTime + deltaTime
if self.tickTime > 2.0 then
if self.tickTime > 5.0 then
defaultAttribute = AWSMetrics_MetricsAttribute()
defaultAttribute:SetName("event_name")
defaultAttribute:SetStrValue("login")
@@ -57,6 +54,9 @@ function metrics:OnTick(deltaTime, timePoint)
if self.numSubmittedMetricsEvents % 2 == 0 then
if AWSMetricsRequestBus.Broadcast.SubmitMetrics(attributeList.attributes, 0, "lua", false) then
Debug.Log("Submitted metrics without buffer.")
AWSMetricsRequestBus.Broadcast.FlushMetrics()
Debug.Log("Flushed the buffered metrics.")
else
Debug.Log("Failed to Submit metrics without buffer.")
end
@@ -67,7 +67,7 @@ function metrics:OnTick(deltaTime, timePoint)
Debug.Log("Failed to Submit metrics with buffer.")
end
end
self.numSubmittedMetricsEvents = self.numSubmittedMetricsEvents + 1
self.tickTime = 0
end
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff