AWSI automation tests support on Linux (#6278)
* AWSI automation tests support on Linux Signed-off-by: Junbo Liang <68558268+junbo75@users.noreply.github.com>
This commit is contained in:
Junbo Liang
GitHub
@@ -13,7 +13,8 @@
|
||||
|
||||
if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS)
|
||||
# Only enable AWS automated tests on Windows
|
||||
if(NOT "${PAL_PLATFORM_NAME}" STREQUAL "Windows")
|
||||
set(SUPPORTED_PLATFORMS "Windows" "Linux")
|
||||
if (NOT "${PAL_PLATFORM_NAME}" IN_LIST SUPPORTED_PLATFORMS)
|
||||
return()
|
||||
endif()
|
||||
|
||||
@@ -23,7 +24,6 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS)
|
||||
TEST_SERIAL
|
||||
PATH ${CMAKE_CURRENT_LIST_DIR}/${PAL_PLATFORM_NAME}/
|
||||
RUNTIME_DEPENDENCIES
|
||||
Legacy::Editor
|
||||
AZ::AssetProcessor
|
||||
AutomatedTesting.GameLauncher
|
||||
AutomatedTesting.Assets
|
||||
|
||||
@@ -2,30 +2,61 @@
|
||||
|
||||
## Prerequisites
|
||||
1. Build the O3DE Editor and AutomatedTesting.GameLauncher in Profile.
|
||||
2. AWS CLI is installed and configured following [Configuration and Credential File Settings](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html).
|
||||
3. [AWS Cloud Development Kit (CDK)](https://docs.aws.amazon.com/cdk/latest/guide/getting_started.html#getting_started_install) is installed.
|
||||
2. Install the latest version of NodeJs.
|
||||
3. AWS CLI is installed and configured following [Configuration and Credential File Settings](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html).
|
||||
4. [AWS Cloud Development Kit (CDK)](https://docs.aws.amazon.com/cdk/latest/guide/getting_started.html#getting_started_install) is installed.
|
||||
|
||||
## Deploy CDK Applications
|
||||
1. Go to the AWS IAM console and create an IAM role called o3de-automation-tests which adds your own account as as a trusted entity and uses the "AdministratorAccess" permissions policy.
|
||||
2. Copy {engine_root}\scripts\build\Platform\Windows\deploy_cdk_applications.cmd to your engine root folder.
|
||||
3. Open a new Command Prompt window at the engine root and set the following environment variables:
|
||||
```
|
||||
Set O3DE_AWS_PROJECT_NAME=AWSAUTO
|
||||
Set O3DE_AWS_DEPLOY_REGION=us-east-1
|
||||
Set O3DE_AWS_DEPLOY_ACCOUNT={your_aws_account_id}
|
||||
Set ASSUME_ROLE_ARN=arn:aws:iam::{your_aws_account_id}:role/o3de-automation-tests
|
||||
Set COMMIT_ID=HEAD
|
||||
```
|
||||
4. In the same Command Prompt window, Deploy the CDK applications for AWS gems by running deploy_cdk_applications.cmd.
|
||||
2. Copy the following deployment script to your engine root folder:
|
||||
* Windows (Command Prompt)
|
||||
```
|
||||
{engine_root}\scripts\build\Platform\Windows\deploy_cdk_applications.cmd
|
||||
```
|
||||
* Linux
|
||||
```
|
||||
{engine_root}/scripts/build/Platform/Linux/deploy_cdk_applications.sh
|
||||
```
|
||||
3. Open a new CLI window at the engine root and set the following environment variables:
|
||||
* Windows
|
||||
```
|
||||
Set O3DE_AWS_PROJECT_NAME=AWSAUTO
|
||||
Set O3DE_AWS_DEPLOY_REGION=us-east-1
|
||||
Set ASSUME_ROLE_ARN=arn:aws:iam::{your_aws_account_id}:role/o3de-automation-tests
|
||||
Set COMMIT_ID=HEAD
|
||||
```
|
||||
* Linux
|
||||
```
|
||||
export O3DE_AWS_PROJECT_NAME=AWSAUTO
|
||||
export O3DE_AWS_DEPLOY_REGION=us-east-1
|
||||
export ASSUME_ROLE_ARN=arn:aws:iam::{your_aws_account_id}:role/o3de-automation-tests
|
||||
export COMMIT_ID=HEAD
|
||||
```
|
||||
4. In the same CLI window, Deploy the CDK applications for AWS gems by running deploy_cdk_applications.cmd.
|
||||
|
||||
## Run Automation Tests
|
||||
### CLI
|
||||
In the same Command Prompt window, run the following CLI command:
|
||||
python\python.cmd -m pytest {path_to_the_test_file} --build-directory {directory_to_the_profile_build}
|
||||
1. In the same CLI window, run the following CLI command:
|
||||
* Windows
|
||||
```
|
||||
python\python.cmd -m pytest {path_to_the_test_file} --build-directory {directory_to_the_profile_build}
|
||||
```
|
||||
* Linux
|
||||
```
|
||||
python/python.sh -m pytest {path_to_the_test_file} --build-directory {directory_to_the_profile_build}
|
||||
```
|
||||
|
||||
### Pycharm
|
||||
You can also run any specific automation test directly from Pycharm by providing the "--build-directory" argument in the Run Configuration.
|
||||
|
||||
## Destroy CDK Applications
|
||||
1. Copy {engine_root}\scripts\build\Platform\Windows\destroy_cdk_applications.cmd to your engine root folder.
|
||||
2. In the same Command Prompt window, destroy the CDK applications for AWS gems by running destroy_cdk_applications.cmd.
|
||||
1. Copy the following destruction script to your engine root folder:
|
||||
* Windows
|
||||
```
|
||||
{engine_root}\scripts\build\Platform\Windows\destroy_cdk_applications.cmd
|
||||
```
|
||||
* Linux
|
||||
```
|
||||
{engine_root}/scripts/build/Platform/Linux/destroy_cdk_applications.sh
|
||||
```
|
||||
2. In the same CLI window, destroy the CDK applications for AWS gems by running destroy_cdk_applications.cmd.
|
||||
|
||||
@@ -1,6 +0,0 @@
|
||||
"""
|
||||
Copyright (c) Contributors to the Open 3D Engine Project.
|
||||
For complete copyright and license terms please see the LICENSE at the root of this distribution.
|
||||
|
||||
SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
"""
|
||||
+289
-289
@@ -1,289 +1,289 @@
|
||||
"""
|
||||
Copyright (c) Contributors to the Open 3D Engine Project.
|
||||
For complete copyright and license terms please see the LICENSE at the root of this distribution.
|
||||
|
||||
SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import pytest
|
||||
import typing
|
||||
from datetime import datetime
|
||||
|
||||
import ly_test_tools.log.log_monitor
|
||||
|
||||
from AWS.common import constants
|
||||
from AWS.common.resource_mappings import AWS_RESOURCE_MAPPINGS_ACCOUNT_ID_KEY
|
||||
from .aws_metrics_custom_thread import AWSMetricsThread
|
||||
|
||||
# fixture imports
|
||||
from assetpipeline.ap_fixtures.asset_processor_fixture import asset_processor
|
||||
from .aws_metrics_utils import aws_metrics_utils
|
||||
|
||||
AWS_METRICS_FEATURE_NAME = 'AWSMetrics'
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def setup(launcher: pytest.fixture,
|
||||
asset_processor: pytest.fixture) -> pytest.fixture:
|
||||
"""
|
||||
Set up the resource mapping configuration and start the log monitor.
|
||||
:param launcher: Client launcher for running the test level.
|
||||
:param asset_processor: asset_processor fixture.
|
||||
:return log monitor object.
|
||||
"""
|
||||
asset_processor.start()
|
||||
asset_processor.wait_for_idle()
|
||||
|
||||
file_to_monitor = os.path.join(launcher.workspace.paths.project_log(), constants.GAME_LOG_NAME)
|
||||
|
||||
# Initialize the log monitor.
|
||||
log_monitor = ly_test_tools.log.log_monitor.LogMonitor(launcher=launcher, log_file_path=file_to_monitor)
|
||||
|
||||
return log_monitor
|
||||
|
||||
|
||||
def monitor_metrics_submission(log_monitor: pytest.fixture) -> None:
|
||||
"""
|
||||
Monitor the messages and notifications for submitting metrics.
|
||||
:param log_monitor: Log monitor to check the log messages.
|
||||
"""
|
||||
expected_lines = [
|
||||
'(Script) - Submitted metrics without buffer.',
|
||||
'(Script) - Submitted metrics with buffer.',
|
||||
'(Script) - Flushed the buffered metrics.',
|
||||
'(Script) - Metrics is sent successfully.'
|
||||
]
|
||||
|
||||
unexpected_lines = [
|
||||
'(Script) - Failed to submit metrics without buffer.',
|
||||
'(Script) - Failed to submit metrics with buffer.',
|
||||
'(Script) - Failed to send metrics.'
|
||||
]
|
||||
|
||||
result = log_monitor.monitor_log_for_lines(
|
||||
expected_lines=expected_lines,
|
||||
unexpected_lines=unexpected_lines,
|
||||
halt_on_unexpected=True)
|
||||
|
||||
# Assert the log monitor detected expected lines and did not detect any unexpected lines.
|
||||
assert result, (
|
||||
f'Log monitoring failed. Used expected_lines values: {expected_lines} & '
|
||||
f'unexpected_lines values: {unexpected_lines}')
|
||||
|
||||
|
||||
def query_metrics_from_s3(aws_metrics_utils: pytest.fixture, resource_mappings: pytest.fixture) -> None:
|
||||
"""
|
||||
Verify that the metrics events are delivered to the S3 bucket and can be queried.
|
||||
:param aws_metrics_utils: aws_metrics_utils fixture.
|
||||
:param resource_mappings: resource_mappings fixture.
|
||||
"""
|
||||
aws_metrics_utils.verify_s3_delivery(
|
||||
resource_mappings.get_resource_name_id('AWSMetrics.AnalyticsBucketName')
|
||||
)
|
||||
logger.info('Metrics are sent to S3.')
|
||||
|
||||
aws_metrics_utils.run_glue_crawler(
|
||||
resource_mappings.get_resource_name_id('AWSMetrics.EventsCrawlerName'))
|
||||
|
||||
# Remove the events_json table if exists so that the sample query can create a table with the same name.
|
||||
aws_metrics_utils.delete_table(resource_mappings.get_resource_name_id('AWSMetrics.EventDatabaseName'), 'events_json')
|
||||
aws_metrics_utils.run_named_queries(resource_mappings.get_resource_name_id('AWSMetrics.AthenaWorkGroupName'))
|
||||
logger.info('Query metrics from S3 successfully.')
|
||||
|
||||
|
||||
def verify_operational_metrics(aws_metrics_utils: pytest.fixture,
|
||||
resource_mappings: pytest.fixture, start_time: datetime) -> None:
|
||||
"""
|
||||
Verify that operational health metrics are delivered to CloudWatch.
|
||||
:param aws_metrics_utils: aws_metrics_utils fixture.
|
||||
:param resource_mappings: resource_mappings fixture.
|
||||
:param start_time: Time when the game launcher starts.
|
||||
"""
|
||||
aws_metrics_utils.verify_cloud_watch_delivery(
|
||||
'AWS/Lambda',
|
||||
'Invocations',
|
||||
[{'Name': 'FunctionName',
|
||||
'Value': resource_mappings.get_resource_name_id('AWSMetrics.AnalyticsProcessingLambdaName')}],
|
||||
start_time)
|
||||
logger.info('AnalyticsProcessingLambda metrics are sent to CloudWatch.')
|
||||
|
||||
aws_metrics_utils.verify_cloud_watch_delivery(
|
||||
'AWS/Lambda',
|
||||
'Invocations',
|
||||
[{'Name': 'FunctionName',
|
||||
'Value': resource_mappings.get_resource_name_id('AWSMetrics.EventProcessingLambdaName')}],
|
||||
start_time)
|
||||
logger.info('EventsProcessingLambda metrics are sent to CloudWatch.')
|
||||
|
||||
|
||||
def update_kinesis_analytics_application_status(aws_metrics_utils: pytest.fixture,
|
||||
resource_mappings: pytest.fixture, start_application: bool) -> None:
|
||||
"""
|
||||
Update the Kinesis analytics application to start or stop it.
|
||||
:param aws_metrics_utils: aws_metrics_utils fixture.
|
||||
:param resource_mappings: resource_mappings fixture.
|
||||
:param start_application: whether to start or stop the application.
|
||||
"""
|
||||
if start_application:
|
||||
aws_metrics_utils.start_kinesis_data_analytics_application(
|
||||
resource_mappings.get_resource_name_id('AWSMetrics.AnalyticsApplicationName'))
|
||||
else:
|
||||
aws_metrics_utils.stop_kinesis_data_analytics_application(
|
||||
resource_mappings.get_resource_name_id('AWSMetrics.AnalyticsApplicationName'))
|
||||
|
||||
@pytest.mark.SUITE_awsi
|
||||
@pytest.mark.usefixtures('automatic_process_killer')
|
||||
@pytest.mark.usefixtures('aws_credentials')
|
||||
@pytest.mark.usefixtures('resource_mappings')
|
||||
@pytest.mark.parametrize('assume_role_arn', [constants.ASSUME_ROLE_ARN])
|
||||
@pytest.mark.parametrize('feature_name', [AWS_METRICS_FEATURE_NAME])
|
||||
@pytest.mark.parametrize('profile_name', ['AWSAutomationTest'])
|
||||
@pytest.mark.parametrize('project', ['AutomatedTesting'])
|
||||
@pytest.mark.parametrize('region_name', [constants.AWS_REGION])
|
||||
@pytest.mark.parametrize('resource_mappings_filename', [constants.AWS_RESOURCE_MAPPING_FILE_NAME])
|
||||
@pytest.mark.parametrize('session_name', [constants.SESSION_NAME])
|
||||
@pytest.mark.parametrize('stacks', [[f'{constants.AWS_PROJECT_NAME}-{AWS_METRICS_FEATURE_NAME}-{constants.AWS_REGION}']])
|
||||
class TestAWSMetricsWindows(object):
|
||||
"""
|
||||
Test class to verify the real-time and batch analytics for metrics.
|
||||
"""
|
||||
@pytest.mark.parametrize('level', ['AWS/Metrics'])
|
||||
def test_realtime_and_batch_analytics(self,
|
||||
level: str,
|
||||
launcher: pytest.fixture,
|
||||
asset_processor: pytest.fixture,
|
||||
workspace: pytest.fixture,
|
||||
aws_utils: pytest.fixture,
|
||||
resource_mappings: pytest.fixture,
|
||||
aws_metrics_utils: pytest.fixture):
|
||||
"""
|
||||
Verify that the metrics events are sent to CloudWatch and S3 for analytics.
|
||||
"""
|
||||
# Start Kinesis analytics application on a separate thread to avoid blocking the test.
|
||||
kinesis_analytics_application_thread = AWSMetricsThread(target=update_kinesis_analytics_application_status,
|
||||
args=(aws_metrics_utils, resource_mappings, True))
|
||||
kinesis_analytics_application_thread.start()
|
||||
|
||||
log_monitor = setup(launcher, asset_processor)
|
||||
|
||||
# Kinesis analytics application needs to be in the running state before we start the game launcher.
|
||||
kinesis_analytics_application_thread.join()
|
||||
launcher.args = ['+LoadLevel', level]
|
||||
launcher.args.extend(['-rhi=null'])
|
||||
start_time = datetime.utcnow()
|
||||
with launcher.start(launch_ap=False):
|
||||
monitor_metrics_submission(log_monitor)
|
||||
|
||||
# Verify that real-time analytics metrics are delivered to CloudWatch.
|
||||
aws_metrics_utils.verify_cloud_watch_delivery(
|
||||
AWS_METRICS_FEATURE_NAME,
|
||||
'TotalLogins',
|
||||
[],
|
||||
start_time)
|
||||
logger.info('Real-time metrics are sent to CloudWatch.')
|
||||
|
||||
# Run time-consuming operations on separate threads to avoid blocking the test.
|
||||
operational_threads = list()
|
||||
operational_threads.append(
|
||||
AWSMetricsThread(target=query_metrics_from_s3,
|
||||
args=(aws_metrics_utils, resource_mappings)))
|
||||
operational_threads.append(
|
||||
AWSMetricsThread(target=verify_operational_metrics,
|
||||
args=(aws_metrics_utils, resource_mappings, start_time)))
|
||||
operational_threads.append(
|
||||
AWSMetricsThread(target=update_kinesis_analytics_application_status,
|
||||
args=(aws_metrics_utils, resource_mappings, False)))
|
||||
for thread in operational_threads:
|
||||
thread.start()
|
||||
for thread in operational_threads:
|
||||
thread.join()
|
||||
|
||||
@pytest.mark.parametrize('level', ['AWS/Metrics'])
|
||||
def test_realtime_and_batch_analytics_no_global_accountid(self,
|
||||
level: str,
|
||||
launcher: pytest.fixture,
|
||||
asset_processor: pytest.fixture,
|
||||
workspace: pytest.fixture,
|
||||
aws_utils: pytest.fixture,
|
||||
resource_mappings: pytest.fixture,
|
||||
aws_metrics_utils: pytest.fixture):
|
||||
"""
|
||||
Verify that the metrics events are sent to CloudWatch and S3 for analytics.
|
||||
"""
|
||||
# Remove top-level account ID from resource mappings
|
||||
resource_mappings.clear_select_keys([AWS_RESOURCE_MAPPINGS_ACCOUNT_ID_KEY])
|
||||
# Start Kinesis analytics application on a separate thread to avoid blocking the test.
|
||||
kinesis_analytics_application_thread = AWSMetricsThread(target=update_kinesis_analytics_application_status,
|
||||
args=(aws_metrics_utils, resource_mappings, True))
|
||||
kinesis_analytics_application_thread.start()
|
||||
|
||||
log_monitor = setup(launcher, asset_processor)
|
||||
|
||||
# Kinesis analytics application needs to be in the running state before we start the game launcher.
|
||||
kinesis_analytics_application_thread.join()
|
||||
launcher.args = ['+LoadLevel', level]
|
||||
launcher.args.extend(['-rhi=null'])
|
||||
start_time = datetime.utcnow()
|
||||
with launcher.start(launch_ap=False):
|
||||
monitor_metrics_submission(log_monitor)
|
||||
|
||||
# Verify that real-time analytics metrics are delivered to CloudWatch.
|
||||
aws_metrics_utils.verify_cloud_watch_delivery(
|
||||
AWS_METRICS_FEATURE_NAME,
|
||||
'TotalLogins',
|
||||
[],
|
||||
start_time)
|
||||
logger.info('Real-time metrics are sent to CloudWatch.')
|
||||
|
||||
# Run time-consuming operations on separate threads to avoid blocking the test.
|
||||
operational_threads = list()
|
||||
operational_threads.append(
|
||||
AWSMetricsThread(target=query_metrics_from_s3,
|
||||
args=(aws_metrics_utils, resource_mappings)))
|
||||
operational_threads.append(
|
||||
AWSMetricsThread(target=verify_operational_metrics,
|
||||
args=(aws_metrics_utils, resource_mappings, start_time)))
|
||||
operational_threads.append(
|
||||
AWSMetricsThread(target=update_kinesis_analytics_application_status,
|
||||
args=(aws_metrics_utils, resource_mappings, False)))
|
||||
for thread in operational_threads:
|
||||
thread.start()
|
||||
for thread in operational_threads:
|
||||
thread.join()
|
||||
|
||||
@pytest.mark.parametrize('level', ['AWS/Metrics'])
|
||||
def test_unauthorized_user_request_rejected(self,
|
||||
level: str,
|
||||
launcher: pytest.fixture,
|
||||
asset_processor: pytest.fixture,
|
||||
workspace: pytest.fixture):
|
||||
"""
|
||||
Verify that unauthorized users cannot send metrics events to the AWS backed backend.
|
||||
"""
|
||||
log_monitor = setup(launcher, asset_processor)
|
||||
|
||||
# Set invalid AWS credentials.
|
||||
launcher.args = ['+LoadLevel', level, '+cl_awsAccessKey', 'AKIAIOSFODNN7EXAMPLE',
|
||||
'+cl_awsSecretKey', 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY']
|
||||
launcher.args.extend(['-rhi=null'])
|
||||
|
||||
with launcher.start(launch_ap=False):
|
||||
result = log_monitor.monitor_log_for_lines(
|
||||
expected_lines=['(Script) - Failed to send metrics.'],
|
||||
unexpected_lines=['(Script) - Metrics is sent successfully.'],
|
||||
halt_on_unexpected=True)
|
||||
assert result, 'Metrics events are sent successfully by unauthorized user'
|
||||
logger.info('Unauthorized user is rejected to send metrics.')
|
||||
|
||||
def test_clean_up_s3_bucket(self,
|
||||
aws_utils: pytest.fixture,
|
||||
resource_mappings: pytest.fixture,
|
||||
aws_metrics_utils: pytest.fixture):
|
||||
"""
|
||||
Clear the analytics bucket objects so that the S3 bucket can be destroyed during tear down.
|
||||
"""
|
||||
aws_metrics_utils.empty_bucket(
|
||||
resource_mappings.get_resource_name_id('AWSMetrics.AnalyticsBucketName'))
|
||||
"""
|
||||
Copyright (c) Contributors to the Open 3D Engine Project.
|
||||
For complete copyright and license terms please see the LICENSE at the root of this distribution.
|
||||
|
||||
SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import pytest
|
||||
import typing
|
||||
from datetime import datetime
|
||||
|
||||
import ly_test_tools.log.log_monitor
|
||||
|
||||
from AWS.common import constants
|
||||
from AWS.common.resource_mappings import AWS_RESOURCE_MAPPINGS_ACCOUNT_ID_KEY
|
||||
from .aws_metrics_custom_thread import AWSMetricsThread
|
||||
|
||||
# fixture imports
|
||||
from assetpipeline.ap_fixtures.asset_processor_fixture import asset_processor
|
||||
from .aws_metrics_utils import aws_metrics_utils
|
||||
|
||||
AWS_METRICS_FEATURE_NAME = 'AWSMetrics'
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def setup(launcher: pytest.fixture,
|
||||
asset_processor: pytest.fixture) -> pytest.fixture:
|
||||
"""
|
||||
Set up the resource mapping configuration and start the log monitor.
|
||||
:param launcher: Client launcher for running the test level.
|
||||
:param asset_processor: asset_processor fixture.
|
||||
:return log monitor object.
|
||||
"""
|
||||
asset_processor.start()
|
||||
asset_processor.wait_for_idle()
|
||||
|
||||
file_to_monitor = os.path.join(launcher.workspace.paths.project_log(), constants.GAME_LOG_NAME)
|
||||
|
||||
# Initialize the log monitor.
|
||||
log_monitor = ly_test_tools.log.log_monitor.LogMonitor(launcher=launcher, log_file_path=file_to_monitor)
|
||||
|
||||
return log_monitor
|
||||
|
||||
|
||||
def monitor_metrics_submission(log_monitor: pytest.fixture) -> None:
|
||||
"""
|
||||
Monitor the messages and notifications for submitting metrics.
|
||||
:param log_monitor: Log monitor to check the log messages.
|
||||
"""
|
||||
expected_lines = [
|
||||
'(Script) - Submitted metrics without buffer.',
|
||||
'(Script) - Submitted metrics with buffer.',
|
||||
'(Script) - Flushed the buffered metrics.',
|
||||
'(Script) - Metrics is sent successfully.'
|
||||
]
|
||||
|
||||
unexpected_lines = [
|
||||
'(Script) - Failed to submit metrics without buffer.',
|
||||
'(Script) - Failed to submit metrics with buffer.',
|
||||
'(Script) - Failed to send metrics.'
|
||||
]
|
||||
|
||||
result = log_monitor.monitor_log_for_lines(
|
||||
expected_lines=expected_lines,
|
||||
unexpected_lines=unexpected_lines,
|
||||
halt_on_unexpected=True)
|
||||
|
||||
# Assert the log monitor detected expected lines and did not detect any unexpected lines.
|
||||
assert result, (
|
||||
f'Log monitoring failed. Used expected_lines values: {expected_lines} & '
|
||||
f'unexpected_lines values: {unexpected_lines}')
|
||||
|
||||
|
||||
def query_metrics_from_s3(aws_metrics_utils: pytest.fixture, resource_mappings: pytest.fixture) -> None:
|
||||
"""
|
||||
Verify that the metrics events are delivered to the S3 bucket and can be queried.
|
||||
:param aws_metrics_utils: aws_metrics_utils fixture.
|
||||
:param resource_mappings: resource_mappings fixture.
|
||||
"""
|
||||
aws_metrics_utils.verify_s3_delivery(
|
||||
resource_mappings.get_resource_name_id('AWSMetrics.AnalyticsBucketName')
|
||||
)
|
||||
logger.info('Metrics are sent to S3.')
|
||||
|
||||
aws_metrics_utils.run_glue_crawler(
|
||||
resource_mappings.get_resource_name_id('AWSMetrics.EventsCrawlerName'))
|
||||
|
||||
# Remove the events_json table if exists so that the sample query can create a table with the same name.
|
||||
aws_metrics_utils.delete_table(resource_mappings.get_resource_name_id('AWSMetrics.EventDatabaseName'), 'events_json')
|
||||
aws_metrics_utils.run_named_queries(resource_mappings.get_resource_name_id('AWSMetrics.AthenaWorkGroupName'))
|
||||
logger.info('Query metrics from S3 successfully.')
|
||||
|
||||
|
||||
def verify_operational_metrics(aws_metrics_utils: pytest.fixture,
|
||||
resource_mappings: pytest.fixture, start_time: datetime) -> None:
|
||||
"""
|
||||
Verify that operational health metrics are delivered to CloudWatch.
|
||||
:param aws_metrics_utils: aws_metrics_utils fixture.
|
||||
:param resource_mappings: resource_mappings fixture.
|
||||
:param start_time: Time when the game launcher starts.
|
||||
"""
|
||||
aws_metrics_utils.verify_cloud_watch_delivery(
|
||||
'AWS/Lambda',
|
||||
'Invocations',
|
||||
[{'Name': 'FunctionName',
|
||||
'Value': resource_mappings.get_resource_name_id('AWSMetrics.AnalyticsProcessingLambdaName')}],
|
||||
start_time)
|
||||
logger.info('AnalyticsProcessingLambda metrics are sent to CloudWatch.')
|
||||
|
||||
aws_metrics_utils.verify_cloud_watch_delivery(
|
||||
'AWS/Lambda',
|
||||
'Invocations',
|
||||
[{'Name': 'FunctionName',
|
||||
'Value': resource_mappings.get_resource_name_id('AWSMetrics.EventProcessingLambdaName')}],
|
||||
start_time)
|
||||
logger.info('EventsProcessingLambda metrics are sent to CloudWatch.')
|
||||
|
||||
|
||||
def update_kinesis_analytics_application_status(aws_metrics_utils: pytest.fixture,
|
||||
resource_mappings: pytest.fixture, start_application: bool) -> None:
|
||||
"""
|
||||
Update the Kinesis analytics application to start or stop it.
|
||||
:param aws_metrics_utils: aws_metrics_utils fixture.
|
||||
:param resource_mappings: resource_mappings fixture.
|
||||
:param start_application: whether to start or stop the application.
|
||||
"""
|
||||
if start_application:
|
||||
aws_metrics_utils.start_kinesis_data_analytics_application(
|
||||
resource_mappings.get_resource_name_id('AWSMetrics.AnalyticsApplicationName'))
|
||||
else:
|
||||
aws_metrics_utils.stop_kinesis_data_analytics_application(
|
||||
resource_mappings.get_resource_name_id('AWSMetrics.AnalyticsApplicationName'))
|
||||
|
||||
@pytest.mark.SUITE_awsi
|
||||
@pytest.mark.usefixtures('automatic_process_killer')
|
||||
@pytest.mark.usefixtures('aws_credentials')
|
||||
@pytest.mark.usefixtures('resource_mappings')
|
||||
@pytest.mark.parametrize('assume_role_arn', [constants.ASSUME_ROLE_ARN])
|
||||
@pytest.mark.parametrize('feature_name', [AWS_METRICS_FEATURE_NAME])
|
||||
@pytest.mark.parametrize('profile_name', ['AWSAutomationTest'])
|
||||
@pytest.mark.parametrize('project', ['AutomatedTesting'])
|
||||
@pytest.mark.parametrize('region_name', [constants.AWS_REGION])
|
||||
@pytest.mark.parametrize('resource_mappings_filename', [constants.AWS_RESOURCE_MAPPING_FILE_NAME])
|
||||
@pytest.mark.parametrize('session_name', [constants.SESSION_NAME])
|
||||
@pytest.mark.parametrize('stacks', [[f'{constants.AWS_PROJECT_NAME}-{AWS_METRICS_FEATURE_NAME}-{constants.AWS_REGION}']])
|
||||
class TestAWSMetricsWindows(object):
|
||||
"""
|
||||
Test class to verify the real-time and batch analytics for metrics.
|
||||
"""
|
||||
@pytest.mark.parametrize('level', ['levels/aws/metrics/metrics.spawnable'])
|
||||
def test_realtime_and_batch_analytics(self,
|
||||
level: str,
|
||||
launcher: pytest.fixture,
|
||||
asset_processor: pytest.fixture,
|
||||
workspace: pytest.fixture,
|
||||
aws_utils: pytest.fixture,
|
||||
resource_mappings: pytest.fixture,
|
||||
aws_metrics_utils: pytest.fixture):
|
||||
"""
|
||||
Verify that the metrics events are sent to CloudWatch and S3 for analytics.
|
||||
"""
|
||||
# Start Kinesis analytics application on a separate thread to avoid blocking the test.
|
||||
kinesis_analytics_application_thread = AWSMetricsThread(target=update_kinesis_analytics_application_status,
|
||||
args=(aws_metrics_utils, resource_mappings, True))
|
||||
kinesis_analytics_application_thread.start()
|
||||
|
||||
log_monitor = setup(launcher, asset_processor)
|
||||
|
||||
# Kinesis analytics application needs to be in the running state before we start the game launcher.
|
||||
kinesis_analytics_application_thread.join()
|
||||
launcher.args = ['+LoadLevel', level]
|
||||
launcher.args.extend(['-rhi=null'])
|
||||
start_time = datetime.utcnow()
|
||||
with launcher.start(launch_ap=False):
|
||||
monitor_metrics_submission(log_monitor)
|
||||
|
||||
# Verify that real-time analytics metrics are delivered to CloudWatch.
|
||||
aws_metrics_utils.verify_cloud_watch_delivery(
|
||||
AWS_METRICS_FEATURE_NAME,
|
||||
'TotalLogins',
|
||||
[],
|
||||
start_time)
|
||||
logger.info('Real-time metrics are sent to CloudWatch.')
|
||||
|
||||
# Run time-consuming operations on separate threads to avoid blocking the test.
|
||||
operational_threads = list()
|
||||
operational_threads.append(
|
||||
AWSMetricsThread(target=query_metrics_from_s3,
|
||||
args=(aws_metrics_utils, resource_mappings)))
|
||||
operational_threads.append(
|
||||
AWSMetricsThread(target=verify_operational_metrics,
|
||||
args=(aws_metrics_utils, resource_mappings, start_time)))
|
||||
operational_threads.append(
|
||||
AWSMetricsThread(target=update_kinesis_analytics_application_status,
|
||||
args=(aws_metrics_utils, resource_mappings, False)))
|
||||
for thread in operational_threads:
|
||||
thread.start()
|
||||
for thread in operational_threads:
|
||||
thread.join()
|
||||
|
||||
@pytest.mark.parametrize('level', ['levels/aws/metrics/metrics.spawnable'])
|
||||
def test_realtime_and_batch_analytics_no_global_accountid(self,
|
||||
level: str,
|
||||
launcher: pytest.fixture,
|
||||
asset_processor: pytest.fixture,
|
||||
workspace: pytest.fixture,
|
||||
aws_utils: pytest.fixture,
|
||||
resource_mappings: pytest.fixture,
|
||||
aws_metrics_utils: pytest.fixture):
|
||||
"""
|
||||
Verify that the metrics events are sent to CloudWatch and S3 for analytics.
|
||||
"""
|
||||
# Remove top-level account ID from resource mappings
|
||||
resource_mappings.clear_select_keys([AWS_RESOURCE_MAPPINGS_ACCOUNT_ID_KEY])
|
||||
# Start Kinesis analytics application on a separate thread to avoid blocking the test.
|
||||
kinesis_analytics_application_thread = AWSMetricsThread(target=update_kinesis_analytics_application_status,
|
||||
args=(aws_metrics_utils, resource_mappings, True))
|
||||
kinesis_analytics_application_thread.start()
|
||||
|
||||
log_monitor = setup(launcher, asset_processor)
|
||||
|
||||
# Kinesis analytics application needs to be in the running state before we start the game launcher.
|
||||
kinesis_analytics_application_thread.join()
|
||||
launcher.args = ['+LoadLevel', level]
|
||||
launcher.args.extend(['-rhi=null'])
|
||||
start_time = datetime.utcnow()
|
||||
with launcher.start(launch_ap=False):
|
||||
monitor_metrics_submission(log_monitor)
|
||||
|
||||
# Verify that real-time analytics metrics are delivered to CloudWatch.
|
||||
aws_metrics_utils.verify_cloud_watch_delivery(
|
||||
AWS_METRICS_FEATURE_NAME,
|
||||
'TotalLogins',
|
||||
[],
|
||||
start_time)
|
||||
logger.info('Real-time metrics are sent to CloudWatch.')
|
||||
|
||||
# Run time-consuming operations on separate threads to avoid blocking the test.
|
||||
operational_threads = list()
|
||||
operational_threads.append(
|
||||
AWSMetricsThread(target=query_metrics_from_s3,
|
||||
args=(aws_metrics_utils, resource_mappings)))
|
||||
operational_threads.append(
|
||||
AWSMetricsThread(target=verify_operational_metrics,
|
||||
args=(aws_metrics_utils, resource_mappings, start_time)))
|
||||
operational_threads.append(
|
||||
AWSMetricsThread(target=update_kinesis_analytics_application_status,
|
||||
args=(aws_metrics_utils, resource_mappings, False)))
|
||||
for thread in operational_threads:
|
||||
thread.start()
|
||||
for thread in operational_threads:
|
||||
thread.join()
|
||||
|
||||
@pytest.mark.parametrize('level', ['levels/aws/metrics/metrics.spawnable'])
|
||||
def test_unauthorized_user_request_rejected(self,
|
||||
level: str,
|
||||
launcher: pytest.fixture,
|
||||
asset_processor: pytest.fixture,
|
||||
workspace: pytest.fixture):
|
||||
"""
|
||||
Verify that unauthorized users cannot send metrics events to the AWS backed backend.
|
||||
"""
|
||||
log_monitor = setup(launcher, asset_processor)
|
||||
|
||||
# Set invalid AWS credentials.
|
||||
launcher.args = ['+LoadLevel', level, '+cl_awsAccessKey', 'AKIAIOSFODNN7EXAMPLE',
|
||||
'+cl_awsSecretKey', 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY']
|
||||
launcher.args.extend(['-rhi=null'])
|
||||
|
||||
with launcher.start(launch_ap=False):
|
||||
result = log_monitor.monitor_log_for_lines(
|
||||
expected_lines=['(Script) - Failed to send metrics.'],
|
||||
unexpected_lines=['(Script) - Metrics is sent successfully.'],
|
||||
halt_on_unexpected=True)
|
||||
assert result, 'Metrics events are sent successfully by unauthorized user'
|
||||
logger.info('Unauthorized user is rejected to send metrics.')
|
||||
|
||||
def test_clean_up_s3_bucket(self,
|
||||
aws_utils: pytest.fixture,
|
||||
resource_mappings: pytest.fixture,
|
||||
aws_metrics_utils: pytest.fixture):
|
||||
"""
|
||||
Clear the analytics bucket objects so that the S3 bucket can be destroyed during tear down.
|
||||
"""
|
||||
aws_metrics_utils.empty_bucket(
|
||||
resource_mappings.get_resource_name_id('AWSMetrics.AnalyticsBucketName'))
|
||||
+29
-29
@@ -1,29 +1,29 @@
|
||||
"""
|
||||
Copyright (c) Contributors to the Open 3D Engine Project.
|
||||
For complete copyright and license terms please see the LICENSE at the root of this distribution.
|
||||
|
||||
SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
"""
|
||||
|
||||
from threading import Thread
|
||||
|
||||
|
||||
class AWSMetricsThread(Thread):
|
||||
"""
|
||||
Custom thread for raising assertion errors on the main thread.
|
||||
"""
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self._error = None
|
||||
|
||||
def run(self) -> None:
|
||||
try:
|
||||
super().run()
|
||||
except AssertionError as e:
|
||||
self._error = e
|
||||
|
||||
def join(self, **kwargs) -> None:
|
||||
super().join(**kwargs)
|
||||
|
||||
if self._error:
|
||||
raise AssertionError(self._error)
|
||||
"""
|
||||
Copyright (c) Contributors to the Open 3D Engine Project.
|
||||
For complete copyright and license terms please see the LICENSE at the root of this distribution.
|
||||
|
||||
SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
"""
|
||||
|
||||
from threading import Thread
|
||||
|
||||
|
||||
class AWSMetricsThread(Thread):
|
||||
"""
|
||||
Custom thread for raising assertion errors on the main thread.
|
||||
"""
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self._error = None
|
||||
|
||||
def run(self) -> None:
|
||||
try:
|
||||
super().run()
|
||||
except AssertionError as e:
|
||||
self._error = e
|
||||
|
||||
def join(self, **kwargs) -> None:
|
||||
super().join(**kwargs)
|
||||
|
||||
if self._error:
|
||||
raise AssertionError(self._error)
|
||||
+239
-239
@@ -1,239 +1,239 @@
|
||||
"""
|
||||
Copyright (c) Contributors to the Open 3D Engine Project.
|
||||
For complete copyright and license terms please see the LICENSE at the root of this distribution.
|
||||
|
||||
SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
"""
|
||||
|
||||
import logging
|
||||
import pathlib
|
||||
import pytest
|
||||
import typing
|
||||
|
||||
from datetime import datetime
|
||||
from botocore.exceptions import WaiterError
|
||||
|
||||
from .aws_metrics_waiters import KinesisAnalyticsApplicationUpdatedWaiter, \
|
||||
CloudWatchMetricsDeliveredWaiter, DataLakeMetricsDeliveredWaiter, GlueCrawlerReadyWaiter
|
||||
|
||||
logging.getLogger('boto').setLevel(logging.CRITICAL)
|
||||
|
||||
# Expected directory and file extension for the S3 objects.
|
||||
EXPECTED_S3_DIRECTORY = 'firehose_events/'
|
||||
EXPECTED_S3_OBJECT_EXTENSION = '.parquet'
|
||||
|
||||
|
||||
class AWSMetricsUtils:
|
||||
"""
|
||||
Provide utils functions for the AWSMetrics gem to interact with the deployed resources.
|
||||
"""
|
||||
|
||||
def __init__(self, aws_utils: pytest.fixture):
|
||||
self._aws_util = aws_utils
|
||||
|
||||
def start_kinesis_data_analytics_application(self, application_name: str) -> None:
|
||||
"""
|
||||
Start the Kenisis Data Analytics application for real-time analytics.
|
||||
:param application_name: Name of the Kenisis Data Analytics application.
|
||||
"""
|
||||
input_id = self.get_kinesis_analytics_application_input_id(application_name)
|
||||
assert input_id, 'invalid Kinesis Data Analytics application input.'
|
||||
|
||||
client = self._aws_util.client('kinesisanalytics')
|
||||
try:
|
||||
client.start_application(
|
||||
ApplicationName=application_name,
|
||||
InputConfigurations=[
|
||||
{
|
||||
'Id': input_id,
|
||||
'InputStartingPositionConfiguration': {
|
||||
'InputStartingPosition': 'NOW'
|
||||
}
|
||||
},
|
||||
]
|
||||
)
|
||||
except client.exceptions.ResourceInUseException:
|
||||
# The application has been started.
|
||||
return
|
||||
|
||||
try:
|
||||
KinesisAnalyticsApplicationUpdatedWaiter(client, 'RUNNING').wait(application_name=application_name)
|
||||
except WaiterError as e:
|
||||
assert False, f'Failed to start the Kinesis Data Analytics application: {str(e)}.'
|
||||
|
||||
def get_kinesis_analytics_application_input_id(self, application_name: str) -> str:
|
||||
"""
|
||||
Get the input ID for the Kenisis Data Analytics application.
|
||||
:param application_name: Name of the Kenisis Data Analytics application.
|
||||
:return: Input ID for the Kenisis Data Analytics application.
|
||||
"""
|
||||
client = self._aws_util.client('kinesisanalytics')
|
||||
response = client.describe_application(
|
||||
ApplicationName=application_name
|
||||
)
|
||||
if not response:
|
||||
return ''
|
||||
input_descriptions = response.get('ApplicationDetail', {}).get('InputDescriptions', [])
|
||||
if len(input_descriptions) != 1:
|
||||
return ''
|
||||
|
||||
return input_descriptions[0].get('InputId', '')
|
||||
|
||||
def stop_kinesis_data_analytics_application(self, application_name: str) -> None:
|
||||
"""
|
||||
Stop the Kenisis Data Analytics application.
|
||||
:param application_name: Name of the Kenisis Data Analytics application.
|
||||
"""
|
||||
client = self._aws_util.client('kinesisanalytics')
|
||||
client.stop_application(
|
||||
ApplicationName=application_name
|
||||
)
|
||||
|
||||
try:
|
||||
KinesisAnalyticsApplicationUpdatedWaiter(client, 'READY').wait(application_name=application_name)
|
||||
except WaiterError as e:
|
||||
assert False, f'Failed to stop the Kinesis Data Analytics application: {str(e)}.'
|
||||
|
||||
def verify_cloud_watch_delivery(self, namespace: str, metrics_name: str,
|
||||
dimensions: typing.List[dict], start_time: datetime) -> None:
|
||||
"""
|
||||
Verify that the expected metrics is delivered to CloudWatch.
|
||||
:param namespace: Namespace of the metrics.
|
||||
:param metrics_name: Name of the metrics.
|
||||
:param dimensions: Dimensions of the metrics.
|
||||
:param start_time: Start time for generating the metrics.
|
||||
"""
|
||||
client = self._aws_util.client('cloudwatch')
|
||||
|
||||
try:
|
||||
CloudWatchMetricsDeliveredWaiter(client).wait(
|
||||
namespace=namespace,
|
||||
metrics_name=metrics_name,
|
||||
dimensions=dimensions,
|
||||
start_time=start_time
|
||||
)
|
||||
except WaiterError as e:
|
||||
assert False, f'Failed to deliver metrics to CloudWatch: {str(e)}.'
|
||||
|
||||
def verify_s3_delivery(self, analytics_bucket_name: str) -> None:
|
||||
"""
|
||||
Verify that metrics are delivered to S3 for batch analytics successfully.
|
||||
:param analytics_bucket_name: Name of the deployed S3 bucket.
|
||||
"""
|
||||
client = self._aws_util.client('s3')
|
||||
bucket_name = analytics_bucket_name
|
||||
|
||||
try:
|
||||
DataLakeMetricsDeliveredWaiter(client).wait(bucket_name=bucket_name, prefix=EXPECTED_S3_DIRECTORY)
|
||||
except WaiterError as e:
|
||||
assert False, f'Failed to find the S3 directory for storing metrics data: {str(e)}.'
|
||||
|
||||
# Check whether the data is converted to the expected data format.
|
||||
response = client.list_objects_v2(
|
||||
Bucket=bucket_name,
|
||||
Prefix=EXPECTED_S3_DIRECTORY
|
||||
)
|
||||
assert response.get('KeyCount', 0) != 0, f'Failed to deliver metrics to the S3 bucket {bucket_name}.'
|
||||
|
||||
s3_objects = response.get('Contents', [])
|
||||
for s3_object in s3_objects:
|
||||
key = s3_object.get('Key', '')
|
||||
assert pathlib.Path(key).suffix == EXPECTED_S3_OBJECT_EXTENSION, \
|
||||
f'Invalid data format is found in the S3 bucket {bucket_name}'
|
||||
|
||||
def run_glue_crawler(self, crawler_name: str) -> None:
|
||||
"""
|
||||
Run the Glue crawler and wait for it to finish.
|
||||
:param crawler_name: Name of the Glue crawler
|
||||
"""
|
||||
client = self._aws_util.client('glue')
|
||||
try:
|
||||
client.start_crawler(
|
||||
Name=crawler_name
|
||||
)
|
||||
except client.exceptions.CrawlerRunningException:
|
||||
# The crawler has already been started.
|
||||
return
|
||||
|
||||
try:
|
||||
GlueCrawlerReadyWaiter(client).wait(crawler_name=crawler_name)
|
||||
except WaiterError as e:
|
||||
assert False, f'Failed to run the Glue crawler: {str(e)}.'
|
||||
|
||||
def run_named_queries(self, work_group: str) -> None:
|
||||
"""
|
||||
Run the named queries under the specific Athena work group.
|
||||
:param work_group: Name of the Athena work group.
|
||||
"""
|
||||
client = self._aws_util.client('athena')
|
||||
# List all the named queries.
|
||||
response = client.list_named_queries(
|
||||
WorkGroup=work_group
|
||||
)
|
||||
named_query_ids = response.get('NamedQueryIds', [])
|
||||
|
||||
# Run each of the queries.
|
||||
for named_query_id in named_query_ids:
|
||||
get_named_query_response = client.get_named_query(
|
||||
NamedQueryId=named_query_id
|
||||
)
|
||||
named_query = get_named_query_response.get('NamedQuery', {})
|
||||
|
||||
start_query_execution_response = client.start_query_execution(
|
||||
QueryString=named_query.get('QueryString', ''),
|
||||
QueryExecutionContext={
|
||||
'Database': named_query.get('Database', '')
|
||||
},
|
||||
WorkGroup=work_group
|
||||
)
|
||||
|
||||
# Wait for the query to finish.
|
||||
state = 'RUNNING'
|
||||
while state == 'QUEUED' or state == 'RUNNING':
|
||||
get_query_execution_response = client.get_query_execution(
|
||||
QueryExecutionId=start_query_execution_response.get('QueryExecutionId', '')
|
||||
)
|
||||
|
||||
state = get_query_execution_response.get('QueryExecution', {}).get('Status', {}).get('State', '')
|
||||
|
||||
assert state == 'SUCCEEDED', f'Failed to run the named query {named_query.get("Name", {})}'
|
||||
|
||||
def empty_bucket(self, bucket_name: str) -> None:
|
||||
"""
|
||||
Empty the S3 bucket following:
|
||||
https://boto3.amazonaws.com/v1/documentation/api/latest/guide/migrations3.html
|
||||
|
||||
:param bucket_name: Name of the S3 bucket.
|
||||
"""
|
||||
s3 = self._aws_util.resource('s3')
|
||||
bucket = s3.Bucket(bucket_name)
|
||||
|
||||
for key in bucket.objects.all():
|
||||
key.delete()
|
||||
|
||||
def delete_table(self, database_name: str, table_name: str) -> None:
|
||||
"""
|
||||
Delete an existing Glue table.
|
||||
|
||||
:param database_name: Name of the Glue database.
|
||||
:param table_name: Name of the table to delete.
|
||||
"""
|
||||
client = self._aws_util.client('glue')
|
||||
client.delete_table(
|
||||
DatabaseName=database_name,
|
||||
Name=table_name
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope='function')
|
||||
def aws_metrics_utils(
|
||||
request: pytest.fixture,
|
||||
aws_utils: pytest.fixture):
|
||||
"""
|
||||
Fixture for the AWS metrics util functions.
|
||||
:param request: _pytest.fixtures.SubRequest class that handles getting
|
||||
a pytest fixture from a pytest function/fixture.
|
||||
:param aws_utils: aws_utils fixture.
|
||||
"""
|
||||
aws_utils_obj = AWSMetricsUtils(aws_utils)
|
||||
return aws_utils_obj
|
||||
"""
|
||||
Copyright (c) Contributors to the Open 3D Engine Project.
|
||||
For complete copyright and license terms please see the LICENSE at the root of this distribution.
|
||||
|
||||
SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
"""
|
||||
|
||||
import logging
|
||||
import pathlib
|
||||
import pytest
|
||||
import typing
|
||||
|
||||
from datetime import datetime
|
||||
from botocore.exceptions import WaiterError
|
||||
|
||||
from .aws_metrics_waiters import KinesisAnalyticsApplicationUpdatedWaiter, \
|
||||
CloudWatchMetricsDeliveredWaiter, DataLakeMetricsDeliveredWaiter, GlueCrawlerReadyWaiter
|
||||
|
||||
logging.getLogger('boto').setLevel(logging.CRITICAL)
|
||||
|
||||
# Expected directory and file extension for the S3 objects.
|
||||
EXPECTED_S3_DIRECTORY = 'firehose_events/'
|
||||
EXPECTED_S3_OBJECT_EXTENSION = '.parquet'
|
||||
|
||||
|
||||
class AWSMetricsUtils:
|
||||
"""
|
||||
Provide utils functions for the AWSMetrics gem to interact with the deployed resources.
|
||||
"""
|
||||
|
||||
def __init__(self, aws_utils: pytest.fixture):
|
||||
self._aws_util = aws_utils
|
||||
|
||||
def start_kinesis_data_analytics_application(self, application_name: str) -> None:
|
||||
"""
|
||||
Start the Kenisis Data Analytics application for real-time analytics.
|
||||
:param application_name: Name of the Kenisis Data Analytics application.
|
||||
"""
|
||||
input_id = self.get_kinesis_analytics_application_input_id(application_name)
|
||||
assert input_id, 'invalid Kinesis Data Analytics application input.'
|
||||
|
||||
client = self._aws_util.client('kinesisanalytics')
|
||||
try:
|
||||
client.start_application(
|
||||
ApplicationName=application_name,
|
||||
InputConfigurations=[
|
||||
{
|
||||
'Id': input_id,
|
||||
'InputStartingPositionConfiguration': {
|
||||
'InputStartingPosition': 'NOW'
|
||||
}
|
||||
},
|
||||
]
|
||||
)
|
||||
except client.exceptions.ResourceInUseException:
|
||||
# The application has been started.
|
||||
return
|
||||
|
||||
try:
|
||||
KinesisAnalyticsApplicationUpdatedWaiter(client, 'RUNNING').wait(application_name=application_name)
|
||||
except WaiterError as e:
|
||||
assert False, f'Failed to start the Kinesis Data Analytics application: {str(e)}.'
|
||||
|
||||
def get_kinesis_analytics_application_input_id(self, application_name: str) -> str:
|
||||
"""
|
||||
Get the input ID for the Kenisis Data Analytics application.
|
||||
:param application_name: Name of the Kenisis Data Analytics application.
|
||||
:return: Input ID for the Kenisis Data Analytics application.
|
||||
"""
|
||||
client = self._aws_util.client('kinesisanalytics')
|
||||
response = client.describe_application(
|
||||
ApplicationName=application_name
|
||||
)
|
||||
if not response:
|
||||
return ''
|
||||
input_descriptions = response.get('ApplicationDetail', {}).get('InputDescriptions', [])
|
||||
if len(input_descriptions) != 1:
|
||||
return ''
|
||||
|
||||
return input_descriptions[0].get('InputId', '')
|
||||
|
||||
def stop_kinesis_data_analytics_application(self, application_name: str) -> None:
|
||||
"""
|
||||
Stop the Kenisis Data Analytics application.
|
||||
:param application_name: Name of the Kenisis Data Analytics application.
|
||||
"""
|
||||
client = self._aws_util.client('kinesisanalytics')
|
||||
client.stop_application(
|
||||
ApplicationName=application_name
|
||||
)
|
||||
|
||||
try:
|
||||
KinesisAnalyticsApplicationUpdatedWaiter(client, 'READY').wait(application_name=application_name)
|
||||
except WaiterError as e:
|
||||
assert False, f'Failed to stop the Kinesis Data Analytics application: {str(e)}.'
|
||||
|
||||
def verify_cloud_watch_delivery(self, namespace: str, metrics_name: str,
|
||||
dimensions: typing.List[dict], start_time: datetime) -> None:
|
||||
"""
|
||||
Verify that the expected metrics is delivered to CloudWatch.
|
||||
:param namespace: Namespace of the metrics.
|
||||
:param metrics_name: Name of the metrics.
|
||||
:param dimensions: Dimensions of the metrics.
|
||||
:param start_time: Start time for generating the metrics.
|
||||
"""
|
||||
client = self._aws_util.client('cloudwatch')
|
||||
|
||||
try:
|
||||
CloudWatchMetricsDeliveredWaiter(client).wait(
|
||||
namespace=namespace,
|
||||
metrics_name=metrics_name,
|
||||
dimensions=dimensions,
|
||||
start_time=start_time
|
||||
)
|
||||
except WaiterError as e:
|
||||
assert False, f'Failed to deliver metrics to CloudWatch: {str(e)}.'
|
||||
|
||||
def verify_s3_delivery(self, analytics_bucket_name: str) -> None:
|
||||
"""
|
||||
Verify that metrics are delivered to S3 for batch analytics successfully.
|
||||
:param analytics_bucket_name: Name of the deployed S3 bucket.
|
||||
"""
|
||||
client = self._aws_util.client('s3')
|
||||
bucket_name = analytics_bucket_name
|
||||
|
||||
try:
|
||||
DataLakeMetricsDeliveredWaiter(client).wait(bucket_name=bucket_name, prefix=EXPECTED_S3_DIRECTORY)
|
||||
except WaiterError as e:
|
||||
assert False, f'Failed to find the S3 directory for storing metrics data: {str(e)}.'
|
||||
|
||||
# Check whether the data is converted to the expected data format.
|
||||
response = client.list_objects_v2(
|
||||
Bucket=bucket_name,
|
||||
Prefix=EXPECTED_S3_DIRECTORY
|
||||
)
|
||||
assert response.get('KeyCount', 0) != 0, f'Failed to deliver metrics to the S3 bucket {bucket_name}.'
|
||||
|
||||
s3_objects = response.get('Contents', [])
|
||||
for s3_object in s3_objects:
|
||||
key = s3_object.get('Key', '')
|
||||
assert pathlib.Path(key).suffix == EXPECTED_S3_OBJECT_EXTENSION, \
|
||||
f'Invalid data format is found in the S3 bucket {bucket_name}'
|
||||
|
||||
def run_glue_crawler(self, crawler_name: str) -> None:
|
||||
"""
|
||||
Run the Glue crawler and wait for it to finish.
|
||||
:param crawler_name: Name of the Glue crawler
|
||||
"""
|
||||
client = self._aws_util.client('glue')
|
||||
try:
|
||||
client.start_crawler(
|
||||
Name=crawler_name
|
||||
)
|
||||
except client.exceptions.CrawlerRunningException:
|
||||
# The crawler has already been started.
|
||||
return
|
||||
|
||||
try:
|
||||
GlueCrawlerReadyWaiter(client).wait(crawler_name=crawler_name)
|
||||
except WaiterError as e:
|
||||
assert False, f'Failed to run the Glue crawler: {str(e)}.'
|
||||
|
||||
def run_named_queries(self, work_group: str) -> None:
|
||||
"""
|
||||
Run the named queries under the specific Athena work group.
|
||||
:param work_group: Name of the Athena work group.
|
||||
"""
|
||||
client = self._aws_util.client('athena')
|
||||
# List all the named queries.
|
||||
response = client.list_named_queries(
|
||||
WorkGroup=work_group
|
||||
)
|
||||
named_query_ids = response.get('NamedQueryIds', [])
|
||||
|
||||
# Run each of the queries.
|
||||
for named_query_id in named_query_ids:
|
||||
get_named_query_response = client.get_named_query(
|
||||
NamedQueryId=named_query_id
|
||||
)
|
||||
named_query = get_named_query_response.get('NamedQuery', {})
|
||||
|
||||
start_query_execution_response = client.start_query_execution(
|
||||
QueryString=named_query.get('QueryString', ''),
|
||||
QueryExecutionContext={
|
||||
'Database': named_query.get('Database', '')
|
||||
},
|
||||
WorkGroup=work_group
|
||||
)
|
||||
|
||||
# Wait for the query to finish.
|
||||
state = 'RUNNING'
|
||||
while state == 'QUEUED' or state == 'RUNNING':
|
||||
get_query_execution_response = client.get_query_execution(
|
||||
QueryExecutionId=start_query_execution_response.get('QueryExecutionId', '')
|
||||
)
|
||||
|
||||
state = get_query_execution_response.get('QueryExecution', {}).get('Status', {}).get('State', '')
|
||||
|
||||
assert state == 'SUCCEEDED', f'Failed to run the named query {named_query.get("Name", {})}'
|
||||
|
||||
def empty_bucket(self, bucket_name: str) -> None:
|
||||
"""
|
||||
Empty the S3 bucket following:
|
||||
https://boto3.amazonaws.com/v1/documentation/api/latest/guide/migrations3.html
|
||||
|
||||
:param bucket_name: Name of the S3 bucket.
|
||||
"""
|
||||
s3 = self._aws_util.resource('s3')
|
||||
bucket = s3.Bucket(bucket_name)
|
||||
|
||||
for key in bucket.objects.all():
|
||||
key.delete()
|
||||
|
||||
def delete_table(self, database_name: str, table_name: str) -> None:
|
||||
"""
|
||||
Delete an existing Glue table.
|
||||
|
||||
:param database_name: Name of the Glue database.
|
||||
:param table_name: Name of the table to delete.
|
||||
"""
|
||||
client = self._aws_util.client('glue')
|
||||
client.delete_table(
|
||||
DatabaseName=database_name,
|
||||
Name=table_name
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope='function')
|
||||
def aws_metrics_utils(
|
||||
request: pytest.fixture,
|
||||
aws_utils: pytest.fixture):
|
||||
"""
|
||||
Fixture for the AWS metrics util functions.
|
||||
:param request: _pytest.fixtures.SubRequest class that handles getting
|
||||
a pytest fixture from a pytest function/fixture.
|
||||
:param aws_utils: aws_utils fixture.
|
||||
"""
|
||||
aws_utils_obj = AWSMetricsUtils(aws_utils)
|
||||
return aws_utils_obj
|
||||
+139
-139
@@ -1,139 +1,139 @@
|
||||
"""
|
||||
Copyright (c) Contributors to the Open 3D Engine Project.
|
||||
For complete copyright and license terms please see the LICENSE at the root of this distribution.
|
||||
|
||||
SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
"""
|
||||
|
||||
import botocore.client
|
||||
import logging
|
||||
|
||||
from datetime import timedelta
|
||||
from AWS.common.custom_waiter import CustomWaiter, WaitState
|
||||
|
||||
logging.getLogger('boto').setLevel(logging.CRITICAL)
|
||||
|
||||
|
||||
class KinesisAnalyticsApplicationUpdatedWaiter(CustomWaiter):
|
||||
"""
|
||||
Subclass of the base custom waiter class.
|
||||
Wait for the Kinesis analytics application being updated to a specific status.
|
||||
"""
|
||||
def __init__(self, client: botocore.client, status: str):
|
||||
"""
|
||||
Initialize the waiter.
|
||||
|
||||
:param client: Boto3 client to use.
|
||||
:param status: Expected status.
|
||||
"""
|
||||
super().__init__(
|
||||
'KinesisAnalyticsApplicationUpdated',
|
||||
'DescribeApplication',
|
||||
'ApplicationDetail.ApplicationStatus',
|
||||
{status: WaitState.SUCCESS},
|
||||
client)
|
||||
|
||||
def wait(self, application_name: str):
|
||||
"""
|
||||
Wait for the expected status.
|
||||
|
||||
:param application_name: Name of the Kinesis analytics application.
|
||||
"""
|
||||
self._wait(ApplicationName=application_name)
|
||||
|
||||
|
||||
class GlueCrawlerReadyWaiter(CustomWaiter):
|
||||
"""
|
||||
Subclass of the base custom waiter class.
|
||||
Wait for the Glue crawler to finish its processing. Return when the crawler is in the "Stopping" status
|
||||
to avoid wasting too much time in the automation tests on its shutdown process.
|
||||
"""
|
||||
def __init__(self, client: botocore.client):
|
||||
"""
|
||||
Initialize the waiter.
|
||||
|
||||
:param client: Boto3 client to use.
|
||||
"""
|
||||
super().__init__(
|
||||
'GlueCrawlerReady',
|
||||
'GetCrawler',
|
||||
'Crawler.State',
|
||||
{'STOPPING': WaitState.SUCCESS},
|
||||
client)
|
||||
|
||||
def wait(self, crawler_name):
|
||||
"""
|
||||
Wait for the expected status.
|
||||
|
||||
:param crawler_name: Name of the Glue crawler.
|
||||
"""
|
||||
self._wait(Name=crawler_name)
|
||||
|
||||
|
||||
class DataLakeMetricsDeliveredWaiter(CustomWaiter):
|
||||
"""
|
||||
Subclass of the base custom waiter class.
|
||||
Wait for the expected directory being created in the S3 bucket.
|
||||
"""
|
||||
def __init__(self, client: botocore.client):
|
||||
"""
|
||||
Initialize the waiter.
|
||||
|
||||
:param client: Boto3 client to use.
|
||||
"""
|
||||
super().__init__(
|
||||
'DataLakeMetricsDelivered',
|
||||
'ListObjectsV2',
|
||||
'KeyCount > `0`',
|
||||
{True: WaitState.SUCCESS},
|
||||
client)
|
||||
|
||||
def wait(self, bucket_name, prefix):
|
||||
"""
|
||||
Wait for the expected directory being created.
|
||||
|
||||
:param bucket_name: Name of the S3 bucket.
|
||||
:param prefix: Name of the expected directory prefix.
|
||||
"""
|
||||
self._wait(Bucket=bucket_name, Prefix=prefix)
|
||||
|
||||
|
||||
class CloudWatchMetricsDeliveredWaiter(CustomWaiter):
|
||||
"""
|
||||
Subclass of the base custom waiter class.
|
||||
Wait for the expected metrics being delivered to CloudWatch.
|
||||
"""
|
||||
def __init__(self, client: botocore.client):
|
||||
"""
|
||||
Initialize the waiter.
|
||||
|
||||
:param client: Boto3 client to use.
|
||||
"""
|
||||
super().__init__(
|
||||
'CloudWatchMetricsDelivered',
|
||||
'GetMetricStatistics',
|
||||
'length(Datapoints) > `0`',
|
||||
{True: WaitState.SUCCESS},
|
||||
client)
|
||||
|
||||
def wait(self, namespace, metrics_name, dimensions, start_time):
|
||||
"""
|
||||
Wait for the expected metrics being delivered.
|
||||
|
||||
:param namespace: Namespace of the metrics.
|
||||
:param metrics_name: Name of the metrics.
|
||||
:param dimensions: Dimensions of the metrics.
|
||||
:param start_time: Start time for generating the metrics.
|
||||
"""
|
||||
self._wait(
|
||||
Namespace=namespace,
|
||||
MetricName=metrics_name,
|
||||
Dimensions=dimensions,
|
||||
StartTime=start_time,
|
||||
EndTime=start_time + timedelta(0, self.timeout),
|
||||
Period=60,
|
||||
Statistics=[
|
||||
'SampleCount'
|
||||
],
|
||||
Unit='Count'
|
||||
)
|
||||
"""
|
||||
Copyright (c) Contributors to the Open 3D Engine Project.
|
||||
For complete copyright and license terms please see the LICENSE at the root of this distribution.
|
||||
|
||||
SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
"""
|
||||
|
||||
import botocore.client
|
||||
import logging
|
||||
|
||||
from datetime import timedelta
|
||||
from AWS.common.custom_waiter import CustomWaiter, WaitState
|
||||
|
||||
logging.getLogger('boto').setLevel(logging.CRITICAL)
|
||||
|
||||
|
||||
class KinesisAnalyticsApplicationUpdatedWaiter(CustomWaiter):
|
||||
"""
|
||||
Subclass of the base custom waiter class.
|
||||
Wait for the Kinesis analytics application being updated to a specific status.
|
||||
"""
|
||||
def __init__(self, client: botocore.client, status: str):
|
||||
"""
|
||||
Initialize the waiter.
|
||||
|
||||
:param client: Boto3 client to use.
|
||||
:param status: Expected status.
|
||||
"""
|
||||
super().__init__(
|
||||
'KinesisAnalyticsApplicationUpdated',
|
||||
'DescribeApplication',
|
||||
'ApplicationDetail.ApplicationStatus',
|
||||
{status: WaitState.SUCCESS},
|
||||
client)
|
||||
|
||||
def wait(self, application_name: str):
|
||||
"""
|
||||
Wait for the expected status.
|
||||
|
||||
:param application_name: Name of the Kinesis analytics application.
|
||||
"""
|
||||
self._wait(ApplicationName=application_name)
|
||||
|
||||
|
||||
class GlueCrawlerReadyWaiter(CustomWaiter):
|
||||
"""
|
||||
Subclass of the base custom waiter class.
|
||||
Wait for the Glue crawler to finish its processing. Return when the crawler is in the "Stopping" status
|
||||
to avoid wasting too much time in the automation tests on its shutdown process.
|
||||
"""
|
||||
def __init__(self, client: botocore.client):
|
||||
"""
|
||||
Initialize the waiter.
|
||||
|
||||
:param client: Boto3 client to use.
|
||||
"""
|
||||
super().__init__(
|
||||
'GlueCrawlerReady',
|
||||
'GetCrawler',
|
||||
'Crawler.State',
|
||||
{'STOPPING': WaitState.SUCCESS},
|
||||
client)
|
||||
|
||||
def wait(self, crawler_name):
|
||||
"""
|
||||
Wait for the expected status.
|
||||
|
||||
:param crawler_name: Name of the Glue crawler.
|
||||
"""
|
||||
self._wait(Name=crawler_name)
|
||||
|
||||
|
||||
class DataLakeMetricsDeliveredWaiter(CustomWaiter):
|
||||
"""
|
||||
Subclass of the base custom waiter class.
|
||||
Wait for the expected directory being created in the S3 bucket.
|
||||
"""
|
||||
def __init__(self, client: botocore.client):
|
||||
"""
|
||||
Initialize the waiter.
|
||||
|
||||
:param client: Boto3 client to use.
|
||||
"""
|
||||
super().__init__(
|
||||
'DataLakeMetricsDelivered',
|
||||
'ListObjectsV2',
|
||||
'KeyCount > `0`',
|
||||
{True: WaitState.SUCCESS},
|
||||
client)
|
||||
|
||||
def wait(self, bucket_name, prefix):
|
||||
"""
|
||||
Wait for the expected directory being created.
|
||||
|
||||
:param bucket_name: Name of the S3 bucket.
|
||||
:param prefix: Name of the expected directory prefix.
|
||||
"""
|
||||
self._wait(Bucket=bucket_name, Prefix=prefix)
|
||||
|
||||
|
||||
class CloudWatchMetricsDeliveredWaiter(CustomWaiter):
|
||||
"""
|
||||
Subclass of the base custom waiter class.
|
||||
Wait for the expected metrics being delivered to CloudWatch.
|
||||
"""
|
||||
def __init__(self, client: botocore.client):
|
||||
"""
|
||||
Initialize the waiter.
|
||||
|
||||
:param client: Boto3 client to use.
|
||||
"""
|
||||
super().__init__(
|
||||
'CloudWatchMetricsDelivered',
|
||||
'GetMetricStatistics',
|
||||
'length(Datapoints) > `0`',
|
||||
{True: WaitState.SUCCESS},
|
||||
client)
|
||||
|
||||
def wait(self, namespace, metrics_name, dimensions, start_time):
|
||||
"""
|
||||
Wait for the expected metrics being delivered.
|
||||
|
||||
:param namespace: Namespace of the metrics.
|
||||
:param metrics_name: Name of the metrics.
|
||||
:param dimensions: Dimensions of the metrics.
|
||||
:param start_time: Start time for generating the metrics.
|
||||
"""
|
||||
self._wait(
|
||||
Namespace=namespace,
|
||||
MetricName=metrics_name,
|
||||
Dimensions=dimensions,
|
||||
StartTime=start_time,
|
||||
EndTime=start_time + timedelta(0, self.timeout),
|
||||
Period=60,
|
||||
Statistics=[
|
||||
'SampleCount'
|
||||
],
|
||||
Unit='Count'
|
||||
)
|
||||
+4
-4
@@ -40,7 +40,7 @@ class TestAWSClientAuthWindows(object):
|
||||
Test class to verify AWS Client Auth gem features on Windows.
|
||||
"""
|
||||
|
||||
@pytest.mark.parametrize('level', ['AWS/ClientAuth'])
|
||||
@pytest.mark.parametrize('level', ['levels/aws/clientauth/clientauth.spawnable'])
|
||||
def test_anonymous_credentials(self,
|
||||
level: str,
|
||||
launcher: pytest.fixture,
|
||||
@@ -72,7 +72,7 @@ class TestAWSClientAuthWindows(object):
|
||||
)
|
||||
assert result, 'Anonymous credentials fetched successfully.'
|
||||
|
||||
@pytest.mark.parametrize('level', ['AWS/ClientAuth'])
|
||||
@pytest.mark.parametrize('level', ['levels/aws/clientauth/clientauth.spawnable'])
|
||||
def test_anonymous_credentials_no_global_accountid(self,
|
||||
level: str,
|
||||
launcher: pytest.fixture,
|
||||
@@ -140,7 +140,7 @@ class TestAWSClientAuthWindows(object):
|
||||
except cognito_idp.exceptions.UserNotFoundException:
|
||||
pass
|
||||
|
||||
launcher.args = ['+LoadLevel', 'AWS/ClientAuthPasswordSignUp']
|
||||
launcher.args = ['+LoadLevel', 'levels/aws/clientauthpasswordsignup/clientauthpasswordsignup.spawnable']
|
||||
launcher.args.extend(['-rhi=null'])
|
||||
|
||||
with launcher.start(launch_ap=False):
|
||||
@@ -158,7 +158,7 @@ class TestAWSClientAuthWindows(object):
|
||||
Username='test1'
|
||||
)
|
||||
|
||||
launcher.args = ['+LoadLevel', 'AWS/ClientAuthPasswordSignIn']
|
||||
launcher.args = ['+LoadLevel', 'levels/aws/clientauthpasswordsignin/clientauthpasswordsignin.spawnable']
|
||||
launcher.args.extend(['-rhi=null'])
|
||||
|
||||
with launcher.start(launch_ap=False):
|
||||
-1
@@ -4,4 +4,3 @@ For complete copyright and license terms please see the LICENSE at the root of t
|
||||
|
||||
SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
"""
|
||||
|
||||
+1
-1
@@ -84,7 +84,7 @@ def write_test_data_to_dynamodb_table(resource_mappings: pytest.fixture, aws_uti
|
||||
@pytest.mark.parametrize('session_name', [constants.SESSION_NAME])
|
||||
@pytest.mark.usefixtures('workspace')
|
||||
@pytest.mark.parametrize('project', ['AutomatedTesting'])
|
||||
@pytest.mark.parametrize('level', ['AWS/Core'])
|
||||
@pytest.mark.parametrize('level', ['levels/aws/core/core.spawnable'])
|
||||
@pytest.mark.usefixtures('resource_mappings')
|
||||
@pytest.mark.parametrize('resource_mappings_filename', [constants.AWS_RESOURCE_MAPPING_FILE_NAME])
|
||||
@pytest.mark.parametrize('stacks', [[f'{constants.AWS_PROJECT_NAME}-{AWS_CORE_FEATURE_NAME}',
|
||||
Reference in New Issue
Block a user