Integrating latest 47acbe8

This commit is contained in:
alexpete
2021-03-25 13:57:57 -07:00
parent 448c549698
commit 75dc720198
10312 changed files with 2711566 additions and 671451 deletions
@@ -0,0 +1,10 @@
"""
All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
its licensors.
For complete copyright and license terms please see the LICENSE at the root of this
distribution (the "License"). All use of this software is governed by the License,
or, if provided, by the license below or the license accompanying this file. Do not
remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
"""
@@ -0,0 +1,194 @@
"""
All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
its licensors.
For complete copyright and license terms please see the LICENSE at the root of this
distribution (the "License"). All use of this software is governed by the License,
or, if provided, by the license below or the license accompanying this file. Do not
remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
"""
import logging
from typing import (Dict, List)
from PySide2.QtCore import (QObject, Signal)
from manager.configuration_manager import ConfigurationManager
from manager.thread_manager import ThreadManager
from manager.view_manager import ViewManager
from model import (constants, error_messages)
from model.configuration import Configuration
from model.basic_resource_attributes import (BasicResourceAttributes, BasicResourceAttributesBuilder)
from model.resource_proxy_model import ResourceProxyModel
from multithread.worker import FunctionWorker
from utils import aws_utils
from view.import_resources_page import (ImportResourcesPage, ImportResourcesPageConstants, ResourceTreeView)
logger = logging.getLogger(__name__)
class ImportResourcesController(QObject):
add_import_resources = Signal(list)
"""
ImportResourcesController is the place to bind ImportResource view with its
corresponding behavior
TODO: add error handling once it is ready
"""
def __init__(self) -> None:
super(ImportResourcesController, self).__init__()
# Initialize manager references
self._configuration_manager: ConfigurationManager = ConfigurationManager.get_instance()
self._view_manager: ViewManager = ViewManager.get_instance()
# Initialize view and model related references
self._import_resources_page: ImportResourcesPage = self._view_manager.get_import_resources_page()
self._tree_view: ResourceTreeView = self._import_resources_page.tree_view
self._proxy_model: ResourceProxyModel = self._tree_view.resource_proxy_model
def _back_to_view_edit_page(self) -> None:
self._view_manager.switch_to_view_edit_page()
self.reset_page()
def _filter_based_on_search_text(self):
self._tree_view.clear_selection()
self._proxy_model.filter_text = self._import_resources_page.search_filter_input.text()
def _import_resources(self) -> None:
unique_resources: List[BasicResourceAttributes] = \
self._proxy_model.deduplicate_selected_import_resources(self._tree_view.selectedIndexes())
if unique_resources:
logger.debug(f"Importing selected resources: {unique_resources} ...")
self.add_import_resources.emit(unique_resources)
self._back_to_view_edit_page()
else:
self._import_resources_page.set_notification_frame_text(
error_messages.IMPORT_RESOURCES_PAGE_NO_RESOURCES_SELECTED_ERROR_MESSAGE)
def _start_search_resources_async(self) -> None:
configuration: Configuration = self._configuration_manager.configuration
async_worker: FunctionWorker
if self._import_resources_page.search_version == constants.SEARCH_TYPED_RESOURCES_VERSION:
async_worker = FunctionWorker(self._request_typed_resources_callback, configuration.region)
async_worker.signals.result.connect(self._load_typed_resources_callback)
elif self._import_resources_page.search_version == constants.SEARCH_CFN_STACKS_VERSION:
async_worker = FunctionWorker(self._request_cfn_resources_callback, configuration.region)
async_worker.signals.result.connect(self._load_cfn_resources_callback)
else:
self._import_resources_page.set_notification_frame_text(
error_messages.IMPORT_RESOURCES_PAGE_SEARCH_VERSION_ERROR_MESSAGE)
return
self._tree_view.reset_view()
self._import_resources_page.set_current_main_view_index(ImportResourcesPageConstants.NOTIFICATION_PAGE_INDEX)
async_worker.signals.finished.connect(self._search_complete_callback)
ThreadManager.get_instance().start(async_worker)
def _request_cfn_resources_callback(self, region: str) -> Dict[str, List[BasicResourceAttributes]]:
resources: Dict[str, List[BasicResourceAttributes]] = {}
logger.debug(f"Requesting CFN stacks with Region={region} ...")
try:
stack_names: List[str] = aws_utils.list_cloudformation_stacks(region)
stack_name: str
for stack_name in stack_names:
logger.debug(f"Requesting CFN stack resources with StackName={stack_name}, Region={region} ...")
resource_type_and_names: List[BasicResourceAttributes] = \
aws_utils.list_cloudformation_stack_resources(stack_name, region)
resources[stack_name] = resource_type_and_names
return resources
except RuntimeError as e:
self._import_resources_page.set_notification_frame_text(str(e))
def _request_typed_resources_callback(self, region: str) -> List[str]:
resource_type_index: int = self._import_resources_page.typed_resources_combobox.currentIndex()
resources: List[str] = []
logger.debug(f"Requesting resources with ResourceTypeIndex={resource_type_index}, Region={region} ...")
try:
if resource_type_index == constants.AWS_RESOURCE_LAMBDA_FUNCTION_INDEX:
resources = aws_utils.list_lambda_functions(region)
elif resource_type_index == constants.AWS_RESOURCE_DYNAMODB_TABLE_INDEX:
resources = aws_utils.list_dynamodb_tables(region)
elif resource_type_index == constants.AWS_RESOURCE_S3_BUCKET_INDEX:
resources = aws_utils.list_s3_buckets(region)
else:
self._import_resources_page.set_notification_frame_text(
error_messages.IMPORT_RESOURCES_PAGE_RESOURCE_TYPE_ERROR_MESSAGE)
return resources
except RuntimeError as e:
self._import_resources_page.set_notification_frame_text(str(e))
def _load_cfn_resources_callback(self, resources: Dict[str, List[BasicResourceAttributes]]) -> None:
if not resources:
logger.debug("No resource found")
return
configuration: Configuration = self._configuration_manager.configuration
stack_name: str
resource_type_and_names: List[BasicResourceAttributes]
for stack_name, resource_type_and_names in resources.items():
if not resource_type_and_names:
continue
# loading cloudformation stack resource data into model
logger.debug(f"Loading CFN stack {stack_name} into resource model ...")
stack_resource_attributes = BasicResourceAttributesBuilder() \
.build_type(constants.AWS_RESOURCE_CLOUDFORMATION_STACK_TYPE) \
.build_name_id(stack_name) \
.build_account_id(configuration.account_id) \
.build_region(configuration.region) \
.build()
self._proxy_model.load_resource(stack_resource_attributes)
# loading all resources data under cloudformation stack into model
resource_entry: BasicResourceAttributes
for resource_entry in resource_type_and_names:
logger.debug(f"Loading resource Type={resource_entry.type}, "
f"NameId={resource_entry.name_id} into resource model ...")
resource_entry.account_id = configuration.account_id
resource_entry.region = configuration.region
self._proxy_model.load_resource(resource_entry)
def _load_typed_resources_callback(self, resources: List[str]) -> None:
if not resources:
logger.debug("No resource found")
return
resource_type_index: int = self._import_resources_page.typed_resources_combobox.currentIndex()
configuration: Configuration = self._configuration_manager.configuration
resource_name_id: str
for resource_name_id in resources:
logger.debug(f"Converting resource {resource_name_id} into resource model ...")
import_resource_attributes: BasicResourceAttributes = BasicResourceAttributesBuilder() \
.build_type(constants.AWS_RESOURCE_TYPES[resource_type_index]) \
.build_name_id(resource_name_id) \
.build_region(configuration.region) \
.build_account_id(configuration.account_id) \
.build()
self._proxy_model.load_resource(import_resource_attributes)
def _search_complete_callback(self) -> None:
self._proxy_model.emit_source_model_layout_changed()
self._import_resources_page.set_current_main_view_index(ImportResourcesPageConstants.TREE_VIEW_PAGE_INDEX)
def reset_page(self):
"""Reset import resources page to its default state"""
self._tree_view.reset_view()
self._import_resources_page.hide_notification_frame()
self._import_resources_page.set_current_main_view_index(ImportResourcesPageConstants.TREE_VIEW_PAGE_INDEX)
self._import_resources_page.typed_resources_combobox.setCurrentIndex(-1)
self._import_resources_page.search_version = None
def setup(self):
"""Binding import resources page interactions with its corresponding behavior"""
self._import_resources_page.back_button.clicked.connect(self._back_to_view_edit_page)
self._import_resources_page.search_filter_input.returnPressed.connect(self._filter_based_on_search_text)
self._import_resources_page.typed_resources_search_button.clicked.connect(self._start_search_resources_async)
self._import_resources_page.typed_resources_import_button.clicked.connect(self._import_resources)
self._import_resources_page.cfn_stacks_search_button.clicked.connect(self._start_search_resources_async)
self._import_resources_page.cfn_stacks_import_button.clicked.connect(self._import_resources)
@@ -0,0 +1,289 @@
"""
All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
its licensors.
For complete copyright and license terms please see the LICENSE at the root of this
distribution (the "License"). All use of this software is governed by the License,
or, if provided, by the license below or the license accompanying this file. Do not
remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
"""
import logging
from PySide2.QtCore import (QCoreApplication, QModelIndex, QObject, Slot)
from PySide2.QtWidgets import QFileDialog
from typing import (Dict, List)
from manager.configuration_manager import ConfigurationManager
from manager.thread_manager import ThreadManager
from manager.view_manager import ViewManager
from model import (constants, error_messages, notification_label_text)
from model.configuration import Configuration
from model.basic_resource_attributes import BasicResourceAttributes
from model.resource_mapping_attributes import (ResourceMappingAttributes, ResourceMappingAttributesBuilder,
ResourceMappingAttributesStatus)
from model.resource_proxy_model import ResourceProxyModel
from multithread.worker import FunctionWorker
from utils import file_utils
from utils import json_utils
from view.view_edit_page import (ResourceTableView, ViewEditPage, ViewEditPageConstants)
logger = logging.getLogger(__name__)
class ViewEditController(QObject):
"""
ViewEditController is the place to bind ViewEdit view with its
corresponding behavior
"""
def __init__(self) -> None:
super(ViewEditController, self).__init__()
# Initialize in memory dict to store content reading from json file
self._config_file_json_source: Dict[str, any] = {}
# Initialize manager references
self._configuration_manager: ConfigurationManager = ConfigurationManager.get_instance()
self._view_manager: ViewManager = ViewManager.get_instance()
# Initialize view and model related references
self._view_edit_page: ViewEditPage = self._view_manager.get_view_edit_page()
self._table_view: ResourceTableView = self._view_edit_page.table_view
self._proxy_model: ResourceProxyModel = self._table_view.resource_proxy_model
def _add_table_row(self) -> None:
configuration: Configuration = self._configuration_manager.configuration
resource_builder: ResourceMappingAttributesBuilder = ResourceMappingAttributesBuilder() \
.build_account_id(configuration.account_id) \
.build_region(configuration.region) \
.build_status(
ResourceMappingAttributesStatus(ResourceMappingAttributesStatus.MODIFIED_STATUS_VALUE,
[ResourceMappingAttributesStatus.MODIFIED_STATUS_DESCRIPTION]))
self._proxy_model.add_resource(resource_builder.build())
def _cancel(self) -> None:
QCoreApplication.instance().quit()
def _convert_and_write_to_json(self, config_file_name: str) -> bool:
# convert model resources into json dict
json_dict: Dict[str, any] = \
json_utils.convert_resources_to_json_dict(self._proxy_model.get_resources(), self._config_file_json_source)
if json_dict == self._config_file_json_source:
# skip because no difference found against existing json file
return True
# try to write in memory json content into json file
configuration: Configuration = self._configuration_manager.configuration
try:
config_file_full_path: str = file_utils.join_path(configuration.config_directory, config_file_name)
json_utils.write_into_json_file(config_file_full_path, json_dict)
self._config_file_json_source = json_dict
return True
except IOError as e:
logger.exception(e)
self._view_edit_page.set_notification_frame_text(str(e))
return False
def _convert_and_load_into_model(self, config_file_name: str) -> None:
# convert json file into resources
configuration: Configuration = self._configuration_manager.configuration
resources: List[ResourceMappingAttributes] = []
try:
config_file_full_path: str = file_utils.join_path(configuration.config_directory, config_file_name)
self._config_file_json_source = json_utils.read_from_json_file(config_file_full_path)
json_utils.validate_json_dict_according_to_json_schema(self._config_file_json_source)
resources = json_utils.convert_json_dict_to_resources(self._config_file_json_source)
except (IOError, ValueError, KeyError) as e:
logger.exception(e)
self._view_edit_page.set_notification_frame_text(str(e))
self._view_edit_page.set_table_view_page_interactions_enabled(False)
# load resources into model
resource: ResourceMappingAttributes
for resource in resources:
self._proxy_model.load_resource(resource)
def _create_new_config_file(self) -> None:
configuration: Configuration = self._configuration_manager.configuration
try:
new_config_file_path: str = file_utils.join_path(
configuration.config_directory, constants.RESOURCE_MAPPING_DEFAULT_CONFIG_FILE_NAME)
json_utils.create_empty_resource_mapping_file(
new_config_file_path, configuration.account_id, configuration.region)
except IOError as e:
logger.exception(e)
self._view_edit_page.set_notification_frame_text(str(e))
return
self._rescan_config_directory()
def _delete_table_row(self) -> None:
indices: List[QModelIndex] = self._table_view.selectedIndexes()
self._proxy_model.remove_resources(indices)
def _filter_based_on_search_text(self):
self._table_view.clear_selection()
self._proxy_model.filter_text = self._view_edit_page.search_filter_input.text()
def _list_config_files_callback(self, config_files: List[str]) -> None:
if config_files:
self._configuration_manager.configuration.config_files = config_files
self._view_edit_page.set_config_files(config_files)
else:
self._configuration_manager.configuration.config_files = []
self._view_edit_page.set_config_files([])
def _open_file_dialog(self) -> None:
configuration: Configuration = self._configuration_manager.configuration
current_config_directory: str = configuration.config_directory
new_config_directory = \
QFileDialog.getExistingDirectory(None, "Config Location",
current_config_directory, QFileDialog.ShowDirsOnly)
if not new_config_directory:
return
if not current_config_directory == new_config_directory:
try:
configuration.config_directory = new_config_directory
self._start_search_config_files_async(new_config_directory)
except RuntimeError as e:
logger.exception(e)
self._view_edit_page.set_notification_frame_text(str(e))
def _rescan_config_directory(self) -> None:
configuration: Configuration = self._configuration_manager.configuration
config_files: List[str] = []
try:
config_files = file_utils.find_files_with_suffix_under_directory(
configuration.config_directory, constants.RESOURCE_MAPPING_CONFIG_FILE_NAME_SUFFIX)
except FileNotFoundError as e:
logger.exception(e)
self._view_edit_page.set_notification_frame_text(str(e))
return
self._configuration_manager.configuration.config_files = config_files
self._view_edit_page.set_config_files(config_files)
def _reset_page(self) -> None:
self._view_edit_page.hide_notification_frame()
self._view_edit_page.set_table_view_page_interactions_enabled(True)
def _save_changes(self) -> None:
if not self._validate_resources_and_post_notification():
# there is invalid resources, stop saving
return
config_file: str = self._view_edit_page.config_file_combobox.currentText()
if not self._convert_and_write_to_json(config_file):
# failed to convert/write into json file, stop saving
return
self._proxy_model.override_all_resources_status(
ResourceMappingAttributesStatus(ResourceMappingAttributesStatus.SUCCESS_STATUS_VALUE,
[ResourceMappingAttributesStatus.SUCCESS_STATUS_VALUE]))
self._view_edit_page.set_notification_frame_text(
notification_label_text.VIEW_EDIT_PAGE_SAVING_SUCCEED_MESSAGE.format(config_file))
def _search_complete_callback(self) -> None:
self._view_edit_page.set_notification_page_text(ViewEditPageConstants.NOTIFICATION_SELECT_CONFIG_FILE_TEXT)
self._reset_page()
def _start_search_config_files_async(self, config_directory: str) -> None:
self._view_edit_page.set_notification_page_text(ViewEditPageConstants.NOTIFICATION_LOADING_TEXT)
self._view_edit_page.set_current_main_view_index(ViewEditPageConstants.NOTIFICATION_PAGE_INDEX)
self._config_file_json_source.clear()
self._table_view.reset_view()
self._view_edit_page.set_config_location(config_directory)
async_worker: FunctionWorker = FunctionWorker(self._search_config_files_callback, config_directory)
async_worker.signals.result.connect(self._list_config_files_callback)
async_worker.signals.finished.connect(self._search_complete_callback)
ThreadManager.get_instance().start(async_worker)
def _search_config_files_callback(self, config_directory: str) -> List[str]:
try:
return file_utils.find_files_with_suffix_under_directory(
config_directory, constants.RESOURCE_MAPPING_CONFIG_FILE_NAME_SUFFIX)
except FileNotFoundError as e:
logger.exception(e)
self._view_edit_page.set_notification_frame_text(str(e))
def _select_config_file(self) -> None:
if self._view_edit_page.config_file_combobox.currentIndex() == -1:
return # return when combobox index is default -1
self._reset_page()
config_file_name: str = self._view_edit_page.config_file_combobox.currentText()
configuration: Configuration = self._configuration_manager.configuration
if config_file_name in configuration.config_files: # sanity check config file name
self._table_view.reset_view()
self._convert_and_load_into_model(config_file_name)
self._proxy_model.emit_source_model_layout_changed()
if not self._validate_resources_and_post_notification():
self._proxy_model.emit_source_model_layout_changed()
else:
self._view_edit_page.set_table_view_page_interactions_enabled(False)
self._view_edit_page.set_notification_frame_text(
error_messages.VIEW_EDIT_PAGE_READ_FROM_JSON_FAILED_WITH_UNEXPECTED_FILE_ERROR_MESSAGE.format(config_file_name))
self._view_edit_page.set_current_main_view_index(ViewEditPageConstants.TABLE_VIEW_PAGE_INDEX)
def _setup_page_interactions_behavior(self) -> None:
self._view_edit_page.config_file_combobox.currentIndexChanged.connect(self._select_config_file)
self._view_edit_page.config_location_button.clicked.connect(self._open_file_dialog)
self._view_edit_page.add_row_button.clicked.connect(self._add_table_row)
self._view_edit_page.delete_row_button.clicked.connect(self._delete_table_row)
self._view_edit_page.import_resources_combobox.currentIndexChanged.connect(self._switch_to_import_resources_page)
self._view_edit_page.save_changes_button.clicked.connect(self._save_changes)
self._view_edit_page.search_filter_input.returnPressed.connect(self._filter_based_on_search_text)
self._view_edit_page.cancel_button.clicked.connect(self._cancel)
self._view_edit_page.notification_prompt_create_new_button.clicked.connect(self._create_new_config_file)
self._view_edit_page.notification_prompt_rescan_button.clicked.connect(self._rescan_config_directory)
def _setup_page_start_state(self) -> None:
configuration: Configuration = self._configuration_manager.configuration
self._view_edit_page.set_notification_page_text(ViewEditPageConstants.NOTIFICATION_SELECT_CONFIG_FILE_TEXT)
self._view_edit_page.set_current_main_view_index(ViewEditPageConstants.NOTIFICATION_PAGE_INDEX)
self._view_edit_page.set_config_files(configuration.config_files)
self._view_edit_page.set_config_location(configuration.config_directory)
def _switch_to_import_resources_page(self) -> None:
if self._view_edit_page.import_resources_combobox.currentIndex() == -1:
return # return when combobox index is default -1
import_resources_search_version: str = self._view_edit_page.import_resources_combobox.currentText()
self._view_manager.switch_to_import_resources_page(import_resources_search_version)
self._view_edit_page.import_resources_combobox.setCurrentIndex(-1)
def _validate_resources_and_post_notification(self) -> bool:
invalid_sources: Dict[int, List[str]] = \
json_utils.validate_resources_according_to_json_schema(self._proxy_model.get_resources())
if invalid_sources:
invalid_row: int
invalid_details: List[str]
for invalid_row, invalid_details in invalid_sources.items():
self._proxy_model.override_resource_status(
invalid_row,
ResourceMappingAttributesStatus(ResourceMappingAttributesStatus.FAILURE_STATUS_VALUE,
invalid_details))
invalid_proxy_rows: List[int] = self._proxy_model.map_from_source_rows(list(invalid_sources.keys()))
self._view_edit_page.set_notification_frame_text(
error_messages.VIEW_EDIT_PAGE_SAVING_FAILED_WITH_INVALID_ROW_ERROR_MESSAGE.format(invalid_proxy_rows))
return False
return True
@Slot(list)
def add_import_resources(self, resources: List[BasicResourceAttributes]) -> None:
resource: BasicResourceAttributes
for resource in resources:
resource_builder: ResourceMappingAttributesBuilder = ResourceMappingAttributesBuilder() \
.build_type(resource.type) \
.build_name_id(resource.name_id) \
.build_account_id(resource.account_id) \
.build_region(resource.region) \
.build_status(
ResourceMappingAttributesStatus(ResourceMappingAttributesStatus.MODIFIED_STATUS_VALUE,
[ResourceMappingAttributesStatus.MODIFIED_STATUS_DESCRIPTION]))
self._proxy_model.add_resource(resource_builder.build())
def setup(self) -> None:
"""Setting view edit page starting state and bind interactions with its corresponding behavior"""
self._setup_page_start_state()
self._setup_page_interactions_behavior()