From 72b55392593e3be865361edead6cdffeab02fc4d Mon Sep 17 00:00:00 2001 From: amzn-phist <52085794+amzn-phist@users.noreply.github.com> Date: Tue, 7 Dec 2021 12:28:17 -0600 Subject: [PATCH] Add verbosity option to some o3de.py sub-commands (#6079) * Add verbosity option to some o3de.py sub-commands - Added the -v/-vv option to the most commonly used o3de sub-commands - Adds a custom argparse Action to make it easy to add verbosity option to other commands as needed. - Sets up parent-child loggers so that when verbosity is changed in a sub-command it will be applied globally to all logger instances. Also: - Fixes up many warnings, PEP8 improvements Signed-off-by: amzn-phist <52085794+amzn-phist@users.noreply.github.com> * Update the log format in o3de script modules - Use a bit nicer format than the default, which was harder to read. Signed-off-by: amzn-phist <52085794+amzn-phist@users.noreply.github.com> --- scripts/o3de.py | 32 +++++---- scripts/o3de/o3de/cmake.py | 6 +- scripts/o3de/o3de/disable_gem.py | 14 ++-- scripts/o3de/o3de/download.py | 6 +- scripts/o3de/o3de/enable_gem.py | 13 ++-- scripts/o3de/o3de/engine_properties.py | 5 +- scripts/o3de/o3de/engine_template.py | 57 ++++++++++------ scripts/o3de/o3de/gem_properties.py | 4 +- scripts/o3de/o3de/get_registration.py | 13 ++-- scripts/o3de/o3de/global_project.py | 7 +- scripts/o3de/o3de/manifest.py | 28 +++++--- scripts/o3de/o3de/print_registration.py | 6 +- scripts/o3de/o3de/project_properties.py | 14 ++-- scripts/o3de/o3de/register.py | 33 +++++----- scripts/o3de/o3de/repo.py | 4 +- scripts/o3de/o3de/sha256.py | 14 ++-- scripts/o3de/o3de/utils.py | 86 +++++++++++++++++++------ scripts/o3de/o3de/validation.py | 29 +++++---- 18 files changed, 236 insertions(+), 135 deletions(-) diff --git a/scripts/o3de.py b/scripts/o3de.py index 05c3b53ba6..bf3bd32a52 100755 --- a/scripts/o3de.py +++ b/scripts/o3de.py @@ -7,20 +7,25 @@ # import argparse +import logging import pathlib import sys +logger = logging.getLogger('o3de') -def add_args(parser, subparsers) -> None: + +def add_args(parser: argparse.ArgumentParser) -> None: """ add_args is called to add expected parser arguments and subparsers arguments to each command such that it can be invoked by o3de.py Ex o3de.py can invoke the register downloadable commands by importing register, call add_args and execute: python o3de.py register --gem-path "C:/TestGem" - :param parser: the caller instantiates a parser and passes it in here - :param subparsers: the caller instantiates subparsers and passes it in here + :param parser: the caller instantiates an ArgumentParser and passes it in here """ + subparsers = parser.add_subparsers(help='To get help on a sub-command:\no3de.py -h', + title='Sub-Commands') + # As o3de.py shares the same name as the o3de package attempting to use a regular # from o3de import line tries to import from the current o3de.py script and not the package # So the {current script directory} / 'o3de' is added to the front of the sys.path @@ -28,24 +33,25 @@ def add_args(parser, subparsers) -> None: o3de_package_dir = (script_dir / 'o3de').resolve() # add the scripts/o3de directory to the front of the sys.path sys.path.insert(0, str(o3de_package_dir)) - from o3de import engine_properties, engine_template, gem_properties, global_project, register, print_registration, get_registration, \ + from o3de import engine_properties, engine_template, gem_properties, \ + global_project, register, print_registration, get_registration, \ enable_gem, disable_gem, project_properties, sha256, download # Remove the temporarily added path sys.path = sys.path[1:] - # global_project + # global project global_project.add_args(subparsers) - # engine templaate + # engine template engine_template.add_args(subparsers) - # register + # registration register.add_args(subparsers) - # show + # show registration print_registration.add_args(subparsers) - # get-registered + # get registration get_registration.add_args(subparsers) # add a gem to a project @@ -74,11 +80,8 @@ if __name__ == "__main__": # parse the command line args the_parser = argparse.ArgumentParser() - # add subparsers - the_subparsers = the_parser.add_subparsers(help='sub-command help') - # add args to the parser - add_args(the_parser, the_subparsers) + add_args(the_parser) # parse args the_args = the_parser.parse_args() @@ -89,7 +92,8 @@ if __name__ == "__main__": sys.exit(1) # run - ret = the_args.func(the_args) + ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1 + logger.info('Success!' if ret == 0 else 'Completed with issues: result {}'.format(ret)) # return sys.exit(ret) diff --git a/scripts/o3de/o3de/cmake.py b/scripts/o3de/o3de/cmake.py index ffac496387..ec970d06f4 100644 --- a/scripts/o3de/o3de/cmake.py +++ b/scripts/o3de/o3de/cmake.py @@ -13,10 +13,10 @@ import logging import os import pathlib -from o3de import manifest +from o3de import manifest, utils -logger = logging.getLogger() -logging.basicConfig() +logger = logging.getLogger('o3de.cmake') +logging.basicConfig(format=utils.LOG_FORMAT) enable_gem_start_marker = 'set(ENABLED_GEMS' enable_gem_end_marker = ')' diff --git a/scripts/o3de/o3de/disable_gem.py b/scripts/o3de/o3de/disable_gem.py index 158507fca1..0b0e465d12 100644 --- a/scripts/o3de/o3de/disable_gem.py +++ b/scripts/o3de/o3de/disable_gem.py @@ -15,10 +15,10 @@ import os import pathlib import sys -from o3de import cmake, manifest +from o3de import cmake, manifest, utils -logger = logging.getLogger() -logging.basicConfig() +logger = logging.getLogger('o3de.disable_gem') +logging.basicConfig(format=utils.LOG_FORMAT) def disable_gem_in_project(gem_name: str = None, @@ -73,7 +73,6 @@ def disable_gem_in_project(gem_name: str = None, logger.error(f'Gem Path {gem_path} does not exist.') return 1 - # Read gem.json from the gem path gem_json_data = manifest.get_gem_json_data(gem_path=gem_path, project_path=project_path) if not gem_json_data: @@ -116,6 +115,10 @@ def add_parser_args(parser): Ex. Directly run from this file alone with: python disable_gem.py --project-path D:/Test --gem-name Atom :param parser: the caller passes an argparse parser like instance to this method """ + + # Sub-commands should declare their own verbosity flag, if desired + utils.add_verbosity_arg(parser) + group = parser.add_mutually_exclusive_group(required=True) group.add_argument('-pp', '--project-path', type=pathlib.Path, required=False, help='The path to the project.') @@ -155,8 +158,6 @@ def main(): # parse the command line args the_parser = argparse.ArgumentParser() - # add subparsers - # add args to the parser add_parser_args(the_parser) @@ -165,6 +166,7 @@ def main(): # run ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1 + logger.info('Success!' if ret == 0 else 'Completed with issues: result {}'.format(ret)) # return sys.exit(ret) diff --git a/scripts/o3de/o3de/download.py b/scripts/o3de/o3de/download.py index e88355bf06..8e28561e50 100644 --- a/scripts/o3de/o3de/download.py +++ b/scripts/o3de/o3de/download.py @@ -24,8 +24,9 @@ from datetime import datetime from o3de import manifest, repo, utils, validation, register -logger = logging.getLogger() -logging.basicConfig() +logger = logging.getLogger('o3de.download') +logging.basicConfig(format=utils.LOG_FORMAT) + def unzip_manifest_json_data(download_zip_path: pathlib.Path, zip_file_name: str) -> dict: json_data = {} @@ -38,6 +39,7 @@ def unzip_manifest_json_data(download_zip_path: pathlib.Path, zip_file_name: str return json_data + def validate_downloaded_zip_sha256(download_uri_json_data: dict, download_zip_path: pathlib.Path, manifest_json_name) -> int: # if the json has a sha256 check it against a sha256 of the zip diff --git a/scripts/o3de/o3de/enable_gem.py b/scripts/o3de/o3de/enable_gem.py index 1614dc4eda..fc4dc6e9cf 100644 --- a/scripts/o3de/o3de/enable_gem.py +++ b/scripts/o3de/o3de/enable_gem.py @@ -16,10 +16,10 @@ import os import pathlib import sys -from o3de import cmake, manifest, register, validation +from o3de import cmake, manifest, register, validation, utils -logger = logging.getLogger() -logging.basicConfig() +logger = logging.getLogger('o3de.enable_gem') +logging.basicConfig(format=utils.LOG_FORMAT) def enable_gem_in_project(gem_name: str = None, @@ -132,6 +132,10 @@ def add_parser_args(parser): Ex. Directly run from this file alone with: python enable_gem.py --project-path "D:/TestProject" --gem-path "D:/TestGem" :param parser: the caller passes an argparse parser like instance to this method """ + + # Sub-commands should declare their own verbosity flag, if desired + utils.add_verbosity_arg(parser) + group = parser.add_mutually_exclusive_group(required=True) group.add_argument('-pp', '--project-path', type=pathlib.Path, required=False, help='The path to the project.') @@ -171,8 +175,6 @@ def main(): # parse the command line args the_parser = argparse.ArgumentParser() - # add subparsers - # add args to the parser add_parser_args(the_parser) @@ -181,6 +183,7 @@ def main(): # run ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1 + logger.info('Success!' if ret == 0 else 'Completed with issues: result {}'.format(ret)) # return sys.exit(ret) diff --git a/scripts/o3de/o3de/engine_properties.py b/scripts/o3de/o3de/engine_properties.py index 92930dbb1c..3ba30e4bf8 100644 --- a/scripts/o3de/o3de/engine_properties.py +++ b/scripts/o3de/o3de/engine_properties.py @@ -15,8 +15,9 @@ import logging from o3de import manifest, utils -logger = logging.getLogger() -logging.basicConfig() +logger = logging.getLogger('o3de.engine_properties') +logging.basicConfig(format=utils.LOG_FORMAT) + def edit_engine_props(engine_path: pathlib.Path = None, engine_name: str = None, diff --git a/scripts/o3de/o3de/engine_template.py b/scripts/o3de/o3de/engine_template.py index e93f972301..369086c9c3 100755 --- a/scripts/o3de/o3de/engine_template.py +++ b/scripts/o3de/o3de/engine_template.py @@ -20,8 +20,8 @@ import re from o3de import manifest, register, validation, utils -logger = logging.getLogger() -logging.basicConfig() +logger = logging.getLogger('o3de.engine_template') +logging.basicConfig(format=utils.LOG_FORMAT) binary_file_ext = { '.pak', @@ -86,6 +86,7 @@ restricted_platforms = { template_file_name = 'template.json' this_script_parent = pathlib.Path(os.path.dirname(os.path.realpath(__file__))) + def _replace_license_text(source_data: str): while '{BEGIN_LICENSE}' in source_data: start = source_data.find('{BEGIN_LICENSE}') @@ -181,7 +182,7 @@ def _execute_template_json(json_data: dict, # regular copy if not templated for copy_file in json_data['copyFiles']: # construct the input file name - in_file = template_path / 'Template' /copy_file['file'] + in_file = template_path / 'Template' / copy_file['file'] # the file can be marked as optional, if it is and it does not exist skip if copy_file['isOptional'] and copy_file['isOptional'] == 'true': @@ -246,7 +247,7 @@ def _execute_restricted_template_json(json_data: dict, for copy_file in json_data['copyFiles']: # construct the input file name in_file = template_restricted_path / restricted_platform / template_restricted_platform_relative_path\ - / template_name / 'Template'/ copy_file['file'] + / template_name / 'Template' / copy_file['file'] # the file can be marked as optional, if it is and it does not exist skip if copy_file['isOptional'] and copy_file['isOptional'] == 'true': @@ -364,7 +365,7 @@ def create_template(source_path: pathlib.Path, keep_restricted_in_template: bool = False, keep_license_text: bool = False, replace: list = None, - force: bool = False) -> int: + force: bool = False) -> int: """ Create a template from a source directory using replacement @@ -1205,12 +1206,12 @@ def create_from_template(destination_path: pathlib.Path, # something is wrong with either the --template-restricted-platform-relative or the template is. if template_restricted_platform_relative_path != template_json_restricted_platform_relative_path: logger.error(f'The supplied --template-restricted-platform-relative-path' - f' "{template_restricted_platform_relative_path}" does not match the' - f' templates.json "restricted_platform_relative_path". Either' - f' --template-restricted-platform-relative-path is incorrect or the templates' - f' "restricted_platform_relative_path" is wrong. Note that since this template' - f' specifies "restricted_platform_relative_path" it need not be supplied and' - f' "{template_json_restricted_platform_relative_path}" will be used.') + f' "{template_restricted_platform_relative_path}" does not match the' + f' templates.json "restricted_platform_relative_path". Either' + f' --template-restricted-platform-relative-path is incorrect or the templates' + f' "restricted_platform_relative_path" is wrong. Note that since this template' + f' specifies "restricted_platform_relative_path" it need not be supplied and' + f' "{template_json_restricted_platform_relative_path}" will be used.') return 1 else: # The user has not supplied --template-restricted-platform-relative-path, try to read it from @@ -1306,7 +1307,7 @@ def create_from_template(destination_path: pathlib.Path, os.makedirs(destination_restricted_path, exist_ok=True) # read the restricted_name from the destination restricted.json - restricted_json = destination_restricted_path / restricted.json + restricted_json = destination_restricted_path / 'restricted.json' if not os.path.isfile(restricted_json): with open(restricted_json, 'w') as s: restricted_json_data = {} @@ -1956,7 +1957,6 @@ def create_gem(gem_path: pathlib.Path, replacements.append(("${NameLower}", gem_name.lower())) replacements.append(("${SanitizedCppName}", sanitized_cpp_name)) - # module id is a uuid with { and - if module_id: replacements.append(("${ModuleClassId}", module_id)) @@ -2157,8 +2157,13 @@ def add_args(subparsers) -> None: call add_args and execute: python o3de.py create-gem --gem-path TestGem :param subparsers: the caller instantiates subparsers and passes it in here """ + # turn a directory into a template create_template_subparser = subparsers.add_parser('create-template') + + # Sub-commands should declare their own verbosity flag, if desired + utils.add_verbosity_arg(create_template_subparser) + create_template_subparser.add_argument('-sp', '--source-path', type=pathlib.Path, required=True, help='The path to the source that you want to make into a template') create_template_subparser.add_argument('-tp', '--template-path', type=pathlib.Path, required=False, @@ -2233,6 +2238,10 @@ def add_args(subparsers) -> None: # create from template create_from_template_subparser = subparsers.add_parser('create-from-template') + + # Sub-commands should declare their own verbosity flag, if desired + utils.add_verbosity_arg(create_from_template_subparser) + create_from_template_subparser.add_argument('-dp', '--destination-path', type=pathlib.Path, required=True, help='The path to where you want the template instantiated,' ' can be absolute or relative to the current working directory.' @@ -2316,6 +2325,10 @@ def add_args(subparsers) -> None: # creation of a project from a template (like create from template but makes project assumptions) create_project_subparser = subparsers.add_parser('create-project') + + # Sub-commands should declare their own verbosity flag, if desired + utils.add_verbosity_arg(create_project_subparser) + create_project_subparser.add_argument('-pp', '--project-path', type=pathlib.Path, required=True, help='The location of the project you wish to create from the template,' ' can be an absolute path or relative to the current working directory.' @@ -2330,7 +2343,7 @@ def add_args(subparsers) -> None: group = create_project_subparser.add_mutually_exclusive_group(required=False) group.add_argument('-tp', '--template-path', type=pathlib.Path, required=False, default=None, - help='the path to the template you want to instance, can be absolute or' + help='The path to the template you want to instance, can be absolute or' ' relative to default templates path') group.add_argument('-tn', '--template-name', type=str, required=False, default=None, @@ -2340,7 +2353,7 @@ def add_args(subparsers) -> None: group = create_project_subparser.add_mutually_exclusive_group(required=False) group.add_argument('-prp', '--project-restricted-path', type=pathlib.Path, required=False, default=None, - help='path to the projects restricted folder, can be absolute or relative to' + help='The path to the projects restricted folder, can be absolute or relative to' ' the default restricted projects directory') group.add_argument('-prn', '--project-restricted-name', type=str, required=False, default=None, @@ -2351,7 +2364,7 @@ def add_args(subparsers) -> None: group.add_argument('-trp', '--template-restricted-path', type=pathlib.Path, required=False, default=None, help='The templates restricted path can be absolute or relative to' - 'the default restricted templates directory') + ' the default restricted templates directory') group.add_argument('-trn', '--template-restricted-name', type=str, required=False, default=None, help='The name of the registered templates restricted path. If supplied this will resolve' @@ -2413,6 +2426,10 @@ def add_args(subparsers) -> None: # creation of a gem from a template (like create from template but makes gem assumptions) create_gem_subparser = subparsers.add_parser('create-gem') + + # Sub-commands should declare their own verbosity flag, if desired + utils.add_verbosity_arg(create_gem_subparser) + create_gem_subparser.add_argument('-gp', '--gem-path', type=pathlib.Path, required=True, help='The gem path, can be absolute or relative to the current working directory') create_gem_subparser.add_argument('-gn', '--gem-name', type=str, @@ -2512,16 +2529,18 @@ if __name__ == "__main__": the_parser = argparse.ArgumentParser() # add subparsers - the_subparsers = the_parser.add_subparsers(help='sub-command help', dest='command', required=True) + subparsers = the_parser.add_subparsers(help='To get help on a sub-command:\nengine_template.py -h', + title='Sub-Commands', dest='command', required=True) - # add args to the parser - add_args(the_subparsers) + # add args to the parsers + add_args(subparsers) # parse args the_args = the_parser.parse_args() # run ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1 + logger.info('Success!' if ret == 0 else 'Completed with issues: result {}'.format(ret)) # return sys.exit(ret) diff --git a/scripts/o3de/o3de/gem_properties.py b/scripts/o3de/o3de/gem_properties.py index faaa4f86ec..423dc47115 100644 --- a/scripts/o3de/o3de/gem_properties.py +++ b/scripts/o3de/o3de/gem_properties.py @@ -15,8 +15,8 @@ import logging from o3de import manifest, utils -logger = logging.getLogger() -logging.basicConfig() +logger = logging.getLogger('o3de.gem_properties') +logging.basicConfig(format=utils.LOG_FORMAT) def update_values_in_key_list(existing_values: list, new_values: list or str, remove_values: list or str, diff --git a/scripts/o3de/o3de/get_registration.py b/scripts/o3de/o3de/get_registration.py index 8b8d53a8b0..675d1d009e 100644 --- a/scripts/o3de/o3de/get_registration.py +++ b/scripts/o3de/o3de/get_registration.py @@ -12,17 +12,18 @@ import sys from o3de import manifest + def _run_get_registered(args: argparse) -> str or pathlib.Path: if args.override_home_folder: manifest.override_home_folder = args.override_home_folder return manifest.get_registered(args.engine_name, - args.project_name, - args.gem_name, - args.template_name, - args.default_folder, - args.repo_name, - args.restricted_name) + args.project_name, + args.gem_name, + args.template_name, + args.default_folder, + args.repo_name, + args.restricted_name) def add_parser_args(parser): diff --git a/scripts/o3de/o3de/global_project.py b/scripts/o3de/o3de/global_project.py index 9463306912..7ff90bead4 100644 --- a/scripts/o3de/o3de/global_project.py +++ b/scripts/o3de/o3de/global_project.py @@ -14,14 +14,15 @@ import re import pathlib import json -from o3de import manifest, validation +from o3de import manifest, validation, utils -logger = logging.getLogger() -logging.basicConfig() +logger = logging.getLogger('o3de.global_project') +logging.basicConfig(format=utils.LOG_FORMAT) DEFAULT_BOOTSTRAP_SETREG = pathlib.Path('~/.o3de/Registry/bootstrap.setreg').expanduser() PROJECT_PATH_KEY = ('Amazon', 'AzCore', 'Bootstrap', 'project_path') + def get_json_data(input_path: pathlib.Path): setreg_json_data = {} # If the output_path exist validate that it is a valid json file diff --git a/scripts/o3de/o3de/manifest.py b/scripts/o3de/o3de/manifest.py index e46b819a7c..ab48915fe4 100644 --- a/scripts/o3de/o3de/manifest.py +++ b/scripts/o3de/o3de/manifest.py @@ -18,8 +18,8 @@ import hashlib from o3de import validation, utils -logger = logging.getLogger() -logging.basicConfig() +logger = logging.getLogger('o3de.manifest') +logging.basicConfig(format=utils.LOG_FORMAT) # Directory methods override_home_folder = None @@ -41,6 +41,7 @@ def get_o3de_folder() -> pathlib.Path: o3de_folder.mkdir(parents=True, exist_ok=True) return o3de_folder + def get_o3de_user_folder() -> pathlib.Path: o3de_user_folder = get_home_folder() / 'O3DE' o3de_user_folder.mkdir(parents=True, exist_ok=True) @@ -177,6 +178,7 @@ def get_default_o3de_manifest_json_data() -> dict: return json_data + def get_o3de_manifest() -> pathlib.Path: manifest_path = get_o3de_folder() / 'o3de_manifest.json' if not manifest_path.is_file(): @@ -228,11 +230,11 @@ def save_o3de_manifest(json_data: dict, manifest_path: pathlib.Path = None) -> b def get_gems_from_subdirectories(external_subdirs: list) -> list: - ''' + """ Helper Method for scanning a set of external subdirectories for gem.json files - ''' + """ def is_gem_subdirectory(subdir_files): - for name in files: + for name in subdir_files: if name == 'gem.json': return True return False @@ -285,13 +287,14 @@ def get_repos() -> list: json_data = load_o3de_manifest() return json_data['repos'] if 'repos' in json_data else [] + # engine.json queries def get_engine_projects() -> list: engine_path = get_this_engine_path() engine_object = get_engine_json_data(engine_path=engine_path) if engine_object: return list(map(lambda rel_path: (pathlib.Path(engine_path) / rel_path).as_posix(), - engine_object['projects'])) if 'projects' in engine_object else [] + engine_object['projects'])) if 'projects' in engine_object else [] return [] @@ -313,7 +316,7 @@ def get_engine_templates() -> list: engine_object = get_engine_json_data(engine_path=engine_path) if engine_object: return list(map(lambda rel_path: (pathlib.Path(engine_path) / rel_path).as_posix(), - engine_object['templates'])) if 'templates' in engine_object else [] + engine_object['templates'])) if 'templates' in engine_object else [] return [] @@ -335,7 +338,7 @@ def get_project_external_subdirectories(project_path: pathlib.Path) -> list: project_object = get_project_json_data(project_path=project_path) if project_object: return list(map(lambda rel_path: (pathlib.Path(project_path) / rel_path).as_posix(), - project_object['external_subdirectories'])) if 'external_subdirectories' in project_object else [] + project_object['external_subdirectories'])) if 'external_subdirectories' in project_object else [] return [] @@ -343,7 +346,7 @@ def get_project_templates(project_path: pathlib.Path) -> list: project_object = get_project_json_data(project_path=project_path) if project_object: return list(map(lambda rel_path: (pathlib.Path(project_path) / rel_path).as_posix(), - project_object['templates'])) if 'templates' in project_object else [] + project_object['templates'])) if 'templates' in project_object else [] return [] @@ -433,6 +436,7 @@ def get_templates_for_generic_creation(): # temporary until we have a better wa return list(filter(filter_project_and_gem_templates_out, get_all_templates())) + def get_json_file_path(object_typename: str, object_path: str or pathlib.Path) -> pathlib.Path: if not object_typename or not object_path: @@ -468,6 +472,7 @@ def get_json_data_file(object_json: pathlib.Path, return None + def get_json_data(object_typename: str, object_path: str or pathlib.Path, object_validator: callable) -> dict or None: @@ -538,6 +543,7 @@ def get_restricted_json_data(restricted_name: str = None, restricted_path: str o return get_json_data('restricted', restricted_path, validation.valid_o3de_restricted_json) + def get_repo_json_data(repo_uri: str) -> dict or None: if not repo_uri: logger.error('Must specify a Repo Uri.') @@ -547,7 +553,8 @@ def get_repo_json_data(repo_uri: str) -> dict or None: return get_json_data_file(repo_json, "Repo", validation.valid_o3de_repo_json) -def get_repo_path(repo_uri: str, cache_folder: str = None) -> pathlib.Path: + +def get_repo_path(repo_uri: str, cache_folder: str or pathlib.Path = None) -> pathlib.Path: if not cache_folder: cache_folder = get_o3de_cache_folder() @@ -555,6 +562,7 @@ def get_repo_path(repo_uri: str, cache_folder: str = None) -> pathlib.Path: repo_sha256 = hashlib.sha256(repo_manifest.encode()) return cache_folder / str(repo_sha256.hexdigest() + '.json') + def get_registered(engine_name: str = None, project_name: str = None, gem_name: str = None, diff --git a/scripts/o3de/o3de/print_registration.py b/scripts/o3de/o3de/print_registration.py index 4b4c9d6515..f81decb6c3 100644 --- a/scripts/o3de/o3de/print_registration.py +++ b/scripts/o3de/o3de/print_registration.py @@ -14,10 +14,10 @@ import pathlib import sys import urllib.parse -from o3de import manifest, validation +from o3de import manifest, validation, utils -logger = logging.getLogger() -logging.basicConfig() +logger = logging.getLogger('o3de.print_registration') +logging.basicConfig(format=utils.LOG_FORMAT) def get_project_path(project_path: pathlib.Path, project_name: str) -> pathlib.Path: diff --git a/scripts/o3de/o3de/project_properties.py b/scripts/o3de/o3de/project_properties.py index 04731ad3de..bbddd5adab 100644 --- a/scripts/o3de/o3de/project_properties.py +++ b/scripts/o3de/o3de/project_properties.py @@ -15,8 +15,9 @@ import logging from o3de import manifest, utils -logger = logging.getLogger() -logging.basicConfig() +logger = logging.getLogger('o3de.project_properties') +logging.basicConfig(format=utils.LOG_FORMAT) + def get_project_props(name: str = None, path: pathlib.Path = None) -> dict: proj_json = manifest.get_project_json_data(project_name=name, project_path=path) @@ -26,6 +27,7 @@ def get_project_props(name: str = None, path: pathlib.Path = None) -> dict: return None return proj_json + def edit_project_props(proj_path: pathlib.Path = None, proj_name: str = None, new_name: str = None, @@ -71,9 +73,9 @@ def edit_project_props(proj_path: pathlib.Path = None, tag_list = replace_tags.split() if isinstance(replace_tags, str) else replace_tags proj_json['user_tags'] = tag_list - return 0 if manifest.save_o3de_manifest(proj_json, pathlib.Path(proj_path) / 'project.json') else 1 + def _edit_project_props(args: argparse) -> int: return edit_project_props(args.project_path, args.project_name, @@ -86,6 +88,7 @@ def _edit_project_props(args: argparse) -> int: args.delete_tags, args.replace_tags) + def add_parser_args(parser): group = parser.add_mutually_exclusive_group(required=True) group.add_argument('-pp', '--project-path', type=pathlib.Path, required=False, @@ -112,10 +115,12 @@ def add_parser_args(parser): help='Replace entirety of user_tags property with space delimited list of values') parser.set_defaults(func=_edit_project_props) + def add_args(subparsers) -> None: enable_project_props_subparser = subparsers.add_parser('edit-project-properties') add_parser_args(enable_project_props_subparser) - + + def main(): the_parser = argparse.ArgumentParser() add_parser_args(the_parser) @@ -123,5 +128,6 @@ def main(): ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1 sys.exit(ret) + if __name__ == "__main__": main() diff --git a/scripts/o3de/o3de/register.py b/scripts/o3de/o3de/register.py index 62a342f948..7625ee8ba9 100644 --- a/scripts/o3de/o3de/register.py +++ b/scripts/o3de/o3de/register.py @@ -23,8 +23,8 @@ import urllib.request from o3de import get_registration, manifest, repo, utils, validation -logger = logging.getLogger() -logging.basicConfig() +logger = logging.getLogger('o3de.register') +logging.basicConfig(format=utils.LOG_FORMAT) def register_shipped_engine_o3de_objects(force: bool = False) -> int: @@ -306,9 +306,9 @@ def register_o3de_object_path(json_data: dict, try: paths_to_remove.append(o3de_object_path.relative_to(save_path.parent)) except ValueError: - pass # It is not an error if a relative path cannot be formed + pass # It is not an error if a relative path cannot be formed manifest_data[o3de_object_key] = list(filter(lambda p: pathlib.Path(p) not in paths_to_remove, - manifest_data.setdefault(o3de_object_key, []))) + manifest_data.setdefault(o3de_object_key, []))) if remove: if save_path: @@ -732,14 +732,14 @@ def register(engine_path: pathlib.Path = None, logger.error(f'Gem path cannot be empty.') return 1 result = result or register_gem_path(json_data, gem_path, remove, - external_subdir_engine_path, external_subdir_project_path) + external_subdir_engine_path, external_subdir_project_path) if isinstance(external_subdir_path, pathlib.PurePath): if not external_subdir_path: logger.error(f'External Subdirectory path is None.') return 1 result = result or register_external_subdirectory(json_data, external_subdir_path, remove, - external_subdir_engine_path, external_subdir_project_path) + external_subdir_engine_path, external_subdir_project_path) if isinstance(template_path, pathlib.PurePath): if not template_path: @@ -841,6 +841,10 @@ def add_parser_args(parser): Ex. Directly run from this file alone with: python register.py --engine-path "C:/o3de" :param parser: the caller passes an argparse parser like instance to this method """ + + # Sub-commands should declare their own verbosity flag, if desired + utils.add_verbosity_arg(parser) + group = parser.add_mutually_exclusive_group(required=True) group.add_argument('--this-engine', action='store_true', required=False, default=False, @@ -888,18 +892,18 @@ def add_parser_args(parser): help='Refresh the repo cache.') parser.add_argument('-ohf', '--override-home-folder', type=pathlib.Path, required=False, - help='By default the home folder is the user folder, override it to this folder.') + help='By default the home folder is the user folder, override it to this folder.') parser.add_argument('-r', '--remove', action='store_true', required=False, - default=False, - help='Remove entry.') + default=False, + help='Remove entry.') parser.add_argument('-f', '--force', action='store_true', default=False, - help='For the update of the registration field being modified.') + help='For the update of the registration field being modified.') - external_subdir_group = parser.add_argument_group(title='external-subdirectory', - description='path arguments to use with the --external-subdirectory option') + external_subdir_group = parser.add_argument_group(title='external-subdirectory', + description='path arguments to use with the --external-subdirectory option') external_subdir_path_group = external_subdir_group.add_mutually_exclusive_group() external_subdir_path_group.add_argument('-esep', '--external-subdirectory-engine-path', type=pathlib.Path, - help='If supplied, registers the external subdirectory with the engine.json at' \ + help='If supplied, registers the external subdirectory with the engine.json at' \ ' the engine-path location') external_subdir_path_group.add_argument('-espp', '--external-subdirectory-project-path', type=pathlib.Path) parser.set_defaults(func=_run_register) @@ -924,8 +928,6 @@ def main(): # parse the command line args the_parser = argparse.ArgumentParser() - # add subparsers - # add args to the parser add_parser_args(the_parser) @@ -934,6 +936,7 @@ def main(): # run ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1 + logger.info('Success!' if ret == 0 else 'Completed with issues: result {}'.format(ret)) # return sys.exit(ret) diff --git a/scripts/o3de/o3de/repo.py b/scripts/o3de/o3de/repo.py index 22c7c54c8f..41c2d235d9 100644 --- a/scripts/o3de/o3de/repo.py +++ b/scripts/o3de/o3de/repo.py @@ -15,8 +15,8 @@ import hashlib from datetime import datetime from o3de import manifest, utils, validation -logger = logging.getLogger() -logging.basicConfig() +logger = logging.getLogger('o3de.repo') +logging.basicConfig(format=utils.LOG_FORMAT) def process_add_o3de_repo(file_name: str or pathlib.Path, diff --git a/scripts/o3de/o3de/sha256.py b/scripts/o3de/o3de/sha256.py index b8f8e522d4..c2eaae073f 100644 --- a/scripts/o3de/o3de/sha256.py +++ b/scripts/o3de/o3de/sha256.py @@ -15,8 +15,8 @@ import sys from o3de import utils -logger = logging.getLogger() -logging.basicConfig() +logger = logging.getLogger('o3de.sha256') +logging.basicConfig(format=utils.LOG_FORMAT) def sha256(file_path: str or pathlib.Path, @@ -35,7 +35,7 @@ def sha256(file_path: str or pathlib.Path, logger.error(f'Json path {json_path} does not exist.') return 1 - sha256 = hashlib.sha256(file_path.open('rb').read()).hexdigest() + the_sha256 = hashlib.sha256(file_path.open('rb').read()).hexdigest() if json_path: with json_path.open('r') as s: @@ -44,7 +44,7 @@ def sha256(file_path: str or pathlib.Path, except json.JSONDecodeError as e: logger.error(f'Failed to read Json path {json_path}: {str(e)}') return 1 - json_data.update({"sha256": sha256}) + json_data.update({"sha256": the_sha256}) utils.backup_file(json_path) with json_path.open('w') as s: try: @@ -53,7 +53,7 @@ def sha256(file_path: str or pathlib.Path, logger.error(f'Failed to write Json path {json_path}: {str(e)}') return 1 else: - print(sha256) + print(the_sha256) return 0 @@ -70,9 +70,9 @@ def add_parser_args(parser): :param parser: the caller passes an argparse parser like instance to this method """ parser.add_argument('-f', '--file-path', type=str, required=True, - help='The path to the file you want to sha256.') + help='The path to the file you want to sha256.') parser.add_argument('-j', '--json-path', type=str, required=False, - help='optional path to an o3de json file to add the "sha256" element to.') + help='Optional path to an o3de json file to add the "sha256" element to.') parser.set_defaults(func=_run_sha256) diff --git a/scripts/o3de/o3de/utils.py b/scripts/o3de/o3de/utils.py index 6f1dd8b2c5..6628e6c946 100644 --- a/scripts/o3de/o3de/utils.py +++ b/scripts/o3de/o3de/utils.py @@ -8,6 +8,7 @@ """ This file contains utility functions """ +import argparse import sys import uuid import os @@ -17,11 +18,54 @@ import urllib.request import logging import zipfile -logger = logging.getLogger() -logging.basicConfig() +LOG_FORMAT = '[%(levelname)s] %(name)s: %(message)s' + +logger = logging.getLogger('o3de.utils') +logging.basicConfig(format=LOG_FORMAT) COPY_BUFSIZE = 64 * 1024 + +class VerbosityAction(argparse.Action): + def __init__(self, + option_strings, + dest, + default=None, + required=False, + help=None): + super().__init__( + option_strings=option_strings, + dest=dest, + nargs=0, + default=default, + required=required, + help=help, + ) + + def __call__(self, parser, namespace, values, option_string=None): + count = getattr(namespace, self.dest, None) + if count is None: + count = self.default + count += 1 + setattr(namespace, self.dest, count) + # get the parent logger instance + log = logging.getLogger('o3de') + if count >= 2: + log.setLevel(logging.DEBUG) + elif count == 1: + log.setLevel(logging.INFO) + + +def add_verbosity_arg(parser: argparse.ArgumentParser) -> None: + """ + Add a consistent/common verbosity option to an arg parser + :param parser: The ArgumentParser to modify + :return: None + """ + parser.add_argument('-v', dest='verbosity', action=VerbosityAction, default=0, + help='Additional logging verbosity, can be -v or -vv') + + def copyfileobj(fsrc, fdst, callback, length=0): # This is functionally the same as the python shutil copyfileobj but # allows for a callback to return the download progress in blocks and allows @@ -43,10 +87,11 @@ def copyfileobj(fsrc, fdst, callback, length=0): return 1 return 0 + def validate_identifier(identifier: str) -> bool: """ - Determine if the identifier supplied is valid. - :param identifier: the name which needs to to checked + Determine if the identifier supplied is valid + :param identifier: the name which needs to be checked :return: bool: if the identifier is valid or not """ if not identifier: @@ -65,7 +110,7 @@ def validate_identifier(identifier: str) -> bool: def sanitize_identifier_for_cpp(identifier: str) -> str: """ Convert the provided identifier to a valid C++ identifier - :param identifier: the name which needs to to sanitized + :param identifier: the name which needs to be sanitized :return: str: sanitized identifier """ if not identifier: @@ -81,8 +126,8 @@ def sanitize_identifier_for_cpp(identifier: str) -> str: def validate_uuid4(uuid_string: str) -> bool: """ - Determine if the uuid supplied is valid. - :param uuid_string: the uuid which needs to to checked + Determine if the uuid supplied is valid + :param uuid_string: the uuid which needs to be checked :return: bool: if the uuid is valid or not """ try: @@ -117,11 +162,13 @@ def backup_folder(folder: str or pathlib.Path) -> None: if backup_folder_name.is_dir(): renamed = True + def download_file(parsed_uri, download_path: pathlib.Path, force_overwrite: bool = False, download_progress_callback = None) -> int: """ + Download file :param parsed_uri: uniform resource identifier to zip file to download :param download_path: location path on disk to download file - :download_progress_callback: callback called with the download progress as a percentage, returns true to request to cancel the download + :param download_progress_callback: callback called with the download progress as a percentage, returns true to request to cancel the download """ if download_path.is_file(): if not force_overwrite: @@ -142,10 +189,12 @@ def download_file(parsed_uri, download_path: pathlib.Path, force_overwrite: bool download_file_size = s.headers['content-length'] except KeyError: pass + def download_progress(downloaded_bytes): if download_progress_callback: return download_progress_callback(int(downloaded_bytes), int(download_file_size)) return False + with download_path.open('wb') as f: download_cancelled = copyfileobj(s, f, download_progress) if download_cancelled: @@ -183,13 +232,12 @@ def download_zip_file(parsed_uri, download_zip_path: pathlib.Path, force_overwri def find_ancestor_file(target_file_name: pathlib.PurePath, start_path: pathlib.Path, max_scan_up_range: int=0) -> pathlib.Path or None: """ - Find a file with the given name in the ancestor directories by walking up the starting path until the file is found. - - :param target_file_name: Name of the file to find. - :param start_path: path to start looking for the file. + Find a file with the given name in the ancestor directories by walking up the starting path until the file is found + :param target_file_name: Name of the file to find + :param start_path: path to start looking for the file :param max_scan_up_range: maximum number of directories to scan upwards when searching for target file if the value is 0, then there is no max - :return: Path to the file or None if not found. + :return: Path to the file or None if not found """ current_path = pathlib.Path(start_path) candidate_path = current_path / target_file_name @@ -211,17 +259,17 @@ def find_ancestor_file(target_file_name: pathlib.PurePath, start_path: pathlib.P return candidate_path if candidate_path.exists() else None + def find_ancestor_dir_containing_file(target_file_name: pathlib.PurePath, start_path: pathlib.Path, max_scan_up_range: int=0) -> pathlib.Path or None: """ - Find nearest ancestor directory that contains the file with the given name by walking up - from the starting path. - - :param target_file_name: Name of the file to find. - :param start_path: path to start looking for the file. + Find the nearest ancestor directory that contains the file with the given name by walking up + from the starting path + :param target_file_name: Name of the file to find + :param start_path: path to start looking for the file :param max_scan_up_range: maximum number of directories to scan upwards when searching for target file if the value is 0, then there is no max - :return: Path to the directory containing file or None if not found. + :return: Path to the directory containing file or None if not found """ ancestor_file = find_ancestor_file(target_file_name, start_path, max_scan_up_range) return ancestor_file.parent if ancestor_file else None diff --git a/scripts/o3de/o3de/validation.py b/scripts/o3de/o3de/validation.py index 5c683f0667..cd0958a250 100644 --- a/scripts/o3de/o3de/validation.py +++ b/scripts/o3de/o3de/validation.py @@ -11,6 +11,7 @@ This file validating o3de object json files import json import pathlib + def valid_o3de_json_dict(json_data: dict, key: str) -> bool: return key in json_data @@ -23,9 +24,9 @@ def valid_o3de_repo_json(file_name: str or pathlib.Path) -> bool: with file_name.open('r') as f: try: json_data = json.load(f) - test = json_data['repo_name'] - test = json_data['origin'] - except (json.JSONDecodeError, KeyError) as e: + _ = json_data['repo_name'] + _ = json_data['origin'] + except (json.JSONDecodeError, KeyError): return False return True @@ -38,8 +39,8 @@ def valid_o3de_engine_json(file_name: str or pathlib.Path) -> bool: with file_name.open('r') as f: try: json_data = json.load(f) - test = json_data['engine_name'] - except (json.JSONDecodeError, KeyError) as e: + _ = json_data['engine_name'] + except (json.JSONDecodeError, KeyError): return False return True @@ -52,8 +53,8 @@ def valid_o3de_project_json(file_name: str or pathlib.Path) -> bool: with file_name.open('r') as f: try: json_data = json.load(f) - test = json_data['project_name'] - except (json.JSONDecodeError, KeyError) as e: + _ = json_data['project_name'] + except (json.JSONDecodeError, KeyError): return False return True @@ -66,8 +67,8 @@ def valid_o3de_gem_json(file_name: str or pathlib.Path) -> bool: with file_name.open('r') as f: try: json_data = json.load(f) - test = json_data['gem_name'] - except (json.JSONDecodeError, KeyError) as e: + _ = json_data['gem_name'] + except (json.JSONDecodeError, KeyError): return False return True @@ -76,11 +77,12 @@ def valid_o3de_template_json(file_name: str or pathlib.Path) -> bool: file_name = pathlib.Path(file_name).resolve() if not file_name.is_file(): return False + with file_name.open('r') as f: try: json_data = json.load(f) - test = json_data['template_name'] - except (json.JSONDecodeError, KeyError) as e: + _ = json_data['template_name'] + except (json.JSONDecodeError, KeyError): return False return True @@ -89,10 +91,11 @@ def valid_o3de_restricted_json(file_name: str or pathlib.Path) -> bool: file_name = pathlib.Path(file_name).resolve() if not file_name.is_file(): return False + with file_name.open('r') as f: try: json_data = json.load(f) - test = json_data['restricted_name'] - except (json.JSONDecodeError, KeyError) as e: + _ = json_data['restricted_name'] + except (json.JSONDecodeError, KeyError): return False return True