From e59b154139ab530e99cda544fcea60eadcdb1302 Mon Sep 17 00:00:00 2001 From: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com> Date: Sun, 23 May 2021 22:10:09 -0500 Subject: [PATCH] Updated the refactored o3de command scripts to be able to run standalone by adding a main section Removed the ability to suppress errors in the add-gem-to-cmake and add-external-subdirectory command Reduced duplicate logic in the download.py, register.py and repo.py scripts Removed the registration.py script and moved the registration of those comamnds directly to the o3de.py script Reduced the exception scope in the o3de command scripts to be as constrained as possible. For example instead of having a block of Exception for catching a dictionary KeyError, the block has been updated to just catch KeyError Added a python test for validating the "register.py --this-engine" functionality --- scripts/o3de.py | 81 ++- .../o3de/o3de/add_external_subdirectory.py | 77 ++- scripts/o3de/o3de/add_gem_cmake.py | 86 ++- scripts/o3de/o3de/add_gem_project.py | 78 ++- scripts/o3de/o3de/download.py | 641 ++++-------------- scripts/o3de/o3de/engine_template.py | 89 ++- scripts/o3de/o3de/get_registration.py | 60 +- scripts/o3de/o3de/global_project.py | 15 +- scripts/o3de/o3de/manifest.py | 26 +- scripts/o3de/o3de/print_registration.py | 72 +- scripts/o3de/o3de/register.py | 398 ++++------- scripts/o3de/o3de/registration.py | 92 --- .../o3de/o3de/remove_external_subdirectory.py | 55 +- scripts/o3de/o3de/remove_gem_cmake.py | 59 +- scripts/o3de/o3de/remove_gem_project.py | 74 +- scripts/o3de/o3de/repo.py | 275 ++------ scripts/o3de/o3de/sha256.py | 63 +- scripts/o3de/o3de/utils.py | 44 +- scripts/o3de/o3de/validation.py | 21 +- scripts/o3de/tests/unit_test_registration.py | 57 +- 20 files changed, 1031 insertions(+), 1332 deletions(-) delete mode 100755 scripts/o3de/o3de/registration.py diff --git a/scripts/o3de.py b/scripts/o3de.py index d3b877620f..dabb83b068 100755 --- a/scripts/o3de.py +++ b/scripts/o3de.py @@ -13,27 +13,70 @@ import argparse import pathlib import sys -# As o3de.py shares the same name as the o3de package attempting to use a regular -# from o3de import line tries to import from the current o3de.py script and not the package -# So the current script directory is removed from the sys.path temporary -SCRIPT_DIR_REMOVED = False -SCRIPT_DIR = pathlib.Path(__file__).parent.resolve() -while str(SCRIPT_DIR) in sys.path: - SCRIPT_DIR_REMOVED = True - sys.path.remove(str(SCRIPT_DIR)) - -from o3de import engine_template -from o3de import global_project -from o3de import registration - -if SCRIPT_DIR_REMOVED: - sys.path.insert(0, str(SCRIPT_DIR)) - def add_args(parser, subparsers) -> None: - global_project.add_args(parser, subparsers) - engine_template.add_args(parser, subparsers) - registration.add_args(parser, subparsers) + """ + add_args is called to add expected parser arguments and subparsers arguments to each command such that it can be + invoked by o3de.py + Ex o3de.py can invoke the register downloadable commands by importing register, + call add_args and execute: python o3de.py register --gem-path "C:/TestGem" + :param parser: the caller instantiates a parser and passes it in here + :param subparsers: the caller instantiates subparsers and passes it in here + """ + + # As o3de.py shares the same name as the o3de package attempting to use a regular + # from o3de import line tries to import from the current o3de.py script and not the package + # So the current script directory is removed from the sys.path temporary + SCRIPT_DIR_REMOVED = False + SCRIPT_DIR = pathlib.Path(__file__).parent.resolve() + while str(SCRIPT_DIR) in sys.path: + SCRIPT_DIR_REMOVED = True + sys.path.remove(str(SCRIPT_DIR)) + + from o3de import engine_template, global_project, register, print_registration, get_registration, download, \ + add_external_subdirectory, remove_external_subdirectory, add_gem_cmake, remove_gem_cmake, add_gem_project, \ + remove_gem_project, sha256 + + if SCRIPT_DIR_REMOVED: + sys.path.insert(0, str(SCRIPT_DIR)) + + # global_project + global_project.add_args(subparsers) + # engine templaate + engine_template.add_args(subparsers) + + # register + register.add_args(subparsers) + + # show + print_registration.add_args(subparsers) + + # get-registered + get_registration.add_args(subparsers) + + # download + download.add_args(subparsers) + + # add external subdirectories + add_external_subdirectory.add_args(subparsers) + + # remove external subdirectories + remove_external_subdirectory.add_args(subparsers) + + # add gems to cmake + add_gem_cmake.add_args(subparsers) + + # remove gems from cmake + remove_gem_cmake.add_args(subparsers) + + # add a gem to a project + add_gem_project.add_args(subparsers) + + # remove a gem from a project + remove_gem_project.add_args(subparsers) + + # sha256 + sha256.add_args(subparsers) if __name__ == "__main__": diff --git a/scripts/o3de/o3de/add_external_subdirectory.py b/scripts/o3de/o3de/add_external_subdirectory.py index 388f0027da..29013d30c8 100644 --- a/scripts/o3de/o3de/add_external_subdirectory.py +++ b/scripts/o3de/o3de/add_external_subdirectory.py @@ -15,6 +15,7 @@ Contains command to add an external_subdirectory to a project's cmake scripts import argparse import logging import pathlib +import sys from o3de import manifest @@ -22,32 +23,27 @@ logger = logging.getLogger() logging.basicConfig() def add_external_subdirectory(external_subdir: str or pathlib.Path, - engine_path: str or pathlib.Path = None, - suppress_errors: bool = False) -> int: + engine_path: str or pathlib.Path = None) -> int: """ add external subdirectory to a cmake :param external_subdir: external subdirectory to add to cmake :param engine_path: optional engine path, defaults to this engine - :param suppress_errors: optional silence errors :return: 0 for success or non 0 failure code """ external_subdir = pathlib.Path(external_subdir).resolve() if not external_subdir.is_dir(): - if not suppress_errors: - logger.error(f'Add External Subdirectory Failed: {external_subdir} does not exist.') + logger.error(f'Add External Subdirectory Failed: {external_subdir} does not exist.') return 1 external_subdir_cmake = external_subdir / 'CMakeLists.txt' if not external_subdir_cmake.is_file(): - if not suppress_errors: - logger.error(f'Add External Subdirectory Failed: {external_subdir} does not contain a CMakeLists.txt.') + logger.error(f'Add External Subdirectory Failed: {external_subdir} does not contain a CMakeLists.txt.') return 1 json_data = manifest.load_o3de_manifest() engine_object = manifest.find_engine_data(json_data, engine_path) if not engine_object: - if not suppress_errors: - logger.error(f'Add External Subdirectory Failed: {engine_path} not registered.') + logger.error(f'Add External Subdirectory Failed: {engine_path} not registered.') return 1 engine_object.setdefault('external_subdirectories', []) @@ -76,7 +72,7 @@ def add_external_subdirectory(external_subdir: str or pathlib.Path, if end > start + len('include('): try: include_cmake_file = pathlib.Path(engine_path / line[start + len('include('): end]).resolve() - except Exception as e: + except FileNotFoundError as e: pass else: parse_cmake_file(include_cmake_file, files) @@ -88,7 +84,7 @@ def add_external_subdirectory(external_subdir: str or pathlib.Path, try: include_cmake_file = pathlib.Path( cmake_path / line[start + len('add_subdirectory('): end]).resolve() - except Exception as e: + except FileNotFoundError as e: pass else: parse_cmake_file(include_cmake_file, files) @@ -100,8 +96,7 @@ def add_external_subdirectory(external_subdir: str or pathlib.Path, if external_subdir in cmake_files: manifest.save_o3de_manifest(json_data) - if not suppress_errors: - logger.error(f'External subdirectory {external_subdir.as_posix()} already included by add_subdirectory().') + logger.warning(f'External subdirectory {external_subdir.as_posix()} already included by add_subdirectory().') return 1 engine_object['external_subdirectories'].insert(0, external_subdir.as_posix()) @@ -119,23 +114,55 @@ def _run_add_external_subdirectory(args: argparse) -> int: return add_external_subdirectory(args.external_subdirectory) -def add_args(parser, subparsers) -> None: +def add_parser_args(parser): """ - add_args is called to add expected parser arguments and subparsers arguments to each command such that it can be + add_parser_args is called to add arguments to each command such that it can be invoked locally or added by a central python file. - Ex. Directly run from this file alone with: python register.py register --gem-path "C:/TestGem" - OR - o3de.py can downloadable commands by importing engine_template, - call add_args and execute: python o3de.py register --gem-path "C:/TestGem" - :param parser: the caller instantiates a parser and passes it in here + Ex. Directly run from this file alone with: python add-external-subdirectory.py "/home/foo/external-subdir" + :param parser: the caller passes an argparse parser like instance to this method + """ + parser.add_argument('external_subdirectory', metavar='external_subdirectory', type=str, + help='add an external subdirectory to cmake') + + parser.add_argument('-ohf', '--override-home-folder', type=str, required=False, + help='By default the home folder is the user folder, override it to this folder.') + + parser.set_defaults(func=_run_add_external_subdirectory) + + +def add_args(subparsers) -> None: + """ + add_args is called to add subparsers arguments to each command such that it can be + a central python file such as o3de.py. + It can be run from the o3de.py script as follows + call add_args and execute: python o3de.py add_external_subdirectory "/home/foo/external-subdir" :param subparsers: the caller instantiates subparsers and passes it in here """ add_external_subdirectory_subparser = subparsers.add_parser('add-external-subdirectory') - add_external_subdirectory_subparser.add_argument('external_subdirectory', metavar='external_subdirectory', type=str, - help='add an external subdirectory to cmake') + add_parser_args(add_external_subdirectory_subparser) - add_external_subdirectory_subparser.add_argument('-ohf', '--override-home-folder', type=str, required=False, - help='By default the home folder is the user folder, override it to this folder.') - add_external_subdirectory_subparser.set_defaults(func=_run_add_external_subdirectory) +def main(): + """ + Runs add_external_subdirectory.py script as standalone script + """ + # parse the command line args + the_parser = argparse.ArgumentParser() + # add subparsers + + # add args to the parser + add_parser_args(the_parser) + + # parse args + the_args = the_parser.parse_args() + + # run + ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1 + + # return + sys.exit(ret) + + +if __name__ == "__main__": + main() diff --git a/scripts/o3de/o3de/add_gem_cmake.py b/scripts/o3de/o3de/add_gem_cmake.py index fa2d2f4bb3..523fb8dce8 100644 --- a/scripts/o3de/o3de/add_gem_cmake.py +++ b/scripts/o3de/o3de/add_gem_cmake.py @@ -15,6 +15,7 @@ Contains command to add a gem to a project's cmake scripts import argparse import logging import pathlib +import sys from o3de import add_external_subdirectory, manifest, validation @@ -24,39 +25,33 @@ logging.basicConfig() def add_gem_to_cmake(gem_name: str = None, gem_path: str or pathlib.Path = None, engine_name: str = None, - engine_path: str or pathlib.Path = None, - suppress_errors: bool = False) -> int: + engine_path: str or pathlib.Path = None) -> int: """ add a gem to a cmake as an external subdirectory for an engine :param gem_name: name of the gem to add to cmake :param gem_path: the path of the gem to add to cmake :param engine_name: name of the engine to add to cmake :param engine_path: the path of the engine to add external subdirectory to, default to this engine - :param suppress_errors: optional silence errors :return: 0 for success or non 0 failure code """ if not gem_name and not gem_path: - if not suppress_errors: - logger.error('Must specify either a Gem name or Gem Path.') + logger.error('Must specify either a Gem name or Gem Path.') return 1 if gem_name and not gem_path: gem_path = manifest.get_registered(gem_name=gem_name) if not gem_path: - if not suppress_errors: - logger.error(f'Gem Path {gem_path} has not been registered.') + logger.error(f'Gem Path {gem_path} has not been registered.') return 1 gem_path = pathlib.Path(gem_path).resolve() gem_json = gem_path / 'gem.json' if not gem_json.is_file(): - if not suppress_errors: - logger.error(f'Gem json {gem_json} is not present.') + logger.error(f'Gem json {gem_json} is not present.') return 1 if not validation.valid_o3de_gem_json(gem_json): - if not suppress_errors: - logger.error(f'Gem json {gem_json} is not valid.') + logger.error(f'Gem json {gem_json} is not valid.') return 1 if not engine_name and not engine_path: @@ -66,22 +61,18 @@ def add_gem_to_cmake(gem_name: str = None, engine_path = manifest.get_registered(engine_name=engine_name) if not engine_path: - if not suppress_errors: - logger.error(f'Engine Path {engine_path} has not been registered.') + logger.error(f'Engine Path {engine_path} has not been registered.') return 1 engine_json = engine_path / 'engine.json' if not engine_json.is_file(): - if not suppress_errors: - logger.error(f'Engine json {engine_json} is not present.') + logger.error(f'Engine json {engine_json} is not present.') return 1 if not validation.valid_o3de_engine_json(engine_json): - if not suppress_errors: - logger.error(f'Engine json {engine_json} is not valid.') + logger.error(f'Engine json {engine_json} is not valid.') return 1 - return add_external_subdirectory.add_external_subdirectory(external_subdir=gem_path, engine_path=engine_path, suppress_errors=suppress_errors) - + return add_external_subdirectory.add_external_subdirectory(external_subdir=gem_path, engine_path=engine_path) def _run_add_gem_to_cmake(args: argparse) -> int: if args.override_home_folder: @@ -90,25 +81,58 @@ def _run_add_gem_to_cmake(args: argparse) -> int: return add_gem_to_cmake(gem_name=args.gem_name, gem_path=args.gem_path) -def add_args(parser, subparsers) -> None: +def add_parser_args(parser): """ - add_args is called to add expected parser arguments and subparsers arguments to each command such that it can be + add_parser_args is called to add arguments to each command such that it can be invoked locally or added by a central python file. - Ex. Directly run from this file alone with: python register.py register --gem-path "C:/TestGem" - OR - o3de.py can downloadable commands by importing engine_template, - call add_args and execute: python o3de.py register --gem-path "C:/TestGem" - :param parser: the caller instantiates a parser and passes it in here - :param subparsers: the caller instantiates subparsers and passes it in here + Ex. Directly run from this file alone with: python add_gem_cmake.py --gem-path "/path/to/gem" + :param parser: the caller passes an argparse parser like instance to this method """ - add_gem_to_cmake_subparser = subparsers.add_parser('add-gem-to-cmake') - group = add_gem_to_cmake_subparser.add_mutually_exclusive_group(required=True) + group = parser.add_mutually_exclusive_group(required=True) group.add_argument('-gp', '--gem-path', type=str, required=False, help='The path to the gem.') group.add_argument('-gn', '--gem-name', type=str, required=False, help='The name of the gem.') - add_gem_to_cmake_subparser.add_argument('-ohf', '--override-home-folder', type=str, required=False, + parser.add_argument('-ohf', '--override-home-folder', type=str, required=False, help='By default the home folder is the user folder, override it to this folder.') - add_gem_to_cmake_subparser.set_defaults(func=_run_add_gem_to_cmake) + parser.set_defaults(func=_run_add_gem_to_cmake) + + +def add_args(subparsers) -> None: + """ + add_args is called to add subparsers arguments to each command such that it can be + a central python file such as o3de.py. + It can be run from the o3de.py script as follows + call add_args and execute: python o3de.py add-gem-to-cmake --gem-path "/path/to/gem" + :param subparsers: the caller instantiates subparsers and passes it in here + """ + add_gem_cmake_subparser = subparsers.add_parser('add-gem-to-cmake') + add_parser_args(add_gem_cmake_subparser) + + +def main(): + """ + Runs add_gem_cmake.py script as standalone script + """ + # parse the command line args + the_parser = argparse.ArgumentParser() + + # add subparsers + + # add args to the parser + add_parser_args(the_parser) + + # parse args + the_args = the_parser.parse_args() + + # run + ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1 + + # return + sys.exit(ret) + + +if __name__ == "__main__": + main() diff --git a/scripts/o3de/o3de/add_gem_project.py b/scripts/o3de/o3de/add_gem_project.py index 933fd019bb..78dffc4477 100644 --- a/scripts/o3de/o3de/add_gem_project.py +++ b/scripts/o3de/o3de/add_gem_project.py @@ -17,6 +17,7 @@ import json import logging import os import pathlib +import sys from o3de import add_gem_cmake, cmake, manifest, validation @@ -126,13 +127,13 @@ def add_gem_to_project(gem_name: str = None, with project_json.open('r') as s: try: project_json_data = json.load(s) - except Exception as e: + except json.JSONDecodeError as e: logger.error(f'Error loading Project json {project_json}: {str(e)}') return 1 else: try: engine_name = project_json_data['engine'] - except Exception as e: + except KeyError as e: logger.error(f'Project json {project_json} "engine" not found: {str(e)}') return 1 else: @@ -261,51 +262,84 @@ def _run_add_gem_to_project(args: argparse) -> int: args.add_to_cmake) -def add_args(parser, subparsers) -> None: +def add_parser_args(parser): """ - add_args is called to add expected parser arguments and subparsers arguments to each command such that it can be + add_parser_args is called to add arguments to each command such that it can be invoked locally or added by a central python file. - Ex. Directly run from this file alone with: python register.py register --gem-path "C:/TestGem" - OR - o3de.py can downloadable commands by importing engine_template, - call add_args and execute: python o3de.py register --gem-path "C:/TestGem" - :param parser: the caller instantiates a parser and passes it in here - :param subparsers: the caller instantiates subparsers and passes it in here + Ex. Directly run from this file alone with: python add_gem_project.py --project-path "D:/TestProject" --gem-path "D:/TestGem" + :param parser: the caller passes an argparse parser like instance to this method """ - add_gem_subparser = subparsers.add_parser('add-gem-to-project') - group = add_gem_subparser.add_mutually_exclusive_group(required=True) + group = parser.add_mutually_exclusive_group(required=True) group.add_argument('-pp', '--project-path', type=str, required=False, help='The path to the project.') group.add_argument('-pn', '--project-name', type=str, required=False, help='The name of the project.') - group = add_gem_subparser.add_mutually_exclusive_group(required=True) + group = parser.add_mutually_exclusive_group(required=True) group.add_argument('-gp', '--gem-path', type=str, required=False, help='The path to the gem.') group.add_argument('-gn', '--gem-name', type=str, required=False, help='The name of the gem.') - add_gem_subparser.add_argument('-gt', '--gem-target', type=str, required=False, + parser.add_argument('-gt', '--gem-target', type=str, required=False, help='The cmake target name to add. If not specified it will assume gem_name') - add_gem_subparser.add_argument('-df', '--dependencies-file', type=str, required=False, + parser.add_argument('-df', '--dependencies-file', type=str, required=False, help='The cmake dependencies file in which the gem dependencies are specified.' 'If not specified it will assume ') - add_gem_subparser.add_argument('-rd', '--runtime-dependency', action='store_true', required=False, + parser.add_argument('-rd', '--runtime-dependency', action='store_true', required=False, default=False, help='Optional toggle if this gem should be added as a runtime dependency') - add_gem_subparser.add_argument('-td', '--tool-dependency', action='store_true', required=False, + parser.add_argument('-td', '--tool-dependency', action='store_true', required=False, default=False, help='Optional toggle if this gem should be added as a tool dependency') - add_gem_subparser.add_argument('-sd', '--server-dependency', action='store_true', required=False, + parser.add_argument('-sd', '--server-dependency', action='store_true', required=False, default=False, help='Optional toggle if this gem should be added as a server dependency') - add_gem_subparser.add_argument('-pl', '--platforms', type=str, required=False, + parser.add_argument('-pl', '--platforms', type=str, required=False, default='Common', help='Optional list of platforms this gem should be added to.' ' Ex. --platforms Mac,Windows,Linux') - add_gem_subparser.add_argument('-a', '--add-to-cmake', type=bool, required=False, + parser.add_argument('-a', '--add-to-cmake', type=bool, required=False, default=True, help='Automatically call add-gem-to-cmake.') - add_gem_subparser.add_argument('-ohf', '--override-home-folder', type=str, required=False, + parser.add_argument('-ohf', '--override-home-folder', type=str, required=False, help='By default the home folder is the user folder, override it to this folder.') - add_gem_subparser.set_defaults(func=_run_add_gem_to_project) + parser.set_defaults(func=_run_add_gem_to_project) + + +def add_args(subparsers) -> None: + """ + add_args is called to add subparsers arguments to each command such that it can be + a central python file such as o3de.py. + It can be run from the o3de.py script as follows + call add_args and execute: python o3de.py add-gem-to-project --project-path "D:/TestProject" --gem-path "D:/TestGem" + :param subparsers: the caller instantiates subparsers and passes it in here + """ + add_gem_project_subparser = subparsers.add_parser('add-gem-to-project') + add_parser_args(add_gem_project_subparser) + + +def main(): + """ + Runs add_gem_project.py script as standalone script + """ + # parse the command line args + the_parser = argparse.ArgumentParser() + + # add subparsers + + # add args to the parser + add_parser_args(the_parser) + + # parse args + the_args = the_parser.parse_args() + + # run + ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1 + + # return + sys.exit(ret) + + +if __name__ == "__main__": + main() diff --git a/scripts/o3de/o3de/download.py b/scripts/o3de/o3de/download.py index 3db2f077cd..1dbb584c92 100644 --- a/scripts/o3de/o3de/download.py +++ b/scripts/o3de/o3de/download.py @@ -18,18 +18,90 @@ import json import logging import pathlib import shutil +import sys import urllib.parse import urllib.request -from o3de import manifest, utils, validation +from o3de import manifest, repo, utils, validation logger = logging.getLogger() logging.basicConfig() -def download_engine(engine_name: str, - dest_path: str) -> int: +def unzip_manifest_json_data(download_zip_path: pathlib.Path, zip_file_name: str) -> dict: + json_data = {} + with zipfile.ZipFile(download_zip_path, 'r') as zip_data: + with zip_data.open(zip_file_name) as manifest_json_file: + try: + json_data = json.load(manifest_json_file) + except json.JSONDecodeError as e: + logger.error(f'UnZip exception:{str(e)}') + + return json_data + +def validate_downloaded_zip_sha256(download_uri_json_data: dict, download_zip_path: pathlib.Path, + manifest_json_name) -> int: + # if the engine.json has a sha256 check it against a sha256 of the zip + try: + sha256A = download_uri_json_data['sha256'] + except KeyError as e: + logger.warn(f'SECURITY WARNING: The advertised o3de object you downloaded has no "sha256"!!! Be VERY careful!!!' + f' We cannot verify this is the actually the advertised object!!!') + else: + sha256B = hashlib.sha256(download_zip_path.open('rb').read()).hexdigest() + if sha256A != sha256B: + logger.error(f'SECURITY VIOLATION: Downloaded zip sha256 {sha256B} does not match' + f' the advertised "sha256":{sha256A} in the f{manifest_json_name}. Deleting unzipped files!!!') + shutil.rmtree(dest_path) + return 1 + + manifest_json_data = unzip_manifest_json_data(download_zip_path, manifest_json_name) + + # remove the sha256 if present in the advertised downloadable manifest json + # then compare it to the json in the zip, they should now be identical + try: + del download_uri_json_data['sha256'] + except KeyError as e: + pass + + sha256A = hashlib.sha256(json.dumps(download_uri_json_data, indent=4).encode('utf8')).hexdigest() + with unzipped_manifest_json.open('r') as s: + try: + unzipped_manifest_json_data = json.load(s) + except json.JSONDecodeError as e: + logger.error(f'Failed to read manifest json {unzipped_manifest_json}. Unable to confirm this' + f' is the same template that was advertised.') + return 1 + sha256B = hashlib.sha256(json.dumps(unzipped_manifest_json_data, indent=4).encode('utf8')).hexdigest() + if sha256A != sha256B: + logger.error(f'SECURITY VIOLATION: Downloaded manifest json does not match' + f' the advertised manifest json. Deleting unzipped files!!!') + shutil.rmtree(dest_path) + return 1 + + return 0 + + +def get_downloadable(engine_name: str = None, + project_name: str = None, + gem_name: str = None, + template_name: str = None, + restricted_name: str = None) -> dict or None: + json_data = manifest.load_o3de_manifest() + try: + o3de_object_uris = json_data['repos'] + except KeyError as key_err: + logger.error(f'Unable to load repos from o3de manifest: {str(key_err)}') + return None + + manifest_json = 'repo.json' + search_func = lambda: repo.search_repo(manifest_json, engine_name, project_name, gem_name, template_name) + return repo.search_o3de_object(manifest_json, o3de_object_uris, search_func) + + +def download_o3de_object(object_name: str, default_folder_name: str, dest_path: str or pathlib.Path, + object_type: str, downloadable_kwarg_key) -> int: if not dest_path: - dest_path = manifest.get_registered(default_folder='engines') + dest_path = manifest.get_registered(default_folder=default_folder_name) if not dest_path: logger.error(f'Destination path not cannot be empty.') return 1 @@ -37,512 +109,50 @@ def download_engine(engine_name: str, dest_path = pathlib.Path(dest_path).resolve() dest_path.mkdir(exist_ok=True) - download_path = manifest.get_o3de_download_folder() / 'engines' / engine_name + download_path = manifest.get_o3de_download_folder() / default_folder_name / object_name download_path.mkdir(exist_ok=True) - download_zip_path = download_path / 'engine.zip' + download_zip_path = download_path / f'{object_type}.zip' - downloadable_engine_data = get_downloadable(engine_name=engine_name) - if not downloadable_engine_data: - logger.error(f'Downloadable engine {engine_name} not found.') + downloadable_object_data = get_downloadable(**{downloadable_kwarg_key : object_name}) + if not downloadable_object_data: + logger.error(f'Downloadable o3de object {object_name} not found.') return 1 - origin = downloadable_engine_data['origin'] - url = f'{origin}/project.zip' + origin = downloadable_json_data['origin'] + url = f'{origin}/object_type.zip' parsed_uri = urllib.parse.urlparse(url) - if download_zip_path.is_file(): - logger.warn(f'Project already downloaded to {download_zip_path}.') - elif parsed_uri.scheme == 'http' or \ - parsed_uri.scheme == 'https' or \ - parsed_uri.scheme == 'ftp' or \ - parsed_uri.scheme == 'ftps': - with urllib.request.urlopen(url) as s: - with download_zip_path.open('wb') as f: - shutil.copyfileobj(s, f) - else: - origin_file = pathlib.Path(url).resolve() - if not origin_file.is_file(): - return 1 - shutil.copy(origin_file, download_zip_path) + download_zip_result = utils.download_zip_file(parsed_uri, download_zip_path) + if download_zip_result != 0: + return download_zip_result - if not zipfile.is_zipfile(download_zip_path): - logger.error(f"Engine zip {download_zip_path} is invalid.") - download_zip_path.unlink() - return 1 + return validate_downloaded_zip_sha256(downloadable_object_data, download_zip_path) - # if the engine.json has a sha256 check it against a sha256 of the zip - try: - sha256A = downloadable_engine_data['sha256'] - except Exception as e: - logger.warn(f'SECURITY WARNING: The advertised engine you downloaded has no "sha256"!!! Be VERY careful!!!' - f' We cannot verify this is the actually the advertised engine!!!') - else: - sha256B = hashlib.sha256(download_zip_path.open('rb').read()).hexdigest() - if sha256A != sha256B: - logger.error(f'SECURITY VIOLATION: Downloaded engine.zip sha256 {sha256B} does not match' - f' the advertised "sha256":{sha256A} in the engine.json. Deleting unzipped files!!!') - shutil.rmtree(dest_path) - return 1 - dest_engine_folder = dest_path / engine_name - if dest_engine_folder.is_dir(): - utils.backup_folder(dest_engine_folder) - with zipfile.ZipFile(download_zip_path, 'r') as project_zip: - try: - project_zip.extractall(dest_path) - except Exception as e: - logger.error(f'UnZip exception:{str(e)}') - shutil.rmtree(dest_path) - return 1 - - unzipped_engine_json = dest_engine_folder / 'engine.json' - if not unzipped_engine_json.is_file(): - logger.error(f'Engine json {unzipped_engine_json} is missing.') - return 1 - - if not validation.valid_o3de_engine_json(unzipped_engine_json): - logger.error(f'Engine json {unzipped_engine_json} is invalid.') - return 1 - - # remove the sha256 if present in the advertised downloadable engine.json - # then compare it to the engine.json in the zip, they should now be identical - try: - del downloadable_engine_data['sha256'] - except Exception as e: - pass - - sha256A = hashlib.sha256(json.dumps(downloadable_engine_data, indent=4).encode('utf8')).hexdigest() - with unzipped_engine_json.open('r') as s: - try: - unzipped_engine_json_data = json.load(s) - except Exception as e: - logger.error(f'Failed to read engine json {unzipped_engine_json}. Unable to confirm this' - f' is the same template that was advertised.') - return 1 - sha256B = hashlib.sha256(json.dumps(unzipped_engine_json_data, indent=4).encode('utf8')).hexdigest() - if sha256A != sha256B: - logger.error(f'SECURITY VIOLATION: Downloaded engine.json does not match' - f' the advertised engine.json. Deleting unzipped files!!!') - shutil.rmtree(dest_path) - return 1 - - return 0 +def download_engine(engine_name: str, + dest_path: str or pathlib.Path) -> int: + return download_o3de_object(engine_name, 'engines', dest_path, 'engine', 'engine_name') def download_project(project_name: str, dest_path: str or pathlib.Path) -> int: - if not dest_path: - dest_path = manifest.get_registered(default_folder='projects') - if not dest_path: - logger.error(f'Destination path not specified and not default projects path.') - return 1 - - dest_path = pathlib.Path(dest_path).resolve() - dest_path.mkdir(exist_ok=True, parents=True) - - download_path = manifest.get_o3de_download_folder() / 'projects' / project_name - download_path.mkdir(exist_ok=True, parents=True) - download_zip_path = download_path / 'project.zip' - - downloadable_project_data = get_downloadable(project_name=project_name) - if not downloadable_project_data: - logger.error(f'Downloadable project {project_name} not found.') - return 1 - - origin = downloadable_project_data['origin'] - url = f'{origin}/project.zip' - parsed_uri = urllib.parse.urlparse(url) - - if download_zip_path.is_file(): - logger.warn(f'Project already downloaded to {download_zip_path}.') - elif parsed_uri.scheme == 'http' or \ - parsed_uri.scheme == 'https' or \ - parsed_uri.scheme == 'ftp' or \ - parsed_uri.scheme == 'ftps': - with urllib.request.urlopen(url) as s: - with download_zip_path.open('wb') as f: - shutil.copyfileobj(s, f) - else: - origin_file = pathlib.Path(url).resolve() - if not origin_file.is_file(): - return 1 - shutil.copy(origin_file, download_zip_path) - - if not zipfile.is_zipfile(download_zip_path): - logger.error(f"Project zip {download_zip_path} is invalid.") - download_zip_path.unlink() - return 1 - - # if the project.json has a sha256 check it against a sha256 of the zip - try: - sha256A = downloadable_project_data['sha256'] - except Exception as e: - logger.warn(f'SECURITY WARNING: The advertised project you downloaded has no "sha256"!!! Be VERY careful!!!' - f' We cannot verify this is the actually the advertised project!!!') - else: - sha256B = hashlib.sha256(download_zip_path.open('rb').read()).hexdigest() - if sha256A != sha256B: - logger.error(f'SECURITY VIOLATION: Downloaded project.zip sha256 {sha256B} does not match' - f' the advertised "sha256":{sha256A} in the project.json. Deleting unzipped files!!!') - shutil.rmtree(dest_path) - return 1 - - dest_project_folder = dest_path / project_name - if dest_project_folder.is_dir(): - utils.backup_folder(dest_project_folder) - with zipfile.ZipFile(download_zip_path, 'r') as project_zip: - try: - project_zip.extractall(dest_project_folder) - except Exception as e: - logger.error(f'UnZip exception:{str(e)}') - shutil.rmtree(dest_path) - return 1 - - unzipped_project_json = dest_project_folder / 'project.json' - if not unzipped_project_json.is_file(): - logger.error(f'Project json {unzipped_project_json} is missing.') - return 1 - - if not validation.valid_o3de_project_json(unzipped_project_json): - logger.error(f'Project json {unzipped_project_json} is invalid.') - return 1 - - # remove the sha256 if present in the advertised downloadable project.json - # then compare it to the project.json in the zip, they should now be identical - try: - del downloadable_project_data['sha256'] - except Exception as e: - pass - - sha256A = hashlib.sha256(json.dumps(downloadable_project_data, indent=4).encode('utf8')).hexdigest() - with unzipped_project_json.open('r') as s: - try: - unzipped_project_json_data = json.load(s) - except Exception as e: - logger.error(f'Failed to read Project json {unzipped_project_json}. Unable to confirm this' - f' is the same project that was advertised.') - return 1 - sha256B = hashlib.sha256(json.dumps(unzipped_project_json_data, indent=4).encode('utf8')).hexdigest() - if sha256A != sha256B: - logger.error(f'SECURITY VIOLATION: Downloaded project.json does not match' - f' the advertised project.json. Deleting unzipped files!!!') - shutil.rmtree(dest_path) - return 1 - - return 0 + return download_o3de_object(project_name, 'projects', dest_path, 'project', 'project_name') def download_gem(gem_name: str, dest_path: str or pathlib.Path) -> int: - if not dest_path: - dest_path = manifest.get_registered(default_folder='gems') - if not dest_path: - logger.error(f'Destination path not cannot be empty.') - return 1 - - dest_path = pathlib.Path(dest_path).resolve() - dest_path.mkdir(exist_ok=True, parents=True) - - download_path = manifest.get_o3de_download_folder() / 'gems' / gem_name - download_path.mkdir(exist_ok=True, parents=True) - download_zip_path = download_path / 'gem.zip' - - downloadable_gem_data = get_downloadable(gem_name=gem_name) - if not downloadable_gem_data: - logger.error(f'Downloadable gem {gem_name} not found.') - return 1 - - origin = downloadable_gem_data['origin'] - url = f'{origin}/gem.zip' - parsed_uri = urllib.parse.urlparse(url) - - if download_zip_path.is_file(): - logger.warn(f'Project already downloaded to {download_zip_path}.') - elif parsed_uri.scheme == 'http' or \ - parsed_uri.scheme == 'https' or \ - parsed_uri.scheme == 'ftp' or \ - parsed_uri.scheme == 'ftps': - with urllib.request.urlopen(url) as s: - with download_zip_path.open('wb') as f: - shutil.copyfileobj(s, f) - else: - origin_file = pathlib.Path(url).resolve() - if not origin_file.is_file(): - return 1 - shutil.copy(origin_file, download_zip_path) - - if not zipfile.is_zipfile(download_zip_path): - logger.error(f"Gem zip {download_zip_path} is invalid.") - download_zip_path.unlink() - return 1 - - # if the gem.json has a sha256 check it against a sha256 of the zip - try: - sha256A = downloadable_gem_data['sha256'] - except Exception as e: - logger.warn(f'SECURITY WARNING: The advertised gem you downloaded has no "sha256"!!! Be VERY careful!!!' - f' We cannot verify this is the actually the advertised gem!!!') - else: - sha256B = hashlib.sha256(download_zip_path.open('rb').read()).hexdigest() - if sha256A != sha256B: - logger.error(f'SECURITY VIOLATION: Downloaded gem.zip sha256 {sha256B} does not match' - f' the advertised "sha256":{sha256A} in the gem.json. Deleting unzipped files!!!') - shutil.rmtree(dest_path) - return 1 - - dest_gem_folder = dest_path / gem_name - if dest_gem_folder.is_dir(): - utils.backup_folder(dest_gem_folder) - with zipfile.ZipFile(download_zip_path, 'r') as gem_zip: - try: - gem_zip.extractall(dest_path) - except Exception as e: - logger.error(f'UnZip exception:{str(e)}') - shutil.rmtree(dest_path) - return 1 - - unzipped_gem_json = dest_gem_folder / 'gem.json' - if not unzipped_gem_json.is_file(): - logger.error(f'Engine json {unzipped_gem_json} is missing.') - return 1 - - if not validation.valid_o3de_engine_json(unzipped_gem_json): - logger.error(f'Engine json {unzipped_gem_json} is invalid.') - return 1 - - # remove the sha256 if present in the advertised downloadable gem.json - # then compare it to the gem.json in the zip, they should now be identical - try: - del downloadable_gem_data['sha256'] - except Exception as e: - pass - - sha256A = hashlib.sha256(json.dumps(downloadable_gem_data, indent=4).encode('utf8')).hexdigest() - with unzipped_gem_json.open('r') as s: - try: - unzipped_gem_json_data = json.load(s) - except Exception as e: - logger.error(f'Failed to read gem json {unzipped_gem_json}. Unable to confirm this' - f' is the same gem that was advertised.') - return 1 - sha256B = hashlib.sha256(json.dumps(unzipped_gem_json_data, indent=4).encode('utf8')).hexdigest() - if sha256A != sha256B: - logger.error(f'SECURITY VIOLATION: Downloaded gem.json does not match' - f' the advertised gem.json. Deleting unzipped files!!!') - shutil.rmtree(dest_path) - return 1 - - return 0 + return download_o3de_object(gem_name, 'gems', dest_path, 'gem', 'gem_name') def download_template(template_name: str, dest_path: str or pathlib.Path) -> int: - if not dest_path: - dest_path = manifest.get_registered(default_folder='templates') - if not dest_path: - logger.error(f'Destination path not cannot be empty.') - return 1 + return download_o3de_object(template_name, 'templates', dest_path, 'template', 'template_name') - dest_path = pathlib.Path(dest_path).resolve() - dest_path.mkdir(exist_ok=True, parents=True) - - download_path = manifest.get_o3de_download_folder() / 'templates' / template_name - download_path.mkdir(exist_ok=True, parents=True) - download_zip_path = download_path / 'template.zip' - - downloadable_template_data = get_downloadable(template_name=template_name) - if not downloadable_template_data: - logger.error(f'Downloadable template {template_name} not found.') - return 1 - - origin = downloadable_template_data['origin'] - url = f'{origin}/project.zip' - parsed_uri = urllib.parse.urlparse(url) - - result = 0 - - if download_zip_path.is_file(): - logger.warn(f'Project already downloaded to {download_zip_path}.') - elif parsed_uri.scheme == 'http' or \ - parsed_uri.scheme == 'https' or \ - parsed_uri.scheme == 'ftp' or \ - parsed_uri.scheme == 'ftps': - with urllib.request.urlopen(url) as s: - with download_zip_path.open('wb') as f: - shutil.copyfileobj(s, f) - else: - origin_file = pathlib.Path(url).resolve() - if not origin_file.is_file(): - return 1 - shutil.copy(origin_file, download_zip_path) - - if not zipfile.is_zipfile(download_zip_path): - logger.error(f"Template zip {download_zip_path} is invalid.") - download_zip_path.unlink() - return 1 - - # if the template.json has a sha256 check it against a sha256 of the zip - try: - sha256A = downloadable_template_data['sha256'] - except Exception as e: - logger.warn(f'SECURITY WARNING: The advertised template you downloaded has no "sha256"!!! Be VERY careful!!!' - f' We cannot verify this is the actually the advertised template!!!') - else: - sha256B = hashlib.sha256(download_zip_path.open('rb').read()).hexdigest() - if sha256A != sha256B: - logger.error(f'SECURITY VIOLATION: Downloaded template.zip sha256 {sha256B} does not match' - f' the advertised "sha256":{sha256A} in the template.json. Deleting unzipped files!!!') - shutil.rmtree(dest_path) - return 1 - - dest_template_folder = dest_path / template_name - if dest_template_folder.is_dir(): - utils.backup_folder(dest_template_folder) - with zipfile.ZipFile(download_zip_path, 'r') as project_zip: - try: - project_zip.extractall(dest_path) - except Exception as e: - logger.error(f'UnZip exception:{str(e)}') - shutil.rmtree(dest_path) - return 1 - - unzipped_template_json = dest_template_folder / 'template.json' - if not unzipped_template_json.is_file(): - logger.error(f'Template json {unzipped_template_json} is missing.') - return 1 - - if not validation.valid_o3de_engine_json(unzipped_template_json): - logger.error(f'Template json {unzipped_template_json} is invalid.') - return 1 - - # remove the sha256 if present in the advertised downloadable template.json - # then compare it to the template.json in the zip, they should now be identical - try: - del downloadable_template_data['sha256'] - except Exception as e: - pass - - sha256A = hashlib.sha256(json.dumps(downloadable_template_data, indent=4).encode('utf8')).hexdigest() - with unzipped_template_json.open('r') as s: - try: - unzipped_template_json_data = json.load(s) - except Exception as e: - logger.error(f'Failed to read Template json {unzipped_template_json}. Unable to confirm this' - f' is the same template that was advertised.') - return 1 - sha256B = hashlib.sha256(json.dumps(unzipped_template_json_data, indent=4).encode('utf8')).hexdigest() - if sha256A != sha256B: - logger.error(f'SECURITY VIOLATION: Downloaded template.json does not match' - f' the advertised template.json. Deleting unzipped files!!!') - shutil.rmtree(dest_path) - return 1 - - return 0 def download_restricted(restricted_name: str, dest_path: str or pathlib.Path) -> int: - if not dest_path: - dest_path = manifest.get_registered(default_folder='restricted') - if not dest_path: - logger.error(f'Destination path not cannot be empty.') - return 1 - - dest_path = pathlib.Path(dest_path).resolve() - dest_path.mkdir(exist_ok=True, parents=True) - - download_path = manifest.get_o3de_download_folder() / 'restricted' / restricted_name - download_path.mkdir(exist_ok=True, parents=True) - download_zip_path = download_path / 'restricted.zip' - - downloadable_restricted_data = get_downloadable(restricted_name=restricted_name) - if not downloadable_restricted_data: - logger.error(f'Downloadable Restricted {restricted_name} not found.') - return 1 - - origin = downloadable_restricted_data['origin'] - url = f'{origin}/restricted.zip' - parsed_uri = urllib.parse.urlparse(url) - - if download_zip_path.is_file(): - logger.warn(f'Restricted already downloaded to {download_zip_path}.') - elif parsed_uri.scheme == 'http' or \ - parsed_uri.scheme == 'https' or \ - parsed_uri.scheme == 'ftp' or \ - parsed_uri.scheme == 'ftps': - with urllib.request.urlopen(url) as s: - with download_zip_path.open('wb') as f: - shutil.copyfileobj(s, f) - else: - origin_file = pathlib.Path(url).resolve() - if not origin_file.is_file(): - return 1 - shutil.copy(origin_file, download_zip_path) - - if not zipfile.is_zipfile(download_zip_path): - logger.error(f"Restricted zip {download_zip_path} is invalid.") - download_zip_path.unlink() - return 1 - - # if the restricted.json has a sha256 check it against a sha256 of the zip - try: - sha256A = downloadable_restricted_data['sha256'] - except Exception as e: - logger.warn(f'SECURITY WARNING: The advertised restricted you downloaded has no "sha256"!!! Be VERY careful!!!' - f' We cannot verify this is the actually the advertised restricted!!!') - else: - sha256B = hashlib.sha256(download_zip_path.open('rb').read()).hexdigest() - if sha256A != sha256B: - logger.error(f'SECURITY VIOLATION: Downloaded restricted.zip sha256 {sha256B} does not match' - f' the advertised "sha256":{sha256A} in the restricted.json. Deleting unzipped files!!!') - shutil.rmtree(dest_path) - return 1 - - dest_restricted_folder = dest_path / restricted_name - if dest_restricted_folder.is_dir(): - utils.backup_folder(dest_restricted_folder) - with zipfile.ZipFile(download_zip_path, 'r') as project_zip: - try: - project_zip.extractall(dest_path) - except Exception as e: - logger.error(f'UnZip exception:{str(e)}') - shutil.rmtree(dest_path) - return 1 - - unzipped_restricted_json = dest_restricted_folder / 'restricted.json' - if not unzipped_restricted_json.is_file(): - logger.error(f'Restricted json {unzipped_restricted_json} is missing.') - return 1 - - if not validation.valid_o3de_engine_json(unzipped_restricted_json): - logger.error(f'Restricted json {unzipped_restricted_json} is invalid.') - return 1 - - # remove the sha256 if present in the advertised downloadable restricted.json - # then compare it to the restricted.json in the zip, they should now be identical - try: - del downloadable_restricted_data['sha256'] - except Exception as e: - pass - - sha256A = hashlib.sha256(json.dumps(downloadable_restricted_data, indent=4).encode('utf8')).hexdigest() - with unzipped_restricted_json.open('r') as s: - try: - unzipped_restricted_json_data = json.load(s) - except Exception as e: - logger.error( - f'Failed to read Restricted json {unzipped_restricted_json}. Unable to confirm this' - f' is the same restricted that was advertised.') - return 1 - sha256B = hashlib.sha256( - json.dumps(unzipped_restricted_json_data, indent=4).encode('utf8')).hexdigest() - if sha256A != sha256B: - logger.error(f'SECURITY VIOLATION: Downloaded restricted.json does not match' - f' the advertised restricted.json. Deleting unzipped files!!!') - shutil.rmtree(dest_path) - return 1 - - return 0 + return download_o3de_object(restricted_name, 'restricted', dest_path, 'restricted', 'restricted_name') def _run_download(args: argparse) -> int: @@ -562,20 +172,16 @@ def _run_download(args: argparse) -> int: return download_template(args.template_name, args.dest_path) + return 1 -def add_args(parser, subparsers) -> None: +def add_parser_args(parser): """ - add_args is called to add expected parser arguments and subparsers arguments to each command such that it can be + add_parser_args is called to add arguments to each command such that it can be invoked locally or added by a central python file. - Ex. Directly run from this file alone with: python register.py register --gem-path "C:/TestGem" - OR - o3de.py can downloadable commands by importing engine_template, - call add_args and execute: python o3de.py register --gem-path "C:/TestGem" - :param parser: the caller instantiates a parser and passes it in here - :param subparsers: the caller instantiates subparsers and passes it in here + Ex. Directly run from this file alone with: python download.py --engine-name "o3de" + :param parser: the caller passes an argparse parser like instance to this method """ - download_subparser = subparsers.add_parser('download') - group = download_subparser.add_mutually_exclusive_group(required=True) + group = parser.add_mutually_exclusive_group(required=True) group.add_argument('-e', '--engine-name', type=str, required=False, help='Downloadable engine name.') group.add_argument('-p', '--project-name', type=str, required=False, @@ -584,15 +190,52 @@ def add_args(parser, subparsers) -> None: help='Downloadable gem name.') group.add_argument('-t', '--template-name', type=str, required=False, help='Downloadable template name.') - download_subparser.add_argument('-dp', '--dest-path', type=str, required=False, + parser.add_argument('-dp', '--dest-path', type=str, required=False, default=None, help='Optional destination folder to download into.' - ' i.e. download --project-name "StarterGame" --dest-path "C:/projects"' - ' will result in C:/projects/StarterGame' + ' i.e. download --project-name "AstomSamplerViewer" --dest-path "C:/projects"' + ' will result in C:/projects/AtomSampleViewer' ' If blank will download to default object type folder') - download_subparser.add_argument('-ohf', '--override-home-folder', type=str, required=False, + parser.add_argument('-ohf', '--override-home-folder', type=str, required=False, help='By default the home folder is the user folder, override it to this folder.') - download_subparser.set_defaults(func=_run_download) + parser.set_defaults(func=_run_download) + +def add_args(subparsers) -> None: + """ + add_args is called to add subparsers arguments to each command such that it can be + a central python file such as o3de.py. + It can be run from the o3de.py script as follows + call add_args and execute: python o3de.py download --engine-name "o3de" + :param subparsers: the caller instantiates subparsers and passes it in here + """ + download_subparser = subparsers.add_parser('download') + add_parser_args(download_subparser) + + +def main(): + """ + Runs download.py script as standalone script + """ + # parse the command line args + the_parser = argparse.ArgumentParser() + + # add subparsers + + # add args to the parser + add_parser_args(the_parser) + + # parse args + the_args = the_parser.parse_args() + + # run + ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1 + + # return + sys.exit(ret) + + +if __name__ == "__main__": + main() diff --git a/scripts/o3de/o3de/engine_template.py b/scripts/o3de/o3de/engine_template.py index 324c204a43..0c63d4e42e 100755 --- a/scripts/o3de/o3de/engine_template.py +++ b/scripts/o3de/o3de/engine_template.py @@ -79,7 +79,7 @@ restricted_platforms = { } template_file_name = 'template.json' - +this_script_parent = os.path.dirname(os.path.realpath(__file__)) def _transform(s_data: str, replacements: list, @@ -329,7 +329,7 @@ def _instantiate_template(template_json_data: dict, with open(platform_json, 'r') as s: try: json_data = json.load(s) - except Exception as e: + except json.JSONDecodeError as e: logger.error(f'Failed to load {platform_json}: ' + str(e)) return 1 else: @@ -407,7 +407,7 @@ def create_template(source_path: str, template_path = f'{default_templates_folder}/{template_path}' logger.info(f'Template path not a full path. Using default templates folder {template_path}') if os.path.isdir(template_path): - logger.error(f'Template path {template_path} is already exists.') + logger.error(f'Template path {template_path} already exists.') return 1 # template name is now the last component of the template_path @@ -432,12 +432,12 @@ def create_template(source_path: str, with open(engine_json) as s: try: engine_json_data = json.load(s) - except Exception as e: + except json.JSONDecodeError as e: logger.error(f"Failed to read engine json {engine_json}: {str(e)}") return 1 try: engine_restricted = engine_json_data['restricted_name'] - except Exception as e: + except KeyError as e: logger.error(f"Engine json {engine_json} restricted not found.") return 1 engine_restricted_folder = manifest.get_registered(restricted_name=engine_restricted) @@ -475,12 +475,12 @@ def create_template(source_path: str, with open(restricted_json, 'r') as s: try: restricted_json_data = json.load(s) - except Exception as e: + except json.JSONDecodeError as e: logger.error(f'Failed to load {restricted_json}: ' + str(e)) return 1 try: template_restricted_name = restricted_json_data['restricted_name'] - except Exception as e: + except KeyError as e: logger.error(f'Failed to read restricted_name from {restricted_json}') return 1 else: @@ -943,8 +943,7 @@ def create_template(source_path: str, s.write(json.dumps(json_data, indent=4)) # copy the default preview.png - this_script_parent = os.path.dirname(os.path.realpath(__file__)) - preview_png_src = f'{this_script_parent}/preview.png' + preview_png_src = f'{this_script_parent}/resources/preview.png' preview_png_dst = f'{template_path}/Template/preview.png' if not os.path.isfile(preview_png_dst): shutil.copy(preview_png_src, preview_png_dst) @@ -1067,14 +1066,14 @@ def create_from_template(destination_path: str, with open(template_json) as s: try: template_json_data = json.load(s) - except Exception as e: + except KeyError as e: logger.error(f'Could read template json {template_json}: {str(e)}.') return 1 # read template name from the json try: template_name = template_json_data['template_name'] - except Exception as e: + except KeyError as e: logger.error(f'Could not read "template_name" from template json {template_json}: {str(e)}.') return 1 @@ -1083,7 +1082,7 @@ def create_from_template(destination_path: str, if not template_restricted_name and not template_restricted_path: try: template_json_restricted_name = template_json_data['restricted_name'] - except Exception as e: + except KeyError as e: # the template json doesn't have a 'restricted_name' element warn and use it logger.info(f'The template does not specify a "restricted_name".') else: @@ -1100,7 +1099,7 @@ def create_from_template(destination_path: str, # The user specified a --template-restricted-name try: template_json_restricted_name = template_json_data['restricted_name'] - except Exception as e: + except KeyError as e: # the template json doesn't have a 'restricted_name' element warn and use it logger.info(f'The template does not specify a "restricted_name".' f' Using supplied {template_restricted_name}') @@ -1120,7 +1119,7 @@ def create_from_template(destination_path: str, template_restricted_path = template_restricted_path.replace('\\', '/') try: template_json_restricted_name = template_json_data['restricted_name'] - except Exception as e: + except KeyError as e: # the template json doesn't have a 'restricted_name' element warn and use it logger.info(f'The template does not specify a "restricted_name".' f' Using supplied {template_restricted_path}') @@ -1154,7 +1153,7 @@ def create_from_template(destination_path: str, try: template_json_restricted_platform_relative_path = template_json_data[ 'restricted_platform_relative_path'] - except Exception as e: + except KeyError as e: # the template json doesn't have a 'restricted_platform_relative_path' element warn and use it logger.info(f'The template does not specify a "restricted_platform_relative_path".' f' Using {template_restricted_platform_relative_path}') @@ -1175,7 +1174,7 @@ def create_from_template(destination_path: str, try: template_restricted_platform_relative_path = template_json_data[ 'restricted_platform_relative_path'] - except Exception as e: + except KeyError as e: # The template json doesn't have a 'restricted_platform_relative_path' element, set empty string. template_restricted_platform_relative_path = '' @@ -1356,14 +1355,14 @@ def create_project(project_path: str, with open(template_json) as s: try: template_json_data = json.load(s) - except Exception as e: + except json.JSONDecodeError as e: logger.error(f'Could read template json {template_json}: {str(e)}.') return 1 # read template name from the json try: template_name = template_json_data['template_name'] - except Exception as e: + except KeyError as e: logger.error(f'Could not read "template_name" from template json {template_json}: {str(e)}.') return 1 @@ -1372,7 +1371,7 @@ def create_project(project_path: str, if not template_restricted_name and not template_restricted_path: try: template_json_restricted_name = template_json_data['restricted_name'] - except Exception as e: + except KeyError as e: # the template json doesn't have a 'restricted_name' element warn and use it logger.info(f'The template does not specify a "restricted_name".') else: @@ -1389,7 +1388,7 @@ def create_project(project_path: str, # The user specified a --template-restricted-name try: template_json_restricted_name = template_json_data['restricted_name'] - except Exception as e: + except KeyError as e: # the template json doesn't have a 'restricted_name' element warn and use it logger.info(f'The template does not specify a "restricted_name".' f' Using supplied {template_restricted_name}') @@ -1409,7 +1408,7 @@ def create_project(project_path: str, template_restricted_path = template_restricted_path.replace('\\', '/') try: template_json_restricted_name = template_json_data['restricted_name'] - except Exception as e: + except KeyError as e: # the template json doesn't have a 'restricted_name' element warn and use it logger.info(f'The template does not specify a "restricted_name".' f' Using supplied {template_restricted_path}') @@ -1442,7 +1441,7 @@ def create_project(project_path: str, try: template_json_restricted_platform_relative_path = template_json_data[ 'restricted_platform_relative_path'] - except Exception as e: + except KeyError as e: # the template json doesn't have a 'restricted_platform_relative_path' element warn and use it logger.info(f'The template does not specify a "restricted_platform_relative_path".' f' Using {template_restricted_platform_relative_path}') @@ -1463,7 +1462,7 @@ def create_project(project_path: str, try: template_restricted_platform_relative_path = template_json_data[ 'restricted_platform_relative_path'] - except Exception as e: + except KeyError as e: # The template json doesn't have a 'restricted_platform_relative_path' element, set empty string. template_restricted_platform_relative_path = '' if not template_restricted_platform_relative_path: @@ -1597,13 +1596,13 @@ def create_project(project_path: str, with open(restricted_json, 'r') as s: try: restricted_json_data = json.load(s) - except Exception as e: + except json.JSONDecodeError as e: logger.error(f'Failed to load restricted json {restricted_json}.') return 1 try: restricted_name = restricted_json_data["restricted_name"] - except Exception as e: + except KeyError as e: logger.error(f'Failed to read "restricted_name" from restricted json {restricted_json}.') return 1 @@ -1616,7 +1615,7 @@ def create_project(project_path: str, with open(project_json, 'r') as s: try: project_json_data = json.load(s) - except Exception as e: + except json.JSONDecodeError as e: logger.error(f'Failed to load project json {project_json}.') return 1 @@ -1625,7 +1624,7 @@ def create_project(project_path: str, with open(project_json, 'w') as s: try: s.write(json.dumps(project_json_data, indent=4)) - except Exception as e: + except OSError as e: logger.error(f'Failed to write project json {project_json}.') return 1 @@ -1656,7 +1655,7 @@ def create_project(project_path: str, engine_json_data = manifest.get_engine_json_data(engine_path=manifest.get_this_engine_path()) try: engine_name = engine_json_data['engine_name'] - except Exception as e: + except KeyError as e: logger.error(f"engine_name for this engine not found in engine.json.") return 1 @@ -1665,7 +1664,7 @@ def create_project(project_path: str, with open(project_json, 'w') as s: try: s.write(json.dumps(project_json_data, indent=4)) - except Exception as e: + except OSError as e: logger.error(f'Failed to write project json at {project_path}.') return 1 @@ -1749,14 +1748,14 @@ def create_gem(gem_path: str, with open(template_json) as s: try: template_json_data = json.load(s) - except Exception as e: + except json.JSONDecodeError as e: logger.error(f'Could read template json {template_json}: {str(e)}.') return 1 # read template name from the json try: template_name = template_json_data['template_name'] - except Exception as e: + except KeyError as e: logger.error(f'Could not read "template_name" from template json {template_json}: {str(e)}.') return 1 @@ -1765,7 +1764,7 @@ def create_gem(gem_path: str, if not template_restricted_name and not template_restricted_path: try: template_json_restricted_name = template_json_data['restricted_name'] - except Exception as e: + except KeyError as e: # the template json doesn't have a 'restricted_name' element warn and use it logger.info(f'The template does not specify a "restricted_name".') else: @@ -1781,7 +1780,7 @@ def create_gem(gem_path: str, # The user specified a --template-restricted-name try: template_json_restricted_name = template_json_data['restricted_name'] - except Exception as e: + except KeyError as e: # the template json doesn't have a 'restricted_name' element warn and use it logger.info(f'The template does not specify a "restricted_name".' f' Using supplied {template_restricted_name}') @@ -1801,7 +1800,7 @@ def create_gem(gem_path: str, template_restricted_path = template_restricted_path.replace('\\', '/') try: template_json_restricted_name = template_json_data['restricted_name'] - except Exception as e: + except KeyError as e: # the template json doesn't have a 'restricted_name' element warn and use it logger.info(f'The template does not specify a "restricted_name".' f' Using supplied {template_restricted_path}') @@ -1833,7 +1832,7 @@ def create_gem(gem_path: str, try: template_json_restricted_platform_relative_path = template_json_data[ 'restricted_platform_relative_path'] - except Exception as e: + except KeyError as e: # the template json doesn't have a 'restricted_platform_relative_path' element warn and use it logger.info(f'The template does not specify a "restricted_platform_relative_path".' f' Using {template_restricted_platform_relative_path}') @@ -1854,7 +1853,7 @@ def create_gem(gem_path: str, try: template_restricted_platform_relative_path = template_json_data[ 'restricted_platform_relative_path'] - except Exception as e: + except KeyError as e: # The template json doesn't have a 'restricted_platform_relative_path' element, set empty string. template_restricted_platform_relative_path = '' if not template_restricted_platform_relative_path: @@ -1988,13 +1987,13 @@ def create_gem(gem_path: str, with open(restricted_json, 'r') as s: try: restricted_json_data = json.load(s) - except Exception as e: + except json.JSONDecodeError as e: logger.error(f'Failed to load restricted json {restricted_json}.') return 1 try: restricted_name = restricted_json_data["restricted_name"] - except Exception as e: + except KeyError as e: logger.error(f'Failed to read "restricted_name" from restricted json {restricted_json}.') return 1 @@ -2007,7 +2006,7 @@ def create_gem(gem_path: str, with open(gem_json, 'r') as s: try: gem_json_data = json.load(s) - except Exception as e: + except json.JSONDecodeError as e: logger.error(f'Failed to load gem json {gem_json}.') return 1 @@ -2016,7 +2015,7 @@ def create_gem(gem_path: str, with open(gem_json, 'w') as s: try: s.write(json.dumps(gem_json_data, indent=4)) - except Exception as e: + except OSError as e: logger.error(f'Failed to write project json {gem_json}.') return 1 @@ -2110,15 +2109,14 @@ def _run_create_gem(args: argparse) -> int: args.module_id) -def add_args(parser, subparsers) -> None: +def add_args(subparsers) -> None: """ add_args is called to add expected parser arguments and subparsers arguments to each command such that it can be invoked locally or aggregated by a central python file. - Ex. Directly run from this file alone with: python engine_template.py create_gem --gem-path TestGem + Ex. Directly run from this file alone with: python engine_template.py create-gem --gem-path TestGem OR o3de.py can aggregate commands by importing engine_template, - call add_args and execute: python o3de.py create_gem --gem-path TestGem - :param parser: the caller instantiates a parser and passes it in here + call add_args and execute: python o3de.py create-gem --gem-path TestGem :param subparsers: the caller instantiates subparsers and passes it in here """ # turn a directory into a template @@ -2438,13 +2436,12 @@ if __name__ == "__main__": the_subparsers = the_parser.add_subparsers(help='sub-command help', dest='command', required=True) # add args to the parser - add_args(the_parser, the_subparsers) + add_args(the_subparsers) # parse args the_args = the_parser.parse_args() # run - ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1 # return diff --git a/scripts/o3de/o3de/get_registration.py b/scripts/o3de/o3de/get_registration.py index c38d4d1cfb..d51600826c 100644 --- a/scripts/o3de/o3de/get_registration.py +++ b/scripts/o3de/o3de/get_registration.py @@ -11,6 +11,7 @@ import argparse import pathlib +import sys from o3de import manifest @@ -27,19 +28,14 @@ def _run_get_registered(args: argparse) -> str or pathlib.Path: args.restricted_name) -def add_args(parser, subparsers) -> None: +def add_parser_args(parser): """ - add_args is called to add expected parser arguments and subparsers arguments to each command such that it can be + add_parser_args is called to add arguments to each command such that it can be invoked locally or added by a central python file. - Ex. Directly run from this file alone with: python register.py register --gem-path "C:/TestGem" - OR - o3de.py can downloadable commands by importing engine_template, - call add_args and execute: python o3de.py register --gem-path "C:/TestGem" - :param parser: the caller instantiates a parser and passes it in here - :param subparsers: the caller instantiates subparsers and passes it in here + Ex. Directly run from this file alone with: python get_registration.py --engine-name "o3de" + :param parser: the caller passes an argparse parser like instance to this method """ - get_registered_subparser = subparsers.add_parser('get-registered') - group = get_registered_subparser.add_mutually_exclusive_group(required=True) + group = parser.add_mutually_exclusive_group(required=True) group.add_argument('-en', '--engine-name', type=str, required=False, help='Engine name.') group.add_argument('-pn', '--project-name', type=str, required=False, @@ -56,7 +52,45 @@ def add_args(parser, subparsers) -> None: group.add_argument('-rsn', '--restricted-name', type=str, required=False, help='Restricted name.') - get_registered_subparser.add_argument('-ohf', '--override-home-folder', type=str, required=False, - help='By default the home folder is the user folder, override it to this folder.') + parser.add_argument('-ohf', '--override-home-folder', type=str, required=False, + help='By default the home folder is the user folder, override it to this folder.') - get_registered_subparser.set_defaults(func=_run_get_registered) + parser.set_defaults(func=_run_get_registered) + + +def add_args(subparsers) -> None: + """ + add_args is called to add subparsers arguments to each command such that it can be + a central python file such as o3de.py. + It can be run from the o3de.py script as follows + call add_args and execute: python o3de.py get-registered --engine-name "o3de" + :param subparsers: the caller instantiates subparsers and passes it in here + """ + get_registered_subparser = subparsers.add_parser('get-registered') + add_parser_args(get_registered_subparser) + + +def main(): + """ + Runs get_registration.py script as standalone script + """ + # parse the command line args + the_parser = argparse.ArgumentParser() + + # add subparsers + + # add args to the parser + add_parser_args(the_parser) + + # parse args + the_args = the_parser.parse_args() + + # run + ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1 + + # return + sys.exit(ret) + + +if __name__ == "__main__": + main() diff --git a/scripts/o3de/o3de/global_project.py b/scripts/o3de/o3de/global_project.py index 1a17e3b79e..787e676a7e 100644 --- a/scripts/o3de/o3de/global_project.py +++ b/scripts/o3de/o3de/global_project.py @@ -52,17 +52,17 @@ def set_global_project(project_name: str or None, with bootstrap_setreg_file.open('r') as f: try: json_data = json.load(f) - except Exception as e: + except json.JSONDecodeError as e: logger.error(f'Bootstrap.setreg failed to load: {str(e)}') else: try: json_data["Amazon"]["AzCore"]["Bootstrap"]["project_path"] = project_path - except Exception as e: + except KeyError as e: logger.error(f'Bootstrap.setreg failed to load: {str(e)}') else: try: os.unlink(bootstrap_setreg_file) - except Exception as e: + except OSError as e: logger.error(f'Failed to unlink bootstrap file {bootstrap_setreg_file}: {str(e)}') return 1 else: @@ -88,12 +88,12 @@ def get_global_project() -> pathlib.Path or None: with bootstrap_setreg_file.open('r') as f: try: json_data = json.load(f) - except Exception as e: + except json.JSONDecodeError as e: logger.error(f'Bootstrap.setreg failed to load: {str(e)}') else: try: project_path = json_data["Amazon"]["AzCore"]["Bootstrap"]["project_path"] - except Exception as e: + except KeyError as e: logger.error(f'Bootstrap.setreg cannot find Amazon:AzCore:Bootstrap:project_path: {str(e)}') else: return pathlib.Path(project_path).resolve() @@ -118,7 +118,7 @@ def _run_set_global_project(args: argparse) -> int: args.project_path) -def add_args(parser, subparsers) -> None: +def add_args(subparsers) -> None: """ add_args is called to add expected parser arguments and subparsers arguments to each command such that it can be invoked locally or aggregated by a central python file. @@ -126,7 +126,6 @@ def add_args(parser, subparsers) -> None: OR o3de.py can aggregate commands by importing global_project, call add_args and execute: python o3de.py set_global_project --project-path C:/TestProject - :param parser: the caller instantiates a parser and passes it in here :param subparsers: the caller instantiates subparsers and passes it in here """ get_global_project_subparser = subparsers.add_parser('get-global-project') @@ -156,7 +155,7 @@ if __name__ == "__main__": the_subparsers = the_parser.add_subparsers(help='sub-command help', dest='command', required=True) # add args to the parser - add_args(the_parser, the_subparsers) + add_args(the_subparsers) # parse args the_args = the_parser.parse_args() diff --git a/scripts/o3de/o3de/manifest.py b/scripts/o3de/o3de/manifest.py index 44d6ff1b61..6c14c2533f 100644 --- a/scripts/o3de/o3de/manifest.py +++ b/scripts/o3de/o3de/manifest.py @@ -176,7 +176,7 @@ def load_o3de_manifest() -> dict: with get_o3de_manifest().open('r') as f: try: json_data = json.load(f) - except Exception as e: + except json.JSONDecodeError as e: logger.error(f'Manifest json failed to load: {str(e)}') return {} else: @@ -187,7 +187,7 @@ def save_o3de_manifest(json_data: dict) -> None: with get_o3de_manifest().open('w') as s: try: s.write(json.dumps(json_data, indent=4)) - except Exception as e: + except OSError as e: logger.error(f'Manifest json failed to save: {str(e)}') @@ -331,7 +331,7 @@ def get_engine_json_data(engine_name: str = None, with engine_json.open('r') as f: try: engine_json_data = json.load(f) - except Exception as e: + except json.JSONDecodeError as e: logger.warn(f'{engine_json} failed to load: {str(e)}') else: return engine_json_data @@ -364,7 +364,7 @@ def get_project_json_data(project_name: str = None, with project_json.open('r') as f: try: project_json_data = json.load(f) - except Exception as e: + except json.JSONDecodeError as e: logger.warn(f'{project_json} failed to load: {str(e)}') else: return project_json_data @@ -397,7 +397,7 @@ def get_gem_json_data(gem_name: str = None, with gem_json.open('r') as f: try: gem_json_data = json.load(f) - except Exception as e: + except json.JSONDecodeError as e: logger.warn(f'{gem_json} failed to load: {str(e)}') else: return gem_json_data @@ -430,7 +430,7 @@ def get_template_json_data(template_name: str = None, with template_json.open('r') as f: try: template_json_data = json.load(f) - except Exception as e: + except json.JSONDecodeError as e: logger.warn(f'{template_json} failed to load: {str(e)}') else: return template_json_data @@ -463,7 +463,7 @@ def get_restricted_data(restricted_name: str = None, with restricted_json.open('r') as f: try: restricted_json_data = json.load(f) - except Exception as e: + except json.JSONDecodeError as e: logger.warn(f'{restricted_json} failed to load: {str(e)}') else: return restricted_json_data @@ -488,7 +488,7 @@ def get_registered(engine_name: str = None, with engine_json.open('r') as f: try: engine_json_data = json.load(f) - except Exception as e: + except json.JSONDecodeError as e: logger.warn(f'{engine_json} failed to load: {str(e)}') else: this_engines_name = engine_json_data['engine_name'] @@ -505,7 +505,7 @@ def get_registered(engine_name: str = None, with project_json.open('r') as f: try: project_json_data = json.load(f) - except Exception as e: + except json.JSONDecodeError as e: logger.warn(f'{project_json} failed to load: {str(e)}') else: this_projects_name = project_json_data['project_name'] @@ -522,7 +522,7 @@ def get_registered(engine_name: str = None, with gem_json.open('r') as f: try: gem_json_data = json.load(f) - except Exception as e: + except json.JSONDecodeError as e: logger.warn(f'{gem_json} failed to load: {str(e)}') else: this_gems_name = gem_json_data['gem_name'] @@ -539,7 +539,7 @@ def get_registered(engine_name: str = None, with template_json.open('r') as f: try: template_json_data = json.load(f) - except Exception as e: + except json.JSONDecodeError as e: logger.warn(f'{template_path} failed to load: {str(e)}') else: this_templates_name = template_json_data['template_name'] @@ -556,7 +556,7 @@ def get_registered(engine_name: str = None, with restricted_json.open('r') as f: try: restricted_json_data = json.load(f) - except Exception as e: + except json.JSONDecodeError as e: logger.warn(f'{restricted_json} failed to load: {str(e)}') else: this_restricted_name = restricted_json_data['restricted_name'] @@ -591,7 +591,7 @@ def get_registered(engine_name: str = None, with repo.open('r') as f: try: repo_json_data = json.load(f) - except Exception as e: + except json.JSONDecodeError as e: logger.warn(f'{cache_file} failed to load: {str(e)}') else: this_repos_name = repo_json_data['repo_name'] diff --git a/scripts/o3de/o3de/print_registration.py b/scripts/o3de/o3de/print_registration.py index 7900fad7e4..292f2224bc 100644 --- a/scripts/o3de/o3de/print_registration.py +++ b/scripts/o3de/o3de/print_registration.py @@ -13,6 +13,7 @@ import argparse import json import hashlib import logging +import sys import urllib.parse from o3de import manifest, validation @@ -143,7 +144,7 @@ def print_engines_data(engines_data: dict) -> None: with engine_json.open('r') as f: try: engine_json_data = json.load(f) - except Exception as e: + except json.JSONDecodeError as e: logger.warn(f'{engine_json} failed to load: {str(e)}') else: print(engine_json) @@ -170,7 +171,7 @@ def print_projects_data(projects_data: dict) -> None: with project_json.open('r') as f: try: project_json_data = json.load(f) - except Exception as e: + except json.JSONDecodeError as e: logger.warn(f'{project_json} failed to load: {str(e)}') else: print(project_json) @@ -197,7 +198,7 @@ def print_gems_data(gems_data: dict) -> None: with gem_json.open('r') as f: try: gem_json_data = json.load(f) - except Exception as e: + except json.JSONDecodeError as e: logger.warn(f'{gem_json} failed to load: {str(e)}') else: print(gem_json) @@ -224,7 +225,7 @@ def print_templates_data(templates_data: dict) -> None: with template_json.open('r') as f: try: template_json_data = json.load(f) - except Exception as e: + except json.JSONDecodeError as e: logger.warn(f'{template_json} failed to load: {str(e)}') else: print(template_json) @@ -243,7 +244,7 @@ def print_repos_data(repos_data: dict) -> None: with cache_file.open('r') as s: try: repo_json_data = json.load(s) - except Exception as e: + except json.JSONDecodeError as e: logger.warn(f'{cache_file} failed to load: {str(e)}') else: print(f'{repo_uri}/repo.json cached as:') @@ -260,7 +261,7 @@ def print_restricted_data(restricted_data: dict) -> None: with restricted_json.open('r') as f: try: restricted_json_data = json.load(f) - except Exception as e: + except json.JSONDecodeError as e: logger.warn(f'{restricted_json} failed to load: {str(e)}') else: print(restricted_json) @@ -365,19 +366,14 @@ def _run_register_show(args: argparse) -> int: return 0 -def add_args(parser, subparsers) -> None: +def add_parser_args(parser): """ - add_args is called to add expected parser arguments and subparsers arguments to each command such that it can be + add_parser_args is called to add arguments to each command such that it can be invoked locally or added by a central python file. - Ex. Directly run from this file alone with: python register.py register --gem-path "C:/TestGem" - OR - o3de.py can downloadable commands by importing engine_template, - call add_args and execute: python o3de.py register --gem-path "C:/TestGem" - :param parser: the caller instantiates a parser and passes it in here - :param subparsers: the caller instantiates subparsers and passes it in here + Ex. Directly run from this file alone with: python print_registration.py --engine-projects + :param parser: the caller passes an argparse parser like instance to this method """ - register_show_subparser = subparsers.add_parser('register-show') - group = register_show_subparser.add_mutually_exclusive_group(required=False) + group = parser.add_mutually_exclusive_group(required=False) group.add_argument('-te', '--this-engine', action='store_true', required=False, default=False, help='Just the local engines.') @@ -446,11 +442,49 @@ def add_args(parser, subparsers) -> None: default=False, help='Combine all repos templates into a single list of resources.') - register_show_subparser.add_argument('-v', '--verbose', action='count', required=False, + parser.add_argument('-v', '--verbose', action='count', required=False, default=0, help='How verbose do you want the output to be.') - register_show_subparser.add_argument('-ohf', '--override-home-folder', type=str, required=False, + parser.add_argument('-ohf', '--override-home-folder', type=str, required=False, help='By default the home folder is the user folder, override it to this folder.') - register_show_subparser.set_defaults(func=_run_register_show) \ No newline at end of file + parser.set_defaults(func=_run_register_show) + + +def add_args(subparsers) -> None: + """ + add_args is called to add subparsers arguments to each command such that it can be + a central python file such as o3de.py. + It can be run from the o3de.py script as follows + call add_args and execute: python o3de.py register-show --engine-projects + :param subparsers: the caller instantiates subparsers and passes it in here + """ + register_show_subparser = subparsers.add_parser('register-show') + add_parser_args(register_show_subparser) + + +def main(): + """ + Runs print_registration.py script as standalone script + """ + # parse the command line args + the_parser = argparse.ArgumentParser() + + # add subparsers + + # add args to the parser + add_parser_args(the_parser) + + # parse args + the_args = the_parser.parse_args() + + # run + ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1 + + # return + sys.exit(ret) + + +if __name__ == "__main__": + main() diff --git a/scripts/o3de/o3de/register.py b/scripts/o3de/o3de/register.py index d6a734e1fd..c44af03b30 100644 --- a/scripts/o3de/o3de/register.py +++ b/scripts/o3de/o3de/register.py @@ -19,6 +19,7 @@ import json import os import pathlib import shutil +import sys import urllib.parse import urllib.request @@ -134,172 +135,74 @@ def register_all_in_folder(folder_path: str or pathlib.Path, return ret_val -def register_all_engines_in_folder(engines_path: str or pathlib.Path, - remove: bool = False, - force: bool = False) -> int: - if not engines_path: +def register_all_o3de_objects_of_type_in_folder(o3de_object_path: str or pathlib.Path, + o3de_object_type: str, + remove: bool, + force: bool, + **register_kwargs) -> int: + if not o3de_object_path: logger.error(f'Engines path cannot be empty.') return 1 - engines_path = pathlib.Path(engines_path).resolve() - if not engines_path.is_dir(): + o3de_object_path = pathlib.Path(o3de_object_path).resolve() + if not o3de_object_path.is_dir(): logger.error(f'Engines path is not dir.') return 1 - engines_set = set() + o3de_object_type_set = set() + register_path_kwarg = f'{o3de_object_type}_path' if o3de_object_type != 'repo' else f'{o3de_object_type}_uri' ret_val = 0 - for root, dirs, files in os.walk(engines_path): - for name in files: - if name == 'engine.json': - engines_set.add(root) + for root, dirs, files in os.walk(o3de_object_path): + if f'{o3de_object_type}.json' in files: + o3de_object_type_set.add(root) + # Stop iteration of any subdirectories + # Nested o3de objects of the same type aren't supported(i.e an engine cannot be inside of a engine). + dirs[:] = [] - for engine in sorted(engines_set, reverse=True): - error_code = register(engine_path=engine, remove=remove, force=force) + for o3de_object_type_root in sorted(o3de_object_type_set, reverse=True): + error_code = register(**{register_path_kwarg: o3de_object_type_root}, + remove=remove, force=force, **register_kwargs) if error_code: ret_val = error_code return ret_val +def register_all_engines_in_folder(engines_path: str or pathlib.Path, + remove: bool = False, + force: bool = False) -> int: + return register_all_o3de_objects_of_type_in_folder(engines_path, 'engine', remove, force) + + def register_all_projects_in_folder(projects_path: str or pathlib.Path, remove: bool = False, engine_path: str or pathlib.Path = None) -> int: - if not projects_path: - logger.error(f'Projects path cannot be empty.') - return 1 - - projects_path = pathlib.Path(projects_path).resolve() - if not projects_path.is_dir(): - logger.error(f'Projects path is not dir.') - return 1 - - projects_set = set() - - ret_val = 0 - for root, dirs, files in os.walk(projects_path): - for name in files: - if name == 'project.json': - projects_set.add(root) - - for project in sorted(projects_set, reverse=True): - error_code = register(engine_path=engine_path, project_path=project, remove=remove) - if error_code: - ret_val = error_code - - return ret_val + return register_all_o3de_objects_of_type_in_folder(projects_path, 'project', remove, False, engine_path=engine_path) def register_all_gems_in_folder(gems_path: str or pathlib.Path, remove: bool = False, engine_path: str or pathlib.Path = None) -> int: - if not gems_path: - logger.error(f'Gems path cannot be empty.') - return 1 - - gems_path = pathlib.Path(gems_path).resolve() - if not gems_path.is_dir(): - logger.error(f'Gems path is not dir.') - return 1 - - gems_set = set() - - ret_val = 0 - for root, dirs, files in os.walk(gems_path): - for name in files: - if name == 'gem.json': - gems_set.add(root) - - for gem in sorted(gems_set, reverse=True): - error_code = register(engine_path=engine_path, gem_path=gem, remove=remove) - if error_code: - ret_val = error_code - - return ret_val + return register_all_o3de_objects_of_type_in_folder(gems_path, 'gem', remove, False, engine_path=engine_path) def register_all_templates_in_folder(templates_path: str or pathlib.Path, remove: bool = False, engine_path: str or pathlib.Path = None) -> int: - if not templates_path: - logger.error(f'Templates path cannot be empty.') - return 1 - - templates_path = pathlib.Path(templates_path).resolve() - if not templates_path.is_dir(): - logger.error(f'Templates path is not dir.') - return 1 - - templates_set = set() - - ret_val = 0 - for root, dirs, files in os.walk(templates_path): - for name in files: - if name == 'template.json': - templates_set.add(root) - - for template in sorted(templates_set, reverse=True): - error_code = register(engine_path=engine_path, template_path=template, remove=remove) - if error_code: - ret_val = error_code - - return ret_val + return register_all_o3de_objects_of_type_in_folder(templates_path, 'template', remove, False, engine_path=engine_path) def register_all_restricted_in_folder(restricted_path: str or pathlib.Path, remove: bool = False, engine_path: str or pathlib.Path = None) -> int: - if not restricted_path: - logger.error(f'Restricted path cannot be empty.') - return 1 - - restricted_path = pathlib.Path(restricted_path).resolve() - if not restricted_path.is_dir(): - logger.error(f'Restricted path is not dir.') - return 1 - - restricted_set = set() - - ret_val = 0 - for root, dirs, files in os.walk(restricted_path): - for name in files: - if name == 'restricted.json': - restricted_set.add(root) - - for restricted in sorted(restricted_set, reverse=True): - error_code = register(engine_path=engine_path, restricted_path=restricted, remove=remove) - if error_code: - ret_val = error_code - - return ret_val + return register_all_o3de_objects_of_type_in_folder(restricted_path, 'restricted', remove, False, engine_path=engine_path) def register_all_repos_in_folder(repos_path: str or pathlib.Path, remove: bool = False, engine_path: str or pathlib.Path = None) -> int: - if not repos_path: - logger.error(f'Repos path cannot be empty.') - return 1 - - repos_path = pathlib.Path(repos_path).resolve() - if not repos_path.is_dir(): - logger.error(f'Repos path is not dir.') - return 1 - - repo_set = set() - - ret_val = 0 - for root, dirs, files in os.walk(repos_path): - for name in files: - if name == 'repo.json': - repo_set.add(root) - - for repo in sorted(repo_set, reverse=True): - error_code = register(engine_path=engine_path, repo_uri=repo, remove=remove) - if error_code: - ret_val = error_code - - return ret_val + return register_all_o3de_objects_of_type_in_folder(repos_path, 'repo', remove, force, engine_path=engine_path) def remove_engine_name_to_path(json_data: dict, @@ -395,21 +298,13 @@ def register_gem_path(json_data: dict, logger.error(f'Engine path {engine_path} is not registered.') return 1 - while gem_path in engine_data['gems']: - engine_data['gems'].remove(gem_path) - - while gem_path.as_posix() in engine_data['gems']: - engine_data['gems'].remove(gem_path.as_posix()) + engine_data['gems'] = list(filter(lambda p: gem_path != pathlib.Path(p), engine_data['gems'])) if remove: logger.warn(f'Removing Gem path {gem_path}.') return 0 else: - while gem_path in json_data['gems']: - json_data['gems'].remove(gem_path) - - while gem_path.as_posix() in json_data['gems']: - json_data['gems'].remove(gem_path.as_posix()) + json_data['gems'] = list(filter(lambda p: gem_path != pathlib.Path(p), json_data['gems'])) if remove: logger.warn(f'Removing Gem path {gem_path}.') @@ -447,21 +342,13 @@ def register_project_path(json_data: dict, logger.error(f'Engine path {engine_path} is not registered.') return 1 - while project_path in engine_data['projects']: - engine_data['projects'].remove(project_path) - - while project_path.as_posix() in engine_data['projects']: - engine_data['projects'].remove(project_path.as_posix()) + engine_data['projects'] = list(filter(lambda p: project_path != pathlib.Path(p), engine_data['projects'])) if remove: logger.warn(f'Engine {engine_path} removing Project path {project_path}.') return 0 else: - while project_path in json_data['projects']: - json_data['projects'].remove(project_path) - - while project_path.as_posix() in json_data['projects']: - json_data['projects'].remove(project_path.as_posix()) + json_data['projects'] = list(filter(lambda p: project_path != pathlib.Path(p), json_data['projects'])) if remove: logger.warn(f'Removing Project path {project_path}.') @@ -486,20 +373,20 @@ def register_project_path(json_data: dict, with this_engine_json.open('r') as f: try: this_engine_json = json.load(f) - except Exception as e: + except json.JSONDecodeError as e: logger.error(f'Engine json failed to load: {str(e)}') return 1 with project_json.open('r') as f: try: project_json_data = json.load(f) - except Exception as e: + except json.JSONDecodeError as e: logger.error(f'Project json failed to load: {str(e)}') return 1 update_project_json = False try: update_project_json = project_json_data['engine'] != this_engine_json['engine_name'] - except Exception as e: + except KeyError as e: update_project_json = True if update_project_json: @@ -508,7 +395,7 @@ def register_project_path(json_data: dict, with project_json.open('w') as s: try: s.write(json.dumps(project_json_data, indent=4)) - except Exception as e: + except OSError as e: logger.error(f'Project json failed to save: {str(e)}') return 1 @@ -530,21 +417,13 @@ def register_template_path(json_data: dict, logger.error(f'Engine path {engine_path} is not registered.') return 1 - while template_path in engine_data['templates']: - engine_data['templates'].remove(template_path) - - while template_path.as_posix() in engine_data['templates']: - engine_data['templates'].remove(template_path.as_posix()) + engine_data['templates'] = list(filter(lambda p: template_path != pathlib.Path(p), engine_data['templates'])) if remove: logger.warn(f'Engine {engine_path} removing Template path {template_path}.') return 0 else: - while template_path in json_data['templates']: - json_data['templates'].remove(template_path) - - while template_path.as_posix() in json_data['templates']: - json_data['templates'].remove(template_path.as_posix()) + json_data['templates'] = list(filter(lambda p: template_path != pathlib.Path(p), json_data['templates'])) if remove: logger.warn(f'Removing Template path {template_path}.') @@ -582,21 +461,13 @@ def register_restricted_path(json_data: dict, logger.error(f'Engine path {engine_path} is not registered.') return 1 - while restricted_path in engine_data['restricted']: - engine_data['restricted'].remove(restricted_path) - - while restricted_path.as_posix() in engine_data['restricted']: - engine_data['restricted'].remove(restricted_path.as_posix()) + engine_data['restricted'] = list(filter(lambda p: restricted_path != pathlib.Path(p), engine_data['restricted'])) if remove: logger.warn(f'Engine {engine_path} removing Restricted path {restricted_path}.') return 0 else: - while restricted_path in json_data['restricted']: - json_data['restricted'].remove(restricted_path) - - while restricted_path.as_posix() in json_data['restricted']: - json_data['restricted'].remove(restricted_path.as_posix()) + json_data['restricted'] = list(filter(lambda p: restricted_path != pathlib.Path(p), json_data['restricted'])) if remove: logger.warn(f'Removing Restricted path {restricted_path}.') @@ -629,10 +500,7 @@ def register_repo(json_data: dict, url = f'{repo_uri}/repo.json' parsed_uri = urllib.parse.urlparse(url) - if parsed_uri.scheme == 'http' or \ - parsed_uri.scheme == 'https' or \ - parsed_uri.scheme == 'ftp' or \ - parsed_uri.scheme == 'ftps': + if parsed_uri.scheme in ['http', 'https', 'ftp', 'ftps']: while repo_uri in json_data['repos']: json_data['repos'].remove(repo_uri) else: @@ -647,118 +515,67 @@ def register_repo(json_data: dict, repo_sha256 = hashlib.sha256(url.encode()) cache_file = manifest.get_o3de_cache_folder() / str(repo_sha256.hexdigest() + '.json') - result = 0 - if parsed_uri.scheme == 'http' or \ - parsed_uri.scheme == 'https' or \ - parsed_uri.scheme == 'ftp' or \ - parsed_uri.scheme == 'ftps': - if not cache_file.is_file(): - with urllib.request.urlopen(url) as s: - with cache_file.open('wb') as f: - shutil.copyfileobj(s, f) - json_data['repos'].insert(0, repo_uri) - else: - if not cache_file.is_file(): - origin_file = pathlib.Path(url).resolve() - if not origin_file.is_file(): - return 1 - shutil.copy(origin_file, origin_file) + result = utils.download_file(url, cache_file) + if result == 0: json_data['repos'].insert(0, repo_uri.as_posix()) - repo_set = set() result = repo.process_add_o3de_repo(cache_file, repo_set) return result +def register_default_o3de_object_folder(json_data: dict, + default_o3de_object_folder: str or pathlib.Path, + o3de_object_key: str) -> int: + # make sure the path exists + default_o3de_object_folder = pathlib.Path(default_o3de_object_folder).resolve() + if not default_o3de_object_folder.is_dir(): + logger.error(f'Default o3de object folder {default_o3de_object_folder} does not exist.') + return 1 + + json_data[o3de_object_key] = default_o3de_object_folder.as_posix() + + return 0 + + def register_default_engines_folder(json_data: dict, default_engines_folder: str or pathlib.Path, remove: bool = False) -> int: - if remove: - default_engines_folder = manifest.get_o3de_engines_folder() - - # make sure the path exists - default_engines_folder = pathlib.Path(default_engines_folder).resolve() - if not default_engines_folder.is_dir(): - logger.error(f'Default engines folder {default_engines_folder} does not exist.') - return 1 - - default_engines_folder = default_engines_folder.as_posix() - json_data['default_engines_folder'] = default_engines_folder - - return 0 + return register_default_o3de_object_folder(json_data, + manifest.get_o3de_engines_folder() if remove else default_engines_folder, + 'default_engines_folder', remove) def register_default_projects_folder(json_data: dict, default_projects_folder: str or pathlib.Path, remove: bool = False) -> int: - if remove: - default_projects_folder = manifest.get_o3de_projects_folder() - - # make sure the path exists - default_projects_folder = pathlib.Path(default_projects_folder).resolve() - if not default_projects_folder.is_dir(): - logger.error(f'Default projects folder {default_projects_folder} does not exist.') - return 1 - - default_projects_folder = default_projects_folder.as_posix() - json_data['default_projects_folder'] = default_projects_folder - - return 0 + return register_default_o3de_object_folder(json_data, + manifest.get_o3de_projects_folder() if remove else default_projects_folder, + 'default_projects_folder', remove) def register_default_gems_folder(json_data: dict, default_gems_folder: str or pathlib.Path, remove: bool = False) -> int: - if remove: - default_gems_folder = manifest.get_o3de_gems_folder() - - # make sure the path exists - default_gems_folder = pathlib.Path(default_gems_folder).resolve() - if not default_gems_folder.is_dir(): - logger.error(f'Default gems folder {default_gems_folder} does not exist.') - return 1 - - default_gems_folder = default_gems_folder.as_posix() - json_data['default_gems_folder'] = default_gems_folder - - return 0 + return register_default_o3de_object_folder(json_data, + manifest.get_o3de_gems_folder() if remove else default_gems_folder, + 'default_gems_folder', remove) def register_default_templates_folder(json_data: dict, default_templates_folder: str or pathlib.Path, remove: bool = False) -> int: - if remove: - default_templates_folder = manifest.get_o3de_templates_folder() - - # make sure the path exists - default_templates_folder = pathlib.Path(default_templates_folder).resolve() - if not default_templates_folder.is_dir(): - logger.error(f'Default templates folder {default_templates_folder} does not exist.') - return 1 - - default_templates_folder = default_templates_folder.as_posix() - json_data['default_templates_folder'] = default_templates_folder - - return 0 + return register_default_o3de_object_folder(json_data, + manifest.get_o3de_templates_folder() if remove else default_templates_folder, + 'default_templates_folder', remove) def register_default_restricted_folder(json_data: dict, default_restricted_folder: str or pathlib.Path, - remove: bool = False) -> int: - if remove: - default_restricted_folder = manifest.get_o3de_restricted_folder() - - # make sure the path exists - default_restricted_folder = pathlib.Path(default_restricted_folder).resolve() - if not default_restricted_folder.is_dir(): - logger.error(f'Default restricted folder {default_restricted_folder} does not exist.') - return 1 - - default_restricted_folder = default_restricted_folder.as_posix() - json_data['default_restricted_folder'] = default_restricted_folder - - return 0 + reset_to_default: bool = False) -> int: + return register_default_o3de_object_folder(json_data, + manifest.get_o3de_restricted_folder() if remove else default_restricted_folder, + 'default_restricted_folder', remove) def register(engine_path: str or pathlib.Path = None, @@ -999,20 +816,14 @@ def _run_register(args: argparse) -> int: force=args.force) -def add_args(parser, subparsers) -> None: +def add_parser_args(parser): """ - add_args is called to add expected parser arguments and subparsers arguments to each command such that it can be + add_parser_args is called to add arguments to each command such that it can be invoked locally or added by a central python file. - Ex. Directly run from this file alone with: python register.py register --gem-path "C:/TestGem" - OR - o3de.py can downloadable commands by importing engine_template, - call add_args and execute: python o3de.py register --gem-path "C:/TestGem" - :param parser: the caller instantiates a parser and passes it in here - :param subparsers: the caller instantiates subparsers and passes it in here + Ex. Directly run from this file alone with: python register.py --engine-path "C:/o3de" + :param parser: the caller passes an argparse parser like instance to this method """ - # register - register_subparser = subparsers.add_parser('register') - group = register_subparser.add_mutually_exclusive_group(required=True) + group = parser.add_mutually_exclusive_group(required=True) group.add_argument('--this-engine', action='store_true', required=False, default=False, help='Registers the engine this script is running from.') @@ -1054,12 +865,49 @@ def add_args(parser, subparsers) -> None: default=False, help='Refresh the repo cache.') - register_subparser.add_argument('-ohf', '--override-home-folder', type=str, required=False, + parser.add_argument('-ohf', '--override-home-folder', type=str, required=False, help='By default the home folder is the user folder, override it to this folder.') - - register_subparser.add_argument('-r', '--remove', action='store_true', required=False, + parser.add_argument('-r', '--remove', action='store_true', required=False, default=False, help='Remove entry.') - register_subparser.add_argument('-f', '--force', action='store_true', default=False, + parser.add_argument('-f', '--force', action='store_true', default=False, help='For the update of the registration field being modified.') - register_subparser.set_defaults(func=_run_register) + parser.set_defaults(func=_run_register) + + +def add_args(subparsers) -> None: + """ + add_args is called to add subparsers arguments to each command such that it can be + a central python file such as o3de.py. + It can be run from the o3de.py script as follows + call add_args and execute: python o3de.py register --engine-path "C:/o3de" + :param subparsers: the caller instantiates subparsers and passes it in here + """ + register_subparser = subparsers.add_parser('register') + add_parser_args(register_subparser) + + +def main(): + """ + Runs register.py script as standalone script + """ + # parse the command line args + the_parser = argparse.ArgumentParser() + + # add subparsers + + # add args to the parser + add_parser_args(the_parser) + + # parse args + the_args = the_parser.parse_args() + + # run + ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1 + + # return + sys.exit(ret) + + +if __name__ == "__main__": + main() diff --git a/scripts/o3de/o3de/registration.py b/scripts/o3de/o3de/registration.py deleted file mode 100755 index 801c698ca4..0000000000 --- a/scripts/o3de/o3de/registration.py +++ /dev/null @@ -1,92 +0,0 @@ -# -# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or -# its licensors. -# -# For complete copyright and license terms please see the LICENSE at the root of this -# distribution (the "License"). All use of this software is governed by the License, -# or, if provided, by the license below or the license accompanying this file. Do not -# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# -""" -This file contains all the code that has to do with registering engines, projects, gems and templates -""" - -import argparse -import sys - - -def add_args(parser, subparsers) -> None: - """ - add_args is called to add expected parser arguments and subparsers arguments to each command such that it can be - invoked locally or added by a central python file. - Ex. Directly run from this file alone with: python register.py register --gem-path "C:/TestGem" - OR - o3de.py can downloadable commands by importing engine_template, - call add_args and execute: python o3de.py register --gem-path "C:/TestGem" - :param parser: the caller instantiates a parser and passes it in here - :param subparsers: the caller instantiates subparsers and passes it in here - """ - # register - from o3de import register - register.add_args(parser, subparsers) - - # show - from o3de import print_registration - print_registration.add_args(parser, subparsers) - - # get-registered - from o3de import get_registration - get_registration.add_args(parser, subparsers) - - # download - from o3de import download - download.add_args(parser, subparsers) - - # add external subdirectories - from o3de import add_external_subdirectory - add_external_subdirectory.add_args(parser, subparsers) - - # remove external subdirectories - from o3de import remove_external_subdirectory - remove_external_subdirectory.add_args(parser, subparsers) - - # add gems to cmake - from o3de import add_gem_cmake - add_gem_cmake.add_args(parser, subparsers) - - # remove gems from cmake - from o3de import remove_gem_cmake - remove_gem_cmake.add_args(parser, subparsers) - - # add a gem to a project - from o3de import add_gem_project - add_gem_project.add_args(parser, subparsers) - - # remove a gem from a project - from o3de import remove_gem_project - remove_gem_project.add_args(parser, subparsers) - - # sha256 - from o3de import sha256 - sha256.add_args(parser, subparsers) - - -if __name__ == "__main__": - # parse the command line args - the_parser = argparse.ArgumentParser() - - # add subparsers - the_subparsers = the_parser.add_subparsers(help='sub-command help', dest='command', required=True) - - # add args to the parser - add_args(the_parser, the_subparsers) - - # parse args - the_args = the_parser.parse_args() - - # run - ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1 - - # return - sys.exit(ret) diff --git a/scripts/o3de/o3de/remove_external_subdirectory.py b/scripts/o3de/o3de/remove_external_subdirectory.py index a636474fba..b433e9c398 100644 --- a/scripts/o3de/o3de/remove_external_subdirectory.py +++ b/scripts/o3de/o3de/remove_external_subdirectory.py @@ -15,6 +15,7 @@ Implemens functinality to remove external_subdirectories from the o3de_manifests import argparse import logging import pathlib +import sys from o3de import manifest @@ -62,12 +63,58 @@ def add_args(parser, subparsers) -> None: :param parser: the caller instantiates a parser and passes it in here :param subparsers: the caller instantiates subparsers and passes it in here """ - remove_external_subdirectory_subparser = subparsers.add_parser('remove-external-subdirectory') - remove_external_subdirectory_subparser.add_argument('external_subdirectory', metavar='external_subdirectory', + + +def add_parser_args(parser): + """ + add_parser_args is called to add arguments to each command such that it can be + invoked locally or added by a central python file. + Ex. Directly run from this file alone with: python remove_external_subdirectory.py "D:/subdir" + :param parser: the caller passes an argparse parser like instance to this method + """ + parser.add_argument('external_subdirectory', metavar='external_subdirectory', type=str, help='remove external subdirectory from cmake') - remove_external_subdirectory_subparser.add_argument('-ohf', '--override-home-folder', type=str, required=False, + parser.add_argument('-ohf', '--override-home-folder', type=str, required=False, help='By default the home folder is the user folder, override it to this folder.') - remove_external_subdirectory_subparser.set_defaults(func=_run_remove_external_subdirectory) + parser.set_defaults(func=_run_remove_external_subdirectory) + + +def add_args(subparsers) -> None: + """ + add_args is called to add subparsers arguments to each command such that it can be + a central python file such as o3de.py. + It can be run from the o3de.py script as follows + call add_args and execute: python o3de.py remove-external-subdirectory "D:/subdir" + :param subparsers: the caller instantiates subparsers and passes it in here + """ + remove_external_subdirectory_subparser = subparsers.add_parser('remove-external-subdirectory') + add_parser_args(remove_external_subdirectory_subparser) + + +def main(): + """ + Runs remove_external_subdirectory.py script as standalone script + """ + # parse the command line args + the_parser = argparse.ArgumentParser() + + # add subparsers + + # add args to the parser + add_parser_args(the_parser) + + # parse args + the_args = the_parser.parse_args() + + # run + ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1 + + # return + sys.exit(ret) + + +if __name__ == "__main__": + main() diff --git a/scripts/o3de/o3de/remove_gem_cmake.py b/scripts/o3de/o3de/remove_gem_cmake.py index 8f73caaad1..3d988a579a 100644 --- a/scripts/o3de/o3de/remove_gem_cmake.py +++ b/scripts/o3de/o3de/remove_gem_cmake.py @@ -15,6 +15,7 @@ Contains methods for removing a gem from a project's cmake scripts import argparse import logging import pathlib +import sys from o3de import manifest, remove_external_subdirectory @@ -64,26 +65,58 @@ def _run_remove_gem_from_cmake(args: argparse) -> int: return remove_gem_from_cmake(args.gem_name, args.gem_path) -def add_args(parser, subparsers) -> None: +def add_parser_args(parser): """ - add_args is called to add expected parser arguments and subparsers arguments to each command such that it can be + add_parser_args is called to add arguments to each command such that it can be invoked locally or added by a central python file. - Ex. Directly run from this file alone with: python register.py register --gem-path "C:/TestGem" - OR - o3de.py can downloadable commands by importing engine_template, - call add_args and execute: python o3de.py register --gem-path "C:/TestGem" - :param parser: the caller instantiates a parser and passes it in here - :param subparsers: the caller instantiates subparsers and passes it in here + Ex. Directly run from this file alone with: python remove_gem_cmake.py --gem-name Atom + :param parser: the caller passes an argparse parser like instance to this method """ - # convenience functions to disambiguate the gem name -> gem_path and call remove-external-subdirectory on gem_path - remove_gem_from_cmake_subparser = subparsers.add_parser('remove-gem-from-cmake') - group = remove_gem_from_cmake_subparser.add_mutually_exclusive_group(required=True) + group = parser.add_mutually_exclusive_group(required=True) group.add_argument('-gp', '--gem-path', type=str, required=False, help='The path to the gem.') group.add_argument('-gn', '--gem-name', type=str, required=False, help='The name of the gem.') - remove_gem_from_cmake_subparser.add_argument('-ohf', '--override-home-folder', type=str, required=False, + parser.add_argument('-ohf', '--override-home-folder', type=str, required=False, help='By default the home folder is the user folder, override it to this folder.') - remove_gem_from_cmake_subparser.set_defaults(func=_run_remove_gem_from_cmake) + parser.set_defaults(func=_run_remove_gem_from_cmake) + + +def add_args(subparsers) -> None: + """ + add_args is called to add subparsers arguments to each command such that it can be + a central python file such as o3de.py. + It can be run from the o3de.py script as follows + call add_args and execute: python o3de.py remove-gem-from-cmake --gem-name Atom + :param subparsers: the caller instantiates subparsers and passes it in here + """ + remove_gem_from_cmake_subparser = subparsers.add_parser('remove-gem-from-cmake') + add_parser_args(remove_gem_from_cmake_subparser) + + +def main(): + """ + Runs remove_gem_cmake.py script as standalone script + """ + # parse the command line args + the_parser = argparse.ArgumentParser() + + # add subparsers + + # add args to the parser + add_parser_args(the_parser) + + # parse args + the_args = the_parser.parse_args() + + # run + ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1 + + # return + sys.exit(ret) + + +if __name__ == "__main__": + main() diff --git a/scripts/o3de/o3de/remove_gem_project.py b/scripts/o3de/o3de/remove_gem_project.py index 7644357042..671427db14 100644 --- a/scripts/o3de/o3de/remove_gem_project.py +++ b/scripts/o3de/o3de/remove_gem_project.py @@ -16,6 +16,7 @@ import argparse import logging import os import pathlib +import sys from o3de import cmake, remove_gem_cmake @@ -220,51 +221,84 @@ def _run_remove_gem_from_project(args: argparse) -> int: args.remove_from_cmake) -def add_args(parser, subparsers) -> None: +def add_parser_args(parser): """ - add_args is called to add expected parser arguments and subparsers arguments to each command such that it can be + add_parser_args is called to add arguments to each command such that it can be invoked locally or added by a central python file. - Ex. Directly run from this file alone with: python register.py register --gem-path "C:/TestGem" - OR - o3de.py can downloadable commands by importing engine_template, - call add_args and execute: python o3de.py register --gem-path "C:/TestGem" - :param parser: the caller instantiates a parser and passes it in here - :param subparsers: the caller instantiates subparsers and passes it in here + Ex. Directly run from this file alone with: python remove_gem_project.py --project-path D:/Test --gem-name Atom + :param parser: the caller passes an argparse parser like instance to this method """ - remove_gem_subparser = subparsers.add_parser('remove-gem-from-project') - group = remove_gem_subparser.add_mutually_exclusive_group(required=True) + group = parser.add_mutually_exclusive_group(required=True) group.add_argument('-pp', '--project-path', type=str, required=False, help='The path to the project.') group.add_argument('-pn', '--project-name', type=str, required=False, help='The name of the project.') - group = remove_gem_subparser.add_mutually_exclusive_group(required=True) + group = parser.add_mutually_exclusive_group(required=True) group.add_argument('-gp', '--gem-path', type=str, required=False, help='The path to the gem.') group.add_argument('-gn', '--gem-name', type=str, required=False, help='The name of the gem.') - remove_gem_subparser.add_argument('-gt', '--gem-target', type=str, required=False, + parser.add_argument('-gt', '--gem-target', type=str, required=False, help='The cmake target name to add. If not specified it will assume gem_name') - remove_gem_subparser.add_argument('-df', '--dependencies-file', type=str, required=False, + parser.add_argument('-df', '--dependencies-file', type=str, required=False, help='The cmake dependencies file in which the gem dependencies are specified.' 'If not specified it will assume ') - remove_gem_subparser.add_argument('-rd', '--runtime-dependency', action='store_true', required=False, + parser.add_argument('-rd', '--runtime-dependency', action='store_true', required=False, default=False, help='Optional toggle if this gem should be removed as a runtime dependency') - remove_gem_subparser.add_argument('-td', '--tool-dependency', action='store_true', required=False, + parser.add_argument('-td', '--tool-dependency', action='store_true', required=False, default=False, help='Optional toggle if this gem should be removed as a server dependency') - remove_gem_subparser.add_argument('-sd', '--server-dependency', action='store_true', required=False, + parser.add_argument('-sd', '--server-dependency', action='store_true', required=False, default=False, help='Optional toggle if this gem should be removed as a server dependency') - remove_gem_subparser.add_argument('-pl', '--platforms', type=str, required=False, + parser.add_argument('-pl', '--platforms', type=str, required=False, default='Common', help='Optional list of platforms this gem should be removed from' ' Ex. --platforms Mac,Windows,Linux') - remove_gem_subparser.add_argument('-r', '--remove-from-cmake', type=bool, required=False, + parser.add_argument('-r', '--remove-from-cmake', type=bool, required=False, default=False, help='Automatically call remove-from-cmake.') - remove_gem_subparser.add_argument('-ohf', '--override-home-folder', type=str, required=False, + parser.add_argument('-ohf', '--override-home-folder', type=str, required=False, help='By default the home folder is the user folder, override it to this folder.') - remove_gem_subparser.set_defaults(func=_run_remove_gem_from_project) + parser.set_defaults(func=_run_remove_gem_from_project) + + +def add_args(subparsers) -> None: + """ + add_args is called to add subparsers arguments to each command such that it can be + a central python file such as o3de.py. + It can be run from the o3de.py script as follows + call add_args and execute: python o3de.py remove-gem-from-project --project-path D:/Test --gem-name Atom + :param subparsers: the caller instantiates subparsers and passes it in here + """ + remove_gem_project_subparser = subparsers.add_parser('remove-gem-from-project') + add_parser_args(remove_gem_project_subparser) + + +def main(): + """ + Runs remove_gem_project.py script as standalone script + """ + # parse the command line args + the_parser = argparse.ArgumentParser() + + # add subparsers + + # add args to the parser + add_parser_args(the_parser) + + # parse args + the_args = the_parser.parse_args() + + # run + ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1 + + # return + sys.exit(ret) + + +if __name__ == "__main__": + main() diff --git a/scripts/o3de/o3de/repo.py b/scripts/o3de/o3de/repo.py index 9cb93d53ae..c6b4874b6a 100644 --- a/scripts/o3de/o3de/repo.py +++ b/scripts/o3de/o3de/repo.py @@ -16,11 +16,12 @@ import shutil import urllib.parse import urllib.request -from o3de import manifest, validation +from o3de import manifest, utils, validation logger = logging.getLogger() logging.basicConfig() + def process_add_o3de_repo(file_name: str or pathlib.Path, repo_set: set) -> int: file_name = pathlib.Path(file_name).resolve() @@ -32,105 +33,26 @@ def process_add_o3de_repo(file_name: str or pathlib.Path, with file_name.open('r') as f: try: repo_data = json.load(f) - except Exception as e: + except json.JSONDecodeError as e: logger.error(f'{file_name} failed to load: {str(e)}') return 1 - for engine_uri in repo_data['engines']: - engine_uri = f'{engine_uri}/engine.json' - engine_sha256 = hashlib.sha256(engine_uri.encode()) - cache_file = cache_folder / str(engine_sha256.hexdigest() + '.json') - if not cache_file.is_file(): - parsed_uri = urllib.parse.urlparse(engine_uri) - if parsed_uri.scheme == 'http' or \ - parsed_uri.scheme == 'https' or \ - parsed_uri.scheme == 'ftp' or \ - parsed_uri.scheme == 'ftps': - with urllib.request.urlopen(engine_uri) as s: - with cache_file.open('wb') as f: - shutil.copyfileobj(s, f) - else: - engine_json = pathlib.Path(engine_uri).resolve() - if not engine_json.is_file(): - return 1 - shutil.copy(engine_json, cache_file) - - for project_uri in repo_data['projects']: - project_uri = f'{project_uri}/project.json' - project_sha256 = hashlib.sha256(project_uri.encode()) - cache_file = cache_folder / str(project_sha256.hexdigest() + '.json') - if not cache_file.is_file(): - parsed_uri = urllib.parse.urlparse(project_uri) - if parsed_uri.scheme == 'http' or \ - parsed_uri.scheme == 'https' or \ - parsed_uri.scheme == 'ftp' or \ - parsed_uri.scheme == 'ftps': - with urllib.request.urlopen(project_uri) as s: - with cache_file.open('wb') as f: - shutil.copyfileobj(s, f) - else: - project_json = pathlib.Path(project_uri).resolve() - if not project_json.is_file(): - return 1 - shutil.copy(project_json, cache_file) - - for gem_uri in repo_data['gems']: - gem_uri = f'{gem_uri}/gem.json' - gem_sha256 = hashlib.sha256(gem_uri.encode()) - cache_file = cache_folder / str(gem_sha256.hexdigest() + '.json') - if not cache_file.is_file(): - parsed_uri = urllib.parse.urlparse(gem_uri) - if parsed_uri.scheme == 'http' or \ - parsed_uri.scheme == 'https' or \ - parsed_uri.scheme == 'ftp' or \ - parsed_uri.scheme == 'ftps': - with urllib.request.urlopen(gem_uri) as s: - with cache_file.open('wb') as f: - shutil.copyfileobj(s, f) - else: - gem_json = pathlib.Path(gem_uri).resolve() - if not gem_json.is_file(): - return 1 - shutil.copy(gem_json, cache_file) - - for template_uri in repo_data['templates']: - template_uri = f'{template_uri}/template.json' - template_sha256 = hashlib.sha256(template_uri.encode()) - cache_file = cache_folder / str(template_sha256.hexdigest() + '.json') - if not cache_file.is_file(): - parsed_uri = urllib.parse.urlparse(template_uri) - if parsed_uri.scheme == 'http' or \ - parsed_uri.scheme == 'https' or \ - parsed_uri.scheme == 'ftp' or \ - parsed_uri.scheme == 'ftps': - with urllib.request.urlopen(template_uri) as s: - with cache_file.open('wb') as f: - shutil.copyfileobj(s, f) - else: - template_json = pathlib.Path(template_uri).resolve() - if not template_json.is_file(): - return 1 - shutil.copy(template_json, cache_file) - - for repo_uri in repo_data['repos']: - if repo_uri not in repo_set: - repo_set.add(repo_uri) - repo_uri = f'{repo_uri}/repo.json' - repo_sha256 = hashlib.sha256(repo_uri.encode()) - cache_file = cache_folder / str(repo_sha256.hexdigest() + '.json') + for o3de_object_uris, manifest_json in [(repo_data['engines'], 'engine.json'), + (repo_data['projects'], 'project.json'), + (repo_data['gems'], 'gem.json'), + (repo_data['template'], 'template.json'), + (repo_data['restricted'], 'restricted.json')]: + for o3de_object_uri in o3de_object_uris: + manifest_json_uri = f'{o3de_object_uri}/{manifest_json}' + manifest_json_sha256 = hashlib.sha256(manifest_json_uri.encode()) + cache_file = cache_folder / str(manifest_json_sha256.hexdigest() + '.json') if not cache_file.is_file(): - if parsed_uri.scheme == 'http' or \ - parsed_uri.scheme == 'https' or \ - parsed_uri.scheme == 'ftp' or \ - parsed_uri.scheme == 'ftps': - with urllib.request.urlopen(repo_uri) as s: - with cache_file.open('wb') as f: - shutil.copyfileobj(s, f) - else: - repo_json = pathlib.Path(repo_uri).resolve() - if not repo_json.is_file(): - return 1 - shutil.copy(repo_json, cache_file) + parsed_uri = urllib.parse.urlparse(manifest_json_uri) + download_file_result = utils.download_file(parsed_uri, cache_file) + if download_file_result != 0: + return download_file_result + + repo_set |= repo_data['repos'] return 0 @@ -156,18 +78,9 @@ def refresh_repos() -> int: cache_file = cache_folder / str(repo_sha256.hexdigest() + '.json') if not cache_file.is_file(): parsed_uri = urllib.parse.urlparse(repo_uri) - if parsed_uri.scheme == 'http' or \ - parsed_uri.scheme == 'https' or \ - parsed_uri.scheme == 'ftp' or \ - parsed_uri.scheme == 'ftps': - with urllib.request.urlopen(repo_uri) as s: - with cache_file.open('wb') as f: - shutil.copyfileobj(s, f) - else: - origin_file = pathlib.Path(repo_uri).resolve() - if not origin_file.is_file(): - return 1 - shutil.copy(origin_file, cache_file) + download_file_result = utils.download_file(parsed_uri, cache_file) + if download_file_result != 0: + return download_file_result if not validation.valid_o3de_repo_json(cache_file): logger.error(f'Repo json {repo_uri} is not valid.') @@ -181,111 +94,67 @@ def refresh_repos() -> int: return result -def search_repo(repo_set: set, - repo_json_data: dict, +def search_repo(repo_json_data: dict, engine_name: str = None, project_name: str = None, gem_name: str = None, template_name: str = None, restricted_name: str = None) -> dict or None: - cache_folder = manifest.get_o3de_cache_folder() if isinstance(engine_name, str) or isinstance(engine_name, pathlib.PurePath): - for engine_uri in repo_json_data['engines']: - engine_uri = f'{engine_uri}/engine.json' - engine_sha256 = hashlib.sha256(engine_uri.encode()) - engine_cache_file = cache_folder / str(engine_sha256.hexdigest() + '.json') - if engine_cache_file.is_file(): - with engine_cache_file.open('r') as f: - try: - engine_json_data = json.load(f) - except Exception as e: - logger.warn(f'{engine_cache_file} failed to load: {str(e)}') - else: - if engine_json_data['engine_name'] == engine_name: - return engine_json_data - + o3de_object_uris = repo_json_data['engines'] + manifest_json = 'engine.json' + json_key = 'engine_name' + search_func = lambda: None if manifest_json_data.get(json_key, '') == engine_name else manifest_json_data elif isinstance(project_name, str) or isinstance(project_name, pathlib.PurePath): - for project_uri in repo_json_data['projects']: - project_uri = f'{project_uri}/project.json' - project_sha256 = hashlib.sha256(project_uri.encode()) - project_cache_file = cache_folder / str(project_sha256.hexdigest() + '.json') - if project_cache_file.is_file(): - with project_cache_file.open('r') as f: - try: - project_json_data = json.load(f) - except Exception as e: - logger.warn(f'{project_cache_file} failed to load: {str(e)}') - else: - if project_json_data['project_name'] == project_name: - return project_json_data - + o3de_object_uris = repo_json_data['projects'] + manifest_json = 'project.json' + json_key = 'project_name' + search_func = lambda: None if manifest_json_data.get(json_key, '') == project_name else manifest_json_data elif isinstance(gem_name, str) or isinstance(gem_name, pathlib.PurePath): - for gem_uri in repo_json_data['gems']: - gem_uri = f'{gem_uri}/gem.json' - gem_sha256 = hashlib.sha256(gem_uri.encode()) - gem_cache_file = cache_folder / str(gem_sha256.hexdigest() + '.json') - if gem_cache_file.is_file(): - with gem_cache_file.open('r') as f: - try: - gem_json_data = json.load(f) - except Exception as e: - logger.warn(f'{gem_cache_file} failed to load: {str(e)}') - else: - if gem_json_data['gem_name'] == gem_name: - return gem_json_data - + o3de_object_uris = repo_json_data['gems'] + manifest_json = 'gem.json' + json_key = 'gem_name' + search_func = lambda: None if manifest_json_data.get(json_key, '') == gem_name else manifest_json_data elif isinstance(template_name, str) or isinstance(template_name, pathlib.PurePath): - for template_uri in repo_json_data['templates']: - template_uri = f'{template_uri}/template.json' - template_sha256 = hashlib.sha256(template_uri.encode()) - template_cache_file = cache_folder / str(template_sha256.hexdigest() + '.json') - if template_cache_file.is_file(): - with template_cache_file.open('r') as f: - try: - template_json_data = json.load(f) - except Exception as e: - logger.warn(f'{template_cache_file} failed to load: {str(e)}') - else: - if template_json_data['template_name'] == template_name: - return template_json_data - + o3de_object_uris = repo_json_data['template'] + manifest_json = 'template.json' + json_key = 'template_name' + search_func = lambda: None if manifest_json_data.get(json_key, '') == template_name_name else manifest_json_data elif isinstance(restricted_name, str) or isinstance(restricted_name, pathlib.PurePath): - for restricted_uri in repo_json_data['restricted']: - restricted_uri = f'{restricted_uri}/restricted.json' - restricted_sha256 = hashlib.sha256(restricted_uri.encode()) - restricted_cache_file = cache_folder / str(restricted_sha256.hexdigest() + '.json') - if restricted_cache_file.is_file(): - with restricted_cache_file.open('r') as f: - try: - restricted_json_data = json.load(f) - except Exception as e: - logger.warn(f'{restricted_cache_file} failed to load: {str(e)}') - else: - if restricted_json_data['restricted_name'] == restricted_name: - return restricted_json_data - # recurse + o3de_object_uris = repo_json_data['restricted'] + manifest_json = 'restricted.json' + json_key = 'restricted_name' + search_func = lambda: None if manifest_json_data.get(json_key, '') == restricted_name else manifest_json_data else: - for repo_repo_uri in repo_json_data['repos']: - if repo_repo_uri not in repo_set: - repo_set.add(repo_repo_uri) - repo_repo_uri = f'{repo_repo_uri}/repo.json' - repo_repo_sha256 = hashlib.sha256(repo_repo_uri.encode()) - repo_repo_cache_file = cache_folder / str(repo_repo_sha256.hexdigest() + '.json') - if repo_repo_cache_file.is_file(): - with repo_repo_cache_file.open('r') as f: - try: - repo_repo_json_data = json.load(f) - except Exception as e: - logger.warn(f'{repo_repo_cache_file} failed to load: {str(e)}') - else: - item = search_repo(repo_set, - repo_repo_json_data, - engine_name, - project_name, - gem_name, - template_name) - if item: - return item - return None + return None + o3de_object = search_o3de_object(manifest_json, o3de_object_uris, search_func) + if o3de_object: + return o3de_object + + # recurse into the repos object to search for the o3de object + o3de_object_uris = repo_json_data['repos'] + manifest_json = 'repo.json' + search_func = lambda: search_repo(manifest_json, engine_name, project_name, gem_name, template_name) + return search_o3de_object(manifest_json, o3de_object_uris, search_func) + + +def search_o3de_object(manifest_json, o3de_object_uris, search_func): + # Search for the o3de object based on the supplied object name in the current repo + cache_folder = manifest.get_o3de_cache_folder() + for o3de_object_uri in o3de_object_uris: + manifest_json_uri = f'{o3de_object_uri}/{manifest_json}' + manifest_json_sha256 = hashlib.sha256(manifest_json_uri.encode()) + cache_file = cache_folder / str(manifest_json_sha256.hexdigest() + '.json') + if cache_file.is_file(): + with cache_file.open('r') as f: + try: + manifest_json_data = json.load(f) + except json.JSONDecodeError as e: + logger.warn(f'{cache_file} failed to load: {str(e)}') + else: + result_json_data = search_func() + if result_json_data: + return result_json_data + return None diff --git a/scripts/o3de/o3de/sha256.py b/scripts/o3de/o3de/sha256.py index bc35919c4e..bbec7696d6 100644 --- a/scripts/o3de/o3de/sha256.py +++ b/scripts/o3de/o3de/sha256.py @@ -13,6 +13,8 @@ import argparse import json import logging import hashlib +import pathlib +import sys from o3de import utils @@ -42,7 +44,7 @@ def sha256(file_path: str or pathlib.Path, with json_path.open('r') as s: try: json_data = json.load(s) - except Exception as e: + except json.JSONDecodeError as e: logger.error(f'Failed to read Json path {json_path}: {str(e)}') return 1 json_data.update({"sha256": sha256}) @@ -50,7 +52,7 @@ def sha256(file_path: str or pathlib.Path, with json_path.open('w') as s: try: s.write(json.dumps(json_data, indent=4)) - except Exception as e: + except OSError as e: logger.error(f'Failed to write Json path {json_path}: {str(e)}') return 1 else: @@ -63,20 +65,53 @@ def _run_sha256(args: argparse) -> int: args.json_path) -def add_args(parser, subparsers) -> None: +def add_parser_args(parser): """ - add_args is called to add expected parser arguments and subparsers arguments to each command such that it can be + add_parser_args is called to add arguments to each command such that it can be invoked locally or added by a central python file. - Ex. Directly run from this file alone with: python register.py register --gem-path "C:/TestGem" - OR - o3de.py can downloadable commands by importing engine_template, - call add_args and execute: python o3de.py register --gem-path "C:/TestGem" - :param parser: the caller instantiates a parser and passes it in here + Ex. Directly run from this file alone with: python sha256.py --file-path "C:/TestGem" + :param parser: the caller passes an argparse parser like instance to this method + """ + parser.add_argument('-f', '--file-path', type=str, required=True, + help='The path to the file you want to sha256.') + parser.add_argument('-j', '--json-path', type=str, required=False, + help='optional path to an o3de json file to add the "sha256" element to.') + parser.set_defaults(func=_run_sha256) + + +def add_args(subparsers) -> None: + """ + add_args is called to add subparsers arguments to each command such that it can be + a central python file such as o3de.py. + It can be run from the o3de.py script as follows + call add_args and execute: python o3de.py sha256 --file-path "C:/TestGem" :param subparsers: the caller instantiates subparsers and passes it in here """ sha256_subparser = subparsers.add_parser('sha256') - sha256_subparser.add_argument('-f', '--file-path', type=str, required=True, - help='The path to the file you want to sha256.') - sha256_subparser.add_argument('-j', '--json-path', type=str, required=False, - help='optional path to an o3de json file to add the "sha256" element to.') - sha256_subparser.set_defaults(func=_run_sha256) + add_parser_args(sha256_subparser) + + +def main(): + """ + Runs sha256.py script as standalone script + """ + # parse the command line args + the_parser = argparse.ArgumentParser() + + # add subparsers + + # add args to the parser + add_parser_args(the_parser) + + # parse args + the_args = the_parser.parse_args() + + # run + ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1 + + # return + sys.exit(ret) + + +if __name__ == "__main__": + main() diff --git a/scripts/o3de/o3de/utils.py b/scripts/o3de/o3de/utils.py index 50a9e5d6dd..4330de25b8 100755 --- a/scripts/o3de/o3de/utils.py +++ b/scripts/o3de/o3de/utils.py @@ -13,7 +13,9 @@ This file contains utility functions """ import uuid - +import pathlib +import shutil +import urllib.request def validate_identifier(identifier: str) -> bool: """ @@ -46,6 +48,7 @@ def validate_uuid4(uuid_string: str) -> bool: return False return str(val) == uuid_string + def backup_file(file_name: str or pathlib.Path) -> None: index = 0 renamed = False @@ -69,4 +72,41 @@ def backup_folder(folder: str or pathlib.Path) -> None: folder = pathlib.Path(folder).resolve() folder.rename(backup_folder_name) if backup_folder_name.is_dir(): - renamed = True \ No newline at end of file + renamed = True + + +def download_file(parsed_uri, download_path: pathlib.Path) -> int: + """ + :param parsed_uri: uniform resource identifier to zip file to download + :param download_path: location path on disk to download file + """ + if download_path.is_file(): + logger.warn(f'File already downloaded to {download_path}.') + elif parsed_uri.scheme in ['http', 'https', 'ftp', 'ftps']: + with urllib.request.urlopen(url) as s: + with download_path.open('wb') as f: + shutil.copyfileobj(s, f) + else: + origin_file = pathlib.Path(url).resolve() + if not origin_file.is_file(): + return 1 + shutil.copy(origin_file, download_path) + + return 0 + + +def download_zip_file(parsed_uri, download_zip_path: pathlib.Path) -> int: + """ + :param parsed_uri: uniform resource identifier to zip file to download + :param download_zip_path: path to output zip file + """ + download_file_result = download_file(parsed_uri, download_zip_path) + if download_file_result != 0: + return download_file_result + + if not zipfile.is_zipfile(download_zip_path): + logger.error(f"File zip {download_zip_path} is invalid.") + download_zip_path.unlink() + return 1 + + return 0 \ No newline at end of file diff --git a/scripts/o3de/o3de/validation.py b/scripts/o3de/o3de/validation.py index f3a5f5e376..721b7eae09 100644 --- a/scripts/o3de/o3de/validation.py +++ b/scripts/o3de/o3de/validation.py @@ -14,6 +14,10 @@ This file validating o3de object json files import json import pathlib +def valid_o3de_json_dict(json_data: dict, key: str) -> bool: + return key in json_data + + def valid_o3de_repo_json(file_name: str or pathlib.Path) -> bool: file_name = pathlib.Path(file_name).resolve() if not file_name.is_file(): @@ -24,7 +28,7 @@ def valid_o3de_repo_json(file_name: str or pathlib.Path) -> bool: json_data = json.load(f) test = json_data['repo_name'] test = json_data['origin'] - except Exception as e: + except (json.JSONDecodeError, KeyError) as e: return False return True @@ -39,8 +43,7 @@ def valid_o3de_engine_json(file_name: str or pathlib.Path) -> bool: try: json_data = json.load(f) test = json_data['engine_name'] - # test = json_data['origin'] # will be required soon - except Exception as e: + except (json.JSONDecodeError, KeyError) as e: return False return True @@ -54,8 +57,7 @@ def valid_o3de_project_json(file_name: str or pathlib.Path) -> bool: try: json_data = json.load(f) test = json_data['project_name'] - # test = json_data['origin'] # will be required soon - except Exception as e: + except (json.JSONDecodeError, KeyError) as e: return False return True @@ -69,8 +71,7 @@ def valid_o3de_gem_json(file_name: str or pathlib.Path) -> bool: try: json_data = json.load(f) test = json_data['gem_name'] - # test = json_data['origin'] # will be required soon - except Exception as e: + except (json.JSONDecodeError, KeyError) as e: return False return True @@ -83,8 +84,7 @@ def valid_o3de_template_json(file_name: str or pathlib.Path) -> bool: try: json_data = json.load(f) test = json_data['template_name'] - # test = json_data['origin'] # will be required soon - except Exception as e: + except (json.JSONDecodeError, KeyError) as e: return False return True @@ -97,7 +97,6 @@ def valid_o3de_restricted_json(file_name: str or pathlib.Path) -> bool: try: json_data = json.load(f) test = json_data['restricted_name'] - # test = json_data['origin'] # will be required soon - except Exception as e: + except (json.JSONDecodeError, KeyError) as e: return False return True diff --git a/scripts/o3de/tests/unit_test_registration.py b/scripts/o3de/tests/unit_test_registration.py index a0abb6cacd..eb866e76d4 100644 --- a/scripts/o3de/tests/unit_test_registration.py +++ b/scripts/o3de/tests/unit_test_registration.py @@ -35,11 +35,10 @@ string_manifest_data = '{}' ) def test_register_engine_path(engine_path, engine_name, force, expected_result): parser = argparse.ArgumentParser() - subparser = parser.add_subparsers(help='sub-command help') # Register the registration script subparsers with the current argument parser - register.add_args(parser, subparser) - arg_list = ['register', '--engine-path', str(engine_path)] + register.add_parser_args(parser) + arg_list = ['--engine-path', str(engine_path)] if force: arg_list += ['--force'] args = parser.parse_args(arg_list) @@ -64,3 +63,55 @@ def test_register_engine_path(engine_path, engine_name, force, expected_result): result = register._run_register(args) assert result == expected_result + +@pytest.fixture(scope='class') +def init_manifest_data(request): + class ManifestData: + def __init__(self): + self.json_string = json.dumps({'default_engines_folder': '', + 'default_projects_folder': '', 'default_gems_folder': '', + 'default_templates_folder': '', 'default_restricted_folder': ''}) + + request.cls.manifest_data = ManifestData() + + +@pytest.mark.usefixtures('init_manifest_data') +class TestRegisterThisEngine: + @pytest.mark.parametrize( + "engine_path, engine_name, force, expected_result", [ + pytest.param(pathlib.PurePath('D:/o3de/o3de'), "o3de", False, 0), + pytest.param(pathlib.PurePath('F:/Open3DEngine'), "o3de", False, 1), + pytest.param(pathlib.PurePath('F:/Open3DEngine'), "o3de", True, 0) + ] + ) + def test_register_this_engine(self, engine_path, engine_name, force, expected_result): + parser = argparse.ArgumentParser() + + # Register the registration script subparsers with the current argument parser + register.add_parser_args(parser) + arg_list = ['--this-engine'] + if force: + arg_list += ['--force'] + args = parser.parse_args(arg_list) + + def load_manifest_from_string() -> dict: + try: + manifest_json = json.loads(self.manifest_data.json_string) + except json.JSONDecodeError as err: + logging.error("Error decoding Json from Manifest file") + else: + return manifest_json + def save_manifest_to_string(manifest_json: dict) -> None: + self.manifest_data.json_string = json.dumps(manifest_json) + + engine_json_data = {'engine_name': engine_name} + + with patch('o3de.manifest.load_o3de_manifest', side_effect=load_manifest_from_string) as load_manifest_mock, \ + patch('o3de.manifest.save_o3de_manifest', side_effect=save_manifest_to_string) as save_manifest_mock, \ + patch('o3de.manifest.get_engine_json_data', return_value=engine_json_data) as engine_paths_mock, \ + patch('o3de.manifest.get_this_engine_path', return_value=engine_path) as engine_paths_mock, \ + patch('o3de.validation.valid_o3de_engine_json', return_value=True) as valid_engine_mock, \ + patch('pathlib.Path.is_dir', return_value=True) as pathlib_is_dir_mock: + result = register._run_register(args) + assert result == expected_result +