diff --git a/scripts/o3de.py b/scripts/o3de.py index d3b877620f..dabb83b068 100755 --- a/scripts/o3de.py +++ b/scripts/o3de.py @@ -13,27 +13,70 @@ import argparse import pathlib import sys -# As o3de.py shares the same name as the o3de package attempting to use a regular -# from o3de import line tries to import from the current o3de.py script and not the package -# So the current script directory is removed from the sys.path temporary -SCRIPT_DIR_REMOVED = False -SCRIPT_DIR = pathlib.Path(__file__).parent.resolve() -while str(SCRIPT_DIR) in sys.path: - SCRIPT_DIR_REMOVED = True - sys.path.remove(str(SCRIPT_DIR)) - -from o3de import engine_template -from o3de import global_project -from o3de import registration - -if SCRIPT_DIR_REMOVED: - sys.path.insert(0, str(SCRIPT_DIR)) - def add_args(parser, subparsers) -> None: - global_project.add_args(parser, subparsers) - engine_template.add_args(parser, subparsers) - registration.add_args(parser, subparsers) + """ + add_args is called to add expected parser arguments and subparsers arguments to each command such that it can be + invoked by o3de.py + Ex o3de.py can invoke the register downloadable commands by importing register, + call add_args and execute: python o3de.py register --gem-path "C:/TestGem" + :param parser: the caller instantiates a parser and passes it in here + :param subparsers: the caller instantiates subparsers and passes it in here + """ + + # As o3de.py shares the same name as the o3de package attempting to use a regular + # from o3de import line tries to import from the current o3de.py script and not the package + # So the current script directory is removed from the sys.path temporary + SCRIPT_DIR_REMOVED = False + SCRIPT_DIR = pathlib.Path(__file__).parent.resolve() + while str(SCRIPT_DIR) in sys.path: + SCRIPT_DIR_REMOVED = True + sys.path.remove(str(SCRIPT_DIR)) + + from o3de import engine_template, global_project, register, print_registration, get_registration, download, \ + add_external_subdirectory, remove_external_subdirectory, add_gem_cmake, remove_gem_cmake, add_gem_project, \ + remove_gem_project, sha256 + + if SCRIPT_DIR_REMOVED: + sys.path.insert(0, str(SCRIPT_DIR)) + + # global_project + global_project.add_args(subparsers) + # engine templaate + engine_template.add_args(subparsers) + + # register + register.add_args(subparsers) + + # show + print_registration.add_args(subparsers) + + # get-registered + get_registration.add_args(subparsers) + + # download + download.add_args(subparsers) + + # add external subdirectories + add_external_subdirectory.add_args(subparsers) + + # remove external subdirectories + remove_external_subdirectory.add_args(subparsers) + + # add gems to cmake + add_gem_cmake.add_args(subparsers) + + # remove gems from cmake + remove_gem_cmake.add_args(subparsers) + + # add a gem to a project + add_gem_project.add_args(subparsers) + + # remove a gem from a project + remove_gem_project.add_args(subparsers) + + # sha256 + sha256.add_args(subparsers) if __name__ == "__main__": diff --git a/scripts/o3de/o3de/add_external_subdirectory.py b/scripts/o3de/o3de/add_external_subdirectory.py index 388f0027da..29013d30c8 100644 --- a/scripts/o3de/o3de/add_external_subdirectory.py +++ b/scripts/o3de/o3de/add_external_subdirectory.py @@ -15,6 +15,7 @@ Contains command to add an external_subdirectory to a project's cmake scripts import argparse import logging import pathlib +import sys from o3de import manifest @@ -22,32 +23,27 @@ logger = logging.getLogger() logging.basicConfig() def add_external_subdirectory(external_subdir: str or pathlib.Path, - engine_path: str or pathlib.Path = None, - suppress_errors: bool = False) -> int: + engine_path: str or pathlib.Path = None) -> int: """ add external subdirectory to a cmake :param external_subdir: external subdirectory to add to cmake :param engine_path: optional engine path, defaults to this engine - :param suppress_errors: optional silence errors :return: 0 for success or non 0 failure code """ external_subdir = pathlib.Path(external_subdir).resolve() if not external_subdir.is_dir(): - if not suppress_errors: - logger.error(f'Add External Subdirectory Failed: {external_subdir} does not exist.') + logger.error(f'Add External Subdirectory Failed: {external_subdir} does not exist.') return 1 external_subdir_cmake = external_subdir / 'CMakeLists.txt' if not external_subdir_cmake.is_file(): - if not suppress_errors: - logger.error(f'Add External Subdirectory Failed: {external_subdir} does not contain a CMakeLists.txt.') + logger.error(f'Add External Subdirectory Failed: {external_subdir} does not contain a CMakeLists.txt.') return 1 json_data = manifest.load_o3de_manifest() engine_object = manifest.find_engine_data(json_data, engine_path) if not engine_object: - if not suppress_errors: - logger.error(f'Add External Subdirectory Failed: {engine_path} not registered.') + logger.error(f'Add External Subdirectory Failed: {engine_path} not registered.') return 1 engine_object.setdefault('external_subdirectories', []) @@ -76,7 +72,7 @@ def add_external_subdirectory(external_subdir: str or pathlib.Path, if end > start + len('include('): try: include_cmake_file = pathlib.Path(engine_path / line[start + len('include('): end]).resolve() - except Exception as e: + except FileNotFoundError as e: pass else: parse_cmake_file(include_cmake_file, files) @@ -88,7 +84,7 @@ def add_external_subdirectory(external_subdir: str or pathlib.Path, try: include_cmake_file = pathlib.Path( cmake_path / line[start + len('add_subdirectory('): end]).resolve() - except Exception as e: + except FileNotFoundError as e: pass else: parse_cmake_file(include_cmake_file, files) @@ -100,8 +96,7 @@ def add_external_subdirectory(external_subdir: str or pathlib.Path, if external_subdir in cmake_files: manifest.save_o3de_manifest(json_data) - if not suppress_errors: - logger.error(f'External subdirectory {external_subdir.as_posix()} already included by add_subdirectory().') + logger.warning(f'External subdirectory {external_subdir.as_posix()} already included by add_subdirectory().') return 1 engine_object['external_subdirectories'].insert(0, external_subdir.as_posix()) @@ -119,23 +114,55 @@ def _run_add_external_subdirectory(args: argparse) -> int: return add_external_subdirectory(args.external_subdirectory) -def add_args(parser, subparsers) -> None: +def add_parser_args(parser): """ - add_args is called to add expected parser arguments and subparsers arguments to each command such that it can be + add_parser_args is called to add arguments to each command such that it can be invoked locally or added by a central python file. - Ex. Directly run from this file alone with: python register.py register --gem-path "C:/TestGem" - OR - o3de.py can downloadable commands by importing engine_template, - call add_args and execute: python o3de.py register --gem-path "C:/TestGem" - :param parser: the caller instantiates a parser and passes it in here + Ex. Directly run from this file alone with: python add-external-subdirectory.py "/home/foo/external-subdir" + :param parser: the caller passes an argparse parser like instance to this method + """ + parser.add_argument('external_subdirectory', metavar='external_subdirectory', type=str, + help='add an external subdirectory to cmake') + + parser.add_argument('-ohf', '--override-home-folder', type=str, required=False, + help='By default the home folder is the user folder, override it to this folder.') + + parser.set_defaults(func=_run_add_external_subdirectory) + + +def add_args(subparsers) -> None: + """ + add_args is called to add subparsers arguments to each command such that it can be + a central python file such as o3de.py. + It can be run from the o3de.py script as follows + call add_args and execute: python o3de.py add_external_subdirectory "/home/foo/external-subdir" :param subparsers: the caller instantiates subparsers and passes it in here """ add_external_subdirectory_subparser = subparsers.add_parser('add-external-subdirectory') - add_external_subdirectory_subparser.add_argument('external_subdirectory', metavar='external_subdirectory', type=str, - help='add an external subdirectory to cmake') + add_parser_args(add_external_subdirectory_subparser) - add_external_subdirectory_subparser.add_argument('-ohf', '--override-home-folder', type=str, required=False, - help='By default the home folder is the user folder, override it to this folder.') - add_external_subdirectory_subparser.set_defaults(func=_run_add_external_subdirectory) +def main(): + """ + Runs add_external_subdirectory.py script as standalone script + """ + # parse the command line args + the_parser = argparse.ArgumentParser() + # add subparsers + + # add args to the parser + add_parser_args(the_parser) + + # parse args + the_args = the_parser.parse_args() + + # run + ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1 + + # return + sys.exit(ret) + + +if __name__ == "__main__": + main() diff --git a/scripts/o3de/o3de/add_gem_cmake.py b/scripts/o3de/o3de/add_gem_cmake.py index fa2d2f4bb3..523fb8dce8 100644 --- a/scripts/o3de/o3de/add_gem_cmake.py +++ b/scripts/o3de/o3de/add_gem_cmake.py @@ -15,6 +15,7 @@ Contains command to add a gem to a project's cmake scripts import argparse import logging import pathlib +import sys from o3de import add_external_subdirectory, manifest, validation @@ -24,39 +25,33 @@ logging.basicConfig() def add_gem_to_cmake(gem_name: str = None, gem_path: str or pathlib.Path = None, engine_name: str = None, - engine_path: str or pathlib.Path = None, - suppress_errors: bool = False) -> int: + engine_path: str or pathlib.Path = None) -> int: """ add a gem to a cmake as an external subdirectory for an engine :param gem_name: name of the gem to add to cmake :param gem_path: the path of the gem to add to cmake :param engine_name: name of the engine to add to cmake :param engine_path: the path of the engine to add external subdirectory to, default to this engine - :param suppress_errors: optional silence errors :return: 0 for success or non 0 failure code """ if not gem_name and not gem_path: - if not suppress_errors: - logger.error('Must specify either a Gem name or Gem Path.') + logger.error('Must specify either a Gem name or Gem Path.') return 1 if gem_name and not gem_path: gem_path = manifest.get_registered(gem_name=gem_name) if not gem_path: - if not suppress_errors: - logger.error(f'Gem Path {gem_path} has not been registered.') + logger.error(f'Gem Path {gem_path} has not been registered.') return 1 gem_path = pathlib.Path(gem_path).resolve() gem_json = gem_path / 'gem.json' if not gem_json.is_file(): - if not suppress_errors: - logger.error(f'Gem json {gem_json} is not present.') + logger.error(f'Gem json {gem_json} is not present.') return 1 if not validation.valid_o3de_gem_json(gem_json): - if not suppress_errors: - logger.error(f'Gem json {gem_json} is not valid.') + logger.error(f'Gem json {gem_json} is not valid.') return 1 if not engine_name and not engine_path: @@ -66,22 +61,18 @@ def add_gem_to_cmake(gem_name: str = None, engine_path = manifest.get_registered(engine_name=engine_name) if not engine_path: - if not suppress_errors: - logger.error(f'Engine Path {engine_path} has not been registered.') + logger.error(f'Engine Path {engine_path} has not been registered.') return 1 engine_json = engine_path / 'engine.json' if not engine_json.is_file(): - if not suppress_errors: - logger.error(f'Engine json {engine_json} is not present.') + logger.error(f'Engine json {engine_json} is not present.') return 1 if not validation.valid_o3de_engine_json(engine_json): - if not suppress_errors: - logger.error(f'Engine json {engine_json} is not valid.') + logger.error(f'Engine json {engine_json} is not valid.') return 1 - return add_external_subdirectory.add_external_subdirectory(external_subdir=gem_path, engine_path=engine_path, suppress_errors=suppress_errors) - + return add_external_subdirectory.add_external_subdirectory(external_subdir=gem_path, engine_path=engine_path) def _run_add_gem_to_cmake(args: argparse) -> int: if args.override_home_folder: @@ -90,25 +81,58 @@ def _run_add_gem_to_cmake(args: argparse) -> int: return add_gem_to_cmake(gem_name=args.gem_name, gem_path=args.gem_path) -def add_args(parser, subparsers) -> None: +def add_parser_args(parser): """ - add_args is called to add expected parser arguments and subparsers arguments to each command such that it can be + add_parser_args is called to add arguments to each command such that it can be invoked locally or added by a central python file. - Ex. Directly run from this file alone with: python register.py register --gem-path "C:/TestGem" - OR - o3de.py can downloadable commands by importing engine_template, - call add_args and execute: python o3de.py register --gem-path "C:/TestGem" - :param parser: the caller instantiates a parser and passes it in here - :param subparsers: the caller instantiates subparsers and passes it in here + Ex. Directly run from this file alone with: python add_gem_cmake.py --gem-path "/path/to/gem" + :param parser: the caller passes an argparse parser like instance to this method """ - add_gem_to_cmake_subparser = subparsers.add_parser('add-gem-to-cmake') - group = add_gem_to_cmake_subparser.add_mutually_exclusive_group(required=True) + group = parser.add_mutually_exclusive_group(required=True) group.add_argument('-gp', '--gem-path', type=str, required=False, help='The path to the gem.') group.add_argument('-gn', '--gem-name', type=str, required=False, help='The name of the gem.') - add_gem_to_cmake_subparser.add_argument('-ohf', '--override-home-folder', type=str, required=False, + parser.add_argument('-ohf', '--override-home-folder', type=str, required=False, help='By default the home folder is the user folder, override it to this folder.') - add_gem_to_cmake_subparser.set_defaults(func=_run_add_gem_to_cmake) + parser.set_defaults(func=_run_add_gem_to_cmake) + + +def add_args(subparsers) -> None: + """ + add_args is called to add subparsers arguments to each command such that it can be + a central python file such as o3de.py. + It can be run from the o3de.py script as follows + call add_args and execute: python o3de.py add-gem-to-cmake --gem-path "/path/to/gem" + :param subparsers: the caller instantiates subparsers and passes it in here + """ + add_gem_cmake_subparser = subparsers.add_parser('add-gem-to-cmake') + add_parser_args(add_gem_cmake_subparser) + + +def main(): + """ + Runs add_gem_cmake.py script as standalone script + """ + # parse the command line args + the_parser = argparse.ArgumentParser() + + # add subparsers + + # add args to the parser + add_parser_args(the_parser) + + # parse args + the_args = the_parser.parse_args() + + # run + ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1 + + # return + sys.exit(ret) + + +if __name__ == "__main__": + main() diff --git a/scripts/o3de/o3de/add_gem_project.py b/scripts/o3de/o3de/add_gem_project.py index 933fd019bb..78dffc4477 100644 --- a/scripts/o3de/o3de/add_gem_project.py +++ b/scripts/o3de/o3de/add_gem_project.py @@ -17,6 +17,7 @@ import json import logging import os import pathlib +import sys from o3de import add_gem_cmake, cmake, manifest, validation @@ -126,13 +127,13 @@ def add_gem_to_project(gem_name: str = None, with project_json.open('r') as s: try: project_json_data = json.load(s) - except Exception as e: + except json.JSONDecodeError as e: logger.error(f'Error loading Project json {project_json}: {str(e)}') return 1 else: try: engine_name = project_json_data['engine'] - except Exception as e: + except KeyError as e: logger.error(f'Project json {project_json} "engine" not found: {str(e)}') return 1 else: @@ -261,51 +262,84 @@ def _run_add_gem_to_project(args: argparse) -> int: args.add_to_cmake) -def add_args(parser, subparsers) -> None: +def add_parser_args(parser): """ - add_args is called to add expected parser arguments and subparsers arguments to each command such that it can be + add_parser_args is called to add arguments to each command such that it can be invoked locally or added by a central python file. - Ex. Directly run from this file alone with: python register.py register --gem-path "C:/TestGem" - OR - o3de.py can downloadable commands by importing engine_template, - call add_args and execute: python o3de.py register --gem-path "C:/TestGem" - :param parser: the caller instantiates a parser and passes it in here - :param subparsers: the caller instantiates subparsers and passes it in here + Ex. Directly run from this file alone with: python add_gem_project.py --project-path "D:/TestProject" --gem-path "D:/TestGem" + :param parser: the caller passes an argparse parser like instance to this method """ - add_gem_subparser = subparsers.add_parser('add-gem-to-project') - group = add_gem_subparser.add_mutually_exclusive_group(required=True) + group = parser.add_mutually_exclusive_group(required=True) group.add_argument('-pp', '--project-path', type=str, required=False, help='The path to the project.') group.add_argument('-pn', '--project-name', type=str, required=False, help='The name of the project.') - group = add_gem_subparser.add_mutually_exclusive_group(required=True) + group = parser.add_mutually_exclusive_group(required=True) group.add_argument('-gp', '--gem-path', type=str, required=False, help='The path to the gem.') group.add_argument('-gn', '--gem-name', type=str, required=False, help='The name of the gem.') - add_gem_subparser.add_argument('-gt', '--gem-target', type=str, required=False, + parser.add_argument('-gt', '--gem-target', type=str, required=False, help='The cmake target name to add. If not specified it will assume gem_name') - add_gem_subparser.add_argument('-df', '--dependencies-file', type=str, required=False, + parser.add_argument('-df', '--dependencies-file', type=str, required=False, help='The cmake dependencies file in which the gem dependencies are specified.' 'If not specified it will assume ') - add_gem_subparser.add_argument('-rd', '--runtime-dependency', action='store_true', required=False, + parser.add_argument('-rd', '--runtime-dependency', action='store_true', required=False, default=False, help='Optional toggle if this gem should be added as a runtime dependency') - add_gem_subparser.add_argument('-td', '--tool-dependency', action='store_true', required=False, + parser.add_argument('-td', '--tool-dependency', action='store_true', required=False, default=False, help='Optional toggle if this gem should be added as a tool dependency') - add_gem_subparser.add_argument('-sd', '--server-dependency', action='store_true', required=False, + parser.add_argument('-sd', '--server-dependency', action='store_true', required=False, default=False, help='Optional toggle if this gem should be added as a server dependency') - add_gem_subparser.add_argument('-pl', '--platforms', type=str, required=False, + parser.add_argument('-pl', '--platforms', type=str, required=False, default='Common', help='Optional list of platforms this gem should be added to.' ' Ex. --platforms Mac,Windows,Linux') - add_gem_subparser.add_argument('-a', '--add-to-cmake', type=bool, required=False, + parser.add_argument('-a', '--add-to-cmake', type=bool, required=False, default=True, help='Automatically call add-gem-to-cmake.') - add_gem_subparser.add_argument('-ohf', '--override-home-folder', type=str, required=False, + parser.add_argument('-ohf', '--override-home-folder', type=str, required=False, help='By default the home folder is the user folder, override it to this folder.') - add_gem_subparser.set_defaults(func=_run_add_gem_to_project) + parser.set_defaults(func=_run_add_gem_to_project) + + +def add_args(subparsers) -> None: + """ + add_args is called to add subparsers arguments to each command such that it can be + a central python file such as o3de.py. + It can be run from the o3de.py script as follows + call add_args and execute: python o3de.py add-gem-to-project --project-path "D:/TestProject" --gem-path "D:/TestGem" + :param subparsers: the caller instantiates subparsers and passes it in here + """ + add_gem_project_subparser = subparsers.add_parser('add-gem-to-project') + add_parser_args(add_gem_project_subparser) + + +def main(): + """ + Runs add_gem_project.py script as standalone script + """ + # parse the command line args + the_parser = argparse.ArgumentParser() + + # add subparsers + + # add args to the parser + add_parser_args(the_parser) + + # parse args + the_args = the_parser.parse_args() + + # run + ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1 + + # return + sys.exit(ret) + + +if __name__ == "__main__": + main() diff --git a/scripts/o3de/o3de/download.py b/scripts/o3de/o3de/download.py index 3db2f077cd..1dbb584c92 100644 --- a/scripts/o3de/o3de/download.py +++ b/scripts/o3de/o3de/download.py @@ -18,18 +18,90 @@ import json import logging import pathlib import shutil +import sys import urllib.parse import urllib.request -from o3de import manifest, utils, validation +from o3de import manifest, repo, utils, validation logger = logging.getLogger() logging.basicConfig() -def download_engine(engine_name: str, - dest_path: str) -> int: +def unzip_manifest_json_data(download_zip_path: pathlib.Path, zip_file_name: str) -> dict: + json_data = {} + with zipfile.ZipFile(download_zip_path, 'r') as zip_data: + with zip_data.open(zip_file_name) as manifest_json_file: + try: + json_data = json.load(manifest_json_file) + except json.JSONDecodeError as e: + logger.error(f'UnZip exception:{str(e)}') + + return json_data + +def validate_downloaded_zip_sha256(download_uri_json_data: dict, download_zip_path: pathlib.Path, + manifest_json_name) -> int: + # if the engine.json has a sha256 check it against a sha256 of the zip + try: + sha256A = download_uri_json_data['sha256'] + except KeyError as e: + logger.warn(f'SECURITY WARNING: The advertised o3de object you downloaded has no "sha256"!!! Be VERY careful!!!' + f' We cannot verify this is the actually the advertised object!!!') + else: + sha256B = hashlib.sha256(download_zip_path.open('rb').read()).hexdigest() + if sha256A != sha256B: + logger.error(f'SECURITY VIOLATION: Downloaded zip sha256 {sha256B} does not match' + f' the advertised "sha256":{sha256A} in the f{manifest_json_name}. Deleting unzipped files!!!') + shutil.rmtree(dest_path) + return 1 + + manifest_json_data = unzip_manifest_json_data(download_zip_path, manifest_json_name) + + # remove the sha256 if present in the advertised downloadable manifest json + # then compare it to the json in the zip, they should now be identical + try: + del download_uri_json_data['sha256'] + except KeyError as e: + pass + + sha256A = hashlib.sha256(json.dumps(download_uri_json_data, indent=4).encode('utf8')).hexdigest() + with unzipped_manifest_json.open('r') as s: + try: + unzipped_manifest_json_data = json.load(s) + except json.JSONDecodeError as e: + logger.error(f'Failed to read manifest json {unzipped_manifest_json}. Unable to confirm this' + f' is the same template that was advertised.') + return 1 + sha256B = hashlib.sha256(json.dumps(unzipped_manifest_json_data, indent=4).encode('utf8')).hexdigest() + if sha256A != sha256B: + logger.error(f'SECURITY VIOLATION: Downloaded manifest json does not match' + f' the advertised manifest json. Deleting unzipped files!!!') + shutil.rmtree(dest_path) + return 1 + + return 0 + + +def get_downloadable(engine_name: str = None, + project_name: str = None, + gem_name: str = None, + template_name: str = None, + restricted_name: str = None) -> dict or None: + json_data = manifest.load_o3de_manifest() + try: + o3de_object_uris = json_data['repos'] + except KeyError as key_err: + logger.error(f'Unable to load repos from o3de manifest: {str(key_err)}') + return None + + manifest_json = 'repo.json' + search_func = lambda: repo.search_repo(manifest_json, engine_name, project_name, gem_name, template_name) + return repo.search_o3de_object(manifest_json, o3de_object_uris, search_func) + + +def download_o3de_object(object_name: str, default_folder_name: str, dest_path: str or pathlib.Path, + object_type: str, downloadable_kwarg_key) -> int: if not dest_path: - dest_path = manifest.get_registered(default_folder='engines') + dest_path = manifest.get_registered(default_folder=default_folder_name) if not dest_path: logger.error(f'Destination path not cannot be empty.') return 1 @@ -37,512 +109,50 @@ def download_engine(engine_name: str, dest_path = pathlib.Path(dest_path).resolve() dest_path.mkdir(exist_ok=True) - download_path = manifest.get_o3de_download_folder() / 'engines' / engine_name + download_path = manifest.get_o3de_download_folder() / default_folder_name / object_name download_path.mkdir(exist_ok=True) - download_zip_path = download_path / 'engine.zip' + download_zip_path = download_path / f'{object_type}.zip' - downloadable_engine_data = get_downloadable(engine_name=engine_name) - if not downloadable_engine_data: - logger.error(f'Downloadable engine {engine_name} not found.') + downloadable_object_data = get_downloadable(**{downloadable_kwarg_key : object_name}) + if not downloadable_object_data: + logger.error(f'Downloadable o3de object {object_name} not found.') return 1 - origin = downloadable_engine_data['origin'] - url = f'{origin}/project.zip' + origin = downloadable_json_data['origin'] + url = f'{origin}/object_type.zip' parsed_uri = urllib.parse.urlparse(url) - if download_zip_path.is_file(): - logger.warn(f'Project already downloaded to {download_zip_path}.') - elif parsed_uri.scheme == 'http' or \ - parsed_uri.scheme == 'https' or \ - parsed_uri.scheme == 'ftp' or \ - parsed_uri.scheme == 'ftps': - with urllib.request.urlopen(url) as s: - with download_zip_path.open('wb') as f: - shutil.copyfileobj(s, f) - else: - origin_file = pathlib.Path(url).resolve() - if not origin_file.is_file(): - return 1 - shutil.copy(origin_file, download_zip_path) + download_zip_result = utils.download_zip_file(parsed_uri, download_zip_path) + if download_zip_result != 0: + return download_zip_result - if not zipfile.is_zipfile(download_zip_path): - logger.error(f"Engine zip {download_zip_path} is invalid.") - download_zip_path.unlink() - return 1 + return validate_downloaded_zip_sha256(downloadable_object_data, download_zip_path) - # if the engine.json has a sha256 check it against a sha256 of the zip - try: - sha256A = downloadable_engine_data['sha256'] - except Exception as e: - logger.warn(f'SECURITY WARNING: The advertised engine you downloaded has no "sha256"!!! Be VERY careful!!!' - f' We cannot verify this is the actually the advertised engine!!!') - else: - sha256B = hashlib.sha256(download_zip_path.open('rb').read()).hexdigest() - if sha256A != sha256B: - logger.error(f'SECURITY VIOLATION: Downloaded engine.zip sha256 {sha256B} does not match' - f' the advertised "sha256":{sha256A} in the engine.json. Deleting unzipped files!!!') - shutil.rmtree(dest_path) - return 1 - dest_engine_folder = dest_path / engine_name - if dest_engine_folder.is_dir(): - utils.backup_folder(dest_engine_folder) - with zipfile.ZipFile(download_zip_path, 'r') as project_zip: - try: - project_zip.extractall(dest_path) - except Exception as e: - logger.error(f'UnZip exception:{str(e)}') - shutil.rmtree(dest_path) - return 1 - - unzipped_engine_json = dest_engine_folder / 'engine.json' - if not unzipped_engine_json.is_file(): - logger.error(f'Engine json {unzipped_engine_json} is missing.') - return 1 - - if not validation.valid_o3de_engine_json(unzipped_engine_json): - logger.error(f'Engine json {unzipped_engine_json} is invalid.') - return 1 - - # remove the sha256 if present in the advertised downloadable engine.json - # then compare it to the engine.json in the zip, they should now be identical - try: - del downloadable_engine_data['sha256'] - except Exception as e: - pass - - sha256A = hashlib.sha256(json.dumps(downloadable_engine_data, indent=4).encode('utf8')).hexdigest() - with unzipped_engine_json.open('r') as s: - try: - unzipped_engine_json_data = json.load(s) - except Exception as e: - logger.error(f'Failed to read engine json {unzipped_engine_json}. Unable to confirm this' - f' is the same template that was advertised.') - return 1 - sha256B = hashlib.sha256(json.dumps(unzipped_engine_json_data, indent=4).encode('utf8')).hexdigest() - if sha256A != sha256B: - logger.error(f'SECURITY VIOLATION: Downloaded engine.json does not match' - f' the advertised engine.json. Deleting unzipped files!!!') - shutil.rmtree(dest_path) - return 1 - - return 0 +def download_engine(engine_name: str, + dest_path: str or pathlib.Path) -> int: + return download_o3de_object(engine_name, 'engines', dest_path, 'engine', 'engine_name') def download_project(project_name: str, dest_path: str or pathlib.Path) -> int: - if not dest_path: - dest_path = manifest.get_registered(default_folder='projects') - if not dest_path: - logger.error(f'Destination path not specified and not default projects path.') - return 1 - - dest_path = pathlib.Path(dest_path).resolve() - dest_path.mkdir(exist_ok=True, parents=True) - - download_path = manifest.get_o3de_download_folder() / 'projects' / project_name - download_path.mkdir(exist_ok=True, parents=True) - download_zip_path = download_path / 'project.zip' - - downloadable_project_data = get_downloadable(project_name=project_name) - if not downloadable_project_data: - logger.error(f'Downloadable project {project_name} not found.') - return 1 - - origin = downloadable_project_data['origin'] - url = f'{origin}/project.zip' - parsed_uri = urllib.parse.urlparse(url) - - if download_zip_path.is_file(): - logger.warn(f'Project already downloaded to {download_zip_path}.') - elif parsed_uri.scheme == 'http' or \ - parsed_uri.scheme == 'https' or \ - parsed_uri.scheme == 'ftp' or \ - parsed_uri.scheme == 'ftps': - with urllib.request.urlopen(url) as s: - with download_zip_path.open('wb') as f: - shutil.copyfileobj(s, f) - else: - origin_file = pathlib.Path(url).resolve() - if not origin_file.is_file(): - return 1 - shutil.copy(origin_file, download_zip_path) - - if not zipfile.is_zipfile(download_zip_path): - logger.error(f"Project zip {download_zip_path} is invalid.") - download_zip_path.unlink() - return 1 - - # if the project.json has a sha256 check it against a sha256 of the zip - try: - sha256A = downloadable_project_data['sha256'] - except Exception as e: - logger.warn(f'SECURITY WARNING: The advertised project you downloaded has no "sha256"!!! Be VERY careful!!!' - f' We cannot verify this is the actually the advertised project!!!') - else: - sha256B = hashlib.sha256(download_zip_path.open('rb').read()).hexdigest() - if sha256A != sha256B: - logger.error(f'SECURITY VIOLATION: Downloaded project.zip sha256 {sha256B} does not match' - f' the advertised "sha256":{sha256A} in the project.json. Deleting unzipped files!!!') - shutil.rmtree(dest_path) - return 1 - - dest_project_folder = dest_path / project_name - if dest_project_folder.is_dir(): - utils.backup_folder(dest_project_folder) - with zipfile.ZipFile(download_zip_path, 'r') as project_zip: - try: - project_zip.extractall(dest_project_folder) - except Exception as e: - logger.error(f'UnZip exception:{str(e)}') - shutil.rmtree(dest_path) - return 1 - - unzipped_project_json = dest_project_folder / 'project.json' - if not unzipped_project_json.is_file(): - logger.error(f'Project json {unzipped_project_json} is missing.') - return 1 - - if not validation.valid_o3de_project_json(unzipped_project_json): - logger.error(f'Project json {unzipped_project_json} is invalid.') - return 1 - - # remove the sha256 if present in the advertised downloadable project.json - # then compare it to the project.json in the zip, they should now be identical - try: - del downloadable_project_data['sha256'] - except Exception as e: - pass - - sha256A = hashlib.sha256(json.dumps(downloadable_project_data, indent=4).encode('utf8')).hexdigest() - with unzipped_project_json.open('r') as s: - try: - unzipped_project_json_data = json.load(s) - except Exception as e: - logger.error(f'Failed to read Project json {unzipped_project_json}. Unable to confirm this' - f' is the same project that was advertised.') - return 1 - sha256B = hashlib.sha256(json.dumps(unzipped_project_json_data, indent=4).encode('utf8')).hexdigest() - if sha256A != sha256B: - logger.error(f'SECURITY VIOLATION: Downloaded project.json does not match' - f' the advertised project.json. Deleting unzipped files!!!') - shutil.rmtree(dest_path) - return 1 - - return 0 + return download_o3de_object(project_name, 'projects', dest_path, 'project', 'project_name') def download_gem(gem_name: str, dest_path: str or pathlib.Path) -> int: - if not dest_path: - dest_path = manifest.get_registered(default_folder='gems') - if not dest_path: - logger.error(f'Destination path not cannot be empty.') - return 1 - - dest_path = pathlib.Path(dest_path).resolve() - dest_path.mkdir(exist_ok=True, parents=True) - - download_path = manifest.get_o3de_download_folder() / 'gems' / gem_name - download_path.mkdir(exist_ok=True, parents=True) - download_zip_path = download_path / 'gem.zip' - - downloadable_gem_data = get_downloadable(gem_name=gem_name) - if not downloadable_gem_data: - logger.error(f'Downloadable gem {gem_name} not found.') - return 1 - - origin = downloadable_gem_data['origin'] - url = f'{origin}/gem.zip' - parsed_uri = urllib.parse.urlparse(url) - - if download_zip_path.is_file(): - logger.warn(f'Project already downloaded to {download_zip_path}.') - elif parsed_uri.scheme == 'http' or \ - parsed_uri.scheme == 'https' or \ - parsed_uri.scheme == 'ftp' or \ - parsed_uri.scheme == 'ftps': - with urllib.request.urlopen(url) as s: - with download_zip_path.open('wb') as f: - shutil.copyfileobj(s, f) - else: - origin_file = pathlib.Path(url).resolve() - if not origin_file.is_file(): - return 1 - shutil.copy(origin_file, download_zip_path) - - if not zipfile.is_zipfile(download_zip_path): - logger.error(f"Gem zip {download_zip_path} is invalid.") - download_zip_path.unlink() - return 1 - - # if the gem.json has a sha256 check it against a sha256 of the zip - try: - sha256A = downloadable_gem_data['sha256'] - except Exception as e: - logger.warn(f'SECURITY WARNING: The advertised gem you downloaded has no "sha256"!!! Be VERY careful!!!' - f' We cannot verify this is the actually the advertised gem!!!') - else: - sha256B = hashlib.sha256(download_zip_path.open('rb').read()).hexdigest() - if sha256A != sha256B: - logger.error(f'SECURITY VIOLATION: Downloaded gem.zip sha256 {sha256B} does not match' - f' the advertised "sha256":{sha256A} in the gem.json. Deleting unzipped files!!!') - shutil.rmtree(dest_path) - return 1 - - dest_gem_folder = dest_path / gem_name - if dest_gem_folder.is_dir(): - utils.backup_folder(dest_gem_folder) - with zipfile.ZipFile(download_zip_path, 'r') as gem_zip: - try: - gem_zip.extractall(dest_path) - except Exception as e: - logger.error(f'UnZip exception:{str(e)}') - shutil.rmtree(dest_path) - return 1 - - unzipped_gem_json = dest_gem_folder / 'gem.json' - if not unzipped_gem_json.is_file(): - logger.error(f'Engine json {unzipped_gem_json} is missing.') - return 1 - - if not validation.valid_o3de_engine_json(unzipped_gem_json): - logger.error(f'Engine json {unzipped_gem_json} is invalid.') - return 1 - - # remove the sha256 if present in the advertised downloadable gem.json - # then compare it to the gem.json in the zip, they should now be identical - try: - del downloadable_gem_data['sha256'] - except Exception as e: - pass - - sha256A = hashlib.sha256(json.dumps(downloadable_gem_data, indent=4).encode('utf8')).hexdigest() - with unzipped_gem_json.open('r') as s: - try: - unzipped_gem_json_data = json.load(s) - except Exception as e: - logger.error(f'Failed to read gem json {unzipped_gem_json}. Unable to confirm this' - f' is the same gem that was advertised.') - return 1 - sha256B = hashlib.sha256(json.dumps(unzipped_gem_json_data, indent=4).encode('utf8')).hexdigest() - if sha256A != sha256B: - logger.error(f'SECURITY VIOLATION: Downloaded gem.json does not match' - f' the advertised gem.json. Deleting unzipped files!!!') - shutil.rmtree(dest_path) - return 1 - - return 0 + return download_o3de_object(gem_name, 'gems', dest_path, 'gem', 'gem_name') def download_template(template_name: str, dest_path: str or pathlib.Path) -> int: - if not dest_path: - dest_path = manifest.get_registered(default_folder='templates') - if not dest_path: - logger.error(f'Destination path not cannot be empty.') - return 1 + return download_o3de_object(template_name, 'templates', dest_path, 'template', 'template_name') - dest_path = pathlib.Path(dest_path).resolve() - dest_path.mkdir(exist_ok=True, parents=True) - - download_path = manifest.get_o3de_download_folder() / 'templates' / template_name - download_path.mkdir(exist_ok=True, parents=True) - download_zip_path = download_path / 'template.zip' - - downloadable_template_data = get_downloadable(template_name=template_name) - if not downloadable_template_data: - logger.error(f'Downloadable template {template_name} not found.') - return 1 - - origin = downloadable_template_data['origin'] - url = f'{origin}/project.zip' - parsed_uri = urllib.parse.urlparse(url) - - result = 0 - - if download_zip_path.is_file(): - logger.warn(f'Project already downloaded to {download_zip_path}.') - elif parsed_uri.scheme == 'http' or \ - parsed_uri.scheme == 'https' or \ - parsed_uri.scheme == 'ftp' or \ - parsed_uri.scheme == 'ftps': - with urllib.request.urlopen(url) as s: - with download_zip_path.open('wb') as f: - shutil.copyfileobj(s, f) - else: - origin_file = pathlib.Path(url).resolve() - if not origin_file.is_file(): - return 1 - shutil.copy(origin_file, download_zip_path) - - if not zipfile.is_zipfile(download_zip_path): - logger.error(f"Template zip {download_zip_path} is invalid.") - download_zip_path.unlink() - return 1 - - # if the template.json has a sha256 check it against a sha256 of the zip - try: - sha256A = downloadable_template_data['sha256'] - except Exception as e: - logger.warn(f'SECURITY WARNING: The advertised template you downloaded has no "sha256"!!! Be VERY careful!!!' - f' We cannot verify this is the actually the advertised template!!!') - else: - sha256B = hashlib.sha256(download_zip_path.open('rb').read()).hexdigest() - if sha256A != sha256B: - logger.error(f'SECURITY VIOLATION: Downloaded template.zip sha256 {sha256B} does not match' - f' the advertised "sha256":{sha256A} in the template.json. Deleting unzipped files!!!') - shutil.rmtree(dest_path) - return 1 - - dest_template_folder = dest_path / template_name - if dest_template_folder.is_dir(): - utils.backup_folder(dest_template_folder) - with zipfile.ZipFile(download_zip_path, 'r') as project_zip: - try: - project_zip.extractall(dest_path) - except Exception as e: - logger.error(f'UnZip exception:{str(e)}') - shutil.rmtree(dest_path) - return 1 - - unzipped_template_json = dest_template_folder / 'template.json' - if not unzipped_template_json.is_file(): - logger.error(f'Template json {unzipped_template_json} is missing.') - return 1 - - if not validation.valid_o3de_engine_json(unzipped_template_json): - logger.error(f'Template json {unzipped_template_json} is invalid.') - return 1 - - # remove the sha256 if present in the advertised downloadable template.json - # then compare it to the template.json in the zip, they should now be identical - try: - del downloadable_template_data['sha256'] - except Exception as e: - pass - - sha256A = hashlib.sha256(json.dumps(downloadable_template_data, indent=4).encode('utf8')).hexdigest() - with unzipped_template_json.open('r') as s: - try: - unzipped_template_json_data = json.load(s) - except Exception as e: - logger.error(f'Failed to read Template json {unzipped_template_json}. Unable to confirm this' - f' is the same template that was advertised.') - return 1 - sha256B = hashlib.sha256(json.dumps(unzipped_template_json_data, indent=4).encode('utf8')).hexdigest() - if sha256A != sha256B: - logger.error(f'SECURITY VIOLATION: Downloaded template.json does not match' - f' the advertised template.json. Deleting unzipped files!!!') - shutil.rmtree(dest_path) - return 1 - - return 0 def download_restricted(restricted_name: str, dest_path: str or pathlib.Path) -> int: - if not dest_path: - dest_path = manifest.get_registered(default_folder='restricted') - if not dest_path: - logger.error(f'Destination path not cannot be empty.') - return 1 - - dest_path = pathlib.Path(dest_path).resolve() - dest_path.mkdir(exist_ok=True, parents=True) - - download_path = manifest.get_o3de_download_folder() / 'restricted' / restricted_name - download_path.mkdir(exist_ok=True, parents=True) - download_zip_path = download_path / 'restricted.zip' - - downloadable_restricted_data = get_downloadable(restricted_name=restricted_name) - if not downloadable_restricted_data: - logger.error(f'Downloadable Restricted {restricted_name} not found.') - return 1 - - origin = downloadable_restricted_data['origin'] - url = f'{origin}/restricted.zip' - parsed_uri = urllib.parse.urlparse(url) - - if download_zip_path.is_file(): - logger.warn(f'Restricted already downloaded to {download_zip_path}.') - elif parsed_uri.scheme == 'http' or \ - parsed_uri.scheme == 'https' or \ - parsed_uri.scheme == 'ftp' or \ - parsed_uri.scheme == 'ftps': - with urllib.request.urlopen(url) as s: - with download_zip_path.open('wb') as f: - shutil.copyfileobj(s, f) - else: - origin_file = pathlib.Path(url).resolve() - if not origin_file.is_file(): - return 1 - shutil.copy(origin_file, download_zip_path) - - if not zipfile.is_zipfile(download_zip_path): - logger.error(f"Restricted zip {download_zip_path} is invalid.") - download_zip_path.unlink() - return 1 - - # if the restricted.json has a sha256 check it against a sha256 of the zip - try: - sha256A = downloadable_restricted_data['sha256'] - except Exception as e: - logger.warn(f'SECURITY WARNING: The advertised restricted you downloaded has no "sha256"!!! Be VERY careful!!!' - f' We cannot verify this is the actually the advertised restricted!!!') - else: - sha256B = hashlib.sha256(download_zip_path.open('rb').read()).hexdigest() - if sha256A != sha256B: - logger.error(f'SECURITY VIOLATION: Downloaded restricted.zip sha256 {sha256B} does not match' - f' the advertised "sha256":{sha256A} in the restricted.json. Deleting unzipped files!!!') - shutil.rmtree(dest_path) - return 1 - - dest_restricted_folder = dest_path / restricted_name - if dest_restricted_folder.is_dir(): - utils.backup_folder(dest_restricted_folder) - with zipfile.ZipFile(download_zip_path, 'r') as project_zip: - try: - project_zip.extractall(dest_path) - except Exception as e: - logger.error(f'UnZip exception:{str(e)}') - shutil.rmtree(dest_path) - return 1 - - unzipped_restricted_json = dest_restricted_folder / 'restricted.json' - if not unzipped_restricted_json.is_file(): - logger.error(f'Restricted json {unzipped_restricted_json} is missing.') - return 1 - - if not validation.valid_o3de_engine_json(unzipped_restricted_json): - logger.error(f'Restricted json {unzipped_restricted_json} is invalid.') - return 1 - - # remove the sha256 if present in the advertised downloadable restricted.json - # then compare it to the restricted.json in the zip, they should now be identical - try: - del downloadable_restricted_data['sha256'] - except Exception as e: - pass - - sha256A = hashlib.sha256(json.dumps(downloadable_restricted_data, indent=4).encode('utf8')).hexdigest() - with unzipped_restricted_json.open('r') as s: - try: - unzipped_restricted_json_data = json.load(s) - except Exception as e: - logger.error( - f'Failed to read Restricted json {unzipped_restricted_json}. Unable to confirm this' - f' is the same restricted that was advertised.') - return 1 - sha256B = hashlib.sha256( - json.dumps(unzipped_restricted_json_data, indent=4).encode('utf8')).hexdigest() - if sha256A != sha256B: - logger.error(f'SECURITY VIOLATION: Downloaded restricted.json does not match' - f' the advertised restricted.json. Deleting unzipped files!!!') - shutil.rmtree(dest_path) - return 1 - - return 0 + return download_o3de_object(restricted_name, 'restricted', dest_path, 'restricted', 'restricted_name') def _run_download(args: argparse) -> int: @@ -562,20 +172,16 @@ def _run_download(args: argparse) -> int: return download_template(args.template_name, args.dest_path) + return 1 -def add_args(parser, subparsers) -> None: +def add_parser_args(parser): """ - add_args is called to add expected parser arguments and subparsers arguments to each command such that it can be + add_parser_args is called to add arguments to each command such that it can be invoked locally or added by a central python file. - Ex. Directly run from this file alone with: python register.py register --gem-path "C:/TestGem" - OR - o3de.py can downloadable commands by importing engine_template, - call add_args and execute: python o3de.py register --gem-path "C:/TestGem" - :param parser: the caller instantiates a parser and passes it in here - :param subparsers: the caller instantiates subparsers and passes it in here + Ex. Directly run from this file alone with: python download.py --engine-name "o3de" + :param parser: the caller passes an argparse parser like instance to this method """ - download_subparser = subparsers.add_parser('download') - group = download_subparser.add_mutually_exclusive_group(required=True) + group = parser.add_mutually_exclusive_group(required=True) group.add_argument('-e', '--engine-name', type=str, required=False, help='Downloadable engine name.') group.add_argument('-p', '--project-name', type=str, required=False, @@ -584,15 +190,52 @@ def add_args(parser, subparsers) -> None: help='Downloadable gem name.') group.add_argument('-t', '--template-name', type=str, required=False, help='Downloadable template name.') - download_subparser.add_argument('-dp', '--dest-path', type=str, required=False, + parser.add_argument('-dp', '--dest-path', type=str, required=False, default=None, help='Optional destination folder to download into.' - ' i.e. download --project-name "StarterGame" --dest-path "C:/projects"' - ' will result in C:/projects/StarterGame' + ' i.e. download --project-name "AstomSamplerViewer" --dest-path "C:/projects"' + ' will result in C:/projects/AtomSampleViewer' ' If blank will download to default object type folder') - download_subparser.add_argument('-ohf', '--override-home-folder', type=str, required=False, + parser.add_argument('-ohf', '--override-home-folder', type=str, required=False, help='By default the home folder is the user folder, override it to this folder.') - download_subparser.set_defaults(func=_run_download) + parser.set_defaults(func=_run_download) + +def add_args(subparsers) -> None: + """ + add_args is called to add subparsers arguments to each command such that it can be + a central python file such as o3de.py. + It can be run from the o3de.py script as follows + call add_args and execute: python o3de.py download --engine-name "o3de" + :param subparsers: the caller instantiates subparsers and passes it in here + """ + download_subparser = subparsers.add_parser('download') + add_parser_args(download_subparser) + + +def main(): + """ + Runs download.py script as standalone script + """ + # parse the command line args + the_parser = argparse.ArgumentParser() + + # add subparsers + + # add args to the parser + add_parser_args(the_parser) + + # parse args + the_args = the_parser.parse_args() + + # run + ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1 + + # return + sys.exit(ret) + + +if __name__ == "__main__": + main() diff --git a/scripts/o3de/o3de/engine_template.py b/scripts/o3de/o3de/engine_template.py index 324c204a43..0c63d4e42e 100755 --- a/scripts/o3de/o3de/engine_template.py +++ b/scripts/o3de/o3de/engine_template.py @@ -79,7 +79,7 @@ restricted_platforms = { } template_file_name = 'template.json' - +this_script_parent = os.path.dirname(os.path.realpath(__file__)) def _transform(s_data: str, replacements: list, @@ -329,7 +329,7 @@ def _instantiate_template(template_json_data: dict, with open(platform_json, 'r') as s: try: json_data = json.load(s) - except Exception as e: + except json.JSONDecodeError as e: logger.error(f'Failed to load {platform_json}: ' + str(e)) return 1 else: @@ -407,7 +407,7 @@ def create_template(source_path: str, template_path = f'{default_templates_folder}/{template_path}' logger.info(f'Template path not a full path. Using default templates folder {template_path}') if os.path.isdir(template_path): - logger.error(f'Template path {template_path} is already exists.') + logger.error(f'Template path {template_path} already exists.') return 1 # template name is now the last component of the template_path @@ -432,12 +432,12 @@ def create_template(source_path: str, with open(engine_json) as s: try: engine_json_data = json.load(s) - except Exception as e: + except json.JSONDecodeError as e: logger.error(f"Failed to read engine json {engine_json}: {str(e)}") return 1 try: engine_restricted = engine_json_data['restricted_name'] - except Exception as e: + except KeyError as e: logger.error(f"Engine json {engine_json} restricted not found.") return 1 engine_restricted_folder = manifest.get_registered(restricted_name=engine_restricted) @@ -475,12 +475,12 @@ def create_template(source_path: str, with open(restricted_json, 'r') as s: try: restricted_json_data = json.load(s) - except Exception as e: + except json.JSONDecodeError as e: logger.error(f'Failed to load {restricted_json}: ' + str(e)) return 1 try: template_restricted_name = restricted_json_data['restricted_name'] - except Exception as e: + except KeyError as e: logger.error(f'Failed to read restricted_name from {restricted_json}') return 1 else: @@ -943,8 +943,7 @@ def create_template(source_path: str, s.write(json.dumps(json_data, indent=4)) # copy the default preview.png - this_script_parent = os.path.dirname(os.path.realpath(__file__)) - preview_png_src = f'{this_script_parent}/preview.png' + preview_png_src = f'{this_script_parent}/resources/preview.png' preview_png_dst = f'{template_path}/Template/preview.png' if not os.path.isfile(preview_png_dst): shutil.copy(preview_png_src, preview_png_dst) @@ -1067,14 +1066,14 @@ def create_from_template(destination_path: str, with open(template_json) as s: try: template_json_data = json.load(s) - except Exception as e: + except KeyError as e: logger.error(f'Could read template json {template_json}: {str(e)}.') return 1 # read template name from the json try: template_name = template_json_data['template_name'] - except Exception as e: + except KeyError as e: logger.error(f'Could not read "template_name" from template json {template_json}: {str(e)}.') return 1 @@ -1083,7 +1082,7 @@ def create_from_template(destination_path: str, if not template_restricted_name and not template_restricted_path: try: template_json_restricted_name = template_json_data['restricted_name'] - except Exception as e: + except KeyError as e: # the template json doesn't have a 'restricted_name' element warn and use it logger.info(f'The template does not specify a "restricted_name".') else: @@ -1100,7 +1099,7 @@ def create_from_template(destination_path: str, # The user specified a --template-restricted-name try: template_json_restricted_name = template_json_data['restricted_name'] - except Exception as e: + except KeyError as e: # the template json doesn't have a 'restricted_name' element warn and use it logger.info(f'The template does not specify a "restricted_name".' f' Using supplied {template_restricted_name}') @@ -1120,7 +1119,7 @@ def create_from_template(destination_path: str, template_restricted_path = template_restricted_path.replace('\\', '/') try: template_json_restricted_name = template_json_data['restricted_name'] - except Exception as e: + except KeyError as e: # the template json doesn't have a 'restricted_name' element warn and use it logger.info(f'The template does not specify a "restricted_name".' f' Using supplied {template_restricted_path}') @@ -1154,7 +1153,7 @@ def create_from_template(destination_path: str, try: template_json_restricted_platform_relative_path = template_json_data[ 'restricted_platform_relative_path'] - except Exception as e: + except KeyError as e: # the template json doesn't have a 'restricted_platform_relative_path' element warn and use it logger.info(f'The template does not specify a "restricted_platform_relative_path".' f' Using {template_restricted_platform_relative_path}') @@ -1175,7 +1174,7 @@ def create_from_template(destination_path: str, try: template_restricted_platform_relative_path = template_json_data[ 'restricted_platform_relative_path'] - except Exception as e: + except KeyError as e: # The template json doesn't have a 'restricted_platform_relative_path' element, set empty string. template_restricted_platform_relative_path = '' @@ -1356,14 +1355,14 @@ def create_project(project_path: str, with open(template_json) as s: try: template_json_data = json.load(s) - except Exception as e: + except json.JSONDecodeError as e: logger.error(f'Could read template json {template_json}: {str(e)}.') return 1 # read template name from the json try: template_name = template_json_data['template_name'] - except Exception as e: + except KeyError as e: logger.error(f'Could not read "template_name" from template json {template_json}: {str(e)}.') return 1 @@ -1372,7 +1371,7 @@ def create_project(project_path: str, if not template_restricted_name and not template_restricted_path: try: template_json_restricted_name = template_json_data['restricted_name'] - except Exception as e: + except KeyError as e: # the template json doesn't have a 'restricted_name' element warn and use it logger.info(f'The template does not specify a "restricted_name".') else: @@ -1389,7 +1388,7 @@ def create_project(project_path: str, # The user specified a --template-restricted-name try: template_json_restricted_name = template_json_data['restricted_name'] - except Exception as e: + except KeyError as e: # the template json doesn't have a 'restricted_name' element warn and use it logger.info(f'The template does not specify a "restricted_name".' f' Using supplied {template_restricted_name}') @@ -1409,7 +1408,7 @@ def create_project(project_path: str, template_restricted_path = template_restricted_path.replace('\\', '/') try: template_json_restricted_name = template_json_data['restricted_name'] - except Exception as e: + except KeyError as e: # the template json doesn't have a 'restricted_name' element warn and use it logger.info(f'The template does not specify a "restricted_name".' f' Using supplied {template_restricted_path}') @@ -1442,7 +1441,7 @@ def create_project(project_path: str, try: template_json_restricted_platform_relative_path = template_json_data[ 'restricted_platform_relative_path'] - except Exception as e: + except KeyError as e: # the template json doesn't have a 'restricted_platform_relative_path' element warn and use it logger.info(f'The template does not specify a "restricted_platform_relative_path".' f' Using {template_restricted_platform_relative_path}') @@ -1463,7 +1462,7 @@ def create_project(project_path: str, try: template_restricted_platform_relative_path = template_json_data[ 'restricted_platform_relative_path'] - except Exception as e: + except KeyError as e: # The template json doesn't have a 'restricted_platform_relative_path' element, set empty string. template_restricted_platform_relative_path = '' if not template_restricted_platform_relative_path: @@ -1597,13 +1596,13 @@ def create_project(project_path: str, with open(restricted_json, 'r') as s: try: restricted_json_data = json.load(s) - except Exception as e: + except json.JSONDecodeError as e: logger.error(f'Failed to load restricted json {restricted_json}.') return 1 try: restricted_name = restricted_json_data["restricted_name"] - except Exception as e: + except KeyError as e: logger.error(f'Failed to read "restricted_name" from restricted json {restricted_json}.') return 1 @@ -1616,7 +1615,7 @@ def create_project(project_path: str, with open(project_json, 'r') as s: try: project_json_data = json.load(s) - except Exception as e: + except json.JSONDecodeError as e: logger.error(f'Failed to load project json {project_json}.') return 1 @@ -1625,7 +1624,7 @@ def create_project(project_path: str, with open(project_json, 'w') as s: try: s.write(json.dumps(project_json_data, indent=4)) - except Exception as e: + except OSError as e: logger.error(f'Failed to write project json {project_json}.') return 1 @@ -1656,7 +1655,7 @@ def create_project(project_path: str, engine_json_data = manifest.get_engine_json_data(engine_path=manifest.get_this_engine_path()) try: engine_name = engine_json_data['engine_name'] - except Exception as e: + except KeyError as e: logger.error(f"engine_name for this engine not found in engine.json.") return 1 @@ -1665,7 +1664,7 @@ def create_project(project_path: str, with open(project_json, 'w') as s: try: s.write(json.dumps(project_json_data, indent=4)) - except Exception as e: + except OSError as e: logger.error(f'Failed to write project json at {project_path}.') return 1 @@ -1749,14 +1748,14 @@ def create_gem(gem_path: str, with open(template_json) as s: try: template_json_data = json.load(s) - except Exception as e: + except json.JSONDecodeError as e: logger.error(f'Could read template json {template_json}: {str(e)}.') return 1 # read template name from the json try: template_name = template_json_data['template_name'] - except Exception as e: + except KeyError as e: logger.error(f'Could not read "template_name" from template json {template_json}: {str(e)}.') return 1 @@ -1765,7 +1764,7 @@ def create_gem(gem_path: str, if not template_restricted_name and not template_restricted_path: try: template_json_restricted_name = template_json_data['restricted_name'] - except Exception as e: + except KeyError as e: # the template json doesn't have a 'restricted_name' element warn and use it logger.info(f'The template does not specify a "restricted_name".') else: @@ -1781,7 +1780,7 @@ def create_gem(gem_path: str, # The user specified a --template-restricted-name try: template_json_restricted_name = template_json_data['restricted_name'] - except Exception as e: + except KeyError as e: # the template json doesn't have a 'restricted_name' element warn and use it logger.info(f'The template does not specify a "restricted_name".' f' Using supplied {template_restricted_name}') @@ -1801,7 +1800,7 @@ def create_gem(gem_path: str, template_restricted_path = template_restricted_path.replace('\\', '/') try: template_json_restricted_name = template_json_data['restricted_name'] - except Exception as e: + except KeyError as e: # the template json doesn't have a 'restricted_name' element warn and use it logger.info(f'The template does not specify a "restricted_name".' f' Using supplied {template_restricted_path}') @@ -1833,7 +1832,7 @@ def create_gem(gem_path: str, try: template_json_restricted_platform_relative_path = template_json_data[ 'restricted_platform_relative_path'] - except Exception as e: + except KeyError as e: # the template json doesn't have a 'restricted_platform_relative_path' element warn and use it logger.info(f'The template does not specify a "restricted_platform_relative_path".' f' Using {template_restricted_platform_relative_path}') @@ -1854,7 +1853,7 @@ def create_gem(gem_path: str, try: template_restricted_platform_relative_path = template_json_data[ 'restricted_platform_relative_path'] - except Exception as e: + except KeyError as e: # The template json doesn't have a 'restricted_platform_relative_path' element, set empty string. template_restricted_platform_relative_path = '' if not template_restricted_platform_relative_path: @@ -1988,13 +1987,13 @@ def create_gem(gem_path: str, with open(restricted_json, 'r') as s: try: restricted_json_data = json.load(s) - except Exception as e: + except json.JSONDecodeError as e: logger.error(f'Failed to load restricted json {restricted_json}.') return 1 try: restricted_name = restricted_json_data["restricted_name"] - except Exception as e: + except KeyError as e: logger.error(f'Failed to read "restricted_name" from restricted json {restricted_json}.') return 1 @@ -2007,7 +2006,7 @@ def create_gem(gem_path: str, with open(gem_json, 'r') as s: try: gem_json_data = json.load(s) - except Exception as e: + except json.JSONDecodeError as e: logger.error(f'Failed to load gem json {gem_json}.') return 1 @@ -2016,7 +2015,7 @@ def create_gem(gem_path: str, with open(gem_json, 'w') as s: try: s.write(json.dumps(gem_json_data, indent=4)) - except Exception as e: + except OSError as e: logger.error(f'Failed to write project json {gem_json}.') return 1 @@ -2110,15 +2109,14 @@ def _run_create_gem(args: argparse) -> int: args.module_id) -def add_args(parser, subparsers) -> None: +def add_args(subparsers) -> None: """ add_args is called to add expected parser arguments and subparsers arguments to each command such that it can be invoked locally or aggregated by a central python file. - Ex. Directly run from this file alone with: python engine_template.py create_gem --gem-path TestGem + Ex. Directly run from this file alone with: python engine_template.py create-gem --gem-path TestGem OR o3de.py can aggregate commands by importing engine_template, - call add_args and execute: python o3de.py create_gem --gem-path TestGem - :param parser: the caller instantiates a parser and passes it in here + call add_args and execute: python o3de.py create-gem --gem-path TestGem :param subparsers: the caller instantiates subparsers and passes it in here """ # turn a directory into a template @@ -2438,13 +2436,12 @@ if __name__ == "__main__": the_subparsers = the_parser.add_subparsers(help='sub-command help', dest='command', required=True) # add args to the parser - add_args(the_parser, the_subparsers) + add_args(the_subparsers) # parse args the_args = the_parser.parse_args() # run - ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1 # return diff --git a/scripts/o3de/o3de/get_registration.py b/scripts/o3de/o3de/get_registration.py index c38d4d1cfb..d51600826c 100644 --- a/scripts/o3de/o3de/get_registration.py +++ b/scripts/o3de/o3de/get_registration.py @@ -11,6 +11,7 @@ import argparse import pathlib +import sys from o3de import manifest @@ -27,19 +28,14 @@ def _run_get_registered(args: argparse) -> str or pathlib.Path: args.restricted_name) -def add_args(parser, subparsers) -> None: +def add_parser_args(parser): """ - add_args is called to add expected parser arguments and subparsers arguments to each command such that it can be + add_parser_args is called to add arguments to each command such that it can be invoked locally or added by a central python file. - Ex. Directly run from this file alone with: python register.py register --gem-path "C:/TestGem" - OR - o3de.py can downloadable commands by importing engine_template, - call add_args and execute: python o3de.py register --gem-path "C:/TestGem" - :param parser: the caller instantiates a parser and passes it in here - :param subparsers: the caller instantiates subparsers and passes it in here + Ex. Directly run from this file alone with: python get_registration.py --engine-name "o3de" + :param parser: the caller passes an argparse parser like instance to this method """ - get_registered_subparser = subparsers.add_parser('get-registered') - group = get_registered_subparser.add_mutually_exclusive_group(required=True) + group = parser.add_mutually_exclusive_group(required=True) group.add_argument('-en', '--engine-name', type=str, required=False, help='Engine name.') group.add_argument('-pn', '--project-name', type=str, required=False, @@ -56,7 +52,45 @@ def add_args(parser, subparsers) -> None: group.add_argument('-rsn', '--restricted-name', type=str, required=False, help='Restricted name.') - get_registered_subparser.add_argument('-ohf', '--override-home-folder', type=str, required=False, - help='By default the home folder is the user folder, override it to this folder.') + parser.add_argument('-ohf', '--override-home-folder', type=str, required=False, + help='By default the home folder is the user folder, override it to this folder.') - get_registered_subparser.set_defaults(func=_run_get_registered) + parser.set_defaults(func=_run_get_registered) + + +def add_args(subparsers) -> None: + """ + add_args is called to add subparsers arguments to each command such that it can be + a central python file such as o3de.py. + It can be run from the o3de.py script as follows + call add_args and execute: python o3de.py get-registered --engine-name "o3de" + :param subparsers: the caller instantiates subparsers and passes it in here + """ + get_registered_subparser = subparsers.add_parser('get-registered') + add_parser_args(get_registered_subparser) + + +def main(): + """ + Runs get_registration.py script as standalone script + """ + # parse the command line args + the_parser = argparse.ArgumentParser() + + # add subparsers + + # add args to the parser + add_parser_args(the_parser) + + # parse args + the_args = the_parser.parse_args() + + # run + ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1 + + # return + sys.exit(ret) + + +if __name__ == "__main__": + main() diff --git a/scripts/o3de/o3de/global_project.py b/scripts/o3de/o3de/global_project.py index 1a17e3b79e..787e676a7e 100644 --- a/scripts/o3de/o3de/global_project.py +++ b/scripts/o3de/o3de/global_project.py @@ -52,17 +52,17 @@ def set_global_project(project_name: str or None, with bootstrap_setreg_file.open('r') as f: try: json_data = json.load(f) - except Exception as e: + except json.JSONDecodeError as e: logger.error(f'Bootstrap.setreg failed to load: {str(e)}') else: try: json_data["Amazon"]["AzCore"]["Bootstrap"]["project_path"] = project_path - except Exception as e: + except KeyError as e: logger.error(f'Bootstrap.setreg failed to load: {str(e)}') else: try: os.unlink(bootstrap_setreg_file) - except Exception as e: + except OSError as e: logger.error(f'Failed to unlink bootstrap file {bootstrap_setreg_file}: {str(e)}') return 1 else: @@ -88,12 +88,12 @@ def get_global_project() -> pathlib.Path or None: with bootstrap_setreg_file.open('r') as f: try: json_data = json.load(f) - except Exception as e: + except json.JSONDecodeError as e: logger.error(f'Bootstrap.setreg failed to load: {str(e)}') else: try: project_path = json_data["Amazon"]["AzCore"]["Bootstrap"]["project_path"] - except Exception as e: + except KeyError as e: logger.error(f'Bootstrap.setreg cannot find Amazon:AzCore:Bootstrap:project_path: {str(e)}') else: return pathlib.Path(project_path).resolve() @@ -118,7 +118,7 @@ def _run_set_global_project(args: argparse) -> int: args.project_path) -def add_args(parser, subparsers) -> None: +def add_args(subparsers) -> None: """ add_args is called to add expected parser arguments and subparsers arguments to each command such that it can be invoked locally or aggregated by a central python file. @@ -126,7 +126,6 @@ def add_args(parser, subparsers) -> None: OR o3de.py can aggregate commands by importing global_project, call add_args and execute: python o3de.py set_global_project --project-path C:/TestProject - :param parser: the caller instantiates a parser and passes it in here :param subparsers: the caller instantiates subparsers and passes it in here """ get_global_project_subparser = subparsers.add_parser('get-global-project') @@ -156,7 +155,7 @@ if __name__ == "__main__": the_subparsers = the_parser.add_subparsers(help='sub-command help', dest='command', required=True) # add args to the parser - add_args(the_parser, the_subparsers) + add_args(the_subparsers) # parse args the_args = the_parser.parse_args() diff --git a/scripts/o3de/o3de/manifest.py b/scripts/o3de/o3de/manifest.py index 44d6ff1b61..6c14c2533f 100644 --- a/scripts/o3de/o3de/manifest.py +++ b/scripts/o3de/o3de/manifest.py @@ -176,7 +176,7 @@ def load_o3de_manifest() -> dict: with get_o3de_manifest().open('r') as f: try: json_data = json.load(f) - except Exception as e: + except json.JSONDecodeError as e: logger.error(f'Manifest json failed to load: {str(e)}') return {} else: @@ -187,7 +187,7 @@ def save_o3de_manifest(json_data: dict) -> None: with get_o3de_manifest().open('w') as s: try: s.write(json.dumps(json_data, indent=4)) - except Exception as e: + except OSError as e: logger.error(f'Manifest json failed to save: {str(e)}') @@ -331,7 +331,7 @@ def get_engine_json_data(engine_name: str = None, with engine_json.open('r') as f: try: engine_json_data = json.load(f) - except Exception as e: + except json.JSONDecodeError as e: logger.warn(f'{engine_json} failed to load: {str(e)}') else: return engine_json_data @@ -364,7 +364,7 @@ def get_project_json_data(project_name: str = None, with project_json.open('r') as f: try: project_json_data = json.load(f) - except Exception as e: + except json.JSONDecodeError as e: logger.warn(f'{project_json} failed to load: {str(e)}') else: return project_json_data @@ -397,7 +397,7 @@ def get_gem_json_data(gem_name: str = None, with gem_json.open('r') as f: try: gem_json_data = json.load(f) - except Exception as e: + except json.JSONDecodeError as e: logger.warn(f'{gem_json} failed to load: {str(e)}') else: return gem_json_data @@ -430,7 +430,7 @@ def get_template_json_data(template_name: str = None, with template_json.open('r') as f: try: template_json_data = json.load(f) - except Exception as e: + except json.JSONDecodeError as e: logger.warn(f'{template_json} failed to load: {str(e)}') else: return template_json_data @@ -463,7 +463,7 @@ def get_restricted_data(restricted_name: str = None, with restricted_json.open('r') as f: try: restricted_json_data = json.load(f) - except Exception as e: + except json.JSONDecodeError as e: logger.warn(f'{restricted_json} failed to load: {str(e)}') else: return restricted_json_data @@ -488,7 +488,7 @@ def get_registered(engine_name: str = None, with engine_json.open('r') as f: try: engine_json_data = json.load(f) - except Exception as e: + except json.JSONDecodeError as e: logger.warn(f'{engine_json} failed to load: {str(e)}') else: this_engines_name = engine_json_data['engine_name'] @@ -505,7 +505,7 @@ def get_registered(engine_name: str = None, with project_json.open('r') as f: try: project_json_data = json.load(f) - except Exception as e: + except json.JSONDecodeError as e: logger.warn(f'{project_json} failed to load: {str(e)}') else: this_projects_name = project_json_data['project_name'] @@ -522,7 +522,7 @@ def get_registered(engine_name: str = None, with gem_json.open('r') as f: try: gem_json_data = json.load(f) - except Exception as e: + except json.JSONDecodeError as e: logger.warn(f'{gem_json} failed to load: {str(e)}') else: this_gems_name = gem_json_data['gem_name'] @@ -539,7 +539,7 @@ def get_registered(engine_name: str = None, with template_json.open('r') as f: try: template_json_data = json.load(f) - except Exception as e: + except json.JSONDecodeError as e: logger.warn(f'{template_path} failed to load: {str(e)}') else: this_templates_name = template_json_data['template_name'] @@ -556,7 +556,7 @@ def get_registered(engine_name: str = None, with restricted_json.open('r') as f: try: restricted_json_data = json.load(f) - except Exception as e: + except json.JSONDecodeError as e: logger.warn(f'{restricted_json} failed to load: {str(e)}') else: this_restricted_name = restricted_json_data['restricted_name'] @@ -591,7 +591,7 @@ def get_registered(engine_name: str = None, with repo.open('r') as f: try: repo_json_data = json.load(f) - except Exception as e: + except json.JSONDecodeError as e: logger.warn(f'{cache_file} failed to load: {str(e)}') else: this_repos_name = repo_json_data['repo_name'] diff --git a/scripts/o3de/o3de/print_registration.py b/scripts/o3de/o3de/print_registration.py index 7900fad7e4..292f2224bc 100644 --- a/scripts/o3de/o3de/print_registration.py +++ b/scripts/o3de/o3de/print_registration.py @@ -13,6 +13,7 @@ import argparse import json import hashlib import logging +import sys import urllib.parse from o3de import manifest, validation @@ -143,7 +144,7 @@ def print_engines_data(engines_data: dict) -> None: with engine_json.open('r') as f: try: engine_json_data = json.load(f) - except Exception as e: + except json.JSONDecodeError as e: logger.warn(f'{engine_json} failed to load: {str(e)}') else: print(engine_json) @@ -170,7 +171,7 @@ def print_projects_data(projects_data: dict) -> None: with project_json.open('r') as f: try: project_json_data = json.load(f) - except Exception as e: + except json.JSONDecodeError as e: logger.warn(f'{project_json} failed to load: {str(e)}') else: print(project_json) @@ -197,7 +198,7 @@ def print_gems_data(gems_data: dict) -> None: with gem_json.open('r') as f: try: gem_json_data = json.load(f) - except Exception as e: + except json.JSONDecodeError as e: logger.warn(f'{gem_json} failed to load: {str(e)}') else: print(gem_json) @@ -224,7 +225,7 @@ def print_templates_data(templates_data: dict) -> None: with template_json.open('r') as f: try: template_json_data = json.load(f) - except Exception as e: + except json.JSONDecodeError as e: logger.warn(f'{template_json} failed to load: {str(e)}') else: print(template_json) @@ -243,7 +244,7 @@ def print_repos_data(repos_data: dict) -> None: with cache_file.open('r') as s: try: repo_json_data = json.load(s) - except Exception as e: + except json.JSONDecodeError as e: logger.warn(f'{cache_file} failed to load: {str(e)}') else: print(f'{repo_uri}/repo.json cached as:') @@ -260,7 +261,7 @@ def print_restricted_data(restricted_data: dict) -> None: with restricted_json.open('r') as f: try: restricted_json_data = json.load(f) - except Exception as e: + except json.JSONDecodeError as e: logger.warn(f'{restricted_json} failed to load: {str(e)}') else: print(restricted_json) @@ -365,19 +366,14 @@ def _run_register_show(args: argparse) -> int: return 0 -def add_args(parser, subparsers) -> None: +def add_parser_args(parser): """ - add_args is called to add expected parser arguments and subparsers arguments to each command such that it can be + add_parser_args is called to add arguments to each command such that it can be invoked locally or added by a central python file. - Ex. Directly run from this file alone with: python register.py register --gem-path "C:/TestGem" - OR - o3de.py can downloadable commands by importing engine_template, - call add_args and execute: python o3de.py register --gem-path "C:/TestGem" - :param parser: the caller instantiates a parser and passes it in here - :param subparsers: the caller instantiates subparsers and passes it in here + Ex. Directly run from this file alone with: python print_registration.py --engine-projects + :param parser: the caller passes an argparse parser like instance to this method """ - register_show_subparser = subparsers.add_parser('register-show') - group = register_show_subparser.add_mutually_exclusive_group(required=False) + group = parser.add_mutually_exclusive_group(required=False) group.add_argument('-te', '--this-engine', action='store_true', required=False, default=False, help='Just the local engines.') @@ -446,11 +442,49 @@ def add_args(parser, subparsers) -> None: default=False, help='Combine all repos templates into a single list of resources.') - register_show_subparser.add_argument('-v', '--verbose', action='count', required=False, + parser.add_argument('-v', '--verbose', action='count', required=False, default=0, help='How verbose do you want the output to be.') - register_show_subparser.add_argument('-ohf', '--override-home-folder', type=str, required=False, + parser.add_argument('-ohf', '--override-home-folder', type=str, required=False, help='By default the home folder is the user folder, override it to this folder.') - register_show_subparser.set_defaults(func=_run_register_show) \ No newline at end of file + parser.set_defaults(func=_run_register_show) + + +def add_args(subparsers) -> None: + """ + add_args is called to add subparsers arguments to each command such that it can be + a central python file such as o3de.py. + It can be run from the o3de.py script as follows + call add_args and execute: python o3de.py register-show --engine-projects + :param subparsers: the caller instantiates subparsers and passes it in here + """ + register_show_subparser = subparsers.add_parser('register-show') + add_parser_args(register_show_subparser) + + +def main(): + """ + Runs print_registration.py script as standalone script + """ + # parse the command line args + the_parser = argparse.ArgumentParser() + + # add subparsers + + # add args to the parser + add_parser_args(the_parser) + + # parse args + the_args = the_parser.parse_args() + + # run + ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1 + + # return + sys.exit(ret) + + +if __name__ == "__main__": + main() diff --git a/scripts/o3de/o3de/register.py b/scripts/o3de/o3de/register.py index d6a734e1fd..c44af03b30 100644 --- a/scripts/o3de/o3de/register.py +++ b/scripts/o3de/o3de/register.py @@ -19,6 +19,7 @@ import json import os import pathlib import shutil +import sys import urllib.parse import urllib.request @@ -134,172 +135,74 @@ def register_all_in_folder(folder_path: str or pathlib.Path, return ret_val -def register_all_engines_in_folder(engines_path: str or pathlib.Path, - remove: bool = False, - force: bool = False) -> int: - if not engines_path: +def register_all_o3de_objects_of_type_in_folder(o3de_object_path: str or pathlib.Path, + o3de_object_type: str, + remove: bool, + force: bool, + **register_kwargs) -> int: + if not o3de_object_path: logger.error(f'Engines path cannot be empty.') return 1 - engines_path = pathlib.Path(engines_path).resolve() - if not engines_path.is_dir(): + o3de_object_path = pathlib.Path(o3de_object_path).resolve() + if not o3de_object_path.is_dir(): logger.error(f'Engines path is not dir.') return 1 - engines_set = set() + o3de_object_type_set = set() + register_path_kwarg = f'{o3de_object_type}_path' if o3de_object_type != 'repo' else f'{o3de_object_type}_uri' ret_val = 0 - for root, dirs, files in os.walk(engines_path): - for name in files: - if name == 'engine.json': - engines_set.add(root) + for root, dirs, files in os.walk(o3de_object_path): + if f'{o3de_object_type}.json' in files: + o3de_object_type_set.add(root) + # Stop iteration of any subdirectories + # Nested o3de objects of the same type aren't supported(i.e an engine cannot be inside of a engine). + dirs[:] = [] - for engine in sorted(engines_set, reverse=True): - error_code = register(engine_path=engine, remove=remove, force=force) + for o3de_object_type_root in sorted(o3de_object_type_set, reverse=True): + error_code = register(**{register_path_kwarg: o3de_object_type_root}, + remove=remove, force=force, **register_kwargs) if error_code: ret_val = error_code return ret_val +def register_all_engines_in_folder(engines_path: str or pathlib.Path, + remove: bool = False, + force: bool = False) -> int: + return register_all_o3de_objects_of_type_in_folder(engines_path, 'engine', remove, force) + + def register_all_projects_in_folder(projects_path: str or pathlib.Path, remove: bool = False, engine_path: str or pathlib.Path = None) -> int: - if not projects_path: - logger.error(f'Projects path cannot be empty.') - return 1 - - projects_path = pathlib.Path(projects_path).resolve() - if not projects_path.is_dir(): - logger.error(f'Projects path is not dir.') - return 1 - - projects_set = set() - - ret_val = 0 - for root, dirs, files in os.walk(projects_path): - for name in files: - if name == 'project.json': - projects_set.add(root) - - for project in sorted(projects_set, reverse=True): - error_code = register(engine_path=engine_path, project_path=project, remove=remove) - if error_code: - ret_val = error_code - - return ret_val + return register_all_o3de_objects_of_type_in_folder(projects_path, 'project', remove, False, engine_path=engine_path) def register_all_gems_in_folder(gems_path: str or pathlib.Path, remove: bool = False, engine_path: str or pathlib.Path = None) -> int: - if not gems_path: - logger.error(f'Gems path cannot be empty.') - return 1 - - gems_path = pathlib.Path(gems_path).resolve() - if not gems_path.is_dir(): - logger.error(f'Gems path is not dir.') - return 1 - - gems_set = set() - - ret_val = 0 - for root, dirs, files in os.walk(gems_path): - for name in files: - if name == 'gem.json': - gems_set.add(root) - - for gem in sorted(gems_set, reverse=True): - error_code = register(engine_path=engine_path, gem_path=gem, remove=remove) - if error_code: - ret_val = error_code - - return ret_val + return register_all_o3de_objects_of_type_in_folder(gems_path, 'gem', remove, False, engine_path=engine_path) def register_all_templates_in_folder(templates_path: str or pathlib.Path, remove: bool = False, engine_path: str or pathlib.Path = None) -> int: - if not templates_path: - logger.error(f'Templates path cannot be empty.') - return 1 - - templates_path = pathlib.Path(templates_path).resolve() - if not templates_path.is_dir(): - logger.error(f'Templates path is not dir.') - return 1 - - templates_set = set() - - ret_val = 0 - for root, dirs, files in os.walk(templates_path): - for name in files: - if name == 'template.json': - templates_set.add(root) - - for template in sorted(templates_set, reverse=True): - error_code = register(engine_path=engine_path, template_path=template, remove=remove) - if error_code: - ret_val = error_code - - return ret_val + return register_all_o3de_objects_of_type_in_folder(templates_path, 'template', remove, False, engine_path=engine_path) def register_all_restricted_in_folder(restricted_path: str or pathlib.Path, remove: bool = False, engine_path: str or pathlib.Path = None) -> int: - if not restricted_path: - logger.error(f'Restricted path cannot be empty.') - return 1 - - restricted_path = pathlib.Path(restricted_path).resolve() - if not restricted_path.is_dir(): - logger.error(f'Restricted path is not dir.') - return 1 - - restricted_set = set() - - ret_val = 0 - for root, dirs, files in os.walk(restricted_path): - for name in files: - if name == 'restricted.json': - restricted_set.add(root) - - for restricted in sorted(restricted_set, reverse=True): - error_code = register(engine_path=engine_path, restricted_path=restricted, remove=remove) - if error_code: - ret_val = error_code - - return ret_val + return register_all_o3de_objects_of_type_in_folder(restricted_path, 'restricted', remove, False, engine_path=engine_path) def register_all_repos_in_folder(repos_path: str or pathlib.Path, remove: bool = False, engine_path: str or pathlib.Path = None) -> int: - if not repos_path: - logger.error(f'Repos path cannot be empty.') - return 1 - - repos_path = pathlib.Path(repos_path).resolve() - if not repos_path.is_dir(): - logger.error(f'Repos path is not dir.') - return 1 - - repo_set = set() - - ret_val = 0 - for root, dirs, files in os.walk(repos_path): - for name in files: - if name == 'repo.json': - repo_set.add(root) - - for repo in sorted(repo_set, reverse=True): - error_code = register(engine_path=engine_path, repo_uri=repo, remove=remove) - if error_code: - ret_val = error_code - - return ret_val + return register_all_o3de_objects_of_type_in_folder(repos_path, 'repo', remove, force, engine_path=engine_path) def remove_engine_name_to_path(json_data: dict, @@ -395,21 +298,13 @@ def register_gem_path(json_data: dict, logger.error(f'Engine path {engine_path} is not registered.') return 1 - while gem_path in engine_data['gems']: - engine_data['gems'].remove(gem_path) - - while gem_path.as_posix() in engine_data['gems']: - engine_data['gems'].remove(gem_path.as_posix()) + engine_data['gems'] = list(filter(lambda p: gem_path != pathlib.Path(p), engine_data['gems'])) if remove: logger.warn(f'Removing Gem path {gem_path}.') return 0 else: - while gem_path in json_data['gems']: - json_data['gems'].remove(gem_path) - - while gem_path.as_posix() in json_data['gems']: - json_data['gems'].remove(gem_path.as_posix()) + json_data['gems'] = list(filter(lambda p: gem_path != pathlib.Path(p), json_data['gems'])) if remove: logger.warn(f'Removing Gem path {gem_path}.') @@ -447,21 +342,13 @@ def register_project_path(json_data: dict, logger.error(f'Engine path {engine_path} is not registered.') return 1 - while project_path in engine_data['projects']: - engine_data['projects'].remove(project_path) - - while project_path.as_posix() in engine_data['projects']: - engine_data['projects'].remove(project_path.as_posix()) + engine_data['projects'] = list(filter(lambda p: project_path != pathlib.Path(p), engine_data['projects'])) if remove: logger.warn(f'Engine {engine_path} removing Project path {project_path}.') return 0 else: - while project_path in json_data['projects']: - json_data['projects'].remove(project_path) - - while project_path.as_posix() in json_data['projects']: - json_data['projects'].remove(project_path.as_posix()) + json_data['projects'] = list(filter(lambda p: project_path != pathlib.Path(p), json_data['projects'])) if remove: logger.warn(f'Removing Project path {project_path}.') @@ -486,20 +373,20 @@ def register_project_path(json_data: dict, with this_engine_json.open('r') as f: try: this_engine_json = json.load(f) - except Exception as e: + except json.JSONDecodeError as e: logger.error(f'Engine json failed to load: {str(e)}') return 1 with project_json.open('r') as f: try: project_json_data = json.load(f) - except Exception as e: + except json.JSONDecodeError as e: logger.error(f'Project json failed to load: {str(e)}') return 1 update_project_json = False try: update_project_json = project_json_data['engine'] != this_engine_json['engine_name'] - except Exception as e: + except KeyError as e: update_project_json = True if update_project_json: @@ -508,7 +395,7 @@ def register_project_path(json_data: dict, with project_json.open('w') as s: try: s.write(json.dumps(project_json_data, indent=4)) - except Exception as e: + except OSError as e: logger.error(f'Project json failed to save: {str(e)}') return 1 @@ -530,21 +417,13 @@ def register_template_path(json_data: dict, logger.error(f'Engine path {engine_path} is not registered.') return 1 - while template_path in engine_data['templates']: - engine_data['templates'].remove(template_path) - - while template_path.as_posix() in engine_data['templates']: - engine_data['templates'].remove(template_path.as_posix()) + engine_data['templates'] = list(filter(lambda p: template_path != pathlib.Path(p), engine_data['templates'])) if remove: logger.warn(f'Engine {engine_path} removing Template path {template_path}.') return 0 else: - while template_path in json_data['templates']: - json_data['templates'].remove(template_path) - - while template_path.as_posix() in json_data['templates']: - json_data['templates'].remove(template_path.as_posix()) + json_data['templates'] = list(filter(lambda p: template_path != pathlib.Path(p), json_data['templates'])) if remove: logger.warn(f'Removing Template path {template_path}.') @@ -582,21 +461,13 @@ def register_restricted_path(json_data: dict, logger.error(f'Engine path {engine_path} is not registered.') return 1 - while restricted_path in engine_data['restricted']: - engine_data['restricted'].remove(restricted_path) - - while restricted_path.as_posix() in engine_data['restricted']: - engine_data['restricted'].remove(restricted_path.as_posix()) + engine_data['restricted'] = list(filter(lambda p: restricted_path != pathlib.Path(p), engine_data['restricted'])) if remove: logger.warn(f'Engine {engine_path} removing Restricted path {restricted_path}.') return 0 else: - while restricted_path in json_data['restricted']: - json_data['restricted'].remove(restricted_path) - - while restricted_path.as_posix() in json_data['restricted']: - json_data['restricted'].remove(restricted_path.as_posix()) + json_data['restricted'] = list(filter(lambda p: restricted_path != pathlib.Path(p), json_data['restricted'])) if remove: logger.warn(f'Removing Restricted path {restricted_path}.') @@ -629,10 +500,7 @@ def register_repo(json_data: dict, url = f'{repo_uri}/repo.json' parsed_uri = urllib.parse.urlparse(url) - if parsed_uri.scheme == 'http' or \ - parsed_uri.scheme == 'https' or \ - parsed_uri.scheme == 'ftp' or \ - parsed_uri.scheme == 'ftps': + if parsed_uri.scheme in ['http', 'https', 'ftp', 'ftps']: while repo_uri in json_data['repos']: json_data['repos'].remove(repo_uri) else: @@ -647,118 +515,67 @@ def register_repo(json_data: dict, repo_sha256 = hashlib.sha256(url.encode()) cache_file = manifest.get_o3de_cache_folder() / str(repo_sha256.hexdigest() + '.json') - result = 0 - if parsed_uri.scheme == 'http' or \ - parsed_uri.scheme == 'https' or \ - parsed_uri.scheme == 'ftp' or \ - parsed_uri.scheme == 'ftps': - if not cache_file.is_file(): - with urllib.request.urlopen(url) as s: - with cache_file.open('wb') as f: - shutil.copyfileobj(s, f) - json_data['repos'].insert(0, repo_uri) - else: - if not cache_file.is_file(): - origin_file = pathlib.Path(url).resolve() - if not origin_file.is_file(): - return 1 - shutil.copy(origin_file, origin_file) + result = utils.download_file(url, cache_file) + if result == 0: json_data['repos'].insert(0, repo_uri.as_posix()) - repo_set = set() result = repo.process_add_o3de_repo(cache_file, repo_set) return result +def register_default_o3de_object_folder(json_data: dict, + default_o3de_object_folder: str or pathlib.Path, + o3de_object_key: str) -> int: + # make sure the path exists + default_o3de_object_folder = pathlib.Path(default_o3de_object_folder).resolve() + if not default_o3de_object_folder.is_dir(): + logger.error(f'Default o3de object folder {default_o3de_object_folder} does not exist.') + return 1 + + json_data[o3de_object_key] = default_o3de_object_folder.as_posix() + + return 0 + + def register_default_engines_folder(json_data: dict, default_engines_folder: str or pathlib.Path, remove: bool = False) -> int: - if remove: - default_engines_folder = manifest.get_o3de_engines_folder() - - # make sure the path exists - default_engines_folder = pathlib.Path(default_engines_folder).resolve() - if not default_engines_folder.is_dir(): - logger.error(f'Default engines folder {default_engines_folder} does not exist.') - return 1 - - default_engines_folder = default_engines_folder.as_posix() - json_data['default_engines_folder'] = default_engines_folder - - return 0 + return register_default_o3de_object_folder(json_data, + manifest.get_o3de_engines_folder() if remove else default_engines_folder, + 'default_engines_folder', remove) def register_default_projects_folder(json_data: dict, default_projects_folder: str or pathlib.Path, remove: bool = False) -> int: - if remove: - default_projects_folder = manifest.get_o3de_projects_folder() - - # make sure the path exists - default_projects_folder = pathlib.Path(default_projects_folder).resolve() - if not default_projects_folder.is_dir(): - logger.error(f'Default projects folder {default_projects_folder} does not exist.') - return 1 - - default_projects_folder = default_projects_folder.as_posix() - json_data['default_projects_folder'] = default_projects_folder - - return 0 + return register_default_o3de_object_folder(json_data, + manifest.get_o3de_projects_folder() if remove else default_projects_folder, + 'default_projects_folder', remove) def register_default_gems_folder(json_data: dict, default_gems_folder: str or pathlib.Path, remove: bool = False) -> int: - if remove: - default_gems_folder = manifest.get_o3de_gems_folder() - - # make sure the path exists - default_gems_folder = pathlib.Path(default_gems_folder).resolve() - if not default_gems_folder.is_dir(): - logger.error(f'Default gems folder {default_gems_folder} does not exist.') - return 1 - - default_gems_folder = default_gems_folder.as_posix() - json_data['default_gems_folder'] = default_gems_folder - - return 0 + return register_default_o3de_object_folder(json_data, + manifest.get_o3de_gems_folder() if remove else default_gems_folder, + 'default_gems_folder', remove) def register_default_templates_folder(json_data: dict, default_templates_folder: str or pathlib.Path, remove: bool = False) -> int: - if remove: - default_templates_folder = manifest.get_o3de_templates_folder() - - # make sure the path exists - default_templates_folder = pathlib.Path(default_templates_folder).resolve() - if not default_templates_folder.is_dir(): - logger.error(f'Default templates folder {default_templates_folder} does not exist.') - return 1 - - default_templates_folder = default_templates_folder.as_posix() - json_data['default_templates_folder'] = default_templates_folder - - return 0 + return register_default_o3de_object_folder(json_data, + manifest.get_o3de_templates_folder() if remove else default_templates_folder, + 'default_templates_folder', remove) def register_default_restricted_folder(json_data: dict, default_restricted_folder: str or pathlib.Path, - remove: bool = False) -> int: - if remove: - default_restricted_folder = manifest.get_o3de_restricted_folder() - - # make sure the path exists - default_restricted_folder = pathlib.Path(default_restricted_folder).resolve() - if not default_restricted_folder.is_dir(): - logger.error(f'Default restricted folder {default_restricted_folder} does not exist.') - return 1 - - default_restricted_folder = default_restricted_folder.as_posix() - json_data['default_restricted_folder'] = default_restricted_folder - - return 0 + reset_to_default: bool = False) -> int: + return register_default_o3de_object_folder(json_data, + manifest.get_o3de_restricted_folder() if remove else default_restricted_folder, + 'default_restricted_folder', remove) def register(engine_path: str or pathlib.Path = None, @@ -999,20 +816,14 @@ def _run_register(args: argparse) -> int: force=args.force) -def add_args(parser, subparsers) -> None: +def add_parser_args(parser): """ - add_args is called to add expected parser arguments and subparsers arguments to each command such that it can be + add_parser_args is called to add arguments to each command such that it can be invoked locally or added by a central python file. - Ex. Directly run from this file alone with: python register.py register --gem-path "C:/TestGem" - OR - o3de.py can downloadable commands by importing engine_template, - call add_args and execute: python o3de.py register --gem-path "C:/TestGem" - :param parser: the caller instantiates a parser and passes it in here - :param subparsers: the caller instantiates subparsers and passes it in here + Ex. Directly run from this file alone with: python register.py --engine-path "C:/o3de" + :param parser: the caller passes an argparse parser like instance to this method """ - # register - register_subparser = subparsers.add_parser('register') - group = register_subparser.add_mutually_exclusive_group(required=True) + group = parser.add_mutually_exclusive_group(required=True) group.add_argument('--this-engine', action='store_true', required=False, default=False, help='Registers the engine this script is running from.') @@ -1054,12 +865,49 @@ def add_args(parser, subparsers) -> None: default=False, help='Refresh the repo cache.') - register_subparser.add_argument('-ohf', '--override-home-folder', type=str, required=False, + parser.add_argument('-ohf', '--override-home-folder', type=str, required=False, help='By default the home folder is the user folder, override it to this folder.') - - register_subparser.add_argument('-r', '--remove', action='store_true', required=False, + parser.add_argument('-r', '--remove', action='store_true', required=False, default=False, help='Remove entry.') - register_subparser.add_argument('-f', '--force', action='store_true', default=False, + parser.add_argument('-f', '--force', action='store_true', default=False, help='For the update of the registration field being modified.') - register_subparser.set_defaults(func=_run_register) + parser.set_defaults(func=_run_register) + + +def add_args(subparsers) -> None: + """ + add_args is called to add subparsers arguments to each command such that it can be + a central python file such as o3de.py. + It can be run from the o3de.py script as follows + call add_args and execute: python o3de.py register --engine-path "C:/o3de" + :param subparsers: the caller instantiates subparsers and passes it in here + """ + register_subparser = subparsers.add_parser('register') + add_parser_args(register_subparser) + + +def main(): + """ + Runs register.py script as standalone script + """ + # parse the command line args + the_parser = argparse.ArgumentParser() + + # add subparsers + + # add args to the parser + add_parser_args(the_parser) + + # parse args + the_args = the_parser.parse_args() + + # run + ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1 + + # return + sys.exit(ret) + + +if __name__ == "__main__": + main() diff --git a/scripts/o3de/o3de/registration.py b/scripts/o3de/o3de/registration.py deleted file mode 100755 index 801c698ca4..0000000000 --- a/scripts/o3de/o3de/registration.py +++ /dev/null @@ -1,92 +0,0 @@ -# -# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or -# its licensors. -# -# For complete copyright and license terms please see the LICENSE at the root of this -# distribution (the "License"). All use of this software is governed by the License, -# or, if provided, by the license below or the license accompanying this file. Do not -# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# -""" -This file contains all the code that has to do with registering engines, projects, gems and templates -""" - -import argparse -import sys - - -def add_args(parser, subparsers) -> None: - """ - add_args is called to add expected parser arguments and subparsers arguments to each command such that it can be - invoked locally or added by a central python file. - Ex. Directly run from this file alone with: python register.py register --gem-path "C:/TestGem" - OR - o3de.py can downloadable commands by importing engine_template, - call add_args and execute: python o3de.py register --gem-path "C:/TestGem" - :param parser: the caller instantiates a parser and passes it in here - :param subparsers: the caller instantiates subparsers and passes it in here - """ - # register - from o3de import register - register.add_args(parser, subparsers) - - # show - from o3de import print_registration - print_registration.add_args(parser, subparsers) - - # get-registered - from o3de import get_registration - get_registration.add_args(parser, subparsers) - - # download - from o3de import download - download.add_args(parser, subparsers) - - # add external subdirectories - from o3de import add_external_subdirectory - add_external_subdirectory.add_args(parser, subparsers) - - # remove external subdirectories - from o3de import remove_external_subdirectory - remove_external_subdirectory.add_args(parser, subparsers) - - # add gems to cmake - from o3de import add_gem_cmake - add_gem_cmake.add_args(parser, subparsers) - - # remove gems from cmake - from o3de import remove_gem_cmake - remove_gem_cmake.add_args(parser, subparsers) - - # add a gem to a project - from o3de import add_gem_project - add_gem_project.add_args(parser, subparsers) - - # remove a gem from a project - from o3de import remove_gem_project - remove_gem_project.add_args(parser, subparsers) - - # sha256 - from o3de import sha256 - sha256.add_args(parser, subparsers) - - -if __name__ == "__main__": - # parse the command line args - the_parser = argparse.ArgumentParser() - - # add subparsers - the_subparsers = the_parser.add_subparsers(help='sub-command help', dest='command', required=True) - - # add args to the parser - add_args(the_parser, the_subparsers) - - # parse args - the_args = the_parser.parse_args() - - # run - ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1 - - # return - sys.exit(ret) diff --git a/scripts/o3de/o3de/remove_external_subdirectory.py b/scripts/o3de/o3de/remove_external_subdirectory.py index a636474fba..b433e9c398 100644 --- a/scripts/o3de/o3de/remove_external_subdirectory.py +++ b/scripts/o3de/o3de/remove_external_subdirectory.py @@ -15,6 +15,7 @@ Implemens functinality to remove external_subdirectories from the o3de_manifests import argparse import logging import pathlib +import sys from o3de import manifest @@ -62,12 +63,58 @@ def add_args(parser, subparsers) -> None: :param parser: the caller instantiates a parser and passes it in here :param subparsers: the caller instantiates subparsers and passes it in here """ - remove_external_subdirectory_subparser = subparsers.add_parser('remove-external-subdirectory') - remove_external_subdirectory_subparser.add_argument('external_subdirectory', metavar='external_subdirectory', + + +def add_parser_args(parser): + """ + add_parser_args is called to add arguments to each command such that it can be + invoked locally or added by a central python file. + Ex. Directly run from this file alone with: python remove_external_subdirectory.py "D:/subdir" + :param parser: the caller passes an argparse parser like instance to this method + """ + parser.add_argument('external_subdirectory', metavar='external_subdirectory', type=str, help='remove external subdirectory from cmake') - remove_external_subdirectory_subparser.add_argument('-ohf', '--override-home-folder', type=str, required=False, + parser.add_argument('-ohf', '--override-home-folder', type=str, required=False, help='By default the home folder is the user folder, override it to this folder.') - remove_external_subdirectory_subparser.set_defaults(func=_run_remove_external_subdirectory) + parser.set_defaults(func=_run_remove_external_subdirectory) + + +def add_args(subparsers) -> None: + """ + add_args is called to add subparsers arguments to each command such that it can be + a central python file such as o3de.py. + It can be run from the o3de.py script as follows + call add_args and execute: python o3de.py remove-external-subdirectory "D:/subdir" + :param subparsers: the caller instantiates subparsers and passes it in here + """ + remove_external_subdirectory_subparser = subparsers.add_parser('remove-external-subdirectory') + add_parser_args(remove_external_subdirectory_subparser) + + +def main(): + """ + Runs remove_external_subdirectory.py script as standalone script + """ + # parse the command line args + the_parser = argparse.ArgumentParser() + + # add subparsers + + # add args to the parser + add_parser_args(the_parser) + + # parse args + the_args = the_parser.parse_args() + + # run + ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1 + + # return + sys.exit(ret) + + +if __name__ == "__main__": + main() diff --git a/scripts/o3de/o3de/remove_gem_cmake.py b/scripts/o3de/o3de/remove_gem_cmake.py index 8f73caaad1..3d988a579a 100644 --- a/scripts/o3de/o3de/remove_gem_cmake.py +++ b/scripts/o3de/o3de/remove_gem_cmake.py @@ -15,6 +15,7 @@ Contains methods for removing a gem from a project's cmake scripts import argparse import logging import pathlib +import sys from o3de import manifest, remove_external_subdirectory @@ -64,26 +65,58 @@ def _run_remove_gem_from_cmake(args: argparse) -> int: return remove_gem_from_cmake(args.gem_name, args.gem_path) -def add_args(parser, subparsers) -> None: +def add_parser_args(parser): """ - add_args is called to add expected parser arguments and subparsers arguments to each command such that it can be + add_parser_args is called to add arguments to each command such that it can be invoked locally or added by a central python file. - Ex. Directly run from this file alone with: python register.py register --gem-path "C:/TestGem" - OR - o3de.py can downloadable commands by importing engine_template, - call add_args and execute: python o3de.py register --gem-path "C:/TestGem" - :param parser: the caller instantiates a parser and passes it in here - :param subparsers: the caller instantiates subparsers and passes it in here + Ex. Directly run from this file alone with: python remove_gem_cmake.py --gem-name Atom + :param parser: the caller passes an argparse parser like instance to this method """ - # convenience functions to disambiguate the gem name -> gem_path and call remove-external-subdirectory on gem_path - remove_gem_from_cmake_subparser = subparsers.add_parser('remove-gem-from-cmake') - group = remove_gem_from_cmake_subparser.add_mutually_exclusive_group(required=True) + group = parser.add_mutually_exclusive_group(required=True) group.add_argument('-gp', '--gem-path', type=str, required=False, help='The path to the gem.') group.add_argument('-gn', '--gem-name', type=str, required=False, help='The name of the gem.') - remove_gem_from_cmake_subparser.add_argument('-ohf', '--override-home-folder', type=str, required=False, + parser.add_argument('-ohf', '--override-home-folder', type=str, required=False, help='By default the home folder is the user folder, override it to this folder.') - remove_gem_from_cmake_subparser.set_defaults(func=_run_remove_gem_from_cmake) + parser.set_defaults(func=_run_remove_gem_from_cmake) + + +def add_args(subparsers) -> None: + """ + add_args is called to add subparsers arguments to each command such that it can be + a central python file such as o3de.py. + It can be run from the o3de.py script as follows + call add_args and execute: python o3de.py remove-gem-from-cmake --gem-name Atom + :param subparsers: the caller instantiates subparsers and passes it in here + """ + remove_gem_from_cmake_subparser = subparsers.add_parser('remove-gem-from-cmake') + add_parser_args(remove_gem_from_cmake_subparser) + + +def main(): + """ + Runs remove_gem_cmake.py script as standalone script + """ + # parse the command line args + the_parser = argparse.ArgumentParser() + + # add subparsers + + # add args to the parser + add_parser_args(the_parser) + + # parse args + the_args = the_parser.parse_args() + + # run + ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1 + + # return + sys.exit(ret) + + +if __name__ == "__main__": + main() diff --git a/scripts/o3de/o3de/remove_gem_project.py b/scripts/o3de/o3de/remove_gem_project.py index 7644357042..671427db14 100644 --- a/scripts/o3de/o3de/remove_gem_project.py +++ b/scripts/o3de/o3de/remove_gem_project.py @@ -16,6 +16,7 @@ import argparse import logging import os import pathlib +import sys from o3de import cmake, remove_gem_cmake @@ -220,51 +221,84 @@ def _run_remove_gem_from_project(args: argparse) -> int: args.remove_from_cmake) -def add_args(parser, subparsers) -> None: +def add_parser_args(parser): """ - add_args is called to add expected parser arguments and subparsers arguments to each command such that it can be + add_parser_args is called to add arguments to each command such that it can be invoked locally or added by a central python file. - Ex. Directly run from this file alone with: python register.py register --gem-path "C:/TestGem" - OR - o3de.py can downloadable commands by importing engine_template, - call add_args and execute: python o3de.py register --gem-path "C:/TestGem" - :param parser: the caller instantiates a parser and passes it in here - :param subparsers: the caller instantiates subparsers and passes it in here + Ex. Directly run from this file alone with: python remove_gem_project.py --project-path D:/Test --gem-name Atom + :param parser: the caller passes an argparse parser like instance to this method """ - remove_gem_subparser = subparsers.add_parser('remove-gem-from-project') - group = remove_gem_subparser.add_mutually_exclusive_group(required=True) + group = parser.add_mutually_exclusive_group(required=True) group.add_argument('-pp', '--project-path', type=str, required=False, help='The path to the project.') group.add_argument('-pn', '--project-name', type=str, required=False, help='The name of the project.') - group = remove_gem_subparser.add_mutually_exclusive_group(required=True) + group = parser.add_mutually_exclusive_group(required=True) group.add_argument('-gp', '--gem-path', type=str, required=False, help='The path to the gem.') group.add_argument('-gn', '--gem-name', type=str, required=False, help='The name of the gem.') - remove_gem_subparser.add_argument('-gt', '--gem-target', type=str, required=False, + parser.add_argument('-gt', '--gem-target', type=str, required=False, help='The cmake target name to add. If not specified it will assume gem_name') - remove_gem_subparser.add_argument('-df', '--dependencies-file', type=str, required=False, + parser.add_argument('-df', '--dependencies-file', type=str, required=False, help='The cmake dependencies file in which the gem dependencies are specified.' 'If not specified it will assume ') - remove_gem_subparser.add_argument('-rd', '--runtime-dependency', action='store_true', required=False, + parser.add_argument('-rd', '--runtime-dependency', action='store_true', required=False, default=False, help='Optional toggle if this gem should be removed as a runtime dependency') - remove_gem_subparser.add_argument('-td', '--tool-dependency', action='store_true', required=False, + parser.add_argument('-td', '--tool-dependency', action='store_true', required=False, default=False, help='Optional toggle if this gem should be removed as a server dependency') - remove_gem_subparser.add_argument('-sd', '--server-dependency', action='store_true', required=False, + parser.add_argument('-sd', '--server-dependency', action='store_true', required=False, default=False, help='Optional toggle if this gem should be removed as a server dependency') - remove_gem_subparser.add_argument('-pl', '--platforms', type=str, required=False, + parser.add_argument('-pl', '--platforms', type=str, required=False, default='Common', help='Optional list of platforms this gem should be removed from' ' Ex. --platforms Mac,Windows,Linux') - remove_gem_subparser.add_argument('-r', '--remove-from-cmake', type=bool, required=False, + parser.add_argument('-r', '--remove-from-cmake', type=bool, required=False, default=False, help='Automatically call remove-from-cmake.') - remove_gem_subparser.add_argument('-ohf', '--override-home-folder', type=str, required=False, + parser.add_argument('-ohf', '--override-home-folder', type=str, required=False, help='By default the home folder is the user folder, override it to this folder.') - remove_gem_subparser.set_defaults(func=_run_remove_gem_from_project) + parser.set_defaults(func=_run_remove_gem_from_project) + + +def add_args(subparsers) -> None: + """ + add_args is called to add subparsers arguments to each command such that it can be + a central python file such as o3de.py. + It can be run from the o3de.py script as follows + call add_args and execute: python o3de.py remove-gem-from-project --project-path D:/Test --gem-name Atom + :param subparsers: the caller instantiates subparsers and passes it in here + """ + remove_gem_project_subparser = subparsers.add_parser('remove-gem-from-project') + add_parser_args(remove_gem_project_subparser) + + +def main(): + """ + Runs remove_gem_project.py script as standalone script + """ + # parse the command line args + the_parser = argparse.ArgumentParser() + + # add subparsers + + # add args to the parser + add_parser_args(the_parser) + + # parse args + the_args = the_parser.parse_args() + + # run + ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1 + + # return + sys.exit(ret) + + +if __name__ == "__main__": + main() diff --git a/scripts/o3de/o3de/repo.py b/scripts/o3de/o3de/repo.py index 9cb93d53ae..c6b4874b6a 100644 --- a/scripts/o3de/o3de/repo.py +++ b/scripts/o3de/o3de/repo.py @@ -16,11 +16,12 @@ import shutil import urllib.parse import urllib.request -from o3de import manifest, validation +from o3de import manifest, utils, validation logger = logging.getLogger() logging.basicConfig() + def process_add_o3de_repo(file_name: str or pathlib.Path, repo_set: set) -> int: file_name = pathlib.Path(file_name).resolve() @@ -32,105 +33,26 @@ def process_add_o3de_repo(file_name: str or pathlib.Path, with file_name.open('r') as f: try: repo_data = json.load(f) - except Exception as e: + except json.JSONDecodeError as e: logger.error(f'{file_name} failed to load: {str(e)}') return 1 - for engine_uri in repo_data['engines']: - engine_uri = f'{engine_uri}/engine.json' - engine_sha256 = hashlib.sha256(engine_uri.encode()) - cache_file = cache_folder / str(engine_sha256.hexdigest() + '.json') - if not cache_file.is_file(): - parsed_uri = urllib.parse.urlparse(engine_uri) - if parsed_uri.scheme == 'http' or \ - parsed_uri.scheme == 'https' or \ - parsed_uri.scheme == 'ftp' or \ - parsed_uri.scheme == 'ftps': - with urllib.request.urlopen(engine_uri) as s: - with cache_file.open('wb') as f: - shutil.copyfileobj(s, f) - else: - engine_json = pathlib.Path(engine_uri).resolve() - if not engine_json.is_file(): - return 1 - shutil.copy(engine_json, cache_file) - - for project_uri in repo_data['projects']: - project_uri = f'{project_uri}/project.json' - project_sha256 = hashlib.sha256(project_uri.encode()) - cache_file = cache_folder / str(project_sha256.hexdigest() + '.json') - if not cache_file.is_file(): - parsed_uri = urllib.parse.urlparse(project_uri) - if parsed_uri.scheme == 'http' or \ - parsed_uri.scheme == 'https' or \ - parsed_uri.scheme == 'ftp' or \ - parsed_uri.scheme == 'ftps': - with urllib.request.urlopen(project_uri) as s: - with cache_file.open('wb') as f: - shutil.copyfileobj(s, f) - else: - project_json = pathlib.Path(project_uri).resolve() - if not project_json.is_file(): - return 1 - shutil.copy(project_json, cache_file) - - for gem_uri in repo_data['gems']: - gem_uri = f'{gem_uri}/gem.json' - gem_sha256 = hashlib.sha256(gem_uri.encode()) - cache_file = cache_folder / str(gem_sha256.hexdigest() + '.json') - if not cache_file.is_file(): - parsed_uri = urllib.parse.urlparse(gem_uri) - if parsed_uri.scheme == 'http' or \ - parsed_uri.scheme == 'https' or \ - parsed_uri.scheme == 'ftp' or \ - parsed_uri.scheme == 'ftps': - with urllib.request.urlopen(gem_uri) as s: - with cache_file.open('wb') as f: - shutil.copyfileobj(s, f) - else: - gem_json = pathlib.Path(gem_uri).resolve() - if not gem_json.is_file(): - return 1 - shutil.copy(gem_json, cache_file) - - for template_uri in repo_data['templates']: - template_uri = f'{template_uri}/template.json' - template_sha256 = hashlib.sha256(template_uri.encode()) - cache_file = cache_folder / str(template_sha256.hexdigest() + '.json') - if not cache_file.is_file(): - parsed_uri = urllib.parse.urlparse(template_uri) - if parsed_uri.scheme == 'http' or \ - parsed_uri.scheme == 'https' or \ - parsed_uri.scheme == 'ftp' or \ - parsed_uri.scheme == 'ftps': - with urllib.request.urlopen(template_uri) as s: - with cache_file.open('wb') as f: - shutil.copyfileobj(s, f) - else: - template_json = pathlib.Path(template_uri).resolve() - if not template_json.is_file(): - return 1 - shutil.copy(template_json, cache_file) - - for repo_uri in repo_data['repos']: - if repo_uri not in repo_set: - repo_set.add(repo_uri) - repo_uri = f'{repo_uri}/repo.json' - repo_sha256 = hashlib.sha256(repo_uri.encode()) - cache_file = cache_folder / str(repo_sha256.hexdigest() + '.json') + for o3de_object_uris, manifest_json in [(repo_data['engines'], 'engine.json'), + (repo_data['projects'], 'project.json'), + (repo_data['gems'], 'gem.json'), + (repo_data['template'], 'template.json'), + (repo_data['restricted'], 'restricted.json')]: + for o3de_object_uri in o3de_object_uris: + manifest_json_uri = f'{o3de_object_uri}/{manifest_json}' + manifest_json_sha256 = hashlib.sha256(manifest_json_uri.encode()) + cache_file = cache_folder / str(manifest_json_sha256.hexdigest() + '.json') if not cache_file.is_file(): - if parsed_uri.scheme == 'http' or \ - parsed_uri.scheme == 'https' or \ - parsed_uri.scheme == 'ftp' or \ - parsed_uri.scheme == 'ftps': - with urllib.request.urlopen(repo_uri) as s: - with cache_file.open('wb') as f: - shutil.copyfileobj(s, f) - else: - repo_json = pathlib.Path(repo_uri).resolve() - if not repo_json.is_file(): - return 1 - shutil.copy(repo_json, cache_file) + parsed_uri = urllib.parse.urlparse(manifest_json_uri) + download_file_result = utils.download_file(parsed_uri, cache_file) + if download_file_result != 0: + return download_file_result + + repo_set |= repo_data['repos'] return 0 @@ -156,18 +78,9 @@ def refresh_repos() -> int: cache_file = cache_folder / str(repo_sha256.hexdigest() + '.json') if not cache_file.is_file(): parsed_uri = urllib.parse.urlparse(repo_uri) - if parsed_uri.scheme == 'http' or \ - parsed_uri.scheme == 'https' or \ - parsed_uri.scheme == 'ftp' or \ - parsed_uri.scheme == 'ftps': - with urllib.request.urlopen(repo_uri) as s: - with cache_file.open('wb') as f: - shutil.copyfileobj(s, f) - else: - origin_file = pathlib.Path(repo_uri).resolve() - if not origin_file.is_file(): - return 1 - shutil.copy(origin_file, cache_file) + download_file_result = utils.download_file(parsed_uri, cache_file) + if download_file_result != 0: + return download_file_result if not validation.valid_o3de_repo_json(cache_file): logger.error(f'Repo json {repo_uri} is not valid.') @@ -181,111 +94,67 @@ def refresh_repos() -> int: return result -def search_repo(repo_set: set, - repo_json_data: dict, +def search_repo(repo_json_data: dict, engine_name: str = None, project_name: str = None, gem_name: str = None, template_name: str = None, restricted_name: str = None) -> dict or None: - cache_folder = manifest.get_o3de_cache_folder() if isinstance(engine_name, str) or isinstance(engine_name, pathlib.PurePath): - for engine_uri in repo_json_data['engines']: - engine_uri = f'{engine_uri}/engine.json' - engine_sha256 = hashlib.sha256(engine_uri.encode()) - engine_cache_file = cache_folder / str(engine_sha256.hexdigest() + '.json') - if engine_cache_file.is_file(): - with engine_cache_file.open('r') as f: - try: - engine_json_data = json.load(f) - except Exception as e: - logger.warn(f'{engine_cache_file} failed to load: {str(e)}') - else: - if engine_json_data['engine_name'] == engine_name: - return engine_json_data - + o3de_object_uris = repo_json_data['engines'] + manifest_json = 'engine.json' + json_key = 'engine_name' + search_func = lambda: None if manifest_json_data.get(json_key, '') == engine_name else manifest_json_data elif isinstance(project_name, str) or isinstance(project_name, pathlib.PurePath): - for project_uri in repo_json_data['projects']: - project_uri = f'{project_uri}/project.json' - project_sha256 = hashlib.sha256(project_uri.encode()) - project_cache_file = cache_folder / str(project_sha256.hexdigest() + '.json') - if project_cache_file.is_file(): - with project_cache_file.open('r') as f: - try: - project_json_data = json.load(f) - except Exception as e: - logger.warn(f'{project_cache_file} failed to load: {str(e)}') - else: - if project_json_data['project_name'] == project_name: - return project_json_data - + o3de_object_uris = repo_json_data['projects'] + manifest_json = 'project.json' + json_key = 'project_name' + search_func = lambda: None if manifest_json_data.get(json_key, '') == project_name else manifest_json_data elif isinstance(gem_name, str) or isinstance(gem_name, pathlib.PurePath): - for gem_uri in repo_json_data['gems']: - gem_uri = f'{gem_uri}/gem.json' - gem_sha256 = hashlib.sha256(gem_uri.encode()) - gem_cache_file = cache_folder / str(gem_sha256.hexdigest() + '.json') - if gem_cache_file.is_file(): - with gem_cache_file.open('r') as f: - try: - gem_json_data = json.load(f) - except Exception as e: - logger.warn(f'{gem_cache_file} failed to load: {str(e)}') - else: - if gem_json_data['gem_name'] == gem_name: - return gem_json_data - + o3de_object_uris = repo_json_data['gems'] + manifest_json = 'gem.json' + json_key = 'gem_name' + search_func = lambda: None if manifest_json_data.get(json_key, '') == gem_name else manifest_json_data elif isinstance(template_name, str) or isinstance(template_name, pathlib.PurePath): - for template_uri in repo_json_data['templates']: - template_uri = f'{template_uri}/template.json' - template_sha256 = hashlib.sha256(template_uri.encode()) - template_cache_file = cache_folder / str(template_sha256.hexdigest() + '.json') - if template_cache_file.is_file(): - with template_cache_file.open('r') as f: - try: - template_json_data = json.load(f) - except Exception as e: - logger.warn(f'{template_cache_file} failed to load: {str(e)}') - else: - if template_json_data['template_name'] == template_name: - return template_json_data - + o3de_object_uris = repo_json_data['template'] + manifest_json = 'template.json' + json_key = 'template_name' + search_func = lambda: None if manifest_json_data.get(json_key, '') == template_name_name else manifest_json_data elif isinstance(restricted_name, str) or isinstance(restricted_name, pathlib.PurePath): - for restricted_uri in repo_json_data['restricted']: - restricted_uri = f'{restricted_uri}/restricted.json' - restricted_sha256 = hashlib.sha256(restricted_uri.encode()) - restricted_cache_file = cache_folder / str(restricted_sha256.hexdigest() + '.json') - if restricted_cache_file.is_file(): - with restricted_cache_file.open('r') as f: - try: - restricted_json_data = json.load(f) - except Exception as e: - logger.warn(f'{restricted_cache_file} failed to load: {str(e)}') - else: - if restricted_json_data['restricted_name'] == restricted_name: - return restricted_json_data - # recurse + o3de_object_uris = repo_json_data['restricted'] + manifest_json = 'restricted.json' + json_key = 'restricted_name' + search_func = lambda: None if manifest_json_data.get(json_key, '') == restricted_name else manifest_json_data else: - for repo_repo_uri in repo_json_data['repos']: - if repo_repo_uri not in repo_set: - repo_set.add(repo_repo_uri) - repo_repo_uri = f'{repo_repo_uri}/repo.json' - repo_repo_sha256 = hashlib.sha256(repo_repo_uri.encode()) - repo_repo_cache_file = cache_folder / str(repo_repo_sha256.hexdigest() + '.json') - if repo_repo_cache_file.is_file(): - with repo_repo_cache_file.open('r') as f: - try: - repo_repo_json_data = json.load(f) - except Exception as e: - logger.warn(f'{repo_repo_cache_file} failed to load: {str(e)}') - else: - item = search_repo(repo_set, - repo_repo_json_data, - engine_name, - project_name, - gem_name, - template_name) - if item: - return item - return None + return None + o3de_object = search_o3de_object(manifest_json, o3de_object_uris, search_func) + if o3de_object: + return o3de_object + + # recurse into the repos object to search for the o3de object + o3de_object_uris = repo_json_data['repos'] + manifest_json = 'repo.json' + search_func = lambda: search_repo(manifest_json, engine_name, project_name, gem_name, template_name) + return search_o3de_object(manifest_json, o3de_object_uris, search_func) + + +def search_o3de_object(manifest_json, o3de_object_uris, search_func): + # Search for the o3de object based on the supplied object name in the current repo + cache_folder = manifest.get_o3de_cache_folder() + for o3de_object_uri in o3de_object_uris: + manifest_json_uri = f'{o3de_object_uri}/{manifest_json}' + manifest_json_sha256 = hashlib.sha256(manifest_json_uri.encode()) + cache_file = cache_folder / str(manifest_json_sha256.hexdigest() + '.json') + if cache_file.is_file(): + with cache_file.open('r') as f: + try: + manifest_json_data = json.load(f) + except json.JSONDecodeError as e: + logger.warn(f'{cache_file} failed to load: {str(e)}') + else: + result_json_data = search_func() + if result_json_data: + return result_json_data + return None diff --git a/scripts/o3de/o3de/sha256.py b/scripts/o3de/o3de/sha256.py index bc35919c4e..bbec7696d6 100644 --- a/scripts/o3de/o3de/sha256.py +++ b/scripts/o3de/o3de/sha256.py @@ -13,6 +13,8 @@ import argparse import json import logging import hashlib +import pathlib +import sys from o3de import utils @@ -42,7 +44,7 @@ def sha256(file_path: str or pathlib.Path, with json_path.open('r') as s: try: json_data = json.load(s) - except Exception as e: + except json.JSONDecodeError as e: logger.error(f'Failed to read Json path {json_path}: {str(e)}') return 1 json_data.update({"sha256": sha256}) @@ -50,7 +52,7 @@ def sha256(file_path: str or pathlib.Path, with json_path.open('w') as s: try: s.write(json.dumps(json_data, indent=4)) - except Exception as e: + except OSError as e: logger.error(f'Failed to write Json path {json_path}: {str(e)}') return 1 else: @@ -63,20 +65,53 @@ def _run_sha256(args: argparse) -> int: args.json_path) -def add_args(parser, subparsers) -> None: +def add_parser_args(parser): """ - add_args is called to add expected parser arguments and subparsers arguments to each command such that it can be + add_parser_args is called to add arguments to each command such that it can be invoked locally or added by a central python file. - Ex. Directly run from this file alone with: python register.py register --gem-path "C:/TestGem" - OR - o3de.py can downloadable commands by importing engine_template, - call add_args and execute: python o3de.py register --gem-path "C:/TestGem" - :param parser: the caller instantiates a parser and passes it in here + Ex. Directly run from this file alone with: python sha256.py --file-path "C:/TestGem" + :param parser: the caller passes an argparse parser like instance to this method + """ + parser.add_argument('-f', '--file-path', type=str, required=True, + help='The path to the file you want to sha256.') + parser.add_argument('-j', '--json-path', type=str, required=False, + help='optional path to an o3de json file to add the "sha256" element to.') + parser.set_defaults(func=_run_sha256) + + +def add_args(subparsers) -> None: + """ + add_args is called to add subparsers arguments to each command such that it can be + a central python file such as o3de.py. + It can be run from the o3de.py script as follows + call add_args and execute: python o3de.py sha256 --file-path "C:/TestGem" :param subparsers: the caller instantiates subparsers and passes it in here """ sha256_subparser = subparsers.add_parser('sha256') - sha256_subparser.add_argument('-f', '--file-path', type=str, required=True, - help='The path to the file you want to sha256.') - sha256_subparser.add_argument('-j', '--json-path', type=str, required=False, - help='optional path to an o3de json file to add the "sha256" element to.') - sha256_subparser.set_defaults(func=_run_sha256) + add_parser_args(sha256_subparser) + + +def main(): + """ + Runs sha256.py script as standalone script + """ + # parse the command line args + the_parser = argparse.ArgumentParser() + + # add subparsers + + # add args to the parser + add_parser_args(the_parser) + + # parse args + the_args = the_parser.parse_args() + + # run + ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1 + + # return + sys.exit(ret) + + +if __name__ == "__main__": + main() diff --git a/scripts/o3de/o3de/utils.py b/scripts/o3de/o3de/utils.py index 50a9e5d6dd..4330de25b8 100755 --- a/scripts/o3de/o3de/utils.py +++ b/scripts/o3de/o3de/utils.py @@ -13,7 +13,9 @@ This file contains utility functions """ import uuid - +import pathlib +import shutil +import urllib.request def validate_identifier(identifier: str) -> bool: """ @@ -46,6 +48,7 @@ def validate_uuid4(uuid_string: str) -> bool: return False return str(val) == uuid_string + def backup_file(file_name: str or pathlib.Path) -> None: index = 0 renamed = False @@ -69,4 +72,41 @@ def backup_folder(folder: str or pathlib.Path) -> None: folder = pathlib.Path(folder).resolve() folder.rename(backup_folder_name) if backup_folder_name.is_dir(): - renamed = True \ No newline at end of file + renamed = True + + +def download_file(parsed_uri, download_path: pathlib.Path) -> int: + """ + :param parsed_uri: uniform resource identifier to zip file to download + :param download_path: location path on disk to download file + """ + if download_path.is_file(): + logger.warn(f'File already downloaded to {download_path}.') + elif parsed_uri.scheme in ['http', 'https', 'ftp', 'ftps']: + with urllib.request.urlopen(url) as s: + with download_path.open('wb') as f: + shutil.copyfileobj(s, f) + else: + origin_file = pathlib.Path(url).resolve() + if not origin_file.is_file(): + return 1 + shutil.copy(origin_file, download_path) + + return 0 + + +def download_zip_file(parsed_uri, download_zip_path: pathlib.Path) -> int: + """ + :param parsed_uri: uniform resource identifier to zip file to download + :param download_zip_path: path to output zip file + """ + download_file_result = download_file(parsed_uri, download_zip_path) + if download_file_result != 0: + return download_file_result + + if not zipfile.is_zipfile(download_zip_path): + logger.error(f"File zip {download_zip_path} is invalid.") + download_zip_path.unlink() + return 1 + + return 0 \ No newline at end of file diff --git a/scripts/o3de/o3de/validation.py b/scripts/o3de/o3de/validation.py index f3a5f5e376..721b7eae09 100644 --- a/scripts/o3de/o3de/validation.py +++ b/scripts/o3de/o3de/validation.py @@ -14,6 +14,10 @@ This file validating o3de object json files import json import pathlib +def valid_o3de_json_dict(json_data: dict, key: str) -> bool: + return key in json_data + + def valid_o3de_repo_json(file_name: str or pathlib.Path) -> bool: file_name = pathlib.Path(file_name).resolve() if not file_name.is_file(): @@ -24,7 +28,7 @@ def valid_o3de_repo_json(file_name: str or pathlib.Path) -> bool: json_data = json.load(f) test = json_data['repo_name'] test = json_data['origin'] - except Exception as e: + except (json.JSONDecodeError, KeyError) as e: return False return True @@ -39,8 +43,7 @@ def valid_o3de_engine_json(file_name: str or pathlib.Path) -> bool: try: json_data = json.load(f) test = json_data['engine_name'] - # test = json_data['origin'] # will be required soon - except Exception as e: + except (json.JSONDecodeError, KeyError) as e: return False return True @@ -54,8 +57,7 @@ def valid_o3de_project_json(file_name: str or pathlib.Path) -> bool: try: json_data = json.load(f) test = json_data['project_name'] - # test = json_data['origin'] # will be required soon - except Exception as e: + except (json.JSONDecodeError, KeyError) as e: return False return True @@ -69,8 +71,7 @@ def valid_o3de_gem_json(file_name: str or pathlib.Path) -> bool: try: json_data = json.load(f) test = json_data['gem_name'] - # test = json_data['origin'] # will be required soon - except Exception as e: + except (json.JSONDecodeError, KeyError) as e: return False return True @@ -83,8 +84,7 @@ def valid_o3de_template_json(file_name: str or pathlib.Path) -> bool: try: json_data = json.load(f) test = json_data['template_name'] - # test = json_data['origin'] # will be required soon - except Exception as e: + except (json.JSONDecodeError, KeyError) as e: return False return True @@ -97,7 +97,6 @@ def valid_o3de_restricted_json(file_name: str or pathlib.Path) -> bool: try: json_data = json.load(f) test = json_data['restricted_name'] - # test = json_data['origin'] # will be required soon - except Exception as e: + except (json.JSONDecodeError, KeyError) as e: return False return True diff --git a/scripts/o3de/tests/unit_test_registration.py b/scripts/o3de/tests/unit_test_registration.py index a0abb6cacd..eb866e76d4 100644 --- a/scripts/o3de/tests/unit_test_registration.py +++ b/scripts/o3de/tests/unit_test_registration.py @@ -35,11 +35,10 @@ string_manifest_data = '{}' ) def test_register_engine_path(engine_path, engine_name, force, expected_result): parser = argparse.ArgumentParser() - subparser = parser.add_subparsers(help='sub-command help') # Register the registration script subparsers with the current argument parser - register.add_args(parser, subparser) - arg_list = ['register', '--engine-path', str(engine_path)] + register.add_parser_args(parser) + arg_list = ['--engine-path', str(engine_path)] if force: arg_list += ['--force'] args = parser.parse_args(arg_list) @@ -64,3 +63,55 @@ def test_register_engine_path(engine_path, engine_name, force, expected_result): result = register._run_register(args) assert result == expected_result + +@pytest.fixture(scope='class') +def init_manifest_data(request): + class ManifestData: + def __init__(self): + self.json_string = json.dumps({'default_engines_folder': '', + 'default_projects_folder': '', 'default_gems_folder': '', + 'default_templates_folder': '', 'default_restricted_folder': ''}) + + request.cls.manifest_data = ManifestData() + + +@pytest.mark.usefixtures('init_manifest_data') +class TestRegisterThisEngine: + @pytest.mark.parametrize( + "engine_path, engine_name, force, expected_result", [ + pytest.param(pathlib.PurePath('D:/o3de/o3de'), "o3de", False, 0), + pytest.param(pathlib.PurePath('F:/Open3DEngine'), "o3de", False, 1), + pytest.param(pathlib.PurePath('F:/Open3DEngine'), "o3de", True, 0) + ] + ) + def test_register_this_engine(self, engine_path, engine_name, force, expected_result): + parser = argparse.ArgumentParser() + + # Register the registration script subparsers with the current argument parser + register.add_parser_args(parser) + arg_list = ['--this-engine'] + if force: + arg_list += ['--force'] + args = parser.parse_args(arg_list) + + def load_manifest_from_string() -> dict: + try: + manifest_json = json.loads(self.manifest_data.json_string) + except json.JSONDecodeError as err: + logging.error("Error decoding Json from Manifest file") + else: + return manifest_json + def save_manifest_to_string(manifest_json: dict) -> None: + self.manifest_data.json_string = json.dumps(manifest_json) + + engine_json_data = {'engine_name': engine_name} + + with patch('o3de.manifest.load_o3de_manifest', side_effect=load_manifest_from_string) as load_manifest_mock, \ + patch('o3de.manifest.save_o3de_manifest', side_effect=save_manifest_to_string) as save_manifest_mock, \ + patch('o3de.manifest.get_engine_json_data', return_value=engine_json_data) as engine_paths_mock, \ + patch('o3de.manifest.get_this_engine_path', return_value=engine_path) as engine_paths_mock, \ + patch('o3de.validation.valid_o3de_engine_json', return_value=True) as valid_engine_mock, \ + patch('pathlib.Path.is_dir', return_value=True) as pathlib_is_dir_mock: + result = register._run_register(args) + assert result == expected_result +