diff --git a/scripts/o3de.py b/scripts/o3de.py index 03f737643f..05c3b53ba6 100755 --- a/scripts/o3de.py +++ b/scripts/o3de.py @@ -29,12 +29,13 @@ def add_args(parser, subparsers) -> None: # add the scripts/o3de directory to the front of the sys.path sys.path.insert(0, str(o3de_package_dir)) from o3de import engine_properties, engine_template, gem_properties, global_project, register, print_registration, get_registration, \ - enable_gem, disable_gem, project_properties, sha256 + enable_gem, disable_gem, project_properties, sha256, download # Remove the temporarily added path sys.path = sys.path[1:] # global_project global_project.add_args(subparsers) + # engine templaate engine_template.add_args(subparsers) @@ -61,10 +62,13 @@ def add_args(parser, subparsers) -> None: # modify gem properties gem_properties.add_args(subparsers) - + # sha256 sha256.add_args(subparsers) + # download + download.add_args(subparsers) + if __name__ == "__main__": # parse the command line args diff --git a/scripts/o3de/o3de/download.py b/scripts/o3de/o3de/download.py index e1e5035ce1..685929b405 100644 --- a/scripts/o3de/o3de/download.py +++ b/scripts/o3de/o3de/download.py @@ -6,20 +6,22 @@ # # """ -Implements functionality for downloading o3de objecs either locally or from a URI +Implements functionality for downloading o3de objects either locally or from a URI """ import argparse import hashlib import json import logging +import os import pathlib import shutil import sys import urllib.parse import urllib.request +import zipfile -from o3de import manifest, repo, utils, validation +from o3de import manifest, repo, utils, validation, register logger = logging.getLogger() logging.basicConfig() @@ -37,21 +39,21 @@ def unzip_manifest_json_data(download_zip_path: pathlib.Path, zip_file_name: str def validate_downloaded_zip_sha256(download_uri_json_data: dict, download_zip_path: pathlib.Path, manifest_json_name) -> int: - # if the engine.json has a sha256 check it against a sha256 of the zip + # if the json has a sha256 check it against a sha256 of the zip try: sha256A = download_uri_json_data['sha256'] except KeyError as e: logger.warn(f'SECURITY WARNING: The advertised o3de object you downloaded has no "sha256"!!! Be VERY careful!!!' f' We cannot verify this is the actually the advertised object!!!') + return 0 else: sha256B = hashlib.sha256(download_zip_path.open('rb').read()).hexdigest() if sha256A != sha256B: logger.error(f'SECURITY VIOLATION: Downloaded zip sha256 {sha256B} does not match' - f' the advertised "sha256":{sha256A} in the f{manifest_json_name}. Deleting unzipped files!!!') - shutil.rmtree(dest_path) + f' the advertised "sha256":{sha256A} in the f{manifest_json_name}.') return 1 - manifest_json_data = unzip_manifest_json_data(download_zip_path, manifest_json_name) + unzipped_manifest_json_data = unzip_manifest_json_data(download_zip_path, manifest_json_name) # remove the sha256 if present in the advertised downloadable manifest json # then compare it to the json in the zip, they should now be identical @@ -61,19 +63,11 @@ def validate_downloaded_zip_sha256(download_uri_json_data: dict, download_zip_pa pass sha256A = hashlib.sha256(json.dumps(download_uri_json_data, indent=4).encode('utf8')).hexdigest() - with unzipped_manifest_json.open('r') as s: - try: - unzipped_manifest_json_data = json.load(s) - except json.JSONDecodeError as e: - logger.error(f'Failed to read manifest json {unzipped_manifest_json}. Unable to confirm this' - f' is the same template that was advertised.') - return 1 sha256B = hashlib.sha256(json.dumps(unzipped_manifest_json_data, indent=4).encode('utf8')).hexdigest() if sha256A != sha256B: logger.error(f'SECURITY VIOLATION: Downloaded manifest json does not match' - f' the advertised manifest json. Deleting unzipped files!!!') - shutil.rmtree(dest_path) - return 1 + f' the advertised manifest json.') + return 0 return 0 @@ -91,23 +85,15 @@ def get_downloadable(engine_name: str = None, return None manifest_json = 'repo.json' - search_func = lambda: repo.search_repo(manifest_json, engine_name, project_name, gem_name, template_name) + search_func = lambda manifest_json_data: repo.search_repo(manifest_json_data, engine_name, project_name, gem_name, template_name) return repo.search_o3de_object(manifest_json, o3de_object_uris, search_func) def download_o3de_object(object_name: str, default_folder_name: str, dest_path: str or pathlib.Path, - object_type: str, downloadable_kwarg_key) -> int: - if not dest_path: - dest_path = manifest.get_registered(default_folder=default_folder_name) - if not dest_path: - logger.error(f'Destination path not cannot be empty.') - return 1 + object_type: str, downloadable_kwarg_key, skip_auto_register: bool) -> int: - dest_path = pathlib.Path(dest_path).resolve() - dest_path.mkdir(exist_ok=True) - - download_path = manifest.get_o3de_download_folder() / default_folder_name / object_name - download_path.mkdir(exist_ok=True) + download_path = manifest.get_o3de_cache_folder() / default_folder_name / object_name + download_path.mkdir(parents=True, exist_ok=True) download_zip_path = download_path / f'{object_type}.zip' downloadable_object_data = get_downloadable(**{downloadable_kwarg_key : object_name}) @@ -115,41 +101,79 @@ def download_o3de_object(object_name: str, default_folder_name: str, dest_path: logger.error(f'Downloadable o3de object {object_name} not found.') return 1 - origin = downloadable_json_data['origin'] - url = f'{origin}/object_type.zip' + url = downloadable_object_data['originuri'] parsed_uri = urllib.parse.urlparse(url) download_zip_result = utils.download_zip_file(parsed_uri, download_zip_path) if download_zip_result != 0: return download_zip_result - return validate_downloaded_zip_sha256(downloadable_object_data, download_zip_path) + if validate_downloaded_zip_sha256(downloadable_object_data, download_zip_path, f'{object_type}.json'): + logger.error(f'Could not validate zip, deleting {download_zip_path}') + os.unlink(download_zip_path) + return 1 + + if not dest_path: + dest_path = manifest.get_registered(default_folder=default_folder_name) + dest_path = pathlib.Path(dest_path).resolve() + dest_path = dest_path / object_name + else: + dest_path = pathlib.Path(dest_path).resolve() + + if not dest_path: + logger.error(f'Destination path not cannot be empty.') + return 1 + if dest_path.exists(): + logger.error(f'Destination path {dest_path} already exists.') + return 1 + + dest_path.mkdir(exist_ok=True) + + # extract zip + with zipfile.ZipFile(download_zip_path, 'r') as zip_file_ref: + try: + zip_file_ref.extractall(dest_path) + except Exception: + logger.error(f'Error unzipping {download_zip_path} to {dest_path}. Deleting {dest_path}.') + shutil.rmtree(dest_path) + return 1 + + if not skip_auto_register: + if object_type == 'gem': + return register.register(gem_path=dest_path) + + return 0 def download_engine(engine_name: str, - dest_path: str or pathlib.Path) -> int: - return download_o3de_object(engine_name, 'engines', dest_path, 'engine', 'engine_name') + dest_path: str or pathlib.Path, + skip_auto_register: bool) -> int: + return download_o3de_object(engine_name, 'engines', dest_path, 'engine', 'engine_name', skip_auto_register) def download_project(project_name: str, - dest_path: str or pathlib.Path) -> int: - return download_o3de_object(project_name, 'projects', dest_path, 'project', 'project_name') + dest_path: str or pathlib.Path, + skip_auto_register: bool) -> int: + return download_o3de_object(project_name, 'projects', dest_path, 'project', 'project_name', skip_auto_register) def download_gem(gem_name: str, - dest_path: str or pathlib.Path) -> int: - return download_o3de_object(gem_name, 'gems', dest_path, 'gem', 'gem_name') + dest_path: str or pathlib.Path, + skip_auto_register: bool) -> int: + return download_o3de_object(gem_name, 'gems', dest_path, 'gem', 'gem_name', skip_auto_register) def download_template(template_name: str, - dest_path: str or pathlib.Path) -> int: - return download_o3de_object(template_name, 'templates', dest_path, 'template', 'template_name') + dest_path: str or pathlib.Path, + skip_auto_register: bool) -> int: + return download_o3de_object(template_name, 'templates', dest_path, 'template', 'template_name', skip_auto_register) def download_restricted(restricted_name: str, - dest_path: str or pathlib.Path) -> int: - return download_o3de_object(restricted_name, 'restricted', dest_path, 'restricted', 'restricted_name') + dest_path: str or pathlib.Path, + skip_auto_register: bool) -> int: + return download_o3de_object(restricted_name, 'restricted', dest_path, 'restricted', 'restricted_name', skip_auto_register) def _run_download(args: argparse) -> int: @@ -158,16 +182,20 @@ def _run_download(args: argparse) -> int: if args.engine_name: return download_engine(args.engine_name, - args.dest_path) + args.dest_path, + args.skip_auto_register) elif args.project_name: return download_project(args.project_name, - args.dest_path) - elif args.gem_nanme: + args.dest_path, + args.skip_auto_register) + elif args.gem_name: return download_gem(args.gem_name, - args.dest_path) + args.dest_path, + args.skip_auto_register) elif args.template_name: return download_template(args.template_name, - args.dest_path) + args.dest_path, + args.skip_auto_register) return 1 @@ -188,14 +216,16 @@ def add_parser_args(parser): group.add_argument('-t', '--template-name', type=str, required=False, help='Downloadable template name.') parser.add_argument('-dp', '--dest-path', type=str, required=False, - default=None, - help='Optional destination folder to download into.' - ' i.e. download --project-name "AstomSamplerViewer" --dest-path "C:/projects"' - ' will result in C:/projects/AtomSampleViewer' - ' If blank will download to default object type folder') - + default=None, + help='Optional destination folder to download into.' + ' i.e. download --project-name "AstomSamplerViewer" --dest-path "C:/projects"' + ' will result in C:/projects/AtomSampleViewer' + ' If blank will download to default object type folder') + parser.add_argument('-sar', '--skip-auto-register', action='store_true', required=False, + default=False, + help = 'Skip the automatic registration of new object download') parser.add_argument('-ohf', '--override-home-folder', type=str, required=False, - help='By default the home folder is the user folder, override it to this folder.') + help='By default the home folder is the user folder, override it to this folder.') parser.set_defaults(func=_run_download) diff --git a/scripts/o3de/o3de/repo.py b/scripts/o3de/o3de/repo.py index 52abfc3386..787c822dd0 100644 --- a/scripts/o3de/o3de/repo.py +++ b/scripts/o3de/o3de/repo.py @@ -129,49 +129,53 @@ def refresh_repos() -> int: return result -def search_repo(repo_json_data: dict, +def search_repo(manifest_json_data: dict, engine_name: str = None, project_name: str = None, gem_name: str = None, template_name: str = None, restricted_name: str = None) -> dict or None: - if isinstance(engine_name, str) or isinstance(engine_name, pathlib.PurePath): - o3de_object_uris = repo_json_data['engines'] + o3de_object_uris = manifest_json_data['engines'] manifest_json = 'engine.json' json_key = 'engine_name' - search_func = lambda: None if manifest_json_data.get(json_key, '') == engine_name else manifest_json_data + search_func = lambda manifest_json_data: manifest_json_data if manifest_json_data.get(json_key, '') == engine_name else None elif isinstance(project_name, str) or isinstance(project_name, pathlib.PurePath): - o3de_object_uris = repo_json_data['projects'] + o3de_object_uris = manifest_json_data['projects'] manifest_json = 'project.json' json_key = 'project_name' - search_func = lambda: None if manifest_json_data.get(json_key, '') == project_name else manifest_json_data + search_func = lambda manifest_json_data: manifest_json_data if manifest_json_data.get(json_key, '') == project_name else None elif isinstance(gem_name, str) or isinstance(gem_name, pathlib.PurePath): - o3de_object_uris = repo_json_data['gems'] + o3de_object_uris = manifest_json_data['gems'] manifest_json = 'gem.json' json_key = 'gem_name' - search_func = lambda: None if manifest_json_data.get(json_key, '') == gem_name else manifest_json_data + search_func = lambda manifest_json_data: manifest_json_data if manifest_json_data.get(json_key, '') == gem_name else None elif isinstance(template_name, str) or isinstance(template_name, pathlib.PurePath): - o3de_object_uris = repo_json_data['template'] + o3de_object_uris = manifest_json_data['template'] manifest_json = 'template.json' json_key = 'template_name' - search_func = lambda: None if manifest_json_data.get(json_key, '') == template_name_name else manifest_json_data + search_func = lambda manifest_json_data: manifest_json_data if manifest_json_data.get(json_key, '') == template_name_name else None elif isinstance(restricted_name, str) or isinstance(restricted_name, pathlib.PurePath): - o3de_object_uris = repo_json_data['restricted'] + o3de_object_uris = manifest_json_data['restricted'] manifest_json = 'restricted.json' json_key = 'restricted_name' - search_func = lambda: None if manifest_json_data.get(json_key, '') == restricted_name else manifest_json_data + search_func = lambda manifest_json_data: manifest_json_data if manifest_json_data.get(json_key, '') == restricted_name else None else: return None - - o3de_object = search_o3de_object(manifest_json, o3de_object_uris, search_func) + o3de_object = search_o3de_object(manifest_json, o3de_object_uris, search_func) if o3de_object: + o3de_object['repo_name'] = manifest_json_data['repo_name'] return o3de_object # recurse into the repos object to search for the o3de object - o3de_object_uris = repo_json_data['repos'] + o3de_object_uris = [] + try: + o3de_object_uris = manifest_json_data['repos'] + except KeyError: + pass + manifest_json = 'repo.json' - search_func = lambda: search_repo(manifest_json, engine_name, project_name, gem_name, template_name) + search_func = lambda manifest_json_data: search_repo(manifest_json_data, engine_name, project_name, gem_name, template_name) return search_o3de_object(manifest_json, o3de_object_uris, search_func) @@ -179,8 +183,8 @@ def search_o3de_object(manifest_json, o3de_object_uris, search_func): # Search for the o3de object based on the supplied object name in the current repo cache_folder = manifest.get_o3de_cache_folder() for o3de_object_uri in o3de_object_uris: - manifest_json_uri = f'{o3de_object_uri}/{manifest_json}' - manifest_json_sha256 = hashlib.sha256(manifest_json_uri.encode()) + parsed_uri = urllib.parse.urlparse(f'{o3de_object_uri}/{manifest_json}') + manifest_json_sha256 = hashlib.sha256(parsed_uri.geturl().encode()) cache_file = cache_folder / str(manifest_json_sha256.hexdigest() + '.json') if cache_file.is_file(): with cache_file.open('r') as f: @@ -189,7 +193,7 @@ def search_o3de_object(manifest_json, o3de_object_uris, search_func): except json.JSONDecodeError as e: logger.warn(f'{cache_file} failed to load: {str(e)}') else: - result_json_data = search_func() + result_json_data = search_func(manifest_json_data) if result_json_data: return result_json_data return None diff --git a/scripts/o3de/o3de/utils.py b/scripts/o3de/o3de/utils.py index 9f9c747390..c5be21c60f 100755 --- a/scripts/o3de/o3de/utils.py +++ b/scripts/o3de/o3de/utils.py @@ -14,6 +14,7 @@ import pathlib import shutil import urllib.request import logging +import zipfile logger = logging.getLogger() logging.basicConfig()