From 09342694c92dd90386b24f3c2f78c0aa3ebdfd89 Mon Sep 17 00:00:00 2001 From: AMZN-Phil Date: Tue, 5 Oct 2021 13:49:27 -0700 Subject: [PATCH] Fix issues with adding a repo and make each object type optional in repo.json Signed-off-by: AMZN-Phil --- scripts/o3de/o3de/register.py | 7 +++-- scripts/o3de/o3de/repo.py | 58 +++++++++++++++++++++++++++++------ scripts/o3de/o3de/utils.py | 8 +++-- 3 files changed, 58 insertions(+), 15 deletions(-) diff --git a/scripts/o3de/o3de/register.py b/scripts/o3de/o3de/register.py index 7db09368ec..c3d201f23c 100644 --- a/scripts/o3de/o3de/register.py +++ b/scripts/o3de/o3de/register.py @@ -487,14 +487,14 @@ def register_repo(json_data: dict, if remove: logger.warn(f'Removing repo uri {repo_uri}.') return 0 - repo_sha256 = hashlib.sha256(url.encode()) cache_file = manifest.get_o3de_cache_folder() / str(repo_sha256.hexdigest() + '.json') - result = utils.download_file(url, cache_file) + result = utils.download_file(parsed_uri, cache_file) if result == 0: - json_data['repos'].insert(0, repo_uri.as_posix()) + json_data['repos'].insert(0, repo_uri) + repo_set = set() result = repo.process_add_o3de_repo(cache_file, repo_set) return result @@ -621,6 +621,7 @@ def register(engine_path: pathlib.Path = None, return 1 result = result or register_gem_path(json_data, gem_path, remove, external_subdir_engine_path, external_subdir_project_path) + if isinstance(external_subdir_path, pathlib.PurePath): if not external_subdir_path: logger.error(f'External Subdirectory path is None.') diff --git a/scripts/o3de/o3de/repo.py b/scripts/o3de/o3de/repo.py index 8492f391e5..39fe79bbc2 100644 --- a/scripts/o3de/o3de/repo.py +++ b/scripts/o3de/o3de/repo.py @@ -12,6 +12,7 @@ import pathlib import shutil import urllib.parse import urllib.request +import hashlib from o3de import manifest, utils, validation @@ -24,7 +25,6 @@ def process_add_o3de_repo(file_name: str or pathlib.Path, file_name = pathlib.Path(file_name).resolve() if not validation.valid_o3de_repo_json(file_name): return 1 - cache_folder = manifest.get_o3de_cache_folder() with file_name.open('r') as f: @@ -34,11 +34,30 @@ def process_add_o3de_repo(file_name: str or pathlib.Path, logger.error(f'{file_name} failed to load: {str(e)}') return 1 - for o3de_object_uris, manifest_json in [(repo_data['engines'], 'engine.json'), - (repo_data['projects'], 'project.json'), - (repo_data['gems'], 'gem.json'), - (repo_data['template'], 'template.json'), - (repo_data['restricted'], 'restricted.json')]: + # A repo may not contain all types of object. + manifest_download_list = [] + try: + manifest_download_list.append((repo_data['engines'], 'engine.json')) + except Exception: + pass + try: + manifest_download_list.append((repo_data['projects'], 'project.json')) + except Exception: + pass + try: + manifest_download_list.append((repo_data['gems'], 'gem.json')) + except Exception: + pass + try: + manifest_download_list.append((repo_data['templates'], 'template.json')) + except Exception: + pass + try: + manifest_download_list.append((repo_data['restricted'], 'restricted.json')) + except Exception: + pass + + for o3de_object_uris, manifest_json in manifest_download_list: for o3de_object_uri in o3de_object_uris: manifest_json_uri = f'{o3de_object_uri}/{manifest_json}' manifest_json_sha256 = hashlib.sha256(manifest_json_uri.encode()) @@ -49,7 +68,27 @@ def process_add_o3de_repo(file_name: str or pathlib.Path, if download_file_result != 0: return download_file_result - repo_set |= repo_data['repos'] + # Having a repo is also optional + repo_list = [] + try: + repo_list.add(repo_data['repos']) + except Exception: + pass + + for repo in repo_list: + if repo not in repo_set: + repo_set.add(repo) + for o3de_object_uri in o3de_object_uris: + parsed_uri = urllib.parse.urlparse(f'{repo}/repo.json') + manifest_json_sha256 = hashlib.sha256(parsed_uri.geturl().encode()) + cache_file = cache_folder / str(manifest_json_sha256.hexdigest() + '.json') + if cache_file.is_file(): + cache_file.unlink() + download_file_result = utils.download_file(parsed_uri, cache_file) + if download_file_result != 0: + return download_file_result + + return process_add_o3de_repo(parsed_uri.geturl(), repo_set) return 0 @@ -70,11 +109,10 @@ def refresh_repos() -> int: if repo_uri not in repo_set: repo_set.add(repo_uri) - repo_uri = f'{repo_uri}/repo.json' - repo_sha256 = hashlib.sha256(repo_uri.encode()) + parsed_uri = urllib.parse.urlparse(f'{repo_uri}/repo.json') + repo_sha256 = hashlib.sha256(parsed_uri.geturl().encode()) cache_file = cache_folder / str(repo_sha256.hexdigest() + '.json') if not cache_file.is_file(): - parsed_uri = urllib.parse.urlparse(repo_uri) download_file_result = utils.download_file(parsed_uri, cache_file) if download_file_result != 0: return download_file_result diff --git a/scripts/o3de/o3de/utils.py b/scripts/o3de/o3de/utils.py index 18f80584cc..9f9c747390 100755 --- a/scripts/o3de/o3de/utils.py +++ b/scripts/o3de/o3de/utils.py @@ -13,6 +13,10 @@ import uuid import pathlib import shutil import urllib.request +import logging + +logger = logging.getLogger() +logging.basicConfig() def validate_identifier(identifier: str) -> bool: """ @@ -97,11 +101,11 @@ def download_file(parsed_uri, download_path: pathlib.Path) -> int: if download_path.is_file(): logger.warn(f'File already downloaded to {download_path}.') elif parsed_uri.scheme in ['http', 'https', 'ftp', 'ftps']: - with urllib.request.urlopen(url) as s: + with urllib.request.urlopen(parsed_uri.geturl()) as s: with download_path.open('wb') as f: shutil.copyfileobj(s, f) else: - origin_file = pathlib.Path(url).resolve() + origin_file = pathlib.Path(parsed_uri.geturl()).resolve() if not origin_file.is_file(): return 1 shutil.copy(origin_file, download_path)