Merge pull request #4505 from aws-lumberyard-dev/CLIAddExternalGemRepo

Fix issues with adding a repo

scripts/o3de/o3de/register.py

@@ -487,14 +487,14 @@ def register_repo(json_data: dict,
     if remove:
         logger.warn(f'Removing repo uri {repo_uri}.')
         return 0
-
     repo_sha256 = hashlib.sha256(url.encode())
     cache_file = manifest.get_o3de_cache_folder() / str(repo_sha256.hexdigest() + '.json')
 
-    result = utils.download_file(url, cache_file)
+    result = utils.download_file(parsed_uri, cache_file)
     if result == 0:
-        json_data['repos'].insert(0, repo_uri.as_posix())
+        json_data['repos'].insert(0, repo_uri)
 
+    repo_set = set()
     result = repo.process_add_o3de_repo(cache_file, repo_set)
 
     return result
@@ -621,6 +621,7 @@ def register(engine_path: pathlib.Path = None,
             return 1
         result = result or register_gem_path(json_data, gem_path, remove,
                                    external_subdir_engine_path, external_subdir_project_path)
+
     if isinstance(external_subdir_path, pathlib.PurePath):
         if not external_subdir_path:
             logger.error(f'External Subdirectory path is None.')

scripts/o3de/o3de/repo.py

@@ -12,6 +12,7 @@ import pathlib
 import shutil
 import urllib.parse
 import urllib.request
+import hashlib
 
 from o3de import manifest, utils, validation
 
@@ -24,7 +25,6 @@ def process_add_o3de_repo(file_name: str or pathlib.Path,
     file_name = pathlib.Path(file_name).resolve()
     if not validation.valid_o3de_repo_json(file_name):
         return 1
-
     cache_folder = manifest.get_o3de_cache_folder()
 
     with file_name.open('r') as f:
@@ -34,11 +34,30 @@ def process_add_o3de_repo(file_name: str or pathlib.Path,
             logger.error(f'{file_name} failed to load: {str(e)}')
             return 1
 
-        for o3de_object_uris, manifest_json in [(repo_data['engines'], 'engine.json'),
-                                                (repo_data['projects'], 'project.json'),
-                                                (repo_data['gems'], 'gem.json'),
-                                                (repo_data['template'], 'template.json'),
-                                                (repo_data['restricted'], 'restricted.json')]:
+        # A repo may not contain all types of object.
+        manifest_download_list = []
+        try:
+            manifest_download_list.append((repo_data['engines'], 'engine.json'))
+        except KeyError:
+            pass
+        try:
+            manifest_download_list.append((repo_data['projects'], 'project.json'))
+        except KeyError:
+            pass
+        try:
+            manifest_download_list.append((repo_data['gems'], 'gem.json'))
+        except KeyError:
+            pass
+        try:
+            manifest_download_list.append((repo_data['templates'], 'template.json'))
+        except KeyError:
+            pass
+        try:
+            manifest_download_list.append((repo_data['restricted'], 'restricted.json'))
+        except KeyError:
+            pass
+
+        for o3de_object_uris, manifest_json in manifest_download_list:
             for o3de_object_uri in o3de_object_uris:
                 manifest_json_uri = f'{o3de_object_uri}/{manifest_json}'
                 manifest_json_sha256 = hashlib.sha256(manifest_json_uri.encode())
@@ -49,7 +68,27 @@ def process_add_o3de_repo(file_name: str or pathlib.Path,
                     if download_file_result != 0:
                         return download_file_result
 
-        repo_set |= repo_data['repos']
+        # Having a repo is also optional
+        repo_list = []
+        try:
+            repo_list.add(repo_data['repos'])
+        except KeyError:
+            pass
+
+        for repo in repo_list:
+            if repo not in repo_set:
+                repo_set.add(repo)
+                for o3de_object_uri in o3de_object_uris:
+                    parsed_uri = urllib.parse.urlparse(f'{repo}/repo.json')
+                    manifest_json_sha256 = hashlib.sha256(parsed_uri.geturl().encode())
+                    cache_file = cache_folder / str(manifest_json_sha256.hexdigest() + '.json')
+                    if cache_file.is_file():
+                        cache_file.unlink()
+                    download_file_result = utils.download_file(parsed_uri, cache_file)
+                    if download_file_result != 0:
+                        return download_file_result
+
+                    return process_add_o3de_repo(parsed_uri.geturl(), repo_set)
     return 0
 
 
@@ -70,11 +109,10 @@ def refresh_repos() -> int:
         if repo_uri not in repo_set:
             repo_set.add(repo_uri)
 
-            repo_uri = f'{repo_uri}/repo.json'
-            repo_sha256 = hashlib.sha256(repo_uri.encode())
+            parsed_uri = urllib.parse.urlparse(f'{repo_uri}/repo.json')
+            repo_sha256 = hashlib.sha256(parsed_uri.geturl().encode())
             cache_file = cache_folder / str(repo_sha256.hexdigest() + '.json')
             if not cache_file.is_file():
-                parsed_uri = urllib.parse.urlparse(repo_uri)
                 download_file_result = utils.download_file(parsed_uri, cache_file)
                 if download_file_result != 0:
                     return download_file_result

scripts/o3de/o3de/utils.py

@@ -13,6 +13,10 @@ import uuid
 import pathlib
 import shutil
 import urllib.request
+import logging
+
+logger = logging.getLogger()
+logging.basicConfig()
 
 def validate_identifier(identifier: str) -> bool:
     """
@@ -97,11 +101,11 @@ def download_file(parsed_uri, download_path: pathlib.Path) -> int:
     if download_path.is_file():
         logger.warn(f'File already downloaded to {download_path}.')
     elif parsed_uri.scheme in ['http', 'https', 'ftp', 'ftps']:
-        with urllib.request.urlopen(url) as s:
+        with urllib.request.urlopen(parsed_uri.geturl()) as s:
             with download_path.open('wb') as f:
                 shutil.copyfileobj(s, f)
     else:
-        origin_file = pathlib.Path(url).resolve()
+        origin_file = pathlib.Path(parsed_uri.geturl()).resolve()
         if not origin_file.is_file():
             return 1
         shutil.copy(origin_file, download_path)