Merge pull request #4736 from aws-lumberyard-dev/CLIDownloadGems

Additional fixes for downloading O3DE objects

scripts/o3de.py

@@ -29,12 +29,13 @@ def add_args(parser, subparsers) -> None:
     # add the scripts/o3de directory to the front of the sys.path
     sys.path.insert(0, str(o3de_package_dir))
     from o3de import engine_properties, engine_template, gem_properties, global_project, register, print_registration, get_registration, \
-        enable_gem, disable_gem, project_properties, sha256
+        enable_gem, disable_gem, project_properties, sha256, download
     # Remove the temporarily added path
     sys.path = sys.path[1:]
 
     # global_project
     global_project.add_args(subparsers)
+
     # engine templaate
     engine_template.add_args(subparsers)
 
@@ -61,10 +62,13 @@ def add_args(parser, subparsers) -> None:
 
     # modify gem properties
     gem_properties.add_args(subparsers)
-    
+
     # sha256
     sha256.add_args(subparsers)
 
+    # download
+    download.add_args(subparsers)
+
 
 if __name__ == "__main__":
     # parse the command line args

scripts/o3de/o3de/download.py

@@ -6,20 +6,22 @@
 #
 #
 """
-Implements functionality for downloading o3de objecs either locally or from a URI
+Implements functionality for downloading o3de objects either locally or from a URI
 """
 
 import argparse
 import hashlib
 import json
 import logging
+import os
 import pathlib
 import shutil
 import sys
 import urllib.parse
 import urllib.request
+import zipfile
 
-from o3de import manifest, repo, utils, validation
+from o3de import manifest, repo, utils, validation, register
 
 logger = logging.getLogger()
 logging.basicConfig()
@@ -37,21 +39,21 @@ def unzip_manifest_json_data(download_zip_path: pathlib.Path, zip_file_name: str
 
 def validate_downloaded_zip_sha256(download_uri_json_data: dict, download_zip_path: pathlib.Path,
                                    manifest_json_name) -> int:
-    # if the engine.json has a sha256 check it against a sha256 of the zip
+    # if the json has a sha256 check it against a sha256 of the zip
     try:
         sha256A = download_uri_json_data['sha256']
     except KeyError as e:
-        logger.warn(f'SECURITY WARNING: The advertised o3de object you downloaded has no "sha256"!!! Be VERY careful!!!'
-                    f' We cannot verify this is the actually the advertised object!!!')
+        logger.warn('SECURITY WARNING: The advertised o3de object you downloaded has no "sha256"!!! Be VERY careful!!!'
+                    ' We cannot verify this is the actually the advertised object!!!')
+        return 1
     else:
         sha256B = hashlib.sha256(download_zip_path.open('rb').read()).hexdigest()
         if sha256A != sha256B:
             logger.error(f'SECURITY VIOLATION: Downloaded zip sha256 {sha256B} does not match'
-                         f' the advertised "sha256":{sha256A} in the f{manifest_json_name}. Deleting unzipped files!!!')
-            shutil.rmtree(dest_path)
-            return 1
+                         f' the advertised "sha256":{sha256A} in the f{manifest_json_name}.')
+            return 0
 
-    manifest_json_data = unzip_manifest_json_data(download_zip_path, manifest_json_name)
+    unzipped_manifest_json_data = unzip_manifest_json_data(download_zip_path, manifest_json_name)
 
     # remove the sha256 if present in the advertised downloadable manifest json
     # then compare it to the json in the zip, they should now be identical
@@ -61,21 +63,13 @@ def validate_downloaded_zip_sha256(download_uri_json_data: dict, download_zip_pa
         pass
 
     sha256A = hashlib.sha256(json.dumps(download_uri_json_data, indent=4).encode('utf8')).hexdigest()
-    with unzipped_manifest_json.open('r') as s:
-        try:
-            unzipped_manifest_json_data = json.load(s)
-        except json.JSONDecodeError as e:
-            logger.error(f'Failed to read manifest json {unzipped_manifest_json}. Unable to confirm this'
-                         f' is the same template that was advertised.')
-            return 1
     sha256B = hashlib.sha256(json.dumps(unzipped_manifest_json_data, indent=4).encode('utf8')).hexdigest()
     if sha256A != sha256B:
-        logger.error(f'SECURITY VIOLATION: Downloaded manifest json does not match'
-                     f' the advertised manifest json. Deleting unzipped files!!!')
-        shutil.rmtree(dest_path)
-        return 1
+        logger.error('SECURITY VIOLATION: Downloaded manifest json does not match'
+                     ' the advertised manifest json.')
+        return 0
 
-    return 0
+    return 1
 
 
 def get_downloadable(engine_name: str = None,
@@ -91,23 +85,15 @@ def get_downloadable(engine_name: str = None,
         return None
 
     manifest_json = 'repo.json'
-    search_func = lambda: repo.search_repo(manifest_json, engine_name, project_name, gem_name, template_name)
+    search_func = lambda manifest_json_data: repo.search_repo(manifest_json_data, engine_name, project_name, gem_name, template_name)
     return repo.search_o3de_object(manifest_json, o3de_object_uris, search_func)
 
 
 def download_o3de_object(object_name: str, default_folder_name: str, dest_path: str or pathlib.Path,
-                         object_type: str, downloadable_kwarg_key) -> int:
-    if not dest_path:
-        dest_path = manifest.get_registered(default_folder=default_folder_name)
-        if not dest_path:
-            logger.error(f'Destination path not cannot be empty.')
-            return 1
+                         object_type: str, downloadable_kwarg_key, skip_auto_register: bool) -> int:
 
-    dest_path = pathlib.Path(dest_path).resolve()
-    dest_path.mkdir(exist_ok=True)
-
-    download_path = manifest.get_o3de_download_folder() / default_folder_name / object_name
-    download_path.mkdir(exist_ok=True)
+    download_path = manifest.get_o3de_cache_folder() / default_folder_name / object_name
+    download_path.mkdir(parents=True, exist_ok=True)
     download_zip_path = download_path / f'{object_type}.zip'
 
     downloadable_object_data = get_downloadable(**{downloadable_kwarg_key : object_name})
@@ -115,41 +101,79 @@ def download_o3de_object(object_name: str, default_folder_name: str, dest_path:
         logger.error(f'Downloadable o3de object {object_name} not found.')
         return 1
 
-    origin = downloadable_json_data['origin']
-    url = f'{origin}/object_type.zip'
-    parsed_uri = urllib.parse.urlparse(url)
+    origin_uri = downloadable_object_data['originuri']
+    parsed_uri = urllib.parse.urlparse(origin_uri)
 
     download_zip_result = utils.download_zip_file(parsed_uri, download_zip_path)
     if download_zip_result != 0:
         return download_zip_result
 
-    return validate_downloaded_zip_sha256(downloadable_object_data, download_zip_path)
+    if not validate_downloaded_zip_sha256(downloadable_object_data, download_zip_path, f'{object_type}.json'):
+        logger.error(f'Could not validate zip, deleting {download_zip_path}')
+        os.unlink(download_zip_path)
+        return 1
+
+    if not dest_path:
+        dest_path = manifest.get_registered(default_folder=default_folder_name)
+        dest_path = pathlib.Path(dest_path).resolve()
+        dest_path = dest_path / object_name
+    else:
+        dest_path = pathlib.Path(dest_path).resolve()
+
+    if not dest_path:
+        logger.error(f'Destination path cannot be empty.')
+        return 1
+    if dest_path.exists():
+        logger.error(f'Destination path {dest_path} already exists.')
+        return 1
+
+    dest_path.mkdir(exist_ok=True)
+
+    # extract zip
+    with zipfile.ZipFile(download_zip_path, 'r') as zip_file_ref:
+        try:
+            zip_file_ref.extractall(dest_path)
+        except Exception:
+            logger.error(f'Error unzipping {download_zip_path} to {dest_path}. Deleting {dest_path}.')
+            shutil.rmtree(dest_path)
+            return 1
+
+    if not skip_auto_register:
+        if object_type == 'gem':
+            return register.register(gem_path=dest_path)
+
+    return 0
 
 
 def download_engine(engine_name: str,
-                    dest_path: str or pathlib.Path) -> int:
-    return download_o3de_object(engine_name, 'engines', dest_path, 'engine', 'engine_name')
+                    dest_path: str or pathlib.Path,
+                    skip_auto_register: bool) -> int:
+    return download_o3de_object(engine_name, 'engines', dest_path, 'engine', 'engine_name', skip_auto_register)
 
 
 def download_project(project_name: str,
-                     dest_path: str or pathlib.Path) -> int:
-    return download_o3de_object(project_name, 'projects', dest_path, 'project', 'project_name')
+                     dest_path: str or pathlib.Path,
+                     skip_auto_register: bool) -> int:
+    return download_o3de_object(project_name, 'projects', dest_path, 'project', 'project_name', skip_auto_register)
 
 
 def download_gem(gem_name: str,
-                 dest_path: str or pathlib.Path) -> int:
-    return download_o3de_object(gem_name, 'gems', dest_path, 'gem', 'gem_name')
+                 dest_path: str or pathlib.Path,
+                 skip_auto_register: bool) -> int:
+    return download_o3de_object(gem_name, 'gems', dest_path, 'gem', 'gem_name', skip_auto_register)
 
 
 def download_template(template_name: str,
-                      dest_path: str or pathlib.Path) -> int:
-    return download_o3de_object(template_name, 'templates', dest_path, 'template', 'template_name')
+                      dest_path: str or pathlib.Path,
+                      skip_auto_register: bool) -> int:
+    return download_o3de_object(template_name, 'templates', dest_path, 'template', 'template_name', skip_auto_register)
 
 
 
 def download_restricted(restricted_name: str,
-                        dest_path: str or pathlib.Path) -> int:
-    return download_o3de_object(restricted_name, 'restricted', dest_path, 'restricted', 'restricted_name')
+                        dest_path: str or pathlib.Path,
+                        skip_auto_register: bool) -> int:
+    return download_o3de_object(restricted_name, 'restricted', dest_path, 'restricted', 'restricted_name', skip_auto_register)
 
 
 def _run_download(args: argparse) -> int:
@@ -158,16 +182,20 @@ def _run_download(args: argparse) -> int:
 
     if args.engine_name:
         return download_engine(args.engine_name,
-                               args.dest_path)
+                               args.dest_path,
+                               args.skip_auto_register)
     elif args.project_name:
         return download_project(args.project_name,
-                                args.dest_path)
-    elif args.gem_nanme:
+                                args.dest_path,
+                               args.skip_auto_register)
+    elif args.gem_name:
         return download_gem(args.gem_name,
-                            args.dest_path)
+                            args.dest_path,
+                               args.skip_auto_register)
     elif args.template_name:
         return download_template(args.template_name,
-                                 args.dest_path)
+                                 args.dest_path,
+                               args.skip_auto_register)
 
     return 1
 
@@ -188,14 +216,16 @@ def add_parser_args(parser):
     group.add_argument('-t', '--template-name', type=str, required=False,
                        help='Downloadable template name.')
     parser.add_argument('-dp', '--dest-path', type=str, required=False,
-                                    default=None,
-                                    help='Optional destination folder to download into.'
-                                         ' i.e. download --project-name "AstomSamplerViewer" --dest-path "C:/projects"'
-                                         ' will result in C:/projects/AtomSampleViewer'
-                                         ' If blank will download to default object type folder')
-
+                            default=None,
+                            help='Optional destination folder to download into.'
+                                 ' i.e. download --project-name "CustomProject" --dest-path "C:/projects"'
+                                 ' will result in C:/projects/CustomProject'
+                                 ' If blank will download to default object type folder')
+    parser.add_argument('-sar', '--skip-auto-register', action='store_true', required=False,
+                            default=False,
+                            help = 'Skip the automatic registration of new object download')
     parser.add_argument('-ohf', '--override-home-folder', type=str, required=False,
-                                    help='By default the home folder is the user folder, override it to this folder.')
+                            help='By default the home folder is the user folder, override it to this folder.')
 
     parser.set_defaults(func=_run_download)
 

scripts/o3de/o3de/repo.py

@@ -129,49 +129,53 @@ def refresh_repos() -> int:
     return result
 
 
-def search_repo(repo_json_data: dict,
+def search_repo(manifest_json_data: dict,
                 engine_name: str = None,
                 project_name: str = None,
                 gem_name: str = None,
                 template_name: str = None,
                 restricted_name: str = None) -> dict or None:
-
     if isinstance(engine_name, str) or isinstance(engine_name, pathlib.PurePath):
-        o3de_object_uris = repo_json_data['engines']
+        o3de_object_uris = manifest_json_data['engines']
         manifest_json = 'engine.json'
         json_key = 'engine_name'
-        search_func = lambda: None if manifest_json_data.get(json_key, '') == engine_name else manifest_json_data
+        search_func = lambda manifest_json_data: manifest_json_data if manifest_json_data.get(json_key, '') == engine_name else None
     elif isinstance(project_name, str) or isinstance(project_name, pathlib.PurePath):
-        o3de_object_uris = repo_json_data['projects']
+        o3de_object_uris = manifest_json_data['projects']
         manifest_json = 'project.json'
         json_key = 'project_name'
-        search_func = lambda: None if manifest_json_data.get(json_key, '') == project_name else manifest_json_data
+        search_func = lambda manifest_json_data: manifest_json_data if manifest_json_data.get(json_key, '') == project_name else None
     elif isinstance(gem_name, str) or isinstance(gem_name, pathlib.PurePath):
-        o3de_object_uris = repo_json_data['gems']
+        o3de_object_uris = manifest_json_data['gems']
         manifest_json = 'gem.json'
         json_key = 'gem_name'
-        search_func = lambda: None if manifest_json_data.get(json_key, '') == gem_name else manifest_json_data
+        search_func = lambda manifest_json_data: manifest_json_data if manifest_json_data.get(json_key, '') == gem_name else None
     elif isinstance(template_name, str) or isinstance(template_name, pathlib.PurePath):
-        o3de_object_uris = repo_json_data['template']
+        o3de_object_uris = manifest_json_data['template']
         manifest_json = 'template.json'
         json_key = 'template_name'
-        search_func = lambda: None if manifest_json_data.get(json_key, '') == template_name_name else manifest_json_data
+        search_func = lambda manifest_json_data: manifest_json_data if manifest_json_data.get(json_key, '') == template_name_name else None
     elif isinstance(restricted_name, str) or isinstance(restricted_name, pathlib.PurePath):
-        o3de_object_uris = repo_json_data['restricted']
+        o3de_object_uris = manifest_json_data['restricted']
         manifest_json = 'restricted.json'
         json_key = 'restricted_name'
-        search_func = lambda: None if manifest_json_data.get(json_key, '') == restricted_name else manifest_json_data
+        search_func = lambda manifest_json_data: manifest_json_data if manifest_json_data.get(json_key, '') == restricted_name else None
     else:
         return None
-
-    o3de_object =  search_o3de_object(manifest_json, o3de_object_uris, search_func)
+    o3de_object = search_o3de_object(manifest_json, o3de_object_uris, search_func)
     if o3de_object:
+        o3de_object['repo_name'] = manifest_json_data['repo_name']
         return o3de_object
 
     # recurse into the repos object to search for the o3de object
-    o3de_object_uris = repo_json_data['repos']
+    o3de_object_uris = []
+    try:
+        o3de_object_uris = manifest_json_data['repos']
+    except KeyError:
+        pass
+
     manifest_json = 'repo.json'
-    search_func = lambda: search_repo(manifest_json, engine_name, project_name, gem_name, template_name)
+    search_func = lambda manifest_json_data: search_repo(manifest_json_data, engine_name, project_name, gem_name, template_name)
     return search_o3de_object(manifest_json, o3de_object_uris, search_func)
 
 
@@ -179,8 +183,8 @@ def search_o3de_object(manifest_json, o3de_object_uris, search_func):
     # Search for the o3de object based on the supplied object name in the current repo
     cache_folder = manifest.get_o3de_cache_folder()
     for o3de_object_uri in o3de_object_uris:
-        manifest_json_uri = f'{o3de_object_uri}/{manifest_json}'
-        manifest_json_sha256 = hashlib.sha256(manifest_json_uri.encode())
+        parsed_uri = urllib.parse.urlparse(f'{o3de_object_uri}/{manifest_json}')
+        manifest_json_sha256 = hashlib.sha256(parsed_uri.geturl().encode())
         cache_file = cache_folder / str(manifest_json_sha256.hexdigest() + '.json')
         if cache_file.is_file():
             with cache_file.open('r') as f:
@@ -189,7 +193,7 @@ def search_o3de_object(manifest_json, o3de_object_uris, search_func):
                 except json.JSONDecodeError as e:
                     logger.warn(f'{cache_file} failed to load: {str(e)}')
                 else:
-                    result_json_data = search_func()
+                    result_json_data = search_func(manifest_json_data)
                     if result_json_data:
                         return result_json_data
     return None

scripts/o3de/o3de/utils.py

@@ -14,6 +14,7 @@ import pathlib
 import shutil
 import urllib.request
 import logging
+import zipfile
 
 logger = logging.getLogger()
 logging.basicConfig()