Merge branch 'development' into Prism/ShowRepoGems

scripts/o3de/o3de/download.py

@@ -90,7 +90,8 @@ def get_downloadable(engine_name: str = None,
 
 
 def download_o3de_object(object_name: str, default_folder_name: str, dest_path: str or pathlib.Path,
-                         object_type: str, downloadable_kwarg_key, skip_auto_register: bool) -> int:
+                         object_type: str, downloadable_kwarg_key, skip_auto_register: bool,
+                         download_progress_callback = None) -> int:
 
     download_path = manifest.get_o3de_cache_folder() / default_folder_name / object_name
     download_path.mkdir(parents=True, exist_ok=True)
@@ -104,7 +105,7 @@ def download_o3de_object(object_name: str, default_folder_name: str, dest_path:
     origin_uri = downloadable_object_data['originuri']
     parsed_uri = urllib.parse.urlparse(origin_uri)
 
-    download_zip_result = utils.download_zip_file(parsed_uri, download_zip_path)
+    download_zip_result = utils.download_zip_file(parsed_uri, download_zip_path, download_progress_callback)
     if download_zip_result != 0:
         return download_zip_result
 
@@ -147,33 +148,38 @@ def download_o3de_object(object_name: str, default_folder_name: str, dest_path:
 
 def download_engine(engine_name: str,
                     dest_path: str or pathlib.Path,
-                    skip_auto_register: bool) -> int:
-    return download_o3de_object(engine_name, 'engines', dest_path, 'engine', 'engine_name', skip_auto_register)
+                    skip_auto_register: bool,
+                    download_progress_callback = None) -> int:
+    return download_o3de_object(engine_name, 'engines', dest_path, 'engine', 'engine_name', skip_auto_register, download_progress_callback)
 
 
 def download_project(project_name: str,
                      dest_path: str or pathlib.Path,
-                     skip_auto_register: bool) -> int:
-    return download_o3de_object(project_name, 'projects', dest_path, 'project', 'project_name', skip_auto_register)
+                     skip_auto_register: bool,
+                     download_progress_callback = None) -> int:
+    return download_o3de_object(project_name, 'projects', dest_path, 'project', 'project_name', skip_auto_register, download_progress_callback)
 
 
 def download_gem(gem_name: str,
                  dest_path: str or pathlib.Path,
-                 skip_auto_register: bool) -> int:
-    return download_o3de_object(gem_name, 'gems', dest_path, 'gem', 'gem_name', skip_auto_register)
+                 skip_auto_register: bool,
+                 download_progress_callback = None) -> int:
+    return download_o3de_object(gem_name, 'gems', dest_path, 'gem', 'gem_name', skip_auto_register, download_progress_callback)
 
 
 def download_template(template_name: str,
                       dest_path: str or pathlib.Path,
-                      skip_auto_register: bool) -> int:
-    return download_o3de_object(template_name, 'templates', dest_path, 'template', 'template_name', skip_auto_register)
+                      skip_auto_register: bool,
+                      download_progress_callback = None) -> int:
+    return download_o3de_object(template_name, 'templates', dest_path, 'template', 'template_name', skip_auto_register, download_progress_callback)
 
 
 
 def download_restricted(restricted_name: str,
                         dest_path: str or pathlib.Path,
-                        skip_auto_register: bool) -> int:
-    return download_o3de_object(restricted_name, 'restricted', dest_path, 'restricted', 'restricted_name', skip_auto_register)
+                        skip_auto_register: bool,
+                        download_progress_callback = None) -> int:
+    return download_o3de_object(restricted_name, 'restricted', dest_path, 'restricted', 'restricted_name', skip_auto_register, download_progress_callback)
 
 
 def _run_download(args: argparse) -> int:

scripts/o3de/o3de/utils.py

@@ -10,6 +10,7 @@ This file contains utility functions
 """
 import sys
 import uuid
+import os
 import pathlib
 import shutil
 import urllib.request
@@ -19,6 +20,29 @@ import zipfile
 logger = logging.getLogger()
 logging.basicConfig()
 
+COPY_BUFSIZE = 64 * 1024
+
+def copyfileobj(fsrc, fdst, callback, length=0):
+    # This is functionally the same as the python shutil copyfileobj but
+    # allows for a callback to return the download progress in blocks and allows
+    # to early out to cancel the copy.
+    if not length:
+        length = COPY_BUFSIZE
+
+    fsrc_read = fsrc.read
+    fdst_write = fdst.write
+
+    copied = 0
+    while True:
+        buf = fsrc_read(length)
+        if not buf:
+            break
+        fdst_write(buf)
+        copied += len(buf)
+        if callback(copied):
+            return 1
+    return 0
+
 def validate_identifier(identifier: str) -> bool:
     """
     Determine if the identifier supplied is valid.
@@ -93,18 +117,29 @@ def backup_folder(folder: str or pathlib.Path) -> None:
             if backup_folder_name.is_dir():
                 renamed = True
 
-
-def download_file(parsed_uri, download_path: pathlib.Path) -> int:
+def download_file(parsed_uri, download_path: pathlib.Path, download_progress_callback = None) -> int:
     """
     :param parsed_uri: uniform resource identifier to zip file to download
     :param download_path: location path on disk to download file
+    :download_progress_callback: callback called with the download progress as a percentage, returns true to request to cancel the download
     """
     if download_path.is_file():
         logger.warn(f'File already downloaded to {download_path}.')
     elif parsed_uri.scheme in ['http', 'https', 'ftp', 'ftps']:
         with urllib.request.urlopen(parsed_uri.geturl()) as s:
+            download_file_size = 0
+            try:
+                download_file_size = s.headers['content-length']
+            except KeyError:
+                pass
+            def download_progress(blocks):
+                if download_progress_callback and download_file_size:
+                    return download_progress_callback(int(blocks/int(download_file_size) * 100))
+                return False
             with download_path.open('wb') as f:
-                shutil.copyfileobj(s, f)
+                download_cancelled = copyfileobj(s, f, download_progress)
+                if download_cancelled:
+                    return 1
     else:
         origin_file = pathlib.Path(parsed_uri.geturl()).resolve()
         if not origin_file.is_file():
@@ -114,12 +149,12 @@ def download_file(parsed_uri, download_path: pathlib.Path) -> int:
     return 0
 
 
-def download_zip_file(parsed_uri, download_zip_path: pathlib.Path) -> int:
+def download_zip_file(parsed_uri, download_zip_path: pathlib.Path, download_progress_callback = None) -> int:
     """
     :param parsed_uri: uniform resource identifier to zip file to download
     :param download_zip_path: path to output zip file
     """
-    download_file_result = download_file(parsed_uri, download_zip_path)
+    download_file_result = download_file(parsed_uri, download_zip_path, download_progress_callback)
     if download_file_result != 0:
         return download_file_result