Add ability to overwrite o3de object when downloading, and to check the last_updated field to check for updates.

Signed-off-by: AMZN-Phil <pconroy@amazon.com>

scripts/o3de/o3de/download.py

@@ -20,6 +20,7 @@ import sys
 import urllib.parse
 import urllib.request
 import zipfile
+from datetime import datetime
 
 from o3de import manifest, repo, utils, validation, register
 
@@ -88,10 +89,9 @@ def get_downloadable(engine_name: str = None,
     search_func = lambda manifest_json_data: repo.search_repo(manifest_json_data, engine_name, project_name, gem_name, template_name)
     return repo.search_o3de_object(manifest_json, o3de_object_uris, search_func)
 
-
 def download_o3de_object(object_name: str, default_folder_name: str, dest_path: str or pathlib.Path,
                          object_type: str, downloadable_kwarg_key, skip_auto_register: bool,
-                         download_progress_callback = None) -> int:
+                         force_overwrite: bool, download_progress_callback = None) -> int:
 
     download_path = manifest.get_o3de_cache_folder() / default_folder_name / object_name
     download_path.mkdir(parents=True, exist_ok=True)
@@ -124,9 +124,11 @@ def download_o3de_object(object_name: str, default_folder_name: str, dest_path:
     if not dest_path:
         logger.error(f'Destination path cannot be empty.')
         return 1
-    if dest_path.exists():
+    if dest_path.exists() and not force_overwrite:
         logger.error(f'Destination path {dest_path} already exists.')
         return 1
+    elif dest_path.exists():
+        shutil.rmtree(dest_path)
 
     dest_path.mkdir(exist_ok=True)
 
@@ -149,38 +151,108 @@ def download_o3de_object(object_name: str, default_folder_name: str, dest_path:
 def download_engine(engine_name: str,
                     dest_path: str or pathlib.Path,
                     skip_auto_register: bool,
+                    force_overwrite: bool,
                     download_progress_callback = None) -> int:
-    return download_o3de_object(engine_name, 'engines', dest_path, 'engine', 'engine_name', skip_auto_register, download_progress_callback)
+    return download_o3de_object(engine_name, 'engines', dest_path, 'engine', 'engine_name', skip_auto_register, force_overwrite, download_progress_callback)
 
 
 def download_project(project_name: str,
                      dest_path: str or pathlib.Path,
                      skip_auto_register: bool,
+                     force_overwrite: bool,
                      download_progress_callback = None) -> int:
-    return download_o3de_object(project_name, 'projects', dest_path, 'project', 'project_name', skip_auto_register, download_progress_callback)
+    return download_o3de_object(project_name, 'projects', dest_path, 'project', 'project_name', skip_auto_register, force_overwrite, download_progress_callback)
 
 
 def download_gem(gem_name: str,
                  dest_path: str or pathlib.Path,
                  skip_auto_register: bool,
+                 force_overwrite: bool,
                  download_progress_callback = None) -> int:
-    return download_o3de_object(gem_name, 'gems', dest_path, 'gem', 'gem_name', skip_auto_register, download_progress_callback)
+    return download_o3de_object(gem_name, 'gems', dest_path, 'gem', 'gem_name', skip_auto_register, force_overwrite, download_progress_callback)
 
 
 def download_template(template_name: str,
                       dest_path: str or pathlib.Path,
                       skip_auto_register: bool,
+                      force_overwrite: bool,
                       download_progress_callback = None) -> int:
-    return download_o3de_object(template_name, 'templates', dest_path, 'template', 'template_name', skip_auto_register, download_progress_callback)
+    return download_o3de_object(template_name, 'templates', dest_path, 'template', 'template_name', skip_auto_register, force_overwrite, download_progress_callback)
 
 
 
 def download_restricted(restricted_name: str,
                         dest_path: str or pathlib.Path,
                         skip_auto_register: bool,
+                        force_overwrite: bool,
                         download_progress_callback = None) -> int:
-    return download_o3de_object(restricted_name, 'restricted', dest_path, 'restricted', 'restricted_name', skip_auto_register, download_progress_callback)
+    return download_o3de_object(restricted_name, 'restricted', dest_path, 'restricted', 'restricted_name', skip_auto_register, force_overwrite, download_progress_callback)
 
+def is_o3de_object_update_available(object_name: str, downloadable_kwarg_key, current_date: datetime) -> bool:
+    downloadable_object_data = get_downloadable(**{downloadable_kwarg_key : object_name})
+    if not downloadable_object_data:
+        logger.error(f'Downloadable o3de object {object_name} not found.')
+        return False
+
+    repo_copy_updated_string = 0
+    try:
+        repo_copy_updated_string = downloadable_object_data['last_updated']
+    except KeyError:
+        logger.warn(f'last_updated field not found for {object_name}.')
+        return False
+
+    try:
+        repo_copy_updated_date = datetime.fromisoformat(repo_copy_updated_string)
+    except ValueError:
+        logger.error(f'last_updated field in incorrect format for {object_name}.')
+        return False
+
+    return repo_copy_updated_date > current_date
+
+def is_o3de_engine_update_available(engine_name: str, local_last_updated: str):
+    try:
+        current_updated_time = datetime.fromisoformat(local_last_updated)
+    except ValueError:
+        logger.warn(f'last_updated field has incorrect format for engine {engine_name}.')
+        # Possible that an earlier version did not have this field so still want to check against cached downloadable version
+        current_updated_time = datetime.min
+    return is_o3de_object_update_available(engine_name, 'engine_name', current_updated_time)
+
+def is_o3de_project_update_available(project_name: str, local_last_updated: str):
+    try:
+        current_updated_time = datetime.fromisoformat(local_last_updated)
+    except ValueError:
+        logger.warn(f'last_updated field has incorrect format for project {project_name}.')
+        # Possible that an earlier version did not have this field so still want to check against cached downloadable version
+        current_updated_time = datetime.min
+    return is_o3de_object_update_available(project_name, 'project_name', current_updated_time)
+
+def is_o3de_gem_update_available(gem_name: str, local_last_updated: str):
+    try:
+        current_updated_time = datetime.fromisoformat(local_last_updated)
+    except ValueError:
+        logger.warn(f'last_updated field has incorrect format for gem {gem_name}.')
+        # Possible that an earlier version did not have this field so still want to check against cached downloadable version
+        current_updated_time = datetime.min
+    return is_o3de_object_update_available(gem_name, 'gem_name', current_updated_time)
+
+def is_o3de_template_update_available(template_name: str, local_last_updated: str):
+    try:
+        current_updated_time = datetime.fromisoformat(local_last_updated)
+    except ValueError:
+        logger.warn(f'last_updated field has incorrect format for template {template_name}.')
+        # Possible that an earlier version did not have this field so still want to check against cached downloadable version
+        current_updated_time = datetime.min
+    return is_o3de_object_update_available(template_name, 'template_name', current_updated_time)
+
+def is_o3de_restricted_update_available(restricted_name: str, local_last_updated: str):
+    try:
+        current_updated_time = datetime.fromisoformat(local_last_updated)
+    except ValueError:
+        logger.warn(f'last_updated field has incorrect format for restricted {restricted_name}.')
+        # Possible that an earlier version did not have this field so still want to check against cached downloadable version
+        current_updated_time = datetime.min
+    return is_o3de_object_update_available(restricted_name, 'restricted_name', current_updated_time)
 
 def _run_download(args: argparse) -> int:
     if args.override_home_folder:
@@ -189,19 +261,23 @@ def _run_download(args: argparse) -> int:
     if args.engine_name:
         return download_engine(args.engine_name,
                                args.dest_path,
-                               args.skip_auto_register)
+                               args.skip_auto_register,
+                               args.force_overwrite)
     elif args.project_name:
         return download_project(args.project_name,
                                 args.dest_path,
-                               args.skip_auto_register)
+                                args.skip_auto_register,
+                                args.force_overwrite)
     elif args.gem_name:
         return download_gem(args.gem_name,
                             args.dest_path,
-                               args.skip_auto_register)
+                            args.skip_auto_register,
+                            args.force_overwrite)
     elif args.template_name:
         return download_template(args.template_name,
                                  args.dest_path,
-                               args.skip_auto_register)
+                                 args.skip_auto_register,
+                                 args.force_overwrite)
 
     return 1
 
@@ -230,6 +306,9 @@ def add_parser_args(parser):
     parser.add_argument('-sar', '--skip-auto-register', action='store_true', required=False,
                             default=False,
                             help = 'Skip the automatic registration of new object download')
+    parser.add_argument('-fce', '--force-overwrite', action='store_true', required=False,
+                            default=False,
+                            help = 'Force overwrite the current object')
     parser.add_argument('-ohf', '--override-home-folder', type=str, required=False,
                             help='By default the home folder is the user folder, override it to this folder.')
 

scripts/o3de/o3de/utils.py

@@ -117,15 +117,18 @@ def backup_folder(folder: str or pathlib.Path) -> None:
             if backup_folder_name.is_dir():
                 renamed = True
 
-def download_file(parsed_uri, download_path: pathlib.Path, download_progress_callback = None) -> int:
+def download_file(parsed_uri, download_path: pathlib.Path, force_overwrite, download_progress_callback = None) -> int:
     """
     :param parsed_uri: uniform resource identifier to zip file to download
     :param download_path: location path on disk to download file
     :download_progress_callback: callback called with the download progress as a percentage, returns true to request to cancel the download
     """
-    if download_path.is_file():
+    if download_path.is_file() and not force_overwrite:
         logger.warn(f'File already downloaded to {download_path}.')
-    elif parsed_uri.scheme in ['http', 'https', 'ftp', 'ftps']:
+    elif download_path.is_file():
+        shutil.rmtree(download_path)
+
+    if parsed_uri.scheme in ['http', 'https', 'ftp', 'ftps']:
         with urllib.request.urlopen(parsed_uri.geturl()) as s:
             download_file_size = 0
             try: