From 047c5f2231dbba742363a2d9396170c23b572374 Mon Sep 17 00:00:00 2001 From: AMZN-Phil Date: Thu, 4 Nov 2021 15:09:11 -0700 Subject: [PATCH 1/3] Add ability to overwrite o3de object when downloading, and to check the last_updated field to check for updates. Signed-off-by: AMZN-Phil --- scripts/o3de/o3de/download.py | 103 ++++++++++++++++++++++++++++++---- scripts/o3de/o3de/utils.py | 9 ++- 2 files changed, 97 insertions(+), 15 deletions(-) diff --git a/scripts/o3de/o3de/download.py b/scripts/o3de/o3de/download.py index 98f5d051a4..5d15c4d0b3 100644 --- a/scripts/o3de/o3de/download.py +++ b/scripts/o3de/o3de/download.py @@ -20,6 +20,7 @@ import sys import urllib.parse import urllib.request import zipfile +from datetime import datetime from o3de import manifest, repo, utils, validation, register @@ -88,10 +89,9 @@ def get_downloadable(engine_name: str = None, search_func = lambda manifest_json_data: repo.search_repo(manifest_json_data, engine_name, project_name, gem_name, template_name) return repo.search_o3de_object(manifest_json, o3de_object_uris, search_func) - def download_o3de_object(object_name: str, default_folder_name: str, dest_path: str or pathlib.Path, object_type: str, downloadable_kwarg_key, skip_auto_register: bool, - download_progress_callback = None) -> int: + force_overwrite: bool, download_progress_callback = None) -> int: download_path = manifest.get_o3de_cache_folder() / default_folder_name / object_name download_path.mkdir(parents=True, exist_ok=True) @@ -124,9 +124,11 @@ def download_o3de_object(object_name: str, default_folder_name: str, dest_path: if not dest_path: logger.error(f'Destination path cannot be empty.') return 1 - if dest_path.exists(): + if dest_path.exists() and not force_overwrite: logger.error(f'Destination path {dest_path} already exists.') return 1 + elif dest_path.exists(): + shutil.rmtree(dest_path) dest_path.mkdir(exist_ok=True) @@ -149,38 +151,108 @@ def download_o3de_object(object_name: str, default_folder_name: str, dest_path: def download_engine(engine_name: str, dest_path: str or pathlib.Path, skip_auto_register: bool, + force_overwrite: bool, download_progress_callback = None) -> int: - return download_o3de_object(engine_name, 'engines', dest_path, 'engine', 'engine_name', skip_auto_register, download_progress_callback) + return download_o3de_object(engine_name, 'engines', dest_path, 'engine', 'engine_name', skip_auto_register, force_overwrite, download_progress_callback) def download_project(project_name: str, dest_path: str or pathlib.Path, skip_auto_register: bool, + force_overwrite: bool, download_progress_callback = None) -> int: - return download_o3de_object(project_name, 'projects', dest_path, 'project', 'project_name', skip_auto_register, download_progress_callback) + return download_o3de_object(project_name, 'projects', dest_path, 'project', 'project_name', skip_auto_register, force_overwrite, download_progress_callback) def download_gem(gem_name: str, dest_path: str or pathlib.Path, skip_auto_register: bool, + force_overwrite: bool, download_progress_callback = None) -> int: - return download_o3de_object(gem_name, 'gems', dest_path, 'gem', 'gem_name', skip_auto_register, download_progress_callback) + return download_o3de_object(gem_name, 'gems', dest_path, 'gem', 'gem_name', skip_auto_register, force_overwrite, download_progress_callback) def download_template(template_name: str, dest_path: str or pathlib.Path, skip_auto_register: bool, + force_overwrite: bool, download_progress_callback = None) -> int: - return download_o3de_object(template_name, 'templates', dest_path, 'template', 'template_name', skip_auto_register, download_progress_callback) + return download_o3de_object(template_name, 'templates', dest_path, 'template', 'template_name', skip_auto_register, force_overwrite, download_progress_callback) def download_restricted(restricted_name: str, dest_path: str or pathlib.Path, skip_auto_register: bool, + force_overwrite: bool, download_progress_callback = None) -> int: - return download_o3de_object(restricted_name, 'restricted', dest_path, 'restricted', 'restricted_name', skip_auto_register, download_progress_callback) + return download_o3de_object(restricted_name, 'restricted', dest_path, 'restricted', 'restricted_name', skip_auto_register, force_overwrite, download_progress_callback) + +def is_o3de_object_update_available(object_name: str, downloadable_kwarg_key, current_date: datetime) -> bool: + downloadable_object_data = get_downloadable(**{downloadable_kwarg_key : object_name}) + if not downloadable_object_data: + logger.error(f'Downloadable o3de object {object_name} not found.') + return False + + repo_copy_updated_string = 0 + try: + repo_copy_updated_string = downloadable_object_data['last_updated'] + except KeyError: + logger.warn(f'last_updated field not found for {object_name}.') + return False + + try: + repo_copy_updated_date = datetime.fromisoformat(repo_copy_updated_string) + except ValueError: + logger.error(f'last_updated field in incorrect format for {object_name}.') + return False + + return repo_copy_updated_date > current_date +def is_o3de_engine_update_available(engine_name: str, local_last_updated: str): + try: + current_updated_time = datetime.fromisoformat(local_last_updated) + except ValueError: + logger.warn(f'last_updated field has incorrect format for engine {engine_name}.') + # Possible that an earlier version did not have this field so still want to check against cached downloadable version + current_updated_time = datetime.min + return is_o3de_object_update_available(engine_name, 'engine_name', current_updated_time) + +def is_o3de_project_update_available(project_name: str, local_last_updated: str): + try: + current_updated_time = datetime.fromisoformat(local_last_updated) + except ValueError: + logger.warn(f'last_updated field has incorrect format for project {project_name}.') + # Possible that an earlier version did not have this field so still want to check against cached downloadable version + current_updated_time = datetime.min + return is_o3de_object_update_available(project_name, 'project_name', current_updated_time) + +def is_o3de_gem_update_available(gem_name: str, local_last_updated: str): + try: + current_updated_time = datetime.fromisoformat(local_last_updated) + except ValueError: + logger.warn(f'last_updated field has incorrect format for gem {gem_name}.') + # Possible that an earlier version did not have this field so still want to check against cached downloadable version + current_updated_time = datetime.min + return is_o3de_object_update_available(gem_name, 'gem_name', current_updated_time) + +def is_o3de_template_update_available(template_name: str, local_last_updated: str): + try: + current_updated_time = datetime.fromisoformat(local_last_updated) + except ValueError: + logger.warn(f'last_updated field has incorrect format for template {template_name}.') + # Possible that an earlier version did not have this field so still want to check against cached downloadable version + current_updated_time = datetime.min + return is_o3de_object_update_available(template_name, 'template_name', current_updated_time) + +def is_o3de_restricted_update_available(restricted_name: str, local_last_updated: str): + try: + current_updated_time = datetime.fromisoformat(local_last_updated) + except ValueError: + logger.warn(f'last_updated field has incorrect format for restricted {restricted_name}.') + # Possible that an earlier version did not have this field so still want to check against cached downloadable version + current_updated_time = datetime.min + return is_o3de_object_update_available(restricted_name, 'restricted_name', current_updated_time) def _run_download(args: argparse) -> int: if args.override_home_folder: @@ -189,19 +261,23 @@ def _run_download(args: argparse) -> int: if args.engine_name: return download_engine(args.engine_name, args.dest_path, - args.skip_auto_register) + args.skip_auto_register, + args.force_overwrite) elif args.project_name: return download_project(args.project_name, args.dest_path, - args.skip_auto_register) + args.skip_auto_register, + args.force_overwrite) elif args.gem_name: return download_gem(args.gem_name, args.dest_path, - args.skip_auto_register) + args.skip_auto_register, + args.force_overwrite) elif args.template_name: return download_template(args.template_name, args.dest_path, - args.skip_auto_register) + args.skip_auto_register, + args.force_overwrite) return 1 @@ -230,6 +306,9 @@ def add_parser_args(parser): parser.add_argument('-sar', '--skip-auto-register', action='store_true', required=False, default=False, help = 'Skip the automatic registration of new object download') + parser.add_argument('-fce', '--force-overwrite', action='store_true', required=False, + default=False, + help = 'Force overwrite the current object') parser.add_argument('-ohf', '--override-home-folder', type=str, required=False, help='By default the home folder is the user folder, override it to this folder.') diff --git a/scripts/o3de/o3de/utils.py b/scripts/o3de/o3de/utils.py index 8663502a3f..df2ea67666 100755 --- a/scripts/o3de/o3de/utils.py +++ b/scripts/o3de/o3de/utils.py @@ -117,15 +117,18 @@ def backup_folder(folder: str or pathlib.Path) -> None: if backup_folder_name.is_dir(): renamed = True -def download_file(parsed_uri, download_path: pathlib.Path, download_progress_callback = None) -> int: +def download_file(parsed_uri, download_path: pathlib.Path, force_overwrite, download_progress_callback = None) -> int: """ :param parsed_uri: uniform resource identifier to zip file to download :param download_path: location path on disk to download file :download_progress_callback: callback called with the download progress as a percentage, returns true to request to cancel the download """ - if download_path.is_file(): + if download_path.is_file() and not force_overwrite: logger.warn(f'File already downloaded to {download_path}.') - elif parsed_uri.scheme in ['http', 'https', 'ftp', 'ftps']: + elif download_path.is_file(): + shutil.rmtree(download_path) + + if parsed_uri.scheme in ['http', 'https', 'ftp', 'ftps']: with urllib.request.urlopen(parsed_uri.geturl()) as s: download_file_size = 0 try: From 31382d080e94548b5d71702e44de5e9951191dd8 Mon Sep 17 00:00:00 2001 From: AMZN-Phil Date: Thu, 4 Nov 2021 17:07:02 -0700 Subject: [PATCH 2/3] Remove duplication, additional readability and added missed file Signed-off-by: AMZN-Phil --- .../ProjectManager/Source/PythonBindings.cpp | 1 + scripts/o3de/o3de/download.py | 79 +++++++------------ scripts/o3de/o3de/utils.py | 13 ++- 3 files changed, 40 insertions(+), 53 deletions(-) diff --git a/Code/Tools/ProjectManager/Source/PythonBindings.cpp b/Code/Tools/ProjectManager/Source/PythonBindings.cpp index 740393fec0..0be0d02ca0 100644 --- a/Code/Tools/ProjectManager/Source/PythonBindings.cpp +++ b/Code/Tools/ProjectManager/Source/PythonBindings.cpp @@ -1173,6 +1173,7 @@ namespace O3DE::ProjectManager QString_To_Py_String(gemName), // gem name pybind11::none(), // destination path false, // skip auto register + false, // force pybind11::cpp_function( [this, gemProgressCallback](int progress) { diff --git a/scripts/o3de/o3de/download.py b/scripts/o3de/o3de/download.py index 5d15c4d0b3..b6d6233559 100644 --- a/scripts/o3de/o3de/download.py +++ b/scripts/o3de/o3de/download.py @@ -124,11 +124,16 @@ def download_o3de_object(object_name: str, default_folder_name: str, dest_path: if not dest_path: logger.error(f'Destination path cannot be empty.') return 1 - if dest_path.exists() and not force_overwrite: - logger.error(f'Destination path {dest_path} already exists.') - return 1 - elif dest_path.exists(): - shutil.rmtree(dest_path) + if dest_path.exists(): + if not force_overwrite: + logger.error(f'Destination path {dest_path} already exists.') + return 1 + else: + try: + shutil.rmtree(dest_path) + except OSError: + logger.error(f'Could not remove existing destination path {dest_path}.') + return 1 dest_path.mkdir(exist_ok=True) @@ -188,71 +193,47 @@ def download_restricted(restricted_name: str, download_progress_callback = None) -> int: return download_o3de_object(restricted_name, 'restricted', dest_path, 'restricted', 'restricted_name', skip_auto_register, force_overwrite, download_progress_callback) -def is_o3de_object_update_available(object_name: str, downloadable_kwarg_key, current_date: datetime) -> bool: +def is_o3de_object_update_available(object_name: str, downloadable_kwarg_key, local_last_updated: str) -> bool: downloadable_object_data = get_downloadable(**{downloadable_kwarg_key : object_name}) if not downloadable_object_data: logger.error(f'Downloadable o3de object {object_name} not found.') return False - repo_copy_updated_string = 0 try: repo_copy_updated_string = downloadable_object_data['last_updated'] except KeyError: logger.warn(f'last_updated field not found for {object_name}.') return False + try: + local_last_updated_time = datetime.fromisoformat(local_last_updated) + except ValueError: + logger.warn(f'last_updated field has incorrect format for local copy of {downloadable_kwarg_key} {object_name}.') + # Possible that an earlier version did not have this field so still want to check against cached downloadable version + local_last_updated_time = datetime.min + try: repo_copy_updated_date = datetime.fromisoformat(repo_copy_updated_string) except ValueError: - logger.error(f'last_updated field in incorrect format for {object_name}.') + logger.error(f'last_updated field in incorrect format for repository copy of {downloadable_kwarg_key} {object_name}.') return False - return repo_copy_updated_date > current_date + return repo_copy_updated_date > local_last_updated_time def is_o3de_engine_update_available(engine_name: str, local_last_updated: str): - try: - current_updated_time = datetime.fromisoformat(local_last_updated) - except ValueError: - logger.warn(f'last_updated field has incorrect format for engine {engine_name}.') - # Possible that an earlier version did not have this field so still want to check against cached downloadable version - current_updated_time = datetime.min - return is_o3de_object_update_available(engine_name, 'engine_name', current_updated_time) + return is_o3de_object_update_available(engine_name, 'engine_name', local_last_updated) def is_o3de_project_update_available(project_name: str, local_last_updated: str): - try: - current_updated_time = datetime.fromisoformat(local_last_updated) - except ValueError: - logger.warn(f'last_updated field has incorrect format for project {project_name}.') - # Possible that an earlier version did not have this field so still want to check against cached downloadable version - current_updated_time = datetime.min - return is_o3de_object_update_available(project_name, 'project_name', current_updated_time) + return is_o3de_object_update_available(project_name, 'project_name', local_last_updated) def is_o3de_gem_update_available(gem_name: str, local_last_updated: str): - try: - current_updated_time = datetime.fromisoformat(local_last_updated) - except ValueError: - logger.warn(f'last_updated field has incorrect format for gem {gem_name}.') - # Possible that an earlier version did not have this field so still want to check against cached downloadable version - current_updated_time = datetime.min - return is_o3de_object_update_available(gem_name, 'gem_name', current_updated_time) + return is_o3de_object_update_available(gem_name, 'gem_name', local_last_updated) def is_o3de_template_update_available(template_name: str, local_last_updated: str): - try: - current_updated_time = datetime.fromisoformat(local_last_updated) - except ValueError: - logger.warn(f'last_updated field has incorrect format for template {template_name}.') - # Possible that an earlier version did not have this field so still want to check against cached downloadable version - current_updated_time = datetime.min - return is_o3de_object_update_available(template_name, 'template_name', current_updated_time) + return is_o3de_object_update_available(template_name, 'template_name', local_last_updated) def is_o3de_restricted_update_available(restricted_name: str, local_last_updated: str): - try: - current_updated_time = datetime.fromisoformat(local_last_updated) - except ValueError: - logger.warn(f'last_updated field has incorrect format for restricted {restricted_name}.') - # Possible that an earlier version did not have this field so still want to check against cached downloadable version - current_updated_time = datetime.min - return is_o3de_object_update_available(restricted_name, 'restricted_name', current_updated_time) + return is_o3de_object_update_available(restricted_name, 'restricted_name', local_last_updated) def _run_download(args: argparse) -> int: if args.override_home_folder: @@ -262,22 +243,22 @@ def _run_download(args: argparse) -> int: return download_engine(args.engine_name, args.dest_path, args.skip_auto_register, - args.force_overwrite) + args.force) elif args.project_name: return download_project(args.project_name, args.dest_path, args.skip_auto_register, - args.force_overwrite) + args.force) elif args.gem_name: return download_gem(args.gem_name, args.dest_path, args.skip_auto_register, - args.force_overwrite) + args.force) elif args.template_name: return download_template(args.template_name, args.dest_path, args.skip_auto_register, - args.force_overwrite) + args.force) return 1 @@ -306,7 +287,7 @@ def add_parser_args(parser): parser.add_argument('-sar', '--skip-auto-register', action='store_true', required=False, default=False, help = 'Skip the automatic registration of new object download') - parser.add_argument('-fce', '--force-overwrite', action='store_true', required=False, + parser.add_argument('-f', '--force', action='store_true', required=False, default=False, help = 'Force overwrite the current object') parser.add_argument('-ohf', '--override-home-folder', type=str, required=False, diff --git a/scripts/o3de/o3de/utils.py b/scripts/o3de/o3de/utils.py index df2ea67666..56f9ae4bcd 100755 --- a/scripts/o3de/o3de/utils.py +++ b/scripts/o3de/o3de/utils.py @@ -123,10 +123,15 @@ def download_file(parsed_uri, download_path: pathlib.Path, force_overwrite, down :param download_path: location path on disk to download file :download_progress_callback: callback called with the download progress as a percentage, returns true to request to cancel the download """ - if download_path.is_file() and not force_overwrite: - logger.warn(f'File already downloaded to {download_path}.') - elif download_path.is_file(): - shutil.rmtree(download_path) + if download_path.is_file(): + if not force_overwrite: + logger.warn(f'File already downloaded to {download_path}.') + else: + try: + shutil.rmtree(download_path) + except OSError: + logger.error(f'Could not remove existing download path {download_path}.') + return 1 if parsed_uri.scheme in ['http', 'https', 'ftp', 'ftps']: with urllib.request.urlopen(parsed_uri.geturl()) as s: From 81515b922f34cc5841a378fea3f47a8b50591e14 Mon Sep 17 00:00:00 2001 From: AMZN-Phil Date: Thu, 4 Nov 2021 17:10:44 -0700 Subject: [PATCH 3/3] Additional readability Signed-off-by: AMZN-Phil --- scripts/o3de/o3de/download.py | 45 +++++++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/scripts/o3de/o3de/download.py b/scripts/o3de/o3de/download.py index b6d6233559..9e576d64b5 100644 --- a/scripts/o3de/o3de/download.py +++ b/scripts/o3de/o3de/download.py @@ -158,7 +158,14 @@ def download_engine(engine_name: str, skip_auto_register: bool, force_overwrite: bool, download_progress_callback = None) -> int: - return download_o3de_object(engine_name, 'engines', dest_path, 'engine', 'engine_name', skip_auto_register, force_overwrite, download_progress_callback) + return download_o3de_object(engine_name, + 'engines', + dest_path, + 'engine', + 'engine_name', + skip_auto_register, + force_overwrite, + download_progress_callback) def download_project(project_name: str, @@ -166,7 +173,14 @@ def download_project(project_name: str, skip_auto_register: bool, force_overwrite: bool, download_progress_callback = None) -> int: - return download_o3de_object(project_name, 'projects', dest_path, 'project', 'project_name', skip_auto_register, force_overwrite, download_progress_callback) + return download_o3de_object(project_name, + 'projects', + dest_path, + 'project', + 'project_name', + skip_auto_register, + force_overwrite, + download_progress_callback) def download_gem(gem_name: str, @@ -174,7 +188,14 @@ def download_gem(gem_name: str, skip_auto_register: bool, force_overwrite: bool, download_progress_callback = None) -> int: - return download_o3de_object(gem_name, 'gems', dest_path, 'gem', 'gem_name', skip_auto_register, force_overwrite, download_progress_callback) + return download_o3de_object(gem_name, + 'gems', + dest_path, + 'gem', + 'gem_name', + skip_auto_register, + force_overwrite, + download_progress_callback) def download_template(template_name: str, @@ -182,7 +203,14 @@ def download_template(template_name: str, skip_auto_register: bool, force_overwrite: bool, download_progress_callback = None) -> int: - return download_o3de_object(template_name, 'templates', dest_path, 'template', 'template_name', skip_auto_register, force_overwrite, download_progress_callback) + return download_o3de_object(template_name, + 'templates', + dest_path, + 'template', + 'template_name', + skip_auto_register, + force_overwrite, + download_progress_callback) @@ -191,7 +219,14 @@ def download_restricted(restricted_name: str, skip_auto_register: bool, force_overwrite: bool, download_progress_callback = None) -> int: - return download_o3de_object(restricted_name, 'restricted', dest_path, 'restricted', 'restricted_name', skip_auto_register, force_overwrite, download_progress_callback) + return download_o3de_object(restricted_name, + 'restricted', + dest_path, + 'restricted', + 'restricted_name', + skip_auto_register, + force_overwrite, + download_progress_callback) def is_o3de_object_update_available(object_name: str, downloadable_kwarg_key, local_last_updated: str) -> bool: downloadable_object_data = get_downloadable(**{downloadable_kwarg_key : object_name})