birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Repos can be refreshed through Project Manager individually or all at once

Browse Source 
Signed-off-by: nggieber <nggieber@amazon.com>
This commit is contained in:
nggieber
2021-10-22 07:38:41 -07:00
parent 74d74050f2
commit b4a85a2ed6
10 changed files with 204 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/o3de/o3de/repo.py
+89 -67
View File
@@ -13,7 +13,7 @@ import shutil
import urllib.parse
import urllib.request
import hashlib
from datetime import datetime
from o3de import manifest, utils, validation
logger = logging.getLogger()
@@ -27,6 +27,7 @@ def process_add_o3de_repo(file_name: str or pathlib.Path,
return 1
cache_folder = manifest.get_o3de_cache_folder()
repo_data = {}
with file_name.open('r') as f:
try:
repo_data = json.load(f)
@@ -34,61 +35,72 @@ def process_add_o3de_repo(file_name: str or pathlib.Path,
logger.error(f'{file_name} failed to load: {str(e)}')
return 1
# A repo may not contain all types of object.
manifest_download_list = []
with file_name.open('w') as f:
try:
manifest_download_list.append((repo_data['engines'], 'engine.json'))
except KeyError:
pass
try:
manifest_download_list.append((repo_data['projects'], 'project.json'))
except KeyError:
pass
try:
manifest_download_list.append((repo_data['gems'], 'gem.json'))
except KeyError:
pass
try:
manifest_download_list.append((repo_data['templates'], 'template.json'))
except KeyError:
pass
try:
manifest_download_list.append((repo_data['restricted'], 'restricted.json'))
except KeyError:
pass
time_now = datetime.now()
# Convert to lower case because AM/PM is capitalized
time_str = time_now.strftime('%d/%m/%Y %I:%M%p').lower()
repo_data.update({'last_updated': time_str})
f.write(json.dumps(repo_data, indent=4) + '\n')
except Exception as e:
logger.error(f'{file_name} failed to save: {str(e)}')
return 1
for o3de_object_uris, manifest_json in manifest_download_list:
# A repo may not contain all types of object.
manifest_download_list = []
try:
manifest_download_list.append((repo_data['engines'], 'engine.json'))
except KeyError:
pass
try:
manifest_download_list.append((repo_data['projects'], 'project.json'))
except KeyError:
pass
try:
manifest_download_list.append((repo_data['gems'], 'gem.json'))
except KeyError:
pass
try:
manifest_download_list.append((repo_data['templates'], 'template.json'))
except KeyError:
pass
try:
manifest_download_list.append((repo_data['restricted'], 'restricted.json'))
except KeyError:
pass
for o3de_object_uris, manifest_json in manifest_download_list:
for o3de_object_uri in o3de_object_uris:
manifest_json_uri = f'{o3de_object_uri}/{manifest_json}'
manifest_json_sha256 = hashlib.sha256(manifest_json_uri.encode())
cache_file = cache_folder / str(manifest_json_sha256.hexdigest() + '.json')
if not cache_file.is_file():
parsed_uri = urllib.parse.urlparse(manifest_json_uri)
download_file_result = utils.download_file(parsed_uri, cache_file)
if download_file_result != 0:
return download_file_result
# Having a repo is also optional
repo_list = []
try:
repo_list.append(repo_data['repos'])
except KeyError:
pass
for repo in repo_list:
if repo not in repo_set:
repo_set.add(repo)
for o3de_object_uri in o3de_object_uris:
manifest_json_uri = f'{o3de_object_uri}/{manifest_json}'
manifest_json_sha256 = hashlib.sha256(manifest_json_uri.encode())
parsed_uri = urllib.parse.urlparse(f'{repo}/repo.json')
manifest_json_sha256 = hashlib.sha256(parsed_uri.geturl().encode())
cache_file = cache_folder / str(manifest_json_sha256.hexdigest() + '.json')
if not cache_file.is_file():
parsed_uri = urllib.parse.urlparse(manifest_json_uri)
download_file_result = utils.download_file(parsed_uri, cache_file)
if download_file_result != 0:
return download_file_result
if cache_file.is_file():
cache_file.unlink()
download_file_result = utils.download_file(parsed_uri, cache_file)
if download_file_result != 0:
return download_file_result
# Having a repo is also optional
repo_list = []
try:
repo_list.append(repo_data['repos'])
except KeyError:
pass
for repo in repo_list:
if repo not in repo_set:
repo_set.add(repo)
for o3de_object_uri in o3de_object_uris:
parsed_uri = urllib.parse.urlparse(f'{repo}/repo.json')
manifest_json_sha256 = hashlib.sha256(parsed_uri.geturl().encode())
cache_file = cache_folder / str(manifest_json_sha256.hexdigest() + '.json')
if cache_file.is_file():
cache_file.unlink()
download_file_result = utils.download_file(parsed_uri, cache_file)
if download_file_result != 0:
return download_file_result
return process_add_o3de_repo(parsed_uri.geturl(), repo_set)
return process_add_o3de_repo(parsed_uri.geturl(), repo_set)
return 0
@@ -130,6 +142,29 @@ def get_gem_json_paths_from_cached_repo(repo_uri: str) -> list:
return gem_list
def refresh_repo(repo_uri: str,
cache_folder: str = None,
repo_set: set = None) -> int:
if not cache_folder:
cache_folder = manifest.get_o3de_cache_folder()
if not repo_set:
repo_set = set()
parsed_uri = urllib.parse.urlparse(f'{repo_uri}/repo.json')
repo_sha256 = hashlib.sha256(parsed_uri.geturl().encode())
cache_file = cache_folder / str(repo_sha256.hexdigest() + '.json')
download_file_result = utils.download_file(parsed_uri, cache_file)
if download_file_result != 0:
return download_file_result
if not validation.valid_o3de_repo_json(cache_file):
logger.error(f'Repo json {repo_uri} is not valid.')
cache_file.unlink()
return 1
return process_add_o3de_repo(cache_file, repo_set)
def refresh_repos() -> int:
json_data = manifest.load_o3de_manifest()
@@ -147,22 +182,9 @@ def refresh_repos() -> int:
if repo_uri not in repo_set:
repo_set.add(repo_uri)
parsed_uri = urllib.parse.urlparse(f'{repo_uri}/repo.json')
repo_sha256 = hashlib.sha256(parsed_uri.geturl().encode())
cache_file = cache_folder / str(repo_sha256.hexdigest() + '.json')
if not cache_file.is_file():
download_file_result = utils.download_file(parsed_uri, cache_file)
if download_file_result != 0:
return download_file_result
if not validation.valid_o3de_repo_json(cache_file):
logger.error(f'Repo json {repo_uri} is not valid.')
cache_file.unlink()
return 1
last_failure = process_add_o3de_repo(cache_file, repo_set)
if last_failure:
result = last_failure
last_failure = refresh_repo(repo_uri, cache_folder, repo_set)
if last_failure:
result = last_failure
return result