From b2bcd6a326d6b5519198d875d38ce5fd92a18a67 Mon Sep 17 00:00:00 2001 From: jromnoa Date: Fri, 11 Jun 2021 18:12:24 -0700 Subject: [PATCH 1/4] adds TEST_SCREENSHOTS boolean toggle to build_config.json & adds subdirectory searching for upload_to_s3.py script --- .../build/Platform/Windows/build_config.json | 3 +- scripts/build/tools/upload_to_s3.py | 67 ++++++++++++++----- 2 files changed, 51 insertions(+), 19 deletions(-) diff --git a/scripts/build/Platform/Windows/build_config.json b/scripts/build/Platform/Windows/build_config.json index 71abf9021f..abf163a2e7 100644 --- a/scripts/build/Platform/Windows/build_config.json +++ b/scripts/build/Platform/Windows/build_config.json @@ -177,7 +177,8 @@ "CMAKE_NATIVE_BUILD_ARGS": "/m /nologo", "CTEST_OPTIONS": "-L \"(SUITE_smoke_REQUIRES_gpu|SUITE_main_REQUIRES_gpu)\" -T Test", "TEST_METRICS": "True", - "TEST_RESULTS": "True" + "TEST_RESULTS": "True", + "TEST_SCREENSHOTS": "True" } }, "asset_profile_vs2019": { diff --git a/scripts/build/tools/upload_to_s3.py b/scripts/build/tools/upload_to_s3.py index 5dfe5eb66e..132929d83e 100755 --- a/scripts/build/tools/upload_to_s3.py +++ b/scripts/build/tools/upload_to_s3.py @@ -17,6 +17,9 @@ python upload_to_s3.py --base_dir %WORKSPACE% --file_regex "(.*zip$|.*MD5$)" --b Use profile to upload all .zip and .MD5 files in %WORKSPACE% folder to bucket ly-packages-mainline: python upload_to_s3.py --base_dir %WORKSPACE% --profile profile --file_regex "(.*zip$|.*MD5$)" --bucket ly-packages-mainline +Another example usage for uploading all .png and .ppm files inside base_dir and only subdirectories within base_dir: +python upload_to_s3.py --base_dir %WORKSPACE%/path/to/files --file_regex "(.*png$|.*ppm$)" --bucket screenshot-test-bucket --search_subdirectories True --key_prefix Test + ''' @@ -34,6 +37,8 @@ def parse_args(): parser.add_option("--profile", dest="profile", default=None, help="The name of a profile to use. If not given, then the default profile is used.") parser.add_option("--bucket", dest="bucket", default=None, help="S3 bucket the files are uploaded to.") parser.add_option("--key_prefix", dest="key_prefix", default='', help="Object key prefix.") + parser.add_option("--search_subdirectories", dest="search_subdirectories", action='store_true', + help="Toggle for searching for files in subdirectories beneath base_dir, defaults to False") ''' ExtraArgs used to call s3.upload_file(), should be in json format. extra_args key must be one of: ACL, CacheControl, ContentDisposition, ContentEncoding, ContentLanguage, ContentType, Expires, GrantFullControl, GrantRead, GrantReadACP, GrantWriteACP, Metadata, RequestPayer, ServerSideEncryption, StorageClass, @@ -62,48 +67,74 @@ def get_client(service_name, profile_name): return client -def get_files_to_upload(base_dir, regex): +def get_files_to_upload(base_dir, regex, search_subdirectories): + """ + Uses a regex expression pattern to return a list of file paths for files to upload to the s3 bucket. + :param base_dir: path for the base directory, if using search_subdirectories=True ensure this is the parent. + :param regex: pattern to use for regex searching, ex. "(.*zip$|.*MD5$)" + :param search_subdirectories: boolean False for only getting files in base_dir, True to get all files in base_dir + and any subdirectory inside base_dir, defaults to False from the parse_args() function. + :return: a list of string file paths for files to upload to the s3 bucket matching the regex expression. + """ # Get all file names in base directory - files = [x for x in os.listdir(base_dir) if os.path.isfile(os.path.join(base_dir, x))] - # strip the surround quotes, if they exist + files = [os.path.join(base_dir, x) for x in os.listdir(base_dir) if os.path.isfile(os.path.join(base_dir, x))] + if search_subdirectories: # Get all file names in base directory and any subdirectories. + for subdirectory in os.walk(base_dir): + # Example output for subdirectory: + # ('C:\path\to\base_dir\', ['Subfolder1', 'Subfolder2'], ['file1', 'file2']) + subdirectory_file_path = subdirectory[0] + subdirectory_files = subdirectory[2] + subdirectory_file_paths = _build_file_paths(subdirectory_file_path, subdirectory_files) + files.extend(subdirectory_file_paths) + try: - regex = json.loads(regex) + regex = json.loads(regex) # strip the surround quotes, if they exist except: pass # Get all file names matching the regular expression, those file will be uploaded to S3 - files_to_upload = [x for x in files if re.match(regex, x)] - return files_to_upload + regex_files_to_upload = [x for x in files if re.match(regex, x)] + + return regex_files_to_upload -def s3_upload_file(client, base_dir, file, bucket, key_prefix=None, extra_args=None, max_retry=1): - print(('Uploading file {} to bucket {}.'.format(file, bucket))) +def s3_upload_file(client, file, bucket, key_prefix=None, extra_args=None, max_retry=1): key = file if key_prefix is None else '{}/{}'.format(key_prefix, file) for x in range(max_retry): try: - client.upload_file( - os.path.join(base_dir, file), bucket, key, - ExtraArgs=extra_args - ) - print('Upload succeeded') + client.upload_file(file, bucket, key, ExtraArgs=extra_args) return True except Exception as err: - print(('exception while uploading: {}'.format(err))) - print('Retrying upload...') - print('Upload failed') + print(('Upload failed: Exception while uploading: {}'.format(err))) return False +def _build_file_paths(path_to_files, files_in_path): + """ + Given a path containing files, returns a list of strings representing complete paths to each file. + :param path_to_files: path to the location storing the files to create string paths for + :param files_in_path: list of files that are inside the path_to_files path string + :return: list of fully parsed file path strings from path_to_files path. + """ + parsed_file_paths = [] + + for file_in_path in files_in_path: + complete_file_path = os.path.join(path_to_files, file_in_path) + parsed_file_paths.append(complete_file_path) + + return parsed_file_paths + + if __name__ == "__main__": options = parse_args() client = get_client('s3', options.profile) - files_to_upload = get_files_to_upload(options.base_dir, options.file_regex) + files_to_upload = get_files_to_upload(options.base_dir, options.file_regex, options.search_subdirectories) extra_args = json.loads(options.extra_args) if options.extra_args else None print(('Uploading {} files to bucket {}.'.format(len(files_to_upload), options.bucket))) failure = [] success = [] for file in files_to_upload: - if not s3_upload_file(client, options.base_dir, file, options.bucket, options.key_prefix, extra_args, 2): + if not s3_upload_file(client, file, options.bucket, options.key_prefix, extra_args, 2): failure.append(file) else: success.append(file) From cac210f25bc7b071903e38a9f15265e4c605c006 Mon Sep 17 00:00:00 2001 From: jromnoa Date: Tue, 15 Jun 2021 11:28:58 -0700 Subject: [PATCH 2/4] add check for files in a given path before creating full paths to the files, add output when json.loads() call fails --- scripts/build/tools/upload_to_s3.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/build/tools/upload_to_s3.py b/scripts/build/tools/upload_to_s3.py index 132929d83e..3694014056 100755 --- a/scripts/build/tools/upload_to_s3.py +++ b/scripts/build/tools/upload_to_s3.py @@ -84,12 +84,14 @@ def get_files_to_upload(base_dir, regex, search_subdirectories): # ('C:\path\to\base_dir\', ['Subfolder1', 'Subfolder2'], ['file1', 'file2']) subdirectory_file_path = subdirectory[0] subdirectory_files = subdirectory[2] - subdirectory_file_paths = _build_file_paths(subdirectory_file_path, subdirectory_files) - files.extend(subdirectory_file_paths) + if subdirectory_files: + subdirectory_file_paths = _build_file_paths(subdirectory_file_path, subdirectory_files) + files.extend(subdirectory_file_paths) try: regex = json.loads(regex) # strip the surround quotes, if they exist except: + print(f'WARNING: failed to call json.loads() for regex: "{regex}"') pass # Get all file names matching the regular expression, those file will be uploaded to S3 regex_files_to_upload = [x for x in files if re.match(regex, x)] From fffff75479b2777f50a7aceeaab3049842875804 Mon Sep 17 00:00:00 2001 From: jromnoa Date: Tue, 15 Jun 2021 13:55:10 -0700 Subject: [PATCH 3/4] add file check for _build_file_paths() function --- scripts/build/tools/upload_to_s3.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/build/tools/upload_to_s3.py b/scripts/build/tools/upload_to_s3.py index 3694014056..80ffcdb8e1 100755 --- a/scripts/build/tools/upload_to_s3.py +++ b/scripts/build/tools/upload_to_s3.py @@ -121,7 +121,8 @@ def _build_file_paths(path_to_files, files_in_path): for file_in_path in files_in_path: complete_file_path = os.path.join(path_to_files, file_in_path) - parsed_file_paths.append(complete_file_path) + if os.path.isfile(complete_file_path): + parsed_file_paths.append(complete_file_path) return parsed_file_paths From cd69b7c9d6c1214d8e4fa4b23dba77971b422eba Mon Sep 17 00:00:00 2001 From: jromnoa Date: Tue, 15 Jun 2021 14:34:44 -0700 Subject: [PATCH 4/4] do 1 print() call for a failed s3_upload_file() call and add a short 100 millisecond sleep between file upload retries --- scripts/build/tools/upload_to_s3.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/build/tools/upload_to_s3.py b/scripts/build/tools/upload_to_s3.py index 80ffcdb8e1..5666e6ec6b 100755 --- a/scripts/build/tools/upload_to_s3.py +++ b/scripts/build/tools/upload_to_s3.py @@ -26,6 +26,7 @@ python upload_to_s3.py --base_dir %WORKSPACE%/path/to/files --file_regex "(.*png import os import re import json +import time import boto3 from optparse import OptionParser @@ -100,13 +101,18 @@ def get_files_to_upload(base_dir, regex, search_subdirectories): def s3_upload_file(client, file, bucket, key_prefix=None, extra_args=None, max_retry=1): - key = file if key_prefix is None else '{}/{}'.format(key_prefix, file) + key = file if key_prefix is None else f'{key_prefix}/{file}' + error_message = None + for x in range(max_retry): try: client.upload_file(file, bucket, key, ExtraArgs=extra_args) return True except Exception as err: - print(('Upload failed: Exception while uploading: {}'.format(err))) + time.sleep(0.1) # Sleep for 100 milliseconds between retries. + error_message = err + + print(f'Upload failed - Exception while uploading: {error_message}') return False