do 1 print() call for a failed s3_upload_file() call and add a short 100 millisecond sleep between file upload retries

This commit is contained in:
jromnoa
2021-06-15 14:34:44 -07:00
parent fffff75479
commit cd69b7c9d6
+8 -2
View File
@@ -26,6 +26,7 @@ python upload_to_s3.py --base_dir %WORKSPACE%/path/to/files --file_regex "(.*png
import os
import re
import json
import time
import boto3
from optparse import OptionParser
@@ -100,13 +101,18 @@ def get_files_to_upload(base_dir, regex, search_subdirectories):
def s3_upload_file(client, file, bucket, key_prefix=None, extra_args=None, max_retry=1):
key = file if key_prefix is None else '{}/{}'.format(key_prefix, file)
key = file if key_prefix is None else f'{key_prefix}/{file}'
error_message = None
for x in range(max_retry):
try:
client.upload_file(file, bucket, key, ExtraArgs=extra_args)
return True
except Exception as err:
print(('Upload failed: Exception while uploading: {}'.format(err)))
time.sleep(0.1) # Sleep for 100 milliseconds between retries.
error_message = err
print(f'Upload failed - Exception while uploading: {error_message}')
return False