do 1 print() call for a failed s3_upload_file() call and add a short 100 millisecond sleep between file upload retries
This commit is contained in:
@@ -26,6 +26,7 @@ python upload_to_s3.py --base_dir %WORKSPACE%/path/to/files --file_regex "(.*png
|
||||
import os
|
||||
import re
|
||||
import json
|
||||
import time
|
||||
import boto3
|
||||
from optparse import OptionParser
|
||||
|
||||
@@ -100,13 +101,18 @@ def get_files_to_upload(base_dir, regex, search_subdirectories):
|
||||
|
||||
|
||||
def s3_upload_file(client, file, bucket, key_prefix=None, extra_args=None, max_retry=1):
|
||||
key = file if key_prefix is None else '{}/{}'.format(key_prefix, file)
|
||||
key = file if key_prefix is None else f'{key_prefix}/{file}'
|
||||
error_message = None
|
||||
|
||||
for x in range(max_retry):
|
||||
try:
|
||||
client.upload_file(file, bucket, key, ExtraArgs=extra_args)
|
||||
return True
|
||||
except Exception as err:
|
||||
print(('Upload failed: Exception while uploading: {}'.format(err)))
|
||||
time.sleep(0.1) # Sleep for 100 milliseconds between retries.
|
||||
error_message = err
|
||||
|
||||
print(f'Upload failed - Exception while uploading: {error_message}')
|
||||
return False
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user