Initial commit

scripts/build/tools/alert_build_failures.py

@@ -0,0 +1,76 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import os
+from incremental_build_util import get_iam_role_credentials
+try:
+    import requests
+except ImportError:
+    import pip
+    pip.main(['install', 'requests', '--ignore-installed', '-q'])
+    import requests
+try:
+    from requests_aws4auth import AWS4Auth
+except ImportError:
+    import pip
+    pip.main(['install', 'requests_aws4auth', '--ignore-installed', '-q'])
+    from requests_aws4auth import AWS4Auth
+
+IAM_ROLE_NAME = 'ec2-jenkins-node'
+TEAM = 'lumberyard-build'
+CHIME_ROOM_WEB_HOOK = "https://hooks.chime.aws/incomingwebhooks/2a6018c9-3bf5-4e03-851c-32ba82c7c4e2?token=YWhTVXZsWVJ8MXxaLTg5RkVZNlA5Q1NiMVdfQndGeS1TSHNnYW5VREVha3pjX1pUUEd5b1JF"
+
+
+def find_curr_oncalls_for_team(team):
+    host = "who-is-oncall-pdx.corp.amazon.com"
+    service_name = "who-is-oncall"
+    aws_region = "us-west-2"
+    headers = {"content-type": "application/json", "host": host}
+
+    credentials = get_iam_role_credentials(IAM_ROLE_NAME)
+    try:
+        aws_access_key_id = credentials['AccessKeyId']
+        aws_secret_access_key = credentials['SecretAccessKey']
+        aws_session_token = credentials['Token']
+    except Exception as e:
+        print(f'ERROR: Cannot get AWS credentials.\n{e}')
+        return ['All']
+
+    auth = AWS4Auth(aws_access_key_id, aws_secret_access_key, aws_region, service_name, session_token=aws_session_token)
+    r = requests.get(f"https://who-is-oncall-pdx.corp.amazon.com/teams/{team}", headers=headers, auth=auth, verify=False)
+    if r.ok:
+        res = r.json()
+        try:
+            return res['currOncalls']
+        except KeyError:
+            return ['All']
+    return ['All']
+
+
+def send_alert_to_chime_room(web_hook, content):
+    data = '{"Content":"' + content + '"}'
+    headers = {'Content-Type': 'application/json'}
+    requests.post(web_hook, headers=headers, data=data)
+
+
+def create_content():
+    content = ''
+    oncalls = find_curr_oncalls_for_team(TEAM)
+    for oncall in oncalls:
+        content += f'@{oncall} '
+    job_name = os.environ['JOB_NAME']
+    build_url = os.environ['BUILD_URL']
+    content += fr'\nJob {job_name} failed\nBuild URL: {build_url}\n'
+    return content
+
+
+send_alert_to_chime_room(CHIME_ROOM_WEB_HOOK, create_content())
+

scripts/build/tools/download_from_s3.py

@@ -0,0 +1,101 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+'''
+Usage:
+Use EC2 role to download files to %WORKSPACE% folder from bucket bucket_name:
+python download_from_s3.py --base_dir %WORKSPACE% --files_to_download "file1,file2" --bucket bucket_name
+
+Use profile to download files to %WORKSPACE% folder from bucket bucket_name:
+python download_from_s3.py --base_dir %WORKSPACE% --profile profile --files_to_download "file1,file2" --bucket bucket_name
+'''
+
+
+import os
+import json
+import boto3
+from optparse import OptionParser
+from util import error
+
+
+def parse_args():
+    parser = OptionParser()
+    parser.add_option("--base_dir", dest="base_dir", default=os.getcwd(), help="Base directory to download files, If not given, then current directory is used.")
+    parser.add_option("--files_to_download", dest="files_to_download", default=None, help="Files to download, separated by comma.")
+    parser.add_option("--profile", dest="profile", default=None, help="The name of a profile to use. If not given, then the default profile is used.")
+    parser.add_option("--bucket", dest="bucket", default=None, help="S3 bucket the files are downloaded from.")
+    parser.add_option("--key_prefix", dest="key_prefix", default='', help="Object key prefix.")
+    '''
+    ExtraArgs used to call s3.download_file(), should be in json format. extra_args key must be one of: ACL, CacheControl, ContentDisposition, ContentEncoding, ContentLanguage, ContentType, Expires,
+    GrantFullControl, GrantRead, GrantReadACP, GrantWriteACP, Metadata, RequestPayer, ServerSideEncryption, StorageClass,
+    SSECustomerAlgorithm, SSECustomerKey, SSECustomerKeyMD5, SSEKMSKeyId, WebsiteRedirectLocation
+    '''
+    parser.add_option("--extra_args", dest="extra_args", default=None, help="Additional parameters used to download file.")
+    parser.add_option("--max_retry", dest="max_retry", default=1, help="Maximum retry times to download file.")
+    (options, args) = parser.parse_args()
+    if not os.path.isdir(options.base_dir):
+        error('{} is not a valid directory'.format(options.base_dir))
+    if not options.files_to_download:
+        error('Use --files_to_download to specify files to download, separated by comma.')
+    if not options.bucket:
+        error('Use --bucket to specify bucket that the files are downloaded from.')
+    return options
+
+
+def get_client(service_name, profile_name=None):
+    session = boto3.session.Session(profile_name=profile_name)
+    client = session.client(service_name)
+    return client
+
+
+def s3_download_file(client, base_dir, file, bucket, key_prefix=None, extra_args=None, max_retry=1):
+    print('Downloading file {} from bucket {}.'.format(file, bucket))
+    key = file if key_prefix is None else '{}/{}'.format(key_prefix, file)
+    for x in range(max_retry):
+        try:
+            client.download_file(
+                bucket, key, os.path.join(base_dir, file),
+                ExtraArgs=extra_args
+            )
+            print('Download succeeded')
+            return True
+        except:
+            print('Retrying download...')
+    print('Download failed')
+    return False
+
+
+def download_files(base_dir, files_to_download, bucket, key_prefix=None, profile=None, extra_args=None, max_retry=1):
+    client = get_client('s3', profile)
+    files_to_download = files_to_download.split(',')
+    extra_args = json.loads(extra_args) if extra_args else None
+
+    print('Downloading {} files from bucket {}.'.format(len(files_to_download), bucket))
+    failure = []
+    success = []
+    for file in files_to_download:
+        if not s3_download_file(client, base_dir, file, bucket, key_prefix, extra_args, max_retry):
+            failure.append(file)
+        else:
+            success.append(file)
+    print('{} files are downloaded successfully:'.format(len(success)))
+    print('\n'.join(success))
+    print('{} files failed to download:'.format(len(failure)))
+    print('\n'.join(failure))
+    # Exit with error code 1 if any file is failed to download
+    if len(failure) > 0:
+        return False
+    return True
+
+
+if __name__ == "__main__":
+    options = parse_args()
+    download_files(options.base_dir, options.files_to_download, options.bucket, options.key_prefix, options.profile, options.extra_args, options.max_retry)

scripts/build/tools/email_to_lionbridge.py

@@ -0,0 +1,128 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+"""
+This script will be used in https://jenkins.agscollab.com/view/%7ESandbox/job/PACKAGE_COPY_S3/
+PACKAGE_COPY_S3 is a downstream job of nightly packaging job, it copies the nightly packages from Infra S3 bucket to Lionbridge S3 bucket based on the INCLUDE_FILTER passed from packaging job
+"""
+import os
+import re
+import json
+import requests
+from requests.auth import HTTPBasicAuth
+import boto3
+from util import error, warn
+
+
+# Write EMAIL_TEMPLATE to a file and inject it into the email sent to Lionbridge
+EMAIL_TEMPLATE = '''Packages are uploaded to S3 bucket {}
+Package List:
+{}
+
+
+Changelists:
+{}
+'''
+
+
+def get_jenkins_env(key):
+    try:
+        return os.environ[key]
+    except KeyError:
+        print('Error: Jenkins parameters {} is not set.'.format(key))
+    return None
+
+
+JENKINS_USERNAME = get_jenkins_env('JENKINS_USERNAME')
+JENKINS_API_TOKEN = get_jenkins_env('JENKINS_API_TOKEN')
+JENKINS_URL = get_jenkins_env('JENKINS_URL')
+WORKSPACE = get_jenkins_env('WORKSPACE')
+S3_TARGET = get_jenkins_env('S3_TARGET')
+INCLUDE_FILTER = get_jenkins_env('INCLUDE_FILTER')
+EMAIL_TEMPLATE_FILE = get_jenkins_env('EMAIL_TEMPLATE_FILE')
+if None in [JENKINS_USERNAME, JENKINS_API_TOKEN, JENKINS_URL, WORKSPACE, S3_TARGET, INCLUDE_FILTER, EMAIL_TEMPLATE_FILE]:
+    error('Please make sure all Jenkins parameters are set correctly.')
+
+
+def parse_include_filter(include_filter):
+    try:
+        res = re.search('^(\w*)-*lumberyard-(\d+)\.(\d+)-(\d+)-(\w+).*\*(\d+)\.\*', include_filter)
+        branch = res.group(1)
+        major_version = int(res.group(2))
+        minor_version = int(res.group(3))
+        changelist_number = res.group(4)
+        platform = res.group(5)
+        build_number = res.group(6)
+        return branch, major_version, minor_version, changelist_number, platform, build_number
+    except (AttributeError, IndexError):
+        error('Unable to parse INCLUDE_FILTER, please make sure the INCLUDE_FILTER is set correctly')
+
+
+# Get the changelists that trigger the build
+def get_changelists(job_name, build_number):
+    changelists = []
+    headers = {'Content-type': 'application/json', 'Accept': 'application/json'}
+    try:
+        res = requests.get('{}/job/{}/{}/api/json'.format(JENKINS_URL, job_name, build_number),
+                           auth=HTTPBasicAuth(JENKINS_USERNAME, JENKINS_API_TOKEN), headers=headers, verify=False)
+        res = json.loads(res.content)
+        changelists = res.get('changeSet').get('items')
+        return changelists
+    except:
+        warn('Error: Failed to get changes from build {} in job {}'.format(build_number, job_name))
+        return []
+
+
+def get_packaging_job_name(branch, major_version, minor_version, platform):
+    if branch == '':
+        branch = 'ML' if major_version + minor_version == 0 else 'v{}_{}'.format(major_version, minor_version)
+    job_name = 'PKG_{}_{}'.format(branch, platform.capitalize())
+    return job_name
+
+
+# Get package names by looking up S3 bucket
+def get_package_names(branch, major_version, minor_version, include_filter, build_number):
+    package_names = []
+    prefix = include_filter[:include_filter.find('*')]
+    pattern = '.*{}.*{}..*'.format(prefix, build_number)
+    if branch == '':
+        bucket_name = 'ly-packages-mainline' if major_version + minor_version == 0 else 'ly-packages-release-candidate'
+        folder = 'lumberyard-packages'
+    else:
+        bucket_name = 'ly-packages-feature-branches'
+        folder = 'lumberyard-packages/{}'.format(branch)
+    s3 = boto3.resource('s3')
+    bucket = s3.Bucket(bucket_name)
+    for obj in bucket.objects.filter(Prefix='{}/{}'.format(folder, prefix)):
+        package_name = obj.key
+        if re.match(pattern, package_name):
+            package_names.append(package_name.replace('{}/'.format(folder), ''))
+    return package_names
+
+
+if __name__ == "__main__":
+    branch, major_version, minor_version, changelist_number, platform, build_number = parse_include_filter(INCLUDE_FILTER)
+    packaging_job_name = get_packaging_job_name(branch, major_version, minor_version, platform)
+    changelists = get_changelists(packaging_job_name, build_number)
+    package_names = get_package_names(branch, major_version, minor_version, INCLUDE_FILTER, build_number)
+    with open(os.path.join(WORKSPACE, EMAIL_TEMPLATE_FILE), 'w+') as output:
+        if len(package_names) > 0:
+            package_list_str = '\n'.join(package_names)
+            changelists_str = ''
+            for item in changelists:
+                changelists_str += '---------------------------------------------------------------------------------------------\n'
+                try:
+                    changelists_str += 'CL{} by {} on {}\n{}\n'.format(item['changeNumber'], item['author']['fullName'], item['changeTime'], item['msg'].encode('utf-8', 'ignore'))
+                except KeyError:
+                    error('Internal error, check the output of Jenkins API.')
+            output.write(EMAIL_TEMPLATE.format(S3_TARGET, package_list_str, changelists_str))
+
+

scripts/build/tools/upload_to_s3.py

@@ -0,0 +1,112 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+'''
+Usage:
+Use EC2 role to upload all .zip and .MD5 files in %WORKSPACE% folder to bucket ly-packages-mainline:
+python upload_to_s3.py --base_dir %WORKSPACE% --file_regex "(.*zip$|.*MD5$)" --bucket ly-packages-mainline
+
+Use profile to upload all .zip and .MD5 files in %WORKSPACE% folder to bucket ly-packages-mainline:
+python upload_to_s3.py --base_dir %WORKSPACE% --profile profile --file_regex "(.*zip$|.*MD5$)" --bucket ly-packages-mainline
+
+'''
+
+
+import os
+import re
+import json
+import boto3
+from optparse import OptionParser
+
+
+def parse_args():
+    parser = OptionParser()
+    parser.add_option("--base_dir", dest="base_dir", default=os.getcwd(), help="Base directory to upload files, If not given, then current directory is used.")
+    parser.add_option("--file_regex", dest="file_regex", default=None, help="Regular expression that used to match file names to upload.")
+    parser.add_option("--profile", dest="profile", default=None, help="The name of a profile to use. If not given, then the default profile is used.")
+    parser.add_option("--bucket", dest="bucket", default=None, help="S3 bucket the files are uploaded to.")
+    parser.add_option("--key_prefix", dest="key_prefix", default='', help="Object key prefix.")
+    '''
+    ExtraArgs used to call s3.upload_file(), should be in json format. extra_args key must be one of: ACL, CacheControl, ContentDisposition, ContentEncoding, ContentLanguage, ContentType, Expires,
+    GrantFullControl, GrantRead, GrantReadACP, GrantWriteACP, Metadata, RequestPayer, ServerSideEncryption, StorageClass,
+    SSECustomerAlgorithm, SSECustomerKey, SSECustomerKeyMD5, SSEKMSKeyId, WebsiteRedirectLocation
+    '''
+    parser.add_option("--extra_args", dest="extra_args", default=None, help="Additional parameters used to upload file.")
+    parser.add_option("--max_retry", dest="max_retry", default=1, help="Maximum retry times to upload file.")
+    (options, args) = parser.parse_args()
+    if not os.path.isdir(options.base_dir):
+        error('{} is not a valid directory'.format(options.base_dir))
+    if not options.file_regex:
+        error('Use --file_regex to specify regular expression that used to match file names to upload.')
+    if not options.bucket:
+        error('Use --bucket to specify bucket that the files are uploaded to.')
+    return options
+
+
+def error(message):
+    print(f'Error: {message}')
+    exit(1)
+
+
+def get_client(service_name, profile_name):
+    session = boto3.session.Session(profile_name=profile_name)
+    client = session.client(service_name)
+    return client
+
+
+def get_files_to_upload(base_dir, regex):
+    # Get all file names in base directory
+    files = [x for x in os.listdir(base_dir) if os.path.isfile(os.path.join(base_dir, x))]
+    # Get all file names matching the regular expression, those file will be uploaded to S3
+    files_to_upload = [x for x in files if re.match(regex, x)]
+    return files_to_upload
+
+
+def s3_upload_file(client, base_dir, file, bucket, key_prefix=None, extra_args=None, max_retry=1):
+    print(('Uploading file {} to bucket {}.'.format(file, bucket)))
+    key = file if key_prefix is None else '{}/{}'.format(key_prefix, file)
+    for x in range(max_retry):
+        try:
+            client.upload_file(
+                os.path.join(base_dir, file), bucket, key,
+                ExtraArgs=extra_args
+            )
+            print('Upload succeeded')
+            return True
+        except Exception as err:
+            print(('exception while uploading: {}'.format(err)))
+            print('Retrying upload...')
+    print('Upload failed')
+    return False
+
+
+if __name__ == "__main__":
+    options = parse_args()
+    client = get_client('s3', options.profile)
+    files_to_upload = get_files_to_upload(options.base_dir, options.file_regex)
+    extra_args = json.loads(options.extra_args) if options.extra_args else None
+
+    print(('Uploading {} files to bucket {}.'.format(len(files_to_upload), options.bucket)))
+    failure = []
+    success = []
+    for file in files_to_upload:
+        if not s3_upload_file(client, options.base_dir, file, options.bucket, options.key_prefix, extra_args, 2):
+            failure.append(file)
+        else:
+            success.append(file)
+    print('Upload finished.')
+    print(('{} files are uploaded successfully:'.format(len(success))))
+    print(('\n'.join(success)))
+    if len(failure) > 0:
+        print(('{} files failed to upload:'.format(len(failure))))
+        print(('\n'.join(failure)))
+        # Exit with error code 1 if any file is failed to upload
+        exit(1)