Initial commit

scripts/build/bootstrap/incremental_build_util.py

@@ -0,0 +1,525 @@
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+
+import argparse
+import ast
+import boto3
+import datetime
+import urllib2
+import os
+import psutil
+import time
+import requests
+import subprocess
+import sys
+import tempfile
+import traceback
+
+IAM_ROLE_NAME = 'ec2-jenkins-node'
+DEFAULT_REGION = 'us-west-2'
+DEFAULT_DISK_SIZE = 300
+DEFAULT_DISK_TYPE = 'gp2'
+DEFAULT_TIMEOUT = 300
+MAX_EBS_MOUNTING_ATTEMPT = 3
+LOW_EBS_DISK_SPACE_LIMIT = 10240
+MAX_EBS_DISK_SIZE = DEFAULT_DISK_SIZE * 2
+
+if os.name == 'nt':
+    MOUNT_PATH = 'D:\\'
+else:
+    MOUNT_PATH = '/data'
+
+if os.name == 'nt':
+    import ctypes
+    import win32api
+    import collections
+    import locale
+
+    locale.setlocale(locale.LC_ALL, '')  # set locale to default to get thousands separators
+
+    PULARGE_INTEGER = ctypes.POINTER(ctypes.c_ulonglong)  # Pointer to large unsigned integer
+    kernel32 = ctypes.WinDLL('kernel32', use_last_error=True)
+    kernel32.GetDiskFreeSpaceExW.argtypes = (ctypes.c_wchar_p,) + (PULARGE_INTEGER,) * 3
+
+    class UsageTuple(collections.namedtuple('UsageTuple', 'total, used, free')):
+        def __str__(self):
+            # Add thousands separator to numbers displayed
+            return self.__class__.__name__ + '(total={:n}, used={:n}, free={:n})'.format(*self)
+
+    def is_dir_symlink(path):
+        FILE_ATTRIBUTE_REPARSE_POINT = 0x0400
+        return os.path.isdir(path) and (ctypes.windll.kernel32.GetFileAttributesW(unicode(path)) & FILE_ATTRIBUTE_REPARSE_POINT)
+
+    def get_free_space_mb(path):
+        if sys.version_info < (3,):  # Python 2?
+            saved_conversion_mode = ctypes.set_conversion_mode('mbcs', 'strict')
+        else:
+            try:
+                path = os.fsdecode(path)  # allows str or bytes (or os.PathLike in Python 3.6+)
+            except AttributeError:  # fsdecode() not added until Python 3.2
+                pass
+
+        # Define variables to receive results when passed as "by reference" arguments
+        _, total, free = ctypes.c_ulonglong(), ctypes.c_ulonglong(), ctypes.c_ulonglong()
+
+        success = kernel32.GetDiskFreeSpaceExW(
+            path, ctypes.byref(_), ctypes.byref(total), ctypes.byref(free))
+        if not success:
+            error_code = ctypes.get_last_error()
+
+        if sys.version_info < (3,):  # Python 2?
+            ctypes.set_conversion_mode(*saved_conversion_mode)  # restore conversion mode
+
+        if not success:
+            windows_error_message = ctypes.FormatError(error_code)
+            raise ctypes.WinError(error_code, '{} {!r}'.format(windows_error_message, path))
+
+        used = total.value - free.value
+
+        return free.value / 1024 / 1024#for now
+else:
+    def get_free_space_mb(dirname):
+        st = os.statvfs(dirname)
+        return st.f_bavail * st.f_frsize / 1024 / 1024
+
+def error(message):
+    print message
+    exit(1)
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-a', '--action', dest="action", help="Action (mount|unmount|delete)")
+    parser.add_argument('-pipe', '--pipeline', dest="pipeline", help="Pipeline")
+    parser.add_argument('-b', '--branch', dest="branch", help="Branch")
+    parser.add_argument('-plat', '--platform', dest="platform", help="Platform")
+    parser.add_argument('-c', '--build_type', dest="build_type", help="Build type")
+    parser.add_argument('-ds', '--disk_size', dest="disk_size", help="Disk size in Gigabytes (defaults to {})".format(DEFAULT_DISK_SIZE), default=DEFAULT_DISK_SIZE)
+    parser.add_argument('-dt', '--disk_type', dest="disk_type", help="Disk type (defaults to {})".format(DEFAULT_DISK_TYPE), default=DEFAULT_DISK_TYPE)
+    args = parser.parse_args()
+
+    # Input validation
+    if args.action is None:
+        error('No action specified')
+    args.action = args.action.lower()
+    if args.action != 'unmount':
+        if args.pipeline is None:
+            error('No pipeline specified')
+        if args.branch is None:
+            error('No branch specified')
+        if args.platform is None:
+            error('No platform specified')
+        if args.build_type is None:
+            error('No build_type specified')
+    
+    return args
+
+def get_mount_name(pipeline, branch, platform, build_type):
+    mount_name = "{}_{}_{}_{}".format(pipeline, branch, platform, build_type)
+    mount_name = mount_name.replace('/','_').replace('\\','_')
+    return mount_name
+
+def get_pipeline_and_branch(pipeline, branch):
+    pipeline_and_branch = "{}_{}".format(pipeline, branch)
+    pipeline_and_branch = pipeline_and_branch.replace('/','_').replace('\\','_')
+    return pipeline_and_branch
+
+def get_iam_role_credentials(role_name):
+    security_metadata = None
+    try:
+        response = urllib2.urlopen(
+            'http://169.254.169.254/latest/meta-data/iam/security-credentials/{0}'.format(role_name)).read()
+        security_metadata = ast.literal_eval(response)
+    except:
+        print 'Unable to get iam role credentials'
+        print traceback.print_exc()
+
+    return security_metadata
+
+
+def get_ec2_client(region):
+    credentials = None
+    try:
+        response = urllib2.urlopen(
+            'http://169.254.169.254/latest/meta-data/iam/security-credentials/{0}'.format(IAM_ROLE_NAME)).read()
+        credentials = ast.literal_eval(response)
+    except Exception as e:
+        print e.message
+        error('Error: Unable to get IAM rols credentials, please contact ly-build@ for help.')
+
+    keys = ['AccessKeyId', 'SecretAccessKey', 'Token']
+    for key in keys:
+        if key not in credentials:
+            error('Error: Unable to find {0} in get_iam_role_credentials response {1}'.format(key, credentials))
+
+    aws_access_key_id = credentials['AccessKeyId']
+    aws_secret_access_key = credentials['SecretAccessKey']
+    aws_session_token = credentials['Token']
+
+    client = boto3.client('ec2', region_name=region, aws_access_key_id=aws_access_key_id,
+                          aws_secret_access_key=aws_secret_access_key,
+                          aws_session_token=aws_session_token)
+    return client
+
+
+def get_ec2_instance_id():
+    try:
+        instance_id = urllib2.urlopen('http://169.254.169.254/latest/meta-data/instance-id').read()
+        return instance_id
+    except Exception as e:
+        print e.message
+        error('No EC2 metadata! Check if you are running this script on an EC2 instance.')
+
+
+def get_availability_zone():
+    try:
+        availability_zone = urllib2.urlopen('http://169.254.169.254/latest/meta-data/placement/availability-zone').read()
+        return availability_zone
+    except Exception as e:
+        print e.message
+        error('No EC2 metadata! Check if you are running this script on an EC2 instance.')
+
+
+def kill_processes(workspace='/dev/'):
+    '''
+    Kills all processes that have open file paths associated with the workspace.
+    Uses PSUtil for cross-platform compatibility
+    '''
+    print 'Checking for any stuck processes...'
+    for proc in psutil.process_iter():
+        try:
+            if workspace in str(proc.open_files()):
+                print "{} has open files in {}. Terminating".format(proc.name(), proc.open_files())
+                proc.kill()
+                time.sleep(1) # Just to make sure a parent process has time to close
+        except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
+            continue
+
+
+def delete_volume(ec2_client, volume_id):
+    response = ec2_client.delete_volume(VolumeId=volume_id)
+    print 'Volume {} deleted'.format(volume_id)
+
+def find_snapshot_id(ec2_client, pipeline, platform, build_type, disk_size):
+    mount_name = get_mount_name(pipeline, 'main', platform, build_type) # we take snapshots out of main
+    response = ec2_client.describe_snapshots(Filters= [{
+        'Name': 'tag:Name', 'Values': [mount_name]
+    }])
+
+    snapshot_id = None
+    if 'Snapshots' in response and len(response['Snapshots']) > 0:
+        snapshot_start_time_max = None # find the latest snapshot
+        for snapshot in response['Snapshots']:
+            if snapshot['State'] == 'completed' and snapshot['VolumeSize'] == disk_size:
+                snapshot_start_time = snapshot['StartTime']
+                if not snapshot_start_time_max or snapshot_start_time > snapshot_start_time_max:
+                    snapshot_start_time_max = snapshot_start_time
+                    snapshot_id = snapshot['SnapshotId']
+    return snapshot_id
+
+def create_volume(ec2_client, availability_zone, pipeline, branch, platform, build_type, disk_size, disk_type):   
+    # The actual EBS default calculation for IOps is a floating point number, the closest approxmiation is 4x of the disk size for simplicity
+    mount_name = get_mount_name(pipeline, branch, platform, build_type)
+    pipeline_and_branch = get_pipeline_and_branch(pipeline, branch) 
+    parameters = dict(
+        AvailabilityZone = availability_zone,
+        VolumeType=disk_type,
+        TagSpecifications= [{
+            'ResourceType': 'volume',
+            'Tags': [
+                { 'Key': 'Name', 'Value': mount_name },
+                { 'Key': 'Pipeline', 'Value': pipeline },
+                { 'Key': 'BranchName', 'Value': branch },
+                { 'Key': 'Platform', 'Value': platform },
+                { 'Key': 'BuildType', 'Value': build_type }, 
+                { 'Key': 'PipelineAndBranch', 'Value': pipeline_and_branch }, # used so the snapshoting easily identifies which volumes to snapshot
+            ]
+        }]
+    )
+    if 'io1' in disk_type.lower(): 
+        parameters['Iops'] = (4 * disk_size)
+
+    snapshot_id = find_snapshot_id(ec2_client, pipeline, platform, build_type, disk_size)
+    if snapshot_id:
+        parameters['SnapshotId'] = snapshot_id
+        created = False
+    else:
+        # If no snapshot id, we need to specify the size
+        parameters['Size'] = disk_size
+        created = True
+
+    response = ec2_client.create_volume(**parameters)
+    volume_id = response['VolumeId']
+
+    # give some time for the creation call to complete
+    time.sleep(1)
+
+    response = ec2_client.describe_volumes(VolumeIds=[volume_id, ])
+    while (response['Volumes'][0]['State'] != 'available'):
+        time.sleep(1)
+        response = ec2_client.describe_volumes(VolumeIds=[volume_id, ])
+
+    print("Volume {} created\n\tSnapshot: {}\n\tPipeline {}\n\tBranch {}\n\tPlatform: {}\n\tBuild type: {}"
+        .format(volume_id, snapshot_id, pipeline, branch, platform, build_type))
+    return volume_id, created
+
+
+def mount_volume(created):
+    print 'Mounting volume...'
+    if os.name == 'nt':
+        f = tempfile.NamedTemporaryFile(delete=False)
+        f.write("""
+      select disk 1
+      online disk
+      attribute disk clear readonly
+      """) # assume disk # for now
+
+        if created:
+            print 'Creating filesystem on new volume'
+            f.write("""create partition primary
+          select partition 1
+          format quick fs=ntfs
+          assign
+          active
+          """)
+
+        f.close()
+        
+        subprocess.call(['diskpart', '/s', f.name])
+
+        time.sleep(5)
+
+        drives_after = win32api.GetLogicalDriveStrings()
+        drives_after = drives_after.split('\000')[:-1]
+
+        print drives_after
+
+        #drive_letter = next(item for item in drives_after if item not in drives_before)
+        drive_letter = MOUNT_PATH
+
+        os.unlink(f.name)
+
+        time.sleep(1)
+
+    else:
+        subprocess.call(['file', '-s', '/dev/xvdf'])
+        if created:
+            subprocess.call(['mkfs', '-t', 'ext4', '/dev/xvdf'])
+        subprocess.call(['mount', '/dev/xvdf', MOUNT_PATH])
+
+
+def attach_volume(volume, volume_id, instance_id, timeout=DEFAULT_TIMEOUT):
+    print 'Attaching volume {} to instance {}'.format(volume_id, instance_id)
+    volume.attach_to_instance(Device='xvdf',
+                              InstanceId=instance_id,
+                              VolumeId=volume_id)
+    # give a little bit of time for the aws call to process
+    time.sleep(2)
+    # reload the volume just in case
+    volume.load()
+    timeout_init = time.clock()
+    while (len(volume.attachments) and volume.attachments[0]['State'] != 'attached'):
+        time.sleep(1)
+        volume.load()
+        if (time.clock() - timeout_init) > timeout:
+            print 'ERROR: Timeout reached trying to mount EBS'
+            exit(1)
+
+    print 'Volume {} has been attached to instance {}'.format(volume_id, instance_id)
+
+
+def unmount_volume():
+    print 'Umounting volume...'
+    if os.name == 'nt':
+        kill_processes(MOUNT_PATH + 'workspace')
+        f = tempfile.NamedTemporaryFile(delete=False)
+        f.write("""
+          select disk 1
+          offline disk
+          """)
+        f.close()
+        subprocess.call('diskpart /s %s' % f.name)
+        os.unlink(f.name)
+    else:
+        kill_processes(MOUNT_PATH)
+        subprocess.call(['umount', '-f', MOUNT_PATH])
+
+
+def detach_volume(volume, ec2_instance_id, force, timeout=DEFAULT_TIMEOUT):
+    print 'Detaching volume {} from instance {}'.format(volume.volume_id, ec2_instance_id)
+    volume.detach_from_instance(Device='xvdf',
+                                Force=force,
+                                InstanceId=ec2_instance_id,
+                                VolumeId=volume.volume_id)
+    timeout_init = time.clock()
+    while len(volume.attachments) and volume.attachments[0]['State'] != 'detached':
+        time.sleep(1)
+        volume.load()
+        if (time.clock() - timeout_init) > timeout:
+            print 'ERROR: Timeout reached trying to unmount EBS.'
+            volume.detach_from_instance(Device='xvdf',Force=True,InstanceId=ec2_instance_id,VolumeId=volume.volume_id)
+            exit(1)
+            
+    print 'Volume {} has been detached from instance {}'.format(volume.volume_id, ec2_instance_id)
+    volume.load()
+    if len(volume.attachments):
+        print 'Volume still has attachments'
+        for attachment in volume.attachments:
+            print 'Volume {} {} to instance {}'.format(attachment['VolumeId'], attachment['State'], attachment['InstanceId'])
+
+
+def attach_ebs_and_create_partition_with_retry(volume, volume_id, ec2_instance_id, created):
+    attach_volume(volume, volume_id, ec2_instance_id)
+    mount_volume(created)
+    attempt = 1
+    while attempt <= MAX_EBS_MOUNTING_ATTEMPT:
+        if os.name == 'nt':
+            drives_after = win32api.GetLogicalDriveStrings()
+            drives_after = drives_after.split('\000')[:-1]
+            if MOUNT_PATH not in drives_after:
+                print('Disk partitioning failed, retrying...')
+                unmount_volume()
+                detach_volume(volume, ec2_instance_id, False)
+                attach_volume(volume, volume_id, ec2_instance_id)
+                mount_volume(created)
+        attempt += 1
+
+def mount_ebs(pipeline, branch, platform, build_type, disk_size, disk_type):
+    session = boto3.session.Session()
+    region = session.region_name
+    if region is None:
+        region = DEFAULT_REGION
+    ec2_client = get_ec2_client(region)
+    ec2_instance_id = get_ec2_instance_id()
+    ec2_availability_zone = get_availability_zone()
+    ec2_resource = boto3.resource('ec2', region_name=region)
+    ec2_instance = ec2_resource.Instance(ec2_instance_id)
+
+    for volume in ec2_instance.volumes.all():
+        for attachment in volume.attachments:
+            print 'attachment device: {}'.format(attachment['Device'])
+            if 'xvdf' in attachment['Device'] and attachment['State'] != 'detached':
+                print 'A device is already attached to xvdf. This likely means a previous build failed to detach its ' \
+                      'build volume. This volume is considered orphaned and will be detached from this instance.'
+                unmount_volume()
+                detach_volume(volume, ec2_instance_id, False) # Force unmounts should not be used, as that will cause the EBS block device driver to fail the remount
+
+    mount_name = get_mount_name(pipeline, branch, platform, build_type)
+    response = ec2_client.describe_volumes(Filters=[{
+        'Name': 'tag:Name', 'Values': [mount_name]
+        }])
+
+    created = False
+    if 'Volumes' in response and not len(response['Volumes']):
+        print 'Volume for {} doesn\'t exist creating it...'.format(mount_name)
+        # volume doesn't exist, create it
+        volume_id, created = create_volume(ec2_client, ec2_availability_zone, pipeline, branch, platform, build_type, disk_size, disk_type)
+    else:
+        volume = response['Volumes'][0]
+        volume_id = volume['VolumeId']
+        print 'Current volume {} is a {} GB {}'.format(volume_id, volume['Size'], volume['VolumeType'])
+        if (volume['Size'] != disk_size or volume['VolumeType'] != disk_type):
+            print 'Override disk attributes does not match the existing volume, deleting {} and replacing the volume'.format(volume_id)
+            delete_volume(ec2_client, volume_id)
+            volume_id, created = create_volume(ec2_client, ec2_availability_zone, pipeline, branch, platform, build_type, disk_size, disk_type)
+        if len(volume['Attachments']):
+            # this is bad we shouldn't be attached, we should have detached at the end of a build
+            attachment = volume['Attachments'][0]
+            print ('Volume already has attachment {}, detaching...'.format(attachment))
+            detach_volume(ec2_resource.Volume(volume_id), attachment['InstanceId'], True)
+
+    volume = ec2_resource.Volume(volume_id)
+
+    if os.name == 'nt':
+        drives_before = win32api.GetLogicalDriveStrings()
+        drives_before = drives_before.split('\000')[:-1]
+
+        print drives_before
+
+    attach_ebs_and_create_partition_with_retry(volume, volume_id, ec2_instance_id, created)
+
+    free_space_mb = get_free_space_mb(MOUNT_PATH)
+    print 'Free disk space {}MB'.format(free_space_mb)
+    
+    if free_space_mb < LOW_EBS_DISK_SPACE_LIMIT:
+        print 'Volume is running below EBS free disk space treshhold {}MB. Recreating volume and running clean build.'.format(LOW_EBS_DISK_SPACE_LIMIT)
+        unmount_volume()
+        detach_volume(volume, ec2_instance_id, False)
+        delete_volume(ec2_client, volume_id)
+        new_disk_size = int(volume.size * 1.25)
+        if new_disk_size > MAX_EBS_DISK_SIZE:
+            print 'Error: EBS disk size reached to the allowed maximum disk size {}MB, please contact ly-infra@ and ly-build@ to investigate.'.format(MAX_EBS_DISK_SIZE)
+            exit(1)
+        print 'Recreating the EBS with disk size {}'.format(new_disk_size)
+        volume_id, created = create_volume(ec2_client, ec2_availability_zone, pipeline, branch, platform, build_type, new_disk_size, disk_type)
+        volume = ec2_resource.Volume(volume_id)
+        attach_ebs_and_create_partition_with_retry(volume, volume_id, ec2_instance_id, created)
+
+def unmount_ebs():
+    session = boto3.session.Session()
+    region = session.region_name
+    if region is None:
+        region = DEFAULT_REGION
+    ec2_client = get_ec2_client(region)
+    ec2_instance_id = get_ec2_instance_id()
+    ec2_resource = boto3.resource('ec2', region_name=region)
+    ec2_instance = ec2_resource.Instance(ec2_instance_id)
+
+    if os.path.isfile('envinject.properties'):
+        os.remove('envinject.properties')
+
+    volume = None
+
+    for attached_volume in ec2_instance.volumes.all():
+        for attachment in attached_volume.attachments:
+            print 'attachment device: {}'.format(attachment['Device'])
+            if attachment['Device'] == 'xvdf':
+                volume = attached_volume
+
+    if not volume:
+        # volume is not mounted
+        print 'Volume is not mounted'
+    else:
+        unmount_volume()
+        detach_volume(volume, ec2_instance_id, False)
+
+def delete_ebs(pipeline, branch, platform, build_type):
+    unmount_ebs()
+
+    session = boto3.session.Session()
+    region = session.region_name
+    if region is None:
+        region = DEFAULT_REGION
+    ec2_client = get_ec2_client(region)
+    ec2_instance_id = get_ec2_instance_id()
+    ec2_resource = boto3.resource('ec2', region_name=region)
+    ec2_instance = ec2_resource.Instance(ec2_instance_id)
+
+    mount_name = get_mount_name(pipeline, branch, platform, build_type)
+    response = ec2_client.describe_volumes(Filters=[
+        { 'Name': 'tag:Name', 'Values': [mount_name] }
+    ])
+
+    if 'Volumes' in response and len(response['Volumes']):
+        volume = response['Volumes'][0]
+        volume_id = volume['VolumeId']
+        delete_volume(ec2_client, volume_id)
+
+
+def main(action, pipeline, branch, platform, build_type, disk_size, disk_type):
+    if action == 'mount':
+        mount_ebs(pipeline, branch, platform, build_type, disk_size, disk_type)
+    elif action == 'unmount':
+        unmount_ebs()
+    elif action == 'delete':
+        delete_ebs(pipeline, branch, platform, build_type)
+
+if __name__ == "__main__":
+    args = parse_args()
+    ret = main(args.action, args.pipeline, args.branch, args.platform, args.build_type, args.disk_size, args.disk_type)
+    sys.exit(ret)

scripts/build/build_node/Platform/Common/requirements.txt

@@ -0,0 +1,185 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --generate-hashes 'requirements.txt'
+#
+
+# These packages are used as part of the pre-SCM sync environment and not actually used in the build
+# The requirements file for build can be found here: dev\Tools\Python\<python version>\requirements.txt
+
+boto3==1.16.18 \
+    --hash=sha256:51c419d890ae216b9b031be31f3182739dc3deb5b64351f286bffca2818ddb35 \
+    --hash=sha256:d70d21ea137d786e84124639a62be42f92f4b09472ebfb761156057c92dc5366 \
+    # via -r requirements.txt
+botocore==1.19.18 \
+    --hash=sha256:288d43e85f12e3c1d6a0535a585a182ca04e8c6e742ebaaf15357a0e3b37ca7a \
+    --hash=sha256:bba18b5c4eef3eb2dc39b1b1f8959ba01ac27e7e12e413e281b0fb242990c0f5 \
+    # via -r requirements.txt, boto3, s3transfer
+certifi==2020.11.8 \
+    --hash=sha256:1f422849db327d534e3d0c5f02a263458c3955ec0aae4ff09b95f195c59f4edd \
+    --hash=sha256:f05def092c44fbf25834a51509ef6e631dc19765ab8a57b4e7ab85531f0a9cf4 \
+    # via requests
+chardet==3.0.4 \
+    --hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \
+    --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 \
+    # via requests
+colorama==0.4.4 \
+    --hash=sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b \
+    --hash=sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2 \
+    # via -r requirements.txt
+docutils==0.16 \
+    --hash=sha256:0c5b78adfbf7762415433f5515cd5c9e762339e23369dbe8000d84a4bf4ab3af \
+    --hash=sha256:c2de3a60e9e7d07be26b7f2b00ca0309c207e06c100f9cc2a94931fc75a478fc \
+    # via -r requirements.txt
+futures==3.3.0 \
+    --hash=sha256:49b3f5b064b6e3afc3316421a3f25f66c137ae88f068abbf72830170033c5e16 \
+    --hash=sha256:7e033af76a5e35f58e56da7a91e687706faf4e7bdfb2cbc3f2cca6b9bcda9794 \
+    # via -r requirements.txt, s3transfer
+idna==2.10 \
+    --hash=sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6 \
+    --hash=sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0 \
+    # via requests
+jinja2==2.11.2 \
+    --hash=sha256:89aab215427ef59c34ad58735269eb58b1a5808103067f7bb9d5836c651b3bb0 \
+    --hash=sha256:f0a4641d3cf955324a89c04f3d94663aa4d638abe8f733ecd3582848e1c37035 \
+    # via -r requirements.txt
+jmespath==0.10.0 \
+    --hash=sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9 \
+    --hash=sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f \
+    # via -r requirements.txt, boto3, botocore
+markupsafe==1.1.1 \
+    --hash=sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473 \
+    --hash=sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161 \
+    --hash=sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235 \
+    --hash=sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5 \
+    --hash=sha256:13d3144e1e340870b25e7b10b98d779608c02016d5184cfb9927a9f10c689f42 \
+    --hash=sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff \
+    --hash=sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b \
+    --hash=sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1 \
+    --hash=sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e \
+    --hash=sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183 \
+    --hash=sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66 \
+    --hash=sha256:596510de112c685489095da617b5bcbbac7dd6384aeebeda4df6025d0256a81b \
+    --hash=sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1 \
+    --hash=sha256:6788b695d50a51edb699cb55e35487e430fa21f1ed838122d722e0ff0ac5ba15 \
+    --hash=sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1 \
+    --hash=sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e \
+    --hash=sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b \
+    --hash=sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905 \
+    --hash=sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735 \
+    --hash=sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d \
+    --hash=sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e \
+    --hash=sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d \
+    --hash=sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c \
+    --hash=sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21 \
+    --hash=sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2 \
+    --hash=sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5 \
+    --hash=sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b \
+    --hash=sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6 \
+    --hash=sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f \
+    --hash=sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f \
+    --hash=sha256:cdb132fc825c38e1aeec2c8aa9338310d29d337bebbd7baa06889d09a60a1fa2 \
+    --hash=sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7 \
+    --hash=sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be \
+    # via -r requirements.txt, jinja2
+p4python==2017.2.1615960 \
+    --hash=sha256:0e6f7a1d654e32a2001a0b39ce360aabdc853e72d4f2ab59e9d1a3b1d1a0f9ed \
+    --hash=sha256:0fec222b4f1b625398630e20b5b63442432d0a029f456169aa8f2e3ba3ea0eff \
+    --hash=sha256:119a9c4c8262a072e8eaa6b70ab00813ff35b8b5a84b50af8308b71847af2166 \
+    --hash=sha256:316499580172a5ed5c873cc361cdd0ecda70bb36cffc2d84c189e53268a06312 \
+    --hash=sha256:31a22054e9a0342932da233c234ae6de54eee4d7797caf79912387a00457e13f \
+    --hash=sha256:3445ef1a1109d887a197aa2f137385747bb75d837fcb5991acaece14567444c2 \
+    --hash=sha256:3df3ebe4fb3bc47f338538dea86f1e45c59e13cb837b9bed29def3f5cfd5327d \
+    --hash=sha256:4e0303a4d482189033fcca0d05702dc91c3c62c3f35acfd7829e750b1905d95f \
+    --hash=sha256:549d5f6d2b094b86e63097650e217c65f65d82b87fdcc3013cf390aa552bb601 \
+    --hash=sha256:55c7c996a5c2897ea5a8fe4c8c9e65dfc6d6f65a09b1f499105d2b9a8a018498 \
+    --hash=sha256:85a083ab4749e3e980286de7eb06e9eae225549c5e5834d28fd2e9109372efaf \
+    --hash=sha256:8d446900216d9d1e42b7faa5f65d5c18d1e15b517f08a7e288631aee7d3cedd0 \
+    --hash=sha256:ae901d370d34097df145b91e754bf5463206c08ef3b3879888d37800d155c1c5 \
+    --hash=sha256:b5c61edda30e58ff1724771e83e03be047905d0708fbab4ef0733e6b1618c8c4 \
+    --hash=sha256:ba8bb0552f8e037a3fed74c6bc64d91dc3c3491015d066218bc91f23bfbcf76f \
+    --hash=sha256:bc3301a0ecf2161128b77bb1b4b23fa37059274a6be2f3ff7a8eb8b4bbf7c15d \
+    --hash=sha256:c5e3c145a2418a4e188f76f8eeb706fb04005887bca04146c202182bba21d2a4 \
+    --hash=sha256:d8917ed4e2ad23b57421cccf7cdfdff0a932cddbf45c53410ee390770beb7270 \
+    --hash=sha256:df99c2dc7e518a22f98fa5a9239bfbfae062c3736c362ca7cc6f9c9c1d3553d6 \
+    --hash=sha256:e539bc34c616a62aaae1d4c50b0f7311eb97c2f0daf5a1eedd4d181249a22d2b \
+    --hash=sha256:f39a7f4aee228f10437876f06a3f013453b04606c8e9c899632dbfeca690bcd4 \
+    # via -r requirements.txt
+psutil==5.7.3 \
+    --hash=sha256:01bc82813fbc3ea304914581954979e637bcc7084e59ac904d870d6eb8bb2bc7 \
+    --hash=sha256:1cd6a0c9fb35ece2ccf2d1dd733c1e165b342604c67454fd56a4c12e0a106787 \
+    --hash=sha256:2cb55ef9591b03ef0104bedf67cc4edb38a3edf015cf8cf24007b99cb8497542 \
+    --hash=sha256:56c85120fa173a5d2ad1d15a0c6e0ae62b388bfb956bb036ac231fbdaf9e4c22 \
+    --hash=sha256:5d9106ff5ec2712e2f659ebbd112967f44e7d33f40ba40530c485cc5904360b8 \
+    --hash=sha256:6a3e1fd2800ca45083d976b5478a2402dd62afdfb719b30ca46cd28bb25a2eb4 \
+    --hash=sha256:ade6af32eb80a536eff162d799e31b7ef92ddcda707c27bbd077238065018df4 \
+    --hash=sha256:af73f7bcebdc538eda9cc81d19db1db7bf26f103f91081d780bbacfcb620dee2 \
+    --hash=sha256:e02c31b2990dcd2431f4524b93491941df39f99619b0d312dfe1d4d530b08b4b \
+    --hash=sha256:fa38ac15dbf161ab1e941ff4ce39abd64b53fec5ddf60c23290daed2bc7d1157 \
+    --hash=sha256:fbcac492cb082fa38d88587d75feb90785d05d7e12d4565cbf1ecc727aff71b7 \
+    # via -r requirements.txt
+pyasn1==0.4.8 \
+    --hash=sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d \
+    --hash=sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba \
+    # via -r requirements.txt, rsa
+dateutil==2.8.1 \
+    --hash=sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c \
+    --hash=sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a \
+    # via -r requirements.txt, botocore
+pytz==2020.4 \
+    --hash=sha256:3e6b7dd2d1e0a59084bcee14a17af60c5c562cdc16d828e8eba2e683d3a7e268 \
+    --hash=sha256:5c55e189b682d420be27c6995ba6edce0c0a77dd67bfbe2ae6607134d5851ffd \
+    # via -r requirements.txt
+pywin32==228 \
+    --hash=sha256:00eaf43dbd05ba6a9b0080c77e161e0b7a601f9a3f660727a952e40140537de7 \
+    --hash=sha256:11cb6610efc2f078c9e6d8f5d0f957620c333f4b23466931a247fb945ed35e89 \
+    --hash=sha256:1f45db18af5d36195447b2cffacd182fe2d296849ba0aecdab24d3852fbf3f80 \
+    --hash=sha256:37dc9935f6a383cc744315ae0c2882ba1768d9b06700a70f35dc1ce73cd4ba9c \
+    --hash=sha256:6e38c44097a834a4707c1b63efa9c2435f5a42afabff634a17f563bc478dfcc8 \
+    --hash=sha256:8319bafdcd90b7202c50d6014efdfe4fde9311b3ff15fd6f893a45c0868de203 \
+    --hash=sha256:9b3466083f8271e1a5eb0329f4e0d61925d46b40b195a33413e0905dccb285e8 \
+    --hash=sha256:a60d795c6590a5b6baeacd16c583d91cce8038f959bd80c53bd9a68f40130f2d \
+    --hash=sha256:af40887b6fc200eafe4d7742c48417529a8702dcc1a60bf89eee152d1d11209f \
+    --hash=sha256:ec16d44b49b5f34e99eb97cf270806fdc560dff6f84d281eb2fcb89a014a56a9 \
+    --hash=sha256:ed74b72d8059a6606f64842e7917aeee99159ebd6b8d6261c518d002837be298 \
+    --hash=sha256:fa6ba028909cfc64ce9e24bcf22f588b14871980d9787f1e2002c99af8f1850c \
+    # via -r requirements.txt
+pyxb==1.2.6 \
+    --hash=sha256:2a00f38dd1d87b88f92d79bc5a09718d730419b88e814545f472bbd5a3bf27b4 \
+    # via -r requirements.txt
+pyyaml==5.3.1 \
+    --hash=sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97 \
+    --hash=sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76 \
+    --hash=sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2 \
+    --hash=sha256:6034f55dab5fea9e53f436aa68fa3ace2634918e8b5994d82f3621c04ff5ed2e \
+    --hash=sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648 \
+    --hash=sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf \
+    --hash=sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f \
+    --hash=sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2 \
+    --hash=sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee \
+    --hash=sha256:ad9c67312c84def58f3c04504727ca879cb0013b2517c85a9a253f0cb6380c0a \
+    --hash=sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d \
+    --hash=sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c \
+    --hash=sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a \
+    # via -r requirements.txt
+requests==2.25.0 \
+    --hash=sha256:7f1a0b932f4a60a1a65caa4263921bb7d9ee911957e0ae4a23a6dd08185ad5f8 \
+    --hash=sha256:e786fa28d8c9154e6a4de5d46a1d921b8749f8b74e28bde23768e5e16eece998 \
+    # via -r requirements.txt
+rsa==4.5 \
+    --hash=sha256:35c5b5f6675ac02120036d97cf96f1fde4d49670543db2822ba5015e21a18032 \
+    --hash=sha256:4d409f5a7d78530a4a2062574c7bd80311bc3af29b364e293aa9b03eea77714f \
+    # via -r requirements.txt
+s3transfer==0.3.3 \
+    --hash=sha256:2482b4259524933a022d59da830f51bd746db62f047d6eb213f2f8855dcb8a13 \
+    --hash=sha256:921a37e2aefc64145e7b73d50c71bb4f26f46e4c9f414dc648c6245ff92cf7db \
+    # via -r requirements.txt, boto3
+six==1.15.0 \
+    --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \
+    --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced \
+    # via -r requirements.txt, dateutil
+urllib3==1.26.2 \
+    --hash=sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08 \
+    --hash=sha256:d8ff90d979214d7b4f8ce956e80f4028fc6860e4431f731ea4a8c08f23f99473 \
+    # via -r requirements.txt, botocore, requests

scripts/build/build_node/Platform/Windows/init_setup.ps1

@@ -0,0 +1,42 @@
+<#
+All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+its licensors.
+
+For complete copyright and license terms please see the LICENSE at the root of this
+distribution (the "License"). All use of this software is governed by the License,
+or, if provided, by the license below or the license accompanying this file. Do not
+remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#>
+
+# Disable Windows Defender
+Set-MpPreference -DisableRealtimeMonitoring $true
+
+# Disable Admin prompts
+New-ItemProperty -Path HKLM:Software\Microsoft\Windows\CurrentVersion\policies\system -Name EnableLUA -PropertyType DWord -Value 0 -Force
+New-ItemProperty -Path HKLM:Software\Microsoft\Windows\CurrentVersion\policies\system -Name ConsentPromptBehaviorAdmin -PropertyType DWord -Value 0 -Force
+
+# Increase port number availablity (for P4) 
+netsh int ipv4 set dynamicport tcp start=1025 num=64511
+
+# Install Chocolatey and Carbon (local user util)
+[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))
+$env:ChocolateyInstall = Convert-Path "$((Get-Command choco).Path)\..\.."   
+Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
+choco install Carbon -y
+choco install awscli -y
+refreshenv
+
+# Grab credentials from parameter store (assumes AWS cli is installed and correct IAM policy is setup)
+$az = curl http://169.254.169.254/latest/meta-data/placement/availability-zone -UseBasicParsing
+$region = $az.Content -replace ".$"
+$username = aws ssm get-parameters --names "shared.builderuser" --region $region --with-decryption | ConvertFrom-Json
+$username = $username.Parameters.Value.ToString()
+$password = aws ssm get-parameters --names "shared.builderpass" --region $region --with-decryption | ConvertFrom-Json
+$password = ConvertTo-SecureString $password.Parameters.Value.ToString() -AsPlainText -Force
+$credential = New-Object System.Management.Automation.PSCredential -ArgumentList $username, $password
+
+Write-Host "Adding builder user"
+Import-Module 'Carbon'
+Install-User -Credential $credential -FullName "$($username)" -Description "Builder account for LY"
+Add-GroupMember -Name "Administrators" -Member "$($username)"

scripts/build/build_node/Platform/Windows/install_android.ps1

@@ -0,0 +1,43 @@
+<#
+All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+its licensors.
+
+For complete copyright and license terms please see the LICENSE at the root of this
+distribution (the "License"). All use of this software is governed by the License,
+or, if provided, by the license below or the license accompanying this file. Do not
+remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#>
+
+choco install -y android-sdk
+
+$Env:ANDROID_HOME = "C:\Android\android-sdk"
+setx ANDROID_HOME "C:\Android\android-sdk"
+
+$apis_packages = '"add-ons;addon-google_apis-google-19" "add-ons;addon-google_apis-google-21" "add-ons;addon-google_apis-google-22" "add-ons;addon-google_apis-google-23" "add-ons;addon-google_apis-google-24"'
+$android_packages = '"platforms;android-19" "platforms;android-21" "platforms;android-22" "platforms;android-23" "platforms;android-24" "platforms;android-25" "platforms;android-26" "platforms;android-27" "platforms;android-28" "platforms;android-29" "platforms;android-30"'
+$googleplay_packages = '"extras;google;market_apk_expansion" "extras;google;market_licensing"'
+$build_tools = '"build-tools;30.0.2" "tools"'
+
+$sdkmanager = "C:\Android\android-sdk\tools\bin\sdkmanager.bat"
+Start-Process -FilePath $sdkmanager -ArgumentList $apis_packages -NoNewWindow -Wait
+Start-Process -FilePath $sdkmanager -ArgumentList $android_packages -NoNewWindow -Wait
+Start-Process -FilePath $sdkmanager -ArgumentList $googleplay_packages -NoNewWindow -Wait
+Start-Process -FilePath $sdkmanager -ArgumentList $build_tools -NoNewWindow -Wait
+
+Write-Host "Installing Gradle and Ninja"
+Import-Module C:\ProgramData\chocolatey\helpers\chocolateyInstaller.psm1 #Grade needs a custom installer due to being hardcoded to C:\Programdata in Chocolatey
+$packageName = 'gradle'
+$version = '5.6.4'
+$checksum = 'ABC10BCEDB58806E8654210F96031DB541BCD2D6FC3161E81CB0572D6A15E821'
+$url = "https://services.gradle.org/distributions/gradle-$version-all.zip"
+$installDir = "C:\Gradle"
+
+Install-ChocolateyZipPackage $packageName $url $installDir -Checksum $checksum -ChecksumType 'sha256'
+
+$gradle_home = Join-Path $installDir "$packageName-$version"
+$gradle_bat = Join-Path $gradle_home 'bin/gradle.bat'
+
+Install-ChocolateyEnvironmentVariable "GRADLE_HOME" $gradle_home 'Machine'
+
+choco install -y ninja --version=1.10.0 --package-parameters="/installDir:C:\Ninja"

scripts/build/build_node/Platform/Windows/install_cygwin.ps1

@@ -0,0 +1,81 @@
+<#
+All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+its licensors.
+
+For complete copyright and license terms please see the LICENSE at the root of this
+distribution (the "License"). All use of this software is governed by the License,
+or, if provided, by the license below or the license accompanying this file. Do not
+remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#>
+
+# Grab credentials from parameter store (assumes AWS cli is installed and correct IAM policy is setup)
+$az = curl http://169.254.169.254/latest/meta-data/placement/availability-zone -UseBasicParsing
+$region = $az.Content -replace ".$"
+
+$username = aws ssm get-parameters --names "shared.builderuser" --region $region --with-decryption | ConvertFrom-Json
+$username = $username.Parameters.Value.ToString()
+$password = aws ssm get-parameters --names "shared.builderpass" --region $region --with-decryption | ConvertFrom-Json
+$password = ConvertTo-SecureString $password.Parameters.Value.ToString() -AsPlainText -Force
+$credential = New-Object System.Management.Automation.PSCredential -ArgumentList $username, $password
+
+$cygwin_packages = "openssh,vim,curl,tar,wget,zip,unzip,diffutils,bzr,nc,procps,ncdu"
+
+# Download cygwin
+New-Item C:\tools\cygwin -Force
+wget https://cygwin.com/setup-x86_64.exe -o C:\tools\cygwin\setup-x86_64.exe
+
+# Install cygwin (this is a specific version not subject to LGPLv3)
+# complete package list: https://cygwin.com/packages/package_list.html
+Write-Host "  * Starting Cygwin install"
+Start-Process "C:\tools\cygwin\setup-x86_64.exe" -ArgumentList ("--quiet-mode " +
+"--wait --root C:\cygwin --site http://cygwin.osuosl.org " +
+"--packages $cygwin_packages") -wait `
+-NoNewWindow -PassThru -RedirectStandardOutput "C:\cygwin_install.log" `
+-RedirectStandardError "C:\cygwin_install.err"
+
+# Open up firewall for ssh daemon
+Write-Host "  * Opening port 22 on firewall"
+New-NetFirewallRule -DisplayName "Allow SSH inbound" -Direction Inbound -LocalPort 22 -Protocol TCP -Action Allow
+
+# Workaround for https://www.cygwin.com/ml/cygwin/2015-10/msg00036.html
+# see:
+#   1) https://www.cygwin.com/ml/cygwin/2015-10/msg00038.html
+#   2) https://goo.gl/EWzeVV
+$env:LOGONSERVER = "\\" + $env:COMPUTERNAME
+
+# Configure sshd
+Write-Host "  * Configuring SSHD"
+Start-Process "C:\cygwin\bin\bash.exe" -ArgumentList "--login
+-c `"ssh-host-config -y -c 'ntsec mintty' -u '$($username)' -w '$($Credential.GetNetworkCredential().Password)'`"" `
+-wait -NoNewWindow -PassThru `
+-RedirectStandardOutput "C:\logs\cygrunsrv.log" -RedirectStandardError "C:\logs\cygrunsrv.err"
+Start-Process "C:\cygwin\bin\bash.exe" -ArgumentList "--login -c 'echo ""KexAlgorithms curve25519-sha256@libssh.org,ecdh-sha2-nistp256,ecdh-sha2-nistp384,ecdh-sha2-nistp521,diffie-hellman-group-exchange-sha256,diffie-hellman-group14-sha1,diffie-hellman-group-exchange-sha1,diffie-hellman-group1-sha1"" >> /etc/sshd_config'"
+
+# Copy bash script to add special permissions to builder account
+aws s3 cp s3://ly-jenkins-node-config/windows/setup.sh c:\cygwin\home\$username\
+
+# Run bash setup script
+echo "  * Configuring Bash"
+Start-Process "C:\cygwin\bin\bash.exe" -ArgumentList "--login -c 'chmod a+x ~/setup.sh; ~/setup.sh $username'" `
+-wait -NoNewWindow -PassThru `
+-RedirectStandardOutput "C:\logs\Administrator_cygwin_setup.log" `
+-RedirectStandardError "C:\logs\Administrator_cygwin_setup.err"
+
+# Start sshd
+echo "  * Starting SSHD"
+Start-Process "net" -ArgumentList "start cygsshd" `
+-wait -NoNewWindow -PassThru `
+-RedirectStandardOutput "C:\logs\net_start_sshd.log" -RedirectStandardError "C:\logs\net_start_sshd.err"
+
+# Add SSH key
+echo "  * Getting SSH key"
+$sshkey = aws ssm get-parameters --names "shared.buildersshkey" --region $region --with-decryption | ConvertFrom-Json
+New-Item "C:\cygwin\home\$username\.ssh" -Force 
+Add-Content "C:\cygwin\home\$username\.ssh\authorized_keys" "$($sshkey.Parameters.Value)"
+Start-Process "C:\cygwin\bin\bash.exe" -ArgumentList "--login -c 'chown -R $username /home/$username; chmod 600 /home/$username/.ssh/authorized_keys; sed -i 's/\r$//' /home/$username/.ssh/authorized_keys'" `
+
+# Clean up secure variables
+Remove-Variable password
+Remove-Variable credential
+Remove-Variable sshkey

scripts/build/build_node/Platform/Windows/install_python.ps1

@@ -0,0 +1,23 @@
+<#
+All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+its licensors.
+
+For complete copyright and license terms please see the LICENSE at the root of this
+distribution (the "License"). All use of this software is governed by the License,
+or, if provided, by the license below or the license accompanying this file. Do not
+remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#>
+
+choco install -y python2 --version=2.7.15
+choco install -y python3 --version=3.7.5
+
+# Ensure Python paths are set
+[Environment]::SetEnvironmentVariable(
+    "Path",
+    [Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarget]::Machine) + ";C:\Python27;C:\Python27\Scripts",
+    [EnvironmentVariableTarget]::Machine)
+
+Write-Host "Installing packages" # requirements.txt hould be in the "Platforms\Common" folder
+pip install -r ..\Common\requirements.txt
+pip3 install -r ..\Common\requirements.txt

scripts/build/build_node/Platform/Windows/install_utiltools.ps1

@@ -0,0 +1,28 @@
+<#
+All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+its licensors.
+
+For complete copyright and license terms please see the LICENSE at the root of this
+distribution (the "License"). All use of this software is governed by the License,
+or, if provided, by the license below or the license accompanying this file. Do not
+remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#>
+
+# Install dependancies
+choco install -y 7zip
+choco install -y procexp
+choco install -y windirstat
+choco install -y sysinternals
+
+# Install source control apps 
+choco install -y git
+choco install -y git-lfs
+choco install -y p4
+
+Write-Host "Configuring Git"
+git config --global "credential.helper" "!aws codecommit credential-helper $@"
+git config --global "credential.UseHttpPath" "true"
+
+# Install Java (for Jenkins)
+choco install corretto8jdk -y --ia INSTALLDIR="c:\jdk8" # Custom directory to handle cases where whitespace in the path is not quote wrapped

scripts/build/build_node/Platform/Windows/install_vsbuildtools.ps1

@@ -0,0 +1,30 @@
+<#
+All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+its licensors.
+
+For complete copyright and license terms please see the LICENSE at the root of this
+distribution (the "License"). All use of this software is governed by the License,
+or, if provided, by the license below or the license accompanying this file. Do not
+remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#>
+
+choco install visualstudio2019buildtools --version=16.8.2 --package-parameters "--config .\vs2019bt.vsconfig" -y
+choco install visualstudio2017buildtools --version=15.9.29 --package-parameters "--config .\vs2017bt.vsconfig" -y
+
+Write-Host "Installing downstream dependancies"
+choco install -y dotnet3.5 
+
+# This is a custom install of the Win10 SDK debugger which was not included in VS2019/2017. This resolves an issue where the older DbgHelp library was being referenced from C:\Windows\System32\
+# More details here: https://docs.microsoft.com/en-us/windows/win32/debug/calling-the-dbghelp-library
+
+Import-Module C:\ProgramData\chocolatey\helpers\chocolateyInstaller.psm1 
+$packageName = 'windows-sdk-10.1'
+$featureName = 'OptionId.WindowsDesktopDebuggers'
+$installerType = 'EXE'
+$url = 'https://download.microsoft.com/download/4/2/2/42245968-6A79-4DA7-A5FB-08C0AD0AE661/windowssdk/winsdksetup.exe'
+$checksum = '2E28117E82B4D02FE30D564B835ACE9976612609271265872F20F2256A9C506B'
+$checksumType = 'sha256'
+$silentArgs = "/Quiet /NoRestart /features $featureName /Log ""$env:temp\${packageName}_$([Guid]::NewGuid().ToString('D')).log"""
+$validExitCodes = @(0,3010)
+Install-ChocolateyPackage $packageName $installerType $silentArgs $url -checksum $checksum -checksumType $checksumType -validExitCodes $validExitCodes

scripts/build/build_node/Platform/Windows/setup.sh

@@ -0,0 +1,70 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+#!/bin/bash
+
+echo "Running setup.sh script..."
+echo
+
+# setup standard groups and user account db for ssh
+mkpasswd -c > /etc/passwd
+mkgroup -c > /etc/group
+
+# give special permissions to the builder user
+editrights -a SeAssignPrimaryTokenPrivilege -u $1
+editrights -a SeCreateTokenPrivilege -u $1
+editrights -a SeTcbPrivilege -u $1
+editrights -a SeServiceLogonRight -u $1
+
+set -e
+
+function add_if_missing {
+    LINE="${1}"
+    FILE="${2}"
+    grep -Fx "${LINE}" "${FILE}" >/dev/null 2>&1 || echo "${LINE}" >> "${FILE}"
+}
+
+echo "  * Setting PS1 environment variable in .bashrc"
+add_if_missing 'export PS1='"'"'\[\033[01;35m\]\u\[\033[34m\]@\[\033[36m\]\h\[\033[00m\]:\[\033[01;33m\]\w\[\033[31m\] \$\[\033[00m\] '"'" ~/.bashrc
+echo "  * Making sure PATH gets sourced in .bashrc"
+add_if_missing 'export PATH="/cygdrive/c/Windows/system32:${PATH}"' ~/.bashrc
+echo "  * Making sure there is a pair of public/private keys"
+[ ! -e ~/.ssh/id_rsa ] || [ ! -e ~/.ssh/id_rsa.pub ] && ssh-keygen -b 1024 -t rsa -N '' -f ~/.ssh/id_rsa
+echo "  * Setting syntax highlighting in vi"
+add_if_missing 'syntax on' ~/.vimrc
+echo "  * Setting colour scheme in vi"
+add_if_missing 'colorscheme murphy' ~/.vimrc
+echo "  * Setting tab to four spaces in vi"
+add_if_missing 'set tabstop=4' ~/.vimrc
+[ ! -e /c ] && ln -s /cygdrive/c /c
+if [ ! -e ~/bin/sudo ]; then
+echo "  * Creating ""Sudo"" command"
+cat > /bin/sudo << 'EOF'
+#!/usr/bin/bash
+cygstart --action=runas "$@"
+EOF
+chmod a+x /bin/sudo
+fi
+echo "  * Installing apt-cyg"
+cd ~
+wget rawgit.com/transcode-open/apt-cyg/master/apt-cyg
+mv apt-cyg.* apt-get
+install apt-get /bin
+apt-get mirror http://mirrors.kernel.org/sourceware/cygwin/
+apt-get update
+
+echo " * Configuring the Cygwin git client to use the AWS codecommit helper"
+git config --global credential.helper "!aws codecommit credential-helper $@"
+git config --global credential.UseHttpPath true
+git config --global filter.lfs.required true
+git config --global filter.lfs.clean "git-lfs clean -- %f"
+git config --global filter.lfs.smudge "git-lfs smudge -- %f"
+git config --global filter.lfs.process "git-lfs filter-process"

scripts/build/build_node/Platform/Windows/vs2017bt.vsconfig

@@ -0,0 +1,28 @@
+{
+  "version": "1.0",
+  "components": [
+    "Microsoft.Component.MSBuild",
+    "Microsoft.Component.NetFX.Native",
+    "Microsoft.Component.VC.Runtime.OSSupport",
+    "Microsoft.Component.VC.Runtime.UCRTSDK",
+    "Microsoft.Net.Component.4.7.1.SDK",
+    "Microsoft.NetCore.Component.SDK",
+    "Microsoft.VisualStudio.Component.CoreBuildTools",
+    "Microsoft.VisualStudio.Component.NuGet.BuildTools",
+    "Microsoft.VisualStudio.Component.Roslyn.Compiler",
+    "Microsoft.VisualStudio.Component.Static.Analysis.Tools",
+    "Microsoft.VisualStudio.Component.TestTools.BuildTools",
+    "Microsoft.VisualStudio.Component.VC.ATL",
+    "Microsoft.VisualStudio.Component.VC.CMake.Project",
+    "Microsoft.VisualStudio.Component.VC.CoreBuildTools",
+    "Microsoft.VisualStudio.Component.VC.Redist.14.Latest",
+    "Microsoft.VisualStudio.Component.VC.Tools.x86.x64",
+    "Microsoft.VisualStudio.Component.Windows10SDK",
+    "Microsoft.VisualStudio.Component.Windows81SDK",
+    "Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Win81",
+    "Microsoft.VisualStudio.ComponentGroup.UWP.BuildTools",
+    "Microsoft.VisualStudio.Workload.MSBuildTools",
+    "Microsoft.VisualStudio.Workload.UniversalBuildTools",
+    "Microsoft.VisualStudio.Workload.VCTools"
+  ]
+}

scripts/build/build_node/Platform/Windows/vs2019bt.vsconfig

@@ -0,0 +1,30 @@
+{
+  "version": "1.0",
+  "components": [
+    "Microsoft.Component.MSBuild",
+    "Microsoft.Component.NetFX.Native",
+    "Microsoft.Net.Component.4.8.SDK",
+    "Microsoft.NetCore.Component.Runtime.3.1",
+    "Microsoft.NetCore.Component.Runtime.5.0",
+    "Microsoft.NetCore.Component.SDK",
+    "Microsoft.VisualStudio.Component.CoreBuildTools",
+    "Microsoft.VisualStudio.Component.NuGet.BuildTools",
+    "Microsoft.VisualStudio.Component.Roslyn.Compiler",
+    "Microsoft.VisualStudio.Component.TestTools.BuildTools",
+    "Microsoft.VisualStudio.Component.TextTemplating",
+    "Microsoft.VisualStudio.Component.VC.ASAN",
+    "Microsoft.VisualStudio.Component.VC.ATL",
+    "Microsoft.VisualStudio.Component.VC.CMake.Project",
+    "Microsoft.VisualStudio.Component.VC.CoreBuildTools",
+    "Microsoft.VisualStudio.Component.VC.CoreIde",
+    "Microsoft.VisualStudio.Component.VC.Redist.14.Latest",
+    "Microsoft.VisualStudio.Component.VC.Tools.x86.x64",
+    "Microsoft.VisualStudio.Component.Windows10SDK",
+    "Microsoft.VisualStudio.Component.Windows10SDK.18362",
+    "Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Core",
+    "Microsoft.VisualStudio.ComponentGroup.UWP.BuildTools",
+    "Microsoft.VisualStudio.Workload.MSBuildTools",
+    "Microsoft.VisualStudio.Workload.UniversalBuildTools",
+    "Microsoft.VisualStudio.Workload.VCTools"
+  ]
+}

scripts/build/lambda/auto_delete_ebs.py

@@ -0,0 +1,57 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import boto3
+import time
+import logging
+
+TIMEOUT = 300
+
+
+def lambda_handler(event, context):
+    log = logging.getLogger(__name__)
+    log.setLevel(logging.INFO)
+    branch_name = event['detail']['referenceName']
+    ec2_client = boto3.resource('ec2')
+    response = ec2_client.volumes.filter(Filters=[
+        {
+            'Name': 'tag:BranchName',
+            'Values': [branch_name]
+        }
+    ])
+    log.info(f'Deleting EBS volumes for remote-branch {branch_name}.')
+    for volume in response:
+        if volume.attachments:
+            ec2_instance_id = volume.attachments[0]['InstanceId']
+            try:
+                log.info(f'Detaching volume {volume.volume_id} from ec2 {ec2_instance_id}.')
+                volume.detach_from_instance(Device='xvdf',
+                                            Force=True,
+                                            InstanceId=ec2_instance_id,
+                                            VolumeId=volume.volume_id)
+            except Exception as e:
+                log.error(f'Failed to detach volume {volume.volume_id} from {ec2_instance_id}.')
+                log.error(e)
+            timeout_init = time.clock()
+            while len(volume.attachments) and volume.attachments[0]['State'] != 'detached':
+                time.sleep(1)
+                volume.load()
+                if (time.clock() - timeout_init) > TIMEOUT:
+                    log.error('Timeout reached trying to detach EBS.')
+        try:
+            log.info(f'Deleting volume {volume.volume_id}')
+            volume.delete()
+        except Exception as e:
+            log.error(f'Failed to delete volume {volume.volume_id}.')
+            log.error(e)
+
+
+lambda_handler(event, context)

scripts/build/lambda/trigger_first_build.py

@@ -0,0 +1,74 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import boto3
+import logging
+import json
+import os
+import requests
+
+from botocore.exceptions import ClientError
+from requests.adapters import HTTPAdapter
+from requests.packages.urllib3.util.retry import Retry
+from requests.exceptions import RetryError
+
+log = logging.getLogger(__name__)
+log.setLevel(logging.INFO)
+
+JENKINS_CREDENTIAL_STORE = os.environ['JENKINS_CREDENTIAL_STORE']
+JENKINS_ENDPOINT = os.environ['JENKINS_ENDPOINT']
+REGION = os.environ['REGION']
+
+
+def get_secret(secret_name):
+    secrets = boto3.client(service_name='secretsmanager', region_name=REGION)
+    try:
+        response = secrets.get_secret_value(SecretId=secret_name)
+        return json.loads(response['SecretString'])
+    except ClientError as e:
+        log.error(f'Unable to get secret value: {e}')
+        return None
+
+
+def lambda_handler(event, context):
+    event_record = event['Records'][0]
+    log.info(f'Received Event: {event_record}')
+
+    ref_prefix = "refs/heads/"
+    branch_name = event_record['codecommit']['references'][0]['ref'][len(ref_prefix):]
+    repo_name = event_record['eventSourceARN'].split(":")[-1]
+
+    credentials = get_secret(JENKINS_CREDENTIAL_STORE)
+
+    if credentials:
+        retries = 3
+        backoff = 30
+        status_list = [404]  # Retry if the branch doesn't exist yet and provide time for Jenkins to discover it.
+        method_list = ['POST']
+        retry_config = Retry(total=retries, backoff_factor=backoff, status_forcelist=status_list, method_whitelist=method_list)
+
+        session = requests.Session()
+        session.mount('https://', HTTPAdapter(max_retries=retry_config))
+
+        encoded_branch_name = requests.utils.quote(branch_name, safe='')
+        build_path = f'/job/{repo_name}/job/{encoded_branch_name}/build'
+        jenkins_url = requests.compat.urljoin(JENKINS_ENDPOINT, build_path)
+
+        try:
+            response = session.post(jenkins_url, auth=(credentials['username'], credentials['apitoken']))
+            if response.status_code == 201:
+                log.info(f'Successfully triggered build on {repo_name}/{branch_name}')
+            elif response.status_code == 400:
+                log.info(f'Initial build already started for {repo_name}/{branch_name}. Parameters are already available.')
+            else:
+                log.error(f'Failed to start build on {repo_name}/{branch_name}. Status code: {response.status_code}')
+        except RetryError as e:
+            log.error(f'Pipeline for {repo_name}/{branch_name} does not exist in Jenkins: {e}')

scripts/build/package/PackageEnv.py

@@ -0,0 +1,194 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+from Params import Params
+from util import *
+
+
+class PackageEnv(Params):
+    def __init__(self, platform, type, json_file):
+        super(PackageEnv, self).__init__()
+        self.__cur_dir = os.path.dirname(os.path.abspath(__file__))
+        global_env_file = os.path.join(self.__cur_dir, json_file)
+        with open(global_env_file, 'r') as source:
+            data = json.load(source)
+            self.__global_env = data.get('global_env')
+        platform_env_file = os.path.join(self.__cur_dir, 'Platform', platform, json_file)
+        if not os.path.exists(platform_env_file):
+            print(f'{platform_env_file} is not found.')
+            # Search restricted platform folders
+            engine_root = self.get('ENGINE_ROOT')
+            # Use real path in case engine root is a symlink path
+            if os.name == 'posix' and os.path.islink(engine_root):
+                engine_root = os.readlink(engine_root)
+            rel_path = os.path.relpath(self.__cur_dir, engine_root)
+            platform_env_file = os.path.join(engine_root, 'restricted', platform, rel_path, json_file)
+            if not os.path.exists(platform_env_file):
+                ly_build_error(f'{platform_env_file} is not found.')
+        with open(platform_env_file, 'r') as source:
+            data = json.load(source)
+            types = data.get('types')
+            if type not in types:
+                ly_build_error(f'Package type {type} is not supported')
+            self.__platform = platform
+            self.__platform_env = data.get('local_env')
+            self.__platform_env.update(self.__global_env)
+            self.__type = type
+            self.__type_env = types.get(type)
+
+    def get_platform(self):
+        return self.__platform
+
+    def get_type(self):
+        return self.__type
+
+    def get_platform_env(self):
+        return self.__platform_env
+
+    def get_type_env(self):
+        return self.__type_env
+
+    def __get_platform_value(self, key):
+        key = key.upper()
+        value = self.__platform_env.get(key)
+        if value is None:
+            ly_build_error(f'{key} is not defined in global env nor in local env')
+        return value
+
+    def __get_type_value(self, key):
+        key = key.upper()
+        value = self.__type_env.get(key)
+        if value is None:
+            ly_build_error(f'{key} is not defined in package type {self.__type} for platform {self.__platform}')
+        return value
+
+    def __evaluate_boolean(self, v):
+        return str(v).lower() in ['1', 'true']
+
+    def __get_engine_root(self):
+        def validate_engine_root(engine_root):
+            if not os.path.isdir(engine_root):
+                return False
+            return os.path.exists(os.path.join(engine_root, 'engineroot.txt'))
+
+        workspace = os.getenv('WORKSPACE')
+        if workspace is not None:
+            print(f'Environment variable WORKSPACE={workspace} detected')
+            if validate_engine_root(workspace):
+                print(f'Setting ENGINE_ROOT to {workspace}')
+                return workspace
+            print('Cannot locate ENGINE_ROOT with Environment variable WORKSPACE')
+
+        engine_root = os.getenv('ENGINE_ROOT', '')
+        if validate_engine_root(engine_root):
+            return engine_root
+
+        print('Environment variable ENGINE_ROOT is not set or invalid, checking ENGINE_ROOT in env json file')
+        engine_root = self.__global_env.get('ENGINE_ROOT')
+        if validate_engine_root(engine_root):
+            return engine_root
+
+        # Set engine_root based on script location
+        engine_root = os.path.dirname(os.path.dirname(os.path.dirname(self.__cur_dir)))
+        print(f'ENGINE_ROOT from env json file is invalid, defaulting to {engine_root}')
+        if validate_engine_root(engine_root):
+            return engine_root
+        else:
+            error('Cannot Locate ENGINE_ROOT')
+
+    def __get_thirdparty_home(self):
+        third_party_home = os.getenv('LY_3RDPARTY_PATH', '')
+        if os.path.exists(third_party_home):
+            print(f'LY_3RDPARTY_PATH found, using {third_party_home} as 3rdParty path.')
+            return third_party_home
+        third_party_home = self.__get_platform_value('THIRDPARTY_HOME')
+        if os.path.isdir(third_party_home):
+            return third_party_home
+
+        # Set engine_root based on script location
+        print('THIRDPARTY_HOME is not valid, looking for THIRD_PARTY_HOME')
+
+        # Finding THIRD_PARTY_HOME
+        cur_dir = self.__get_engine_root()
+        last_dir = None
+        while last_dir != cur_dir:
+            third_party_home = os.path.join(cur_dir, '3rdParty')
+            print(f'Cheking THIRDPARTY_HOME {third_party_home}')
+            if os.path.exists(os.path.join(third_party_home, '3rdParty.txt')):
+                print(f'Setting THIRDPARTY_HOME to {third_party_home}')
+                return third_party_home
+            last_dir = cur_dir
+            cur_dir = os.path.dirname(cur_dir)
+        error('Cannot locate THIRDPARTY_HOME')
+
+    def __get_package_name_pattern(self):
+        package_name_pattern = self.__get_platform_value('PACKAGE_NAME_PATTERN')
+        if os.getenv('PACKAGE_NAME_PATTERN') is not None:
+            package_name_pattern = os.getenv('PACKAGE_NAME_PATTERN')
+        return package_name_pattern
+
+    def __get_branch_name(self):
+        branch_name = self.__get_platform_value('BRANCH_NAME')
+        if os.getenv('BRANCH_NAME') is not None:
+            branch_name = os.getenv('BRANCH_NAME')
+        branch_name = branch_name.replace('/', '_').replace('\\', '_')
+        return branch_name
+
+    def __get_build_number(self):
+        build_number = self.__get_platform_value('BUILD_NUMBER')
+        if os.getenv('BUILD_NUMBER') is not None:
+            build_number = os.getenv('BUILD_NUMBER')
+        return build_number
+
+    def __get_scrub_params(self):
+        return self.__get_type_value('SCRUB_PARAMS')
+
+    def __get_validator_platforms(self):
+        return self.__get_type_value('VALIDATOR_PLATFORMS')
+
+    def __get_package_targets(self):
+        return self.__get_type_value('PACKAGE_TARGETS')
+
+    def __get_build_targets(self):
+        return self.__get_type_value('BUILD_TARGETS')
+
+    def __get_asset_processor_path(self):
+        return self.__get_type_value('ASSET_PROCESSOR_PATH')
+
+    def __get_asset_game_folders(self):
+        return self.__get_type_value('ASSET_GAME_FOLDERS')
+
+    def __get_asset_platform(self):
+        return self.__get_type_value('ASSET_PLATFORM')
+
+    def __get_bootstrap_cfg_game_folder(self):
+        return self.__get_type_value('BOOTSTRAP_CFG_GAME_FOLDER')
+
+    def __get_skip_build(self):
+        skip_build = os.getenv('SKIP_BUILD')
+        if skip_build is None:
+            skip_build = self.__get_type_value('SKIP_BUILD')
+        return self.__evaluate_boolean(skip_build)
+
+    def __get_skip_scrubbing(self):
+        skip_scrubbing = os.getenv('SKIP_SCRUBBING')
+        if skip_scrubbing is None:
+            skip_scrubbing = self.__type_env.get('SKIP_SCRUBBING', 'False')
+        return self.__evaluate_boolean(skip_scrubbing)
+
+    def __get_internal_s3_bucket(self):
+        return self.__get_platform_value('INTERNAL_S3_BUCKET')
+
+    def __get_qa_s3_bucket(self):
+        return self.__get_platform_value('QA_S3_BUCKET')
+
+    def __get_s3_prefix(self):
+        return self.__get_platform_value('S3_PREFIX')

scripts/build/package/Params.py

@@ -0,0 +1,83 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+import os
+import sys
+import re
+from util import ly_build_error
+
+
+class Params(object):
+    def __init__(self):
+        # Cache params
+        self.__params = {}
+
+    def get(self, param_name):
+        param_value = self.__params.get(param_name)
+        if param_value is not None:
+            return param_value
+        # Call __get_${param_name} function
+        func = getattr(self, '_{}__get_{}'.format(self.__class__.__name__, param_name.lower()), None)
+        if func is not None:
+            param_value = func()
+            # Replace all ${env} in value
+            if isinstance(param_value, str):
+                param_value = self.__process_string(param_name, param_value)
+            elif isinstance(param_value, list):
+                param_value = self.__process_list(param_name, param_value)
+            elif isinstance(param_value, dict):
+                param_value = self.__process_dict(param_name, param_value)
+            # Cache param
+            self.__params[param_name] = param_value
+            return param_value
+        ly_build_error('method __get_{} is not defined in class {}'.format(param_name.lower(), self.__class__.__name__))
+
+    def set(self, param_name, param_value):
+        self.__params[param_name] = param_value
+
+    def exists(self, param_name):
+        try:
+            self.get(param_name)
+        except LyBuildError:
+            return False
+        return True
+
+    def __process_string(self, param_name, param_value):
+        # Find all param with format ${param}
+        params = re.findall('\${(\w+)}', param_value)
+        # Avoid using the same param name in value, like 'WORKSPACE': '${WORKSPACE} some string'
+        if param_name in params:
+            ly_build_error('The use of same parameter name({}) in value is not allowed'.format(param_name))
+        # Replace ${param} with actual value
+        for param in params:
+            param_value = param_value.replace('${' + param + '}', self.get(param))
+        return param_value
+
+    def __process_list(self, param_name, param_value):
+        processed_list = []
+        for entry in param_value:
+            if isinstance(entry, str):
+                entry = self.__process_string(param_name, entry)
+            elif isinstance(entry, list):
+                entry = self.__process_list(param_name, entry)
+            elif isinstance(entry, dict):
+                entry = self.__process_dict(param_name, entry)
+            processed_list.append(entry)
+        return processed_list
+
+    def __process_dict(self, param_name, param_value):
+        for key in param_value:
+            if isinstance(param_value[key], str):
+                param_value[key] = self.__process_string(param_name, param_value[key])
+            elif isinstance(param_value[key], list):
+                param_value[key] = self.__process_list(param_name, param_value[key])
+            elif isinstance(param_value[key], dict):
+                param_value[key] = self.__process_dict(param_name, param_value[key])
+        return param_value

scripts/build/package/Platform/3rdParty/package_env.json

@@ -0,0 +1,19 @@
+{
+    "local_env": {
+        "S3_PREFIX": "${BRANCH_NAME}/3rdParty"
+    },
+    "types": {
+        "3rdParty_all": {
+            "PACKAGE_TARGETS":[
+                {
+                    "FILE_LIST": "3rdParty.json",
+                    "FILE_LIST_TYPE": "3rdParty",
+                    "PACKAGE_NAME": "${PACKAGE_NAME_PATTERN}-3rdParty-all-${BUILD_NUMBER}.zip"
+                }
+            ],
+            "BOOTSTRAP_CFG_GAME_FOLDER":"CMakeTestbed",
+            "SKIP_BUILD": 1,
+            "SKIP_SCRUBBING": 1
+        }
+    }
+}

scripts/build/package/Platform/3rdParty/package_filelists/3rdParty.json

@@ -0,0 +1,45 @@
+{
+    "@3rdParty": {
+        "3rdParty.txt": "#include",
+        "AWS/AWSNativeSDK/1.7.167-az.2/**": "#include",
+        "AWS/GameLift/3.4.0/**": "#include",
+        "Blast/1.1.6-az.1/**": "#include",
+        "benchmark/1.5.0/**": "#include",
+        "cityhash/1.1-az.1/**": "#include",
+        "civetweb/civetweb-20160922-az.2/**": "#include",
+        "Clang/6.0.1-az/**": "#include",
+        "DirectXShaderCompiler/1.0.1-az.1/**": "#include",
+        "dyad/0.2.0-17-amazon/**": "#include",
+        "etc2comp/2017_04_24-az.2/**": "#include",
+        "expat/2.1.0-pkg.3/**": "#include",
+        "FbxSdk/2016.1.2-az.1/**": "#include",
+        "glad/2.0.0-beta/**": "#include",
+        "googletest/1.8.1-az.3/**": "#include",
+        "jsmn/78b1dca/**": "#include",
+        "libav/11.7/**": "#include",
+        "LibTomCrypt/1.17-az.2/**": "#include",
+        "LibTomMath/0.42.0-az.2/**": "#include",
+        "lz4/r128-pkg.3/**": "#include",
+        "Lzma/unknown-pkg.3/**": "#include",
+        "LZSS/unknown-pkg.3/**": "#include",
+        "md5/2.0-pkg.3/**": "#include",
+        "mikkelsen/1.0.0-az.2/**": "#include",
+        "nvapi/R361-developer_V2/**": "#include",
+        "NvCloth/1.1.6-az.4/**": "#include",
+        "OpenSSL/1.1.1b-noasm-az/**": "#include",
+        "p4api/2019.1/**": "#include",
+        "PhysX/4.1.0.25992954-az.1/**": "#include",
+        "poly2tri/0.3.3-az.2/**": "#include",
+        "PVRTexTool/2016_r1.1/**": "#include",
+        "Qt/5.15.1.2-az/**": "#include",
+        "RadTelemetry/3.5.0.17/**": "#include",
+        "rapidjson/rapidjson-1.1.0/**": "#include",
+        "rapidxml/1.13-modified.1/**": "#include",
+        "SQLite/v3.32.2/**": "#include",
+        "tiff/3.9.5-az.3/**": "#include",
+        "unwind/1.2.1/**": "#include",
+        "Wwise/2019.2.8.7432/**": "#include",
+        "xxhash/0.7.4/**": "#include",
+        "zstd/1.35-pkg.1/**": "#include"
+    }
+}

scripts/build/package/Platform/Mac/package_env.json

@@ -0,0 +1,28 @@
+{
+    "local_env": {},
+    "types":{
+        "all":{
+            "PACKAGE_TARGETS":[
+                {
+                    "FILE_LIST": "all.json",
+                    "FILE_LIST_TYPE": "All",
+                    "PACKAGE_NAME": "${PACKAGE_NAME_PATTERN}-mac-all-${BUILD_NUMBER}.zip"
+                }
+            ],
+            "BOOTSTRAP_CFG_GAME_FOLDER":"CMakeTestbed",
+            "SKIP_BUILD": 0,
+            "BUILD_TARGETS":[
+                {
+                    "BUILD_CONFIG_FILENAME": "build_config.json",
+                    "PLATFORM": "Mac",
+                    "TYPE": "profile"
+                },
+                {
+                    "BUILD_CONFIG_FILENAME": "build_config.json",
+                    "PLATFORM": "iOS",
+                    "TYPE": "profile"
+                }
+            ]
+        }
+    }
+}

scripts/build/package/Platform/Windows/package_env.json

@@ -0,0 +1,63 @@
+{
+    "local_env": {
+        "S3_PREFIX": "${BRANCH_NAME}/Windows"
+    },
+    "types":{
+        "all":{
+            "PACKAGE_TARGETS":[
+                {
+                    "FILE_LIST": "all.json",
+                    "FILE_LIST_TYPE": "All",
+                    "PACKAGE_NAME": "${PACKAGE_NAME_PATTERN}-windows-all-${BUILD_NUMBER}.zip"
+                },
+                {
+                    "FILE_LIST": "symbols.json",
+                    "FILE_LIST_TYPE": "All",
+                    "PACKAGE_NAME": "${PACKAGE_NAME_PATTERN}-windows-all-symbols-${BUILD_NUMBER}.zip"
+                }
+            ],
+            "BOOTSTRAP_CFG_GAME_FOLDER":"CMakeTestbed",
+            "SKIP_BUILD": 0,
+            "BUILD_TARGETS":[
+                {
+                    "BUILD_CONFIG_FILENAME": "build_config.json",
+                    "PLATFORM": "Windows",
+                    "TYPE": "profile_vs2017"
+                },
+                {
+                    "BUILD_CONFIG_FILENAME": "build_config.json",
+                    "PLATFORM": "Windows",
+                    "TYPE": "profile_vs2019"
+                }
+            ]
+        },
+        "atom":{
+            "PACKAGE_TARGETS":[
+                {
+                    "FILE_LIST": "atom.json",
+                    "FILE_LIST_TYPE": "Windows",
+                    "PACKAGE_NAME": "${PACKAGE_NAME_PATTERN}-windows-atom-${BUILD_NUMBER}.zip"
+                },
+                {
+                    "FILE_LIST": "symbols.json",
+                    "FILE_LIST_TYPE": "All",
+                    "PACKAGE_NAME": "${PACKAGE_NAME_PATTERN}-windows-atom-symbols-${BUILD_NUMBER}.zip"
+                }
+            ],
+            "BOOTSTRAP_CFG_GAME_FOLDER":"AtomSampleViewer;AtomTest",
+            "SKIP_BUILD": 1,
+            "BUILD_TARGETS":[
+                {
+                    "BUILD_CONFIG_FILENAME": "build_config.json",
+                    "PLATFORM": "Windows",
+                    "TYPE": "profile_vs2017_atom"
+                },
+                {
+                    "BUILD_CONFIG_FILENAME": "build_config.json",
+                    "PLATFORM": "Windows",
+                    "TYPE": "profile_vs2019_atom"
+                }
+            ]
+        }
+    }
+}

scripts/build/package/Platform/Windows/package_filelist/atom.json

@@ -0,0 +1,198 @@
+{
+    "@3rdParty": {
+        "**/.owner": "#exclude",
+        "3rdParty.txt": "#move:3rdParty",
+        "OpenEXR/**": "#move:3rdParty",
+        "CMake/3.19.1/**": "#move:3rdParty",
+        "Redistributables":{
+            "WwiseLTX": {
+                "LTX_2018.1.2.6762": {
+                    "**": "#move:dev/Tools/Redistributables/WwiseLTX/LTX_2018.1.2.6762",
+                    "*.app.zip": "#exclude",
+                    "WwiseLauncher.pkg": "#exclude"
+                }
+            },
+            "FbxSdk": {
+                "2016.1.2": {
+                    "*win*": "#move:dev/Tools/Redistributables/FbxSdk/2016.1.2-az.1",
+                    "*vs2013*": "#exclude"
+                }
+            }
+        }
+    },
+    "@lyengine": {
+        "**/*.pyc": "#exclude",
+        "*": "#include",
+        "AtomTest":
+        {
+            "**":"#include",
+            "**/*.ma":"#exclude",
+            "**/*.max":"#exclude",
+            "**/*.mb":"#exclude",
+            "**/*.psd":"#exclude"
+        },
+        "AtomSampleViewer":
+        {
+            "**":"#include",
+            "**/*.ma":"#exclude",
+            "**/*.max":"#exclude",
+            "**/*.mb":"#exclude",
+            "**/*.psd":"#exclude"
+        },
+        "cmake/**": "#include",
+        "Code": {
+            "CryEngine/**": "#include",
+            "Framework/**": "#include",
+            "LauncherUnified/**": "#include",
+            "Sandbox/**": "#include",
+            "Tools": {
+                "Android/**": "#include",
+                "AWSNativeSDKInit/**": "#include",
+                "AssetProcessor*/**": "#include",
+                "AssetBundler/**": "#include",
+                "AzTestRunner/**": "#include",
+                "CrashHandler/**": "#include",
+                "CryCommonTools/**": "#include",
+                "CrySCompileServer/**": "#include",
+                "CryXML/**": "#include",
+                "DeltaCataloger/**": "#include",
+                "GemRegistry/**": "#include",
+                "GridHub/**": "#include",
+                "HLSLCrossCompiler/**": "#include",
+                "HLSLCrossCompilerMETAL/**": "#include",
+                "LyIdentity/**": "#include",
+                "LyMetrics/**": "#include",
+                "News/**": "#include",
+                "PythonBindingsExample/**": "#include",
+                "RC/**": "#include",
+                "RemoteConsole/**": "#include",
+                "SceneAPI/**": "#include",
+                "SerializeContextTools/**": "#include",
+                "ShaderCacheGen/**": "#include",
+                "SharedQMLResource/**": "#include",
+                "Woodpecker/**": "#include",
+                "CMakeLists.txt": "#include"
+            },
+            "CMakeLists.txt": "#include"
+        },
+        "ctest_scripts/**": "#include",
+        "Editor/**": "#include",
+        "Engine/**": "#include",
+        "Gems": {
+            "Achievements": "#include",
+            "AssetMemoryAnalyzer": "#include",
+            "AssetValidation": "#include",
+            "Atom": "#include",
+            "AtomLyIntegration": "#include",
+            "AudioEngineWwise": "#include",
+            "AudioSystem": "#include",
+            "AutomatedLauncherTesting": "#include",
+            "Blast": "#include",
+            "Camera": "#include",
+            "CameraFramework": "#include",
+            "CertificateManager": "#include",
+            "ChatPlay": "#include",
+            "Clouds": "#include",
+            "CrashReporting": "#include",
+            "CustomAssetExample": "#include",
+            "DebugDraw": "#include",
+            "EditorPythonBindings": "#include",
+            "EMotionFX": "#include",
+            "ExpressionEvaluation": "#include",
+            "FastNoise": "#include",
+            "GameEffectSystem": "#include",
+            "GameLift": "#include",
+            "GameState": "#include",
+            "GameStateSamples": "#include",
+            "Gestures": "#include",
+            "GradientSignal": "#include",
+            "GraphCanvas": "#include",
+            "GraphModel": "#include",
+            "HttpRequestor": "#include",
+            "ImageProcessing": "#include",
+            "ImGui": "#include",
+            "InAppPurchases": "#include",
+            "LandscapeCanvas": "#include",
+            "LegacyTerrain": "#include",
+            "LmbrCentral": "#include",
+            "LocalUser": "#include",
+            "LyShine": "#include",
+            "LyShineExamples": "#include",
+            "Maestro": "#include",
+            "MessagePopup": "#include",
+            "Metastream": "#include",
+            "Microphone": "#include",
+            "Multiplayer": "#include",
+            "MultiplayerImGui": "#include",
+            "NvCloth": "#include",
+            "PhysX": "#include",
+            "PhysXDebug": "#include",
+            "Presence": "#include",
+            "QtForPython": "#include",
+            "RADTelemetry": "#include",
+            "RenderToTexture": "#include",
+            "SaveData": "#include",
+            "SceneLoggingExample": "#include",
+            "SceneProcessing": "#include",
+            "ScriptCanvas": "#include",
+            "ScriptCanvasDeveloper": "#include",
+            "ScriptCanvasDiagnosticLibrary": "#include",
+            "ScriptCanvasPhysics": "#include",
+            "ScriptCanvasTesting": "#include",
+            "ScriptedEntityTweener": "#include",
+            "ScriptEvents": "#include",
+            "SliceFavorites": "#include",
+            "StartingPointCamera": "#include",
+            "StartingPointInput": "#include",
+            "StartingPointMovement": "#include",
+            "Substance": "#include",
+            "SurfaceData": "#include",
+            "SVOGI": "#include",
+            "TestAssetBuilder": "#include",
+            "TextureAtlas": "#include",
+            "TickBusOrderViewer": "#include",
+            "TouchBending": "#include",
+            "Twitch": "#include",
+            "Vegetation": "#include",
+            "VideoPlayback": "#include",
+            "VideoPlaybackBink": "#include",
+            "VideoPlaybackFramework": "#include",
+            "VirtualGamepad": "#include",
+            "Visibility": "#include",
+            "Water": "#include",
+            "WhiteBox": "#include",
+            "CMakeLists.txt": "#include"
+        },
+        "Tools": {
+            "3dsmax/**": "#include",
+            "7za.exe": "#include",
+            "7za_legal_notice.txt": "#include",
+            "AWSNativeSDK": {
+                "**": "#include",
+                "Upgrader/restricted_platforms.py": "#exclude"
+            },
+            "AWSPythonSDK/**": "#include",
+            "Crashpad/**": "#include",
+            "CrySCompileServer/**": "#include",
+            "PakShaders/**": "#include",
+            "Python/**": "#include",
+            "Redistributables": {
+                "**": "#include",
+                "ANGLE/**": "#exclude",
+                "D3DCompiler/**": "#exclude",
+                "DbgHelp/**": "#exclude",
+                "FFMpeg/**": "#exclude",
+                "LuaCompiler/**": "#exclude",
+                "MSVC90/**": "#exclude",
+                "OpenGL32/**": "#exclude",
+                "SSLEAY/**": "#exclude"
+            },
+            "RemoteConsole/**": "#include",
+            "__init__.py": "#include",
+            "lmbr_aws/**": "#include",
+            "maxscript/**": "#include",
+            "maya/**": "#include",
+            "photoshop/**": "#include"
+        }
+    }
+}

scripts/build/package/glob3.py

@@ -0,0 +1,166 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import os
+import re
+import fnmatch
+
+__all__ = ["glob", "iglob", "escape"]
+
+def glob(pathname, recursive=False):
+    """Return a list of paths matching a pathname pattern.
+    The pattern may contain simple shell-style wildcards a la
+    fnmatch. However, unlike fnmatch, filenames starting with a
+    dot are special cases that are not matched by '*' and '?'
+    patterns.
+    If recursive is true, the pattern '**' will match any files and
+    zero or more directories and subdirectories.
+    """
+    return list(iglob(pathname, recursive=recursive))
+
+def iglob(pathname, recursive=False):
+    """Return an iterator which yields the paths matching a pathname pattern.
+    The pattern may contain simple shell-style wildcards a la
+    fnmatch. However, unlike fnmatch, filenames starting with a
+    dot are special cases that are not matched by '*' and '?'
+    patterns.
+    If recursive is true, the pattern '**' will match any files and
+    zero or more directories and subdirectories.
+    """
+    it = _iglob(pathname, recursive, False)
+    if recursive and _isrecursive(pathname):
+        s = next(it)  # skip empty string
+        assert not s
+    return it
+
+def _iglob(pathname, recursive, dironly):
+    dirname, basename = os.path.split(pathname)
+    if not has_magic(pathname):
+        assert not dironly
+        if basename:
+            if os.path.lexists(pathname):
+                yield pathname
+        else:
+            # Patterns ending with a slash should match only directories
+            if os.path.isdir(dirname):
+                yield pathname
+        return
+    if not dirname:
+        if recursive and _isrecursive(basename):
+            yield _glob2(dirname, basename, dironly)
+        else:
+            yield _glob1(dirname, basename, dironly)
+        return
+    # `os.path.split()` returns the argument itself as a dirname if it is a
+    # drive or UNC path.  Prevent an infinite recursion if a drive or UNC path
+    # contains magic characters (i.e. r'\\?\C:').
+    if dirname != pathname and has_magic(dirname):
+        dirs = _iglob(dirname, recursive, True)
+    else:
+        dirs = [dirname]
+    if has_magic(basename):
+        if recursive and _isrecursive(basename):
+            glob_in_dir = _glob2
+        else:
+            glob_in_dir = _glob1
+    else:
+        glob_in_dir = _glob0
+    for dirname in dirs:
+        for name in glob_in_dir(dirname, basename, dironly):
+            yield os.path.join(dirname, name)
+
+# These 2 helper functions non-recursively glob inside a literal directory.
+# They return a list of basenames.  _glob1 accepts a pattern while _glob0
+# takes a literal basename (so it only has to check for its existence).
+
+def _glob1(dirname, pattern, dironly):
+    names = list(_iterdir(dirname, dironly))
+    return fnmatch.filter(names, pattern)
+
+def _glob0(dirname, basename, dironly):
+    if not basename:
+        # `os.path.split()` returns an empty basename for paths ending with a
+        # directory separator.  'q*x/' should match only directories.
+        if os.path.isdir(dirname):
+            return [basename]
+    else:
+        if os.path.lexists(os.path.join(dirname, basename)):
+            return [basename]
+    return []
+
+# Following functions are not public but can be used by third-party code.
+
+def glob0(dirname, pattern):
+    return _glob0(dirname, pattern, False)
+
+def glob1(dirname, pattern):
+    return _glob1(dirname, pattern, False)
+
+# This helper function recursively yields relative pathnames inside a literal
+# directory.
+
+def _glob2(dirname, pattern, dironly):
+    assert _isrecursive(pattern)
+    return [pattern[:0]] + list(_rlistdir(dirname, dironly))
+
+# If dironly is false, yields all file names inside a directory.
+# If dironly is true, yields only directory names.
+def _iterdir(dirname, dironly):
+    if not dirname:
+        if isinstance(dirname, bytes):
+            dirname = bytes(os.curdir, 'ASCII')
+        else:
+            dirname = os.curdir
+    try:
+        for entry in os.listdir(dirname):
+            yield entry
+    except OSError:
+        return
+
+# Recursively yields relative pathnames inside a literal directory.
+def _rlistdir(dirname, dironly):
+    if not os.path.islink(dirname):
+        names = list(_iterdir(dirname, dironly))
+        for x in names:
+            yield x
+            path = os.path.join(dirname, x) if dirname else x
+            for y in _rlistdir(path, dironly):
+                yield os.path.join(x, y)
+magic_check = re.compile('([*?[])')
+magic_check_bytes = re.compile(b'([*?[])')
+
+def has_magic(s):
+    if isinstance(s, bytes):
+        match = magic_check_bytes.search(s)
+    else:
+        match = magic_check.search(s)
+    return match is not None
+
+def _ishidden(path):
+    return path[0] in ('.', b'.'[0])
+
+def _isrecursive(pattern):
+    if isinstance(pattern, bytes):
+        return pattern == b'**'
+    else:
+        return pattern == '**'
+
+def escape(pathname):
+    """Escape all special characters.
+    """
+    # Escaping is done by wrapping any of "*?[" between square brackets.
+    # Metacharacters do not work in the drive part and shouldn't be escaped.
+    drive, pathname = os.path.splitdrive(pathname)
+    if isinstance(pathname, bytes):
+        pathname = magic_check_bytes.sub(br'[\1]', pathname)
+    else:
+        pathname = magic_check.sub(r'[\1]', pathname)
+    return drive + pathname

scripts/build/package/glob_to_regex.py

@@ -0,0 +1,127 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+from __future__ import absolute_import
+import os
+import re
+import json
+import sys
+try:
+    import six
+except ImportError:
+    import pip
+    pip.main(['install', 'six', '--ignore-installed', '-q'])
+    import six
+from pathlib import Path
+
+this_file_path = os.path.dirname(os.path.realpath(__file__))
+    
+def convert_glob_pattern_to_regex_pattern(glob_pattern):
+    # switch to forward slashes because way easier to pattern match against
+    pattern = re.sub(r'\\', r'/', glob_pattern)
+
+    # Replace the dots and question marks
+    pattern = re.sub(r'\.', r'\\.', pattern)
+    pattern = re.sub(r'\?', r'.', pattern)
+
+    # Handle the * vs ** expansions
+    pattern = re.sub(r'([^*])\*($|[^*])', r'\1[^/\\\\]*\2', pattern)
+    pattern = re.sub(r'\*\*/', r'(.*/)?', pattern)
+    pattern = re.sub(r'\*\*', r'.*', pattern)
+
+    # replace the forward slashes with [/\\] so it works on PC/unix
+    pattern = re.sub(r'([^^])/', r'\1[/\\\\]', pattern)
+    return pattern
+
+# Convert the package json into a pair of regexes we can use to look for includes and excludes
+def convert_glob_list_to_regex_list(filelist, prefix):
+    includes = []
+    excludes = []
+    for key, value in six.iteritems(filelist):
+        glob_pattern = os.path.join(prefix, key)
+        if isinstance(value, dict):
+            (sub_includes, sub_excludes) = convert_glob_list_to_regex_list(value, glob_pattern)
+            includes.extend(sub_includes)
+            excludes.extend(sub_excludes)
+        else:
+            # Simulate what glob would do with file walking to scope the * within a directory
+            # and ** across directories
+            regex_pattern = convert_glob_pattern_to_regex_pattern(os.path.normpath(glob_pattern))
+
+            # Deal with the commands. include/exclude are straight forward. Moves/renames are to be considered
+            # includes, and we will stick with validating the original contents for now
+            if value == "#include":
+                includes.append(regex_pattern)
+            elif value == "#exclude":
+                excludes.append(regex_pattern)
+            elif value.startswith('#move:'):
+                includes.append(regex_pattern)
+            elif value.startswith('#rename:'):
+                includes.append(regex_pattern)
+            else:
+                pass
+    return (includes, excludes)
+
+def generate_excludes_for_platform(root, platform):
+    if platform == 'all':
+        platform_exclusions_filename = os.path.join(this_file_path, 'platform_exclusions.json')
+        with open(platform_exclusions_filename, 'r') as platform_exclusions_file:
+            platform_exclusions = json.load(platform_exclusions_file)
+    else:
+        # Use real path in case engine root is a symlink path
+        if os.name == 'posix' and os.path.islink(root):
+            root = os.readlink(root)
+        cur_dir = os.path.dirname(os.path.abspath(__file__))
+        relative_folder = os.path.relpath(cur_dir, root)
+        platform_exclusions_filename = os.path.join(root, 'restricted', platform, relative_folder, platform.lower() + '_exclusions.json')
+        with open(platform_exclusions_filename, 'r') as platform_exclusions_file:
+            platform_exclusions = json.load(platform_exclusions_file)
+        
+    if platform not in platform_exclusions:
+        raise KeyError('No {} found in {}'.format(platform, platform_exclusions_filename))
+    if '@lyengine' not in platform_exclusions[platform]:
+        raise KeyError('No {}/@lyengine found in {}'.format(platform, package_file_list))
+    (_, excludes) = convert_glob_list_to_regex_list(platform_exclusions[platform]['@lyengine'], root)
+    del _
+    return excludes
+
+def generate_include_exclude_regexes(package_platform, package_type, root, prohibited_platforms):
+    # The general contents will be indicated by the package file
+    if package_type == 'all':
+        package_file_list = os.path.join(this_file_path, 'package_filelists', 'all.json')
+    else:
+        # Search non-restricted platform first
+        package_file_list = os.path.join(this_file_path, 'Platform', package_platform, 'package_filelists', f'{package_type}.json')
+        if not os.path.exists(filelist):
+            # Use real path in case engine root is a symlink path
+            if os.name == 'posix' and os.path.islink(root):
+                root = os.readlink(root)
+            rel_path = os.path.relpath(cur_dir, root)
+            package_file_list = os.path.join(root, 'restricted', package_platform, rel_path, 'package_filelists',
+                                    f'{package_type}.json')
+    with open(package_file_list, 'r') as package_file:
+        package = json.load(package_file)
+
+    if '@lyengine' not in package:
+        raise KeyError('No @lyengine found in {}'.format(package_file_list))
+
+    (includes_list, excludes_list) = convert_glob_list_to_regex_list(package['@lyengine'], root)
+    prohibited_platforms.append('all')
+
+    # Add the exclusions of each prohibited platform
+    for p in prohibited_platforms:
+        excludes_list.extend(generate_excludes_for_platform(root, p))
+
+    includes = re.compile('|'.join(includes_list), re.IGNORECASE)
+    excludes = re.compile('|'.join(excludes_list), re.IGNORECASE)
+    return (includes, excludes)
+
+def generate_exclude_regexes_for_platform(root, platform):
+    return re.compile('|'.join(generate_excludes_for_platform(root, platform)), re.IGNORECASE)

scripts/build/package/package.py

@@ -0,0 +1,302 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+import os
+import sys
+import glob_to_regex
+import zipfile
+import timeit
+import stat
+import progressbar
+from optparse import OptionParser
+from PackageEnv import PackageEnv
+cur_dir = cur_dir = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, f'{cur_dir}/../../../Tools/build/JenkinsScripts/build')
+from ci_build import build
+from util import *
+from glob3 import glob
+
+
+def package(options):
+    package_env = PackageEnv(options.platform, options.type, options.package_env)
+    engine_root = package_env.get('ENGINE_ROOT')
+
+    if not package_env.get('SKIP_SCRUBBING'):
+        # Ask the validator code to tell us which files need to be removed from the package
+        prohibited_file_mask = get_prohibited_file_mask(options.platform, engine_root)
+
+        # Scrub files. This is destructive, but is necessary to allow the current file existance checks to work properly. Better to copy and then build, or to
+        # mask on sync, but this is what we have for now
+        scrub_files(package_env, prohibited_file_mask)
+
+        # validate files
+        validate_restricted_files(options.platform, options.type, package_env)
+
+    # Override values in bootstrap.cfg for PC package
+    override_bootstrap_cfg(package_env)
+
+    if not package_env.get('SKIP_BUILD'):
+        print(package_env.get('SKIP_BUILD'))
+        print('SKIP_BUILD is False, running CMake build...')
+        cmake_build(package_env)
+
+    # TODO Compile Assets
+    #if package_env.exists('ASSET_PROCESSOR_PATH'):
+    #    compile_assets(package_env)
+
+    #create packages
+    package_targets = package_env.get('PACKAGE_TARGETS')
+    for package_target in package_targets:
+        create_package(package_env, package_target)
+        upload_package(package_env, package_target)
+
+
+def get_python_path(package_env):
+    if sys.platform == 'win32':
+        return os.path.join(package_env.get('ENGINE_ROOT'), 'python', 'python.cmd')
+    else:
+        return os.path.join(package_env.get('ENGINE_ROOT'), 'python', 'python.sh')
+
+
+def override_bootstrap_cfg(package_env):
+    print('Override values in bootstrap.cfg')
+    engine_root = package_env.get('ENGINE_ROOT')
+    bootstrap_path = os.path.join(engine_root, 'bootstrap.cfg')
+    replace_values = {'sys_game_folder':'{}'.format(package_env.get('BOOTSTRAP_CFG_GAME_FOLDER'))}
+    try:
+        with open(bootstrap_path, 'r') as bootstrap_cfg:
+            content = bootstrap_cfg.read()
+    except:
+        error('Cannot read file {}'.format(bootstrap_path))
+    content = content.split('\n')
+    new_content = []
+    for line in content:
+        if not line.startswith('--'):
+            strs = line.split('=')
+            if len(strs):
+                key = strs[0].strip(' ')
+                if key in replace_values:
+                    line = '{}={}'.format(key, replace_values[key])
+        new_content.append(line)
+    try:
+        with open(bootstrap_path, 'w') as out:
+            out.write('\n'.join(new_content))
+    except:
+        error('Cannot write to file {}'.format(bootstrap_path))
+    print('{} updated with value {}'.format(bootstrap_path, replace_values))
+
+
+def get_prohibited_file_mask(platform, engine_root):
+    sys.path.append(os.path.join(engine_root, 'Tools', 'build', 'JenkinsScripts', 'distribution', 'scrubbing'))
+    from validator_data_LEGAL_REVIEW_REQUIRED import get_prohibited_platforms_for_package
+
+    # The list of prohibited platforms is controlled by the validator on a per-package basis
+    prohibited_platforms = get_prohibited_platforms_for_package(platform)
+    prohibited_platforms.append('all')
+    excludes_list = []
+    for p in prohibited_platforms:
+        platform_excludes = glob_to_regex.generate_excludes_for_platform(engine_root, p)
+        excludes_list.extend(platform_excludes)
+    prohibited_file_mask = re.compile('|'.join(excludes_list), re.IGNORECASE)
+    return prohibited_file_mask
+
+
+def scrub_files(package_env, prohibited_file_mask):
+    print('Perform the Code Scrubbing')
+    engine_root = package_env.get('ENGINE_ROOT')
+
+    success = True
+    for dirname, subFolders, files in os.walk(engine_root):
+        for filename in files:
+            full_path = os.path.join(dirname, filename)
+            if prohibited_file_mask.match(full_path):
+                try:
+                    print('Deleting: {}'.format(full_path))
+                    os.chmod(full_path, stat.S_IWRITE)
+                    os.unlink(full_path)
+                except:
+                    e = sys.exc_info()[0]
+                    sys.stderr.write('Error: could not delete {} ... aborting.\n'.format(full_path))
+                    sys.stderr.write('{}\n'.format(str(e)))
+                    success = False
+    if not success:
+        sys.stderr.write('ERROR: scrub_files failed\n')
+        sys.exit(1)
+
+
+def validate_restricted_files(package_platform, package_type, package_env):
+    print('Perform the Code Scrubbing')
+    engine_root = package_env.get('ENGINE_ROOT')
+
+    # Run validator
+    success = True
+    validator_path = os.path.join(engine_root, 'Tools/build/JenkinsScripts/distribution/scrubbing/validator.py')
+    python = get_python_path(package_env)
+    args = [python, validator_path, '--package_platform', package_platform, '--package_type', package_type, engine_root]
+    return_code = safe_execute_system_call(args)
+    if return_code != 0:
+        success = False
+    if not success:
+        error('Restricted file validator failed.')
+    print('Restricted file validator completed successfully.')
+
+
+def cmake_build(package_env):
+    build_targets = package_env.get('BUILD_TARGETS')
+    for build_target in build_targets:
+        build(build_target['BUILD_CONFIG_FILENAME'], build_target['PLATFORM'], build_target['TYPE'])
+
+
+def create_package(package_env, package_target):
+    print('Creating zipfile for package target {}'.format(package_target))
+    cur_dir = os.path.dirname(os.path.abspath(__file__))
+    file_list_type = package_target['FILE_LIST_TYPE']
+    if file_list_type == 'All':
+        filelist = os.path.join(cur_dir, 'package_filelists', package_target['FILE_LIST'])
+    else:
+        # Search non-restricted platform first
+        filelist = os.path.join(cur_dir, 'Platform', file_list_type, 'package_filelists', package_target['FILE_LIST'])
+        if not os.path.exists(filelist):
+            engine_root = package_env.get('ENGINE_ROOT')
+            # Use real path in case engine root is a symlink path
+            if os.name == 'posix' and os.path.islink(engine_root):
+                engine_root = os.readlink(engine_root)
+            rel_path = os.path.relpath(cur_dir, engine_root)
+            filelist = os.path.join(engine_root, 'restricted', file_list_type, rel_path, 'package_filelists', package_target['FILE_LIST'])
+    with open(filelist, 'r') as source:
+        data = json.load(source)
+    lyengine = package_env.get('ENGINE_ROOT')
+    print('Calculating filelists...')
+    files = {}
+
+    if '@lyengine' in data:
+        files.update(filter_files(data['@lyengine'], lyengine))
+    if '@3rdParty' in data:
+        files.update(filter_files(data['@3rdParty'], package_env.get('THIRDPARTY_HOME')))
+    package_path = os.path.join(lyengine, package_target['PACKAGE_NAME'])
+    print('Creating zipfile at {}'.format(package_path))
+    start = timeit.default_timer()
+    with progressbar.ProgressBar(max_value=len(files), redirect_stderr=True) as bar:
+        with zipfile.ZipFile(package_path, 'w', compression=zipfile.ZIP_DEFLATED, allowZip64=True) as myzip:
+            i = 0
+            bar.update(i)
+            last_bar_update = timeit.default_timer()
+            for f in files:
+                if os.path.islink(f):
+                    zipInfo = zipfile.ZipInfo(files[f])
+                    zipInfo.create_system = 3
+                    # long type of hex val of '0xA1ED0000L',
+                    # say, symlink attr magic...
+                    zipInfo.external_attr |= 0xA0000000
+                    myzip.writestr(zipInfo, os.readlink(f))
+                else:
+                    myzip.write(f, files[f])
+                i += 1
+                # Update progress bar every 2 minutes
+                if int(timeit.default_timer() - last_bar_update) > 120:
+                    last_bar_update = timeit.default_timer()
+                    bar.update(i)
+            bar.update(i)
+
+    stop = timeit.default_timer()
+    total_time = int(stop - start)
+    print('{} is created. Total time: {} seconds.'.format(package_path, total_time))
+
+    def get_MD5(file_path):
+        from hashlib import md5
+        chunk_size = 200 * 1024
+        h = md5()
+        with open(file_path, 'rb') as f:
+            while True:
+                chunk = f.read(chunk_size)
+                if len(chunk):
+                    h.update(chunk)
+                else:
+                    break
+        return h.hexdigest()
+
+    md5_file = '{}.MD5'.format(package_path)
+    print('Creating MD5 file at {}'.format(md5_file))
+    start = timeit.default_timer()
+    with open(md5_file, 'w') as output:
+        output.write(get_MD5(package_path))
+    stop = timeit.default_timer()
+    total_time = int(stop - start)
+    print('{} is created. Total time: {} seconds.'.format(md5_file, total_time))
+
+
+def upload_package(package_env, package_target):
+    package_name = package_target['PACKAGE_NAME']
+    engine_root = package_env.get('ENGINE_ROOT')
+    internal_s3_bucket = package_env.get('INTERNAL_S3_BUCKET')
+    qa_s3_bucket = package_env.get('QA_S3_BUCKET')
+    s3_prefix = package_env.get('S3_PREFIX')
+    print(f'Uploading {package_name} to S3://{internal_s3_bucket}/{s3_prefix}/{package_name}')
+    cmd = ['aws', 's3', 'cp', os.path.join(engine_root, package_name), f's3://{internal_s3_bucket}/{s3_prefix}/{package_name}']
+    execute_system_call(cmd, stdout=subprocess.DEVNULL)
+    print(f'Uploading {package_name} to S3://{qa_s3_bucket}/{s3_prefix}/{package_name}')
+    cmd = ['aws', 's3', 'cp', os.path.join(engine_root, package_name), f's3://{qa_s3_bucket}/{s3_prefix}/{package_name}', '--acl', 'bucket-owner-full-control']
+    execute_system_call(cmd, stdout=subprocess.DEVNULL)
+
+
+def filter_files(data, base, prefix='', support_symlinks=True):
+    includes = {}
+    excludes = set()
+    for key, value in data.items():
+        pattern = os.path.join(base, prefix, key)
+        if not isinstance(value, dict):
+            pattern = os.path.normpath(pattern)
+            result = glob(pattern, recursive=True)
+            files = [x for x in result if os.path.isfile(x) or (support_symlinks and os.path.islink(x))]
+            if value == "#exclude":
+                excludes.update(files)
+            elif value == "#include":
+                for file in files:
+                    includes[file] = os.path.relpath(file, base)
+            else:
+                if value.startswith('#move:'):
+                    for file in files:
+                        file_name = os.path.relpath(file, os.path.join(base, prefix))
+                        dst_dir = value.replace('#move:', '').strip(' ')
+                        includes[file] = os.path.join(dst_dir, file_name)
+                elif value.startswith('#rename:'):
+                    for file in files:
+                        dst_file = value.replace('#rename:', '').strip(' ')
+                        includes[file] = dst_file
+                else:
+                    warn('Unknown directive {} for pattern {}'.format(value, pattern))
+        else:
+            includes.update(filter_files(value, base, os.path.join(prefix, key), support_symlinks))
+
+    for exclude in excludes:
+        try:
+            includes.pop(exclude)
+        except KeyError:
+            pass
+    return includes
+
+
+def parse_args():
+    parser = OptionParser()
+    parser.add_option("--platform", dest="platform", default='consoles', help="Target platform to package")
+    parser.add_option("--type", dest="type", default='consoles', help="Package type")
+    parser.add_option("--package_env", dest="package_env", default="package_env.json",
+                      help="JSON file that defines package environment variables")
+    (options, args) = parser.parse_args()
+    return options, args
+
+
+if __name__ == "__main__":
+    (options, args) = parse_args()
+    package(options)
+
+
+
+

scripts/build/package/package_env.json

@@ -0,0 +1,11 @@
+{
+    "global_env":{
+        "ENGINE_ROOT":"",
+        "THIRDPARTY_HOME":"",
+        "BRANCH_NAME":"",
+        "PACKAGE_NAME_PATTERN":"${BRANCH_NAME}-spectra",
+        "BUILD_NUMBER":"0",
+        "INTERNAL_S3_BUCKET": "ly-spectra-packages",
+        "QA_S3_BUCKET": "amazon.ly.lionbridgeshare/ly-spectra-packages"
+    }
+}

scripts/build/package/package_filelists/all.json

@@ -0,0 +1,7 @@
+{
+    "@lyengine": {
+        "**": "#include",
+        "**/*.pyc": "#exclude",
+        "**/*.pdb": "#exclude"
+    }
+}

scripts/build/package/package_filelists/symbols.json

@@ -0,0 +1,6 @@
+{
+    "@lyengine": {
+        "**/*.pdb": "#include",
+        "Tools/Crashpad/**": "#exclude"
+    }
+}

scripts/build/package/platform_exclusions.json

@@ -0,0 +1,12 @@
+{ 
+    "all": {
+        "@lyengine": {
+            "**/Gems/Atom/RHI/DX12/External/pix/**": "#exclude",
+            "**/.idea/**": "#exclude",
+            "**/*.csproj*": "#exclude",
+            "**/.owner": "#exclude",
+            "**/WinPixEventRuntime.dll": "#exclude",
+            "**/XenonConsole.exe": "#exclude"
+        }
+    }
+}

scripts/build/package/util.py

@@ -0,0 +1,62 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import json
+import os
+import re
+import sys
+import subprocess
+
+
+class LyBuildError(Exception):
+    def __init__(self, message):
+        super(LyBuildError, self).__init__(message)
+
+
+def ly_build_error(message):
+    raise LyBuildError(message)
+
+
+def error(message):
+    print(('Error: {}'.format(message)))
+    exit(1)
+
+
+# Exit with status code 0 means it won't fail the whole build process
+def safe_exit_with_error(message):
+    print(('Error: {}'.format(message)))
+    exit(0)
+
+
+def warn(message):
+    print(('Warning: {}'.format(message)))
+
+
+def execute_system_call(command, **kwargs):
+    print(('Executing subprocess.check_call({})'.format(command)))
+    try:
+        subprocess.check_call(command, **kwargs)
+    except subprocess.CalledProcessError as e:
+        print((e.output))
+        error('Executing subprocess.check_call({}) failed with error {}'.format(command, e))
+    except FileNotFoundError as e:
+        error("File Not Found - Failed to call {} with error {}".format(command, e))
+
+
+def safe_execute_system_call(command, **kwargs):
+    print(('Executing subprocess.check_call({})'.format(command)))
+    try:
+        subprocess.check_call(command, **kwargs)
+    except subprocess.CalledProcessError as e:
+        print((e.output))
+        warn('Executing subprocess.check_call({}) failed'.format(command))
+        return e.returncode
+    return 0

scripts/build/tools/alert_build_failures.py

@@ -0,0 +1,76 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import os
+from incremental_build_util import get_iam_role_credentials
+try:
+    import requests
+except ImportError:
+    import pip
+    pip.main(['install', 'requests', '--ignore-installed', '-q'])
+    import requests
+try:
+    from requests_aws4auth import AWS4Auth
+except ImportError:
+    import pip
+    pip.main(['install', 'requests_aws4auth', '--ignore-installed', '-q'])
+    from requests_aws4auth import AWS4Auth
+
+IAM_ROLE_NAME = 'ec2-jenkins-node'
+TEAM = 'lumberyard-build'
+CHIME_ROOM_WEB_HOOK = "https://hooks.chime.aws/incomingwebhooks/2a6018c9-3bf5-4e03-851c-32ba82c7c4e2?token=YWhTVXZsWVJ8MXxaLTg5RkVZNlA5Q1NiMVdfQndGeS1TSHNnYW5VREVha3pjX1pUUEd5b1JF"
+
+
+def find_curr_oncalls_for_team(team):
+    host = "who-is-oncall-pdx.corp.amazon.com"
+    service_name = "who-is-oncall"
+    aws_region = "us-west-2"
+    headers = {"content-type": "application/json", "host": host}
+
+    credentials = get_iam_role_credentials(IAM_ROLE_NAME)
+    try:
+        aws_access_key_id = credentials['AccessKeyId']
+        aws_secret_access_key = credentials['SecretAccessKey']
+        aws_session_token = credentials['Token']
+    except Exception as e:
+        print(f'ERROR: Cannot get AWS credentials.\n{e}')
+        return ['All']
+
+    auth = AWS4Auth(aws_access_key_id, aws_secret_access_key, aws_region, service_name, session_token=aws_session_token)
+    r = requests.get(f"https://who-is-oncall-pdx.corp.amazon.com/teams/{team}", headers=headers, auth=auth, verify=False)
+    if r.ok:
+        res = r.json()
+        try:
+            return res['currOncalls']
+        except KeyError:
+            return ['All']
+    return ['All']
+
+
+def send_alert_to_chime_room(web_hook, content):
+    data = '{"Content":"' + content + '"}'
+    headers = {'Content-Type': 'application/json'}
+    requests.post(web_hook, headers=headers, data=data)
+
+
+def create_content():
+    content = ''
+    oncalls = find_curr_oncalls_for_team(TEAM)
+    for oncall in oncalls:
+        content += f'@{oncall} '
+    job_name = os.environ['JOB_NAME']
+    build_url = os.environ['BUILD_URL']
+    content += fr'\nJob {job_name} failed\nBuild URL: {build_url}\n'
+    return content
+
+
+send_alert_to_chime_room(CHIME_ROOM_WEB_HOOK, create_content())
+

scripts/build/tools/download_from_s3.py

@@ -0,0 +1,101 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+'''
+Usage:
+Use EC2 role to download files to %WORKSPACE% folder from bucket bucket_name:
+python download_from_s3.py --base_dir %WORKSPACE% --files_to_download "file1,file2" --bucket bucket_name
+
+Use profile to download files to %WORKSPACE% folder from bucket bucket_name:
+python download_from_s3.py --base_dir %WORKSPACE% --profile profile --files_to_download "file1,file2" --bucket bucket_name
+'''
+
+
+import os
+import json
+import boto3
+from optparse import OptionParser
+from util import error
+
+
+def parse_args():
+    parser = OptionParser()
+    parser.add_option("--base_dir", dest="base_dir", default=os.getcwd(), help="Base directory to download files, If not given, then current directory is used.")
+    parser.add_option("--files_to_download", dest="files_to_download", default=None, help="Files to download, separated by comma.")
+    parser.add_option("--profile", dest="profile", default=None, help="The name of a profile to use. If not given, then the default profile is used.")
+    parser.add_option("--bucket", dest="bucket", default=None, help="S3 bucket the files are downloaded from.")
+    parser.add_option("--key_prefix", dest="key_prefix", default='', help="Object key prefix.")
+    '''
+    ExtraArgs used to call s3.download_file(), should be in json format. extra_args key must be one of: ACL, CacheControl, ContentDisposition, ContentEncoding, ContentLanguage, ContentType, Expires,
+    GrantFullControl, GrantRead, GrantReadACP, GrantWriteACP, Metadata, RequestPayer, ServerSideEncryption, StorageClass,
+    SSECustomerAlgorithm, SSECustomerKey, SSECustomerKeyMD5, SSEKMSKeyId, WebsiteRedirectLocation
+    '''
+    parser.add_option("--extra_args", dest="extra_args", default=None, help="Additional parameters used to download file.")
+    parser.add_option("--max_retry", dest="max_retry", default=1, help="Maximum retry times to download file.")
+    (options, args) = parser.parse_args()
+    if not os.path.isdir(options.base_dir):
+        error('{} is not a valid directory'.format(options.base_dir))
+    if not options.files_to_download:
+        error('Use --files_to_download to specify files to download, separated by comma.')
+    if not options.bucket:
+        error('Use --bucket to specify bucket that the files are downloaded from.')
+    return options
+
+
+def get_client(service_name, profile_name=None):
+    session = boto3.session.Session(profile_name=profile_name)
+    client = session.client(service_name)
+    return client
+
+
+def s3_download_file(client, base_dir, file, bucket, key_prefix=None, extra_args=None, max_retry=1):
+    print('Downloading file {} from bucket {}.'.format(file, bucket))
+    key = file if key_prefix is None else '{}/{}'.format(key_prefix, file)
+    for x in range(max_retry):
+        try:
+            client.download_file(
+                bucket, key, os.path.join(base_dir, file),
+                ExtraArgs=extra_args
+            )
+            print('Download succeeded')
+            return True
+        except:
+            print('Retrying download...')
+    print('Download failed')
+    return False
+
+
+def download_files(base_dir, files_to_download, bucket, key_prefix=None, profile=None, extra_args=None, max_retry=1):
+    client = get_client('s3', profile)
+    files_to_download = files_to_download.split(',')
+    extra_args = json.loads(extra_args) if extra_args else None
+
+    print('Downloading {} files from bucket {}.'.format(len(files_to_download), bucket))
+    failure = []
+    success = []
+    for file in files_to_download:
+        if not s3_download_file(client, base_dir, file, bucket, key_prefix, extra_args, max_retry):
+            failure.append(file)
+        else:
+            success.append(file)
+    print('{} files are downloaded successfully:'.format(len(success)))
+    print('\n'.join(success))
+    print('{} files failed to download:'.format(len(failure)))
+    print('\n'.join(failure))
+    # Exit with error code 1 if any file is failed to download
+    if len(failure) > 0:
+        return False
+    return True
+
+
+if __name__ == "__main__":
+    options = parse_args()
+    download_files(options.base_dir, options.files_to_download, options.bucket, options.key_prefix, options.profile, options.extra_args, options.max_retry)

scripts/build/tools/email_to_lionbridge.py

@@ -0,0 +1,128 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+"""
+This script will be used in https://jenkins.agscollab.com/view/%7ESandbox/job/PACKAGE_COPY_S3/
+PACKAGE_COPY_S3 is a downstream job of nightly packaging job, it copies the nightly packages from Infra S3 bucket to Lionbridge S3 bucket based on the INCLUDE_FILTER passed from packaging job
+"""
+import os
+import re
+import json
+import requests
+from requests.auth import HTTPBasicAuth
+import boto3
+from util import error, warn
+
+
+# Write EMAIL_TEMPLATE to a file and inject it into the email sent to Lionbridge
+EMAIL_TEMPLATE = '''Packages are uploaded to S3 bucket {}
+Package List:
+{}
+
+
+Changelists:
+{}
+'''
+
+
+def get_jenkins_env(key):
+    try:
+        return os.environ[key]
+    except KeyError:
+        print('Error: Jenkins parameters {} is not set.'.format(key))
+    return None
+
+
+JENKINS_USERNAME = get_jenkins_env('JENKINS_USERNAME')
+JENKINS_API_TOKEN = get_jenkins_env('JENKINS_API_TOKEN')
+JENKINS_URL = get_jenkins_env('JENKINS_URL')
+WORKSPACE = get_jenkins_env('WORKSPACE')
+S3_TARGET = get_jenkins_env('S3_TARGET')
+INCLUDE_FILTER = get_jenkins_env('INCLUDE_FILTER')
+EMAIL_TEMPLATE_FILE = get_jenkins_env('EMAIL_TEMPLATE_FILE')
+if None in [JENKINS_USERNAME, JENKINS_API_TOKEN, JENKINS_URL, WORKSPACE, S3_TARGET, INCLUDE_FILTER, EMAIL_TEMPLATE_FILE]:
+    error('Please make sure all Jenkins parameters are set correctly.')
+
+
+def parse_include_filter(include_filter):
+    try:
+        res = re.search('^(\w*)-*lumberyard-(\d+)\.(\d+)-(\d+)-(\w+).*\*(\d+)\.\*', include_filter)
+        branch = res.group(1)
+        major_version = int(res.group(2))
+        minor_version = int(res.group(3))
+        changelist_number = res.group(4)
+        platform = res.group(5)
+        build_number = res.group(6)
+        return branch, major_version, minor_version, changelist_number, platform, build_number
+    except (AttributeError, IndexError):
+        error('Unable to parse INCLUDE_FILTER, please make sure the INCLUDE_FILTER is set correctly')
+
+
+# Get the changelists that trigger the build
+def get_changelists(job_name, build_number):
+    changelists = []
+    headers = {'Content-type': 'application/json', 'Accept': 'application/json'}
+    try:
+        res = requests.get('{}/job/{}/{}/api/json'.format(JENKINS_URL, job_name, build_number),
+                           auth=HTTPBasicAuth(JENKINS_USERNAME, JENKINS_API_TOKEN), headers=headers, verify=False)
+        res = json.loads(res.content)
+        changelists = res.get('changeSet').get('items')
+        return changelists
+    except:
+        warn('Error: Failed to get changes from build {} in job {}'.format(build_number, job_name))
+        return []
+
+
+def get_packaging_job_name(branch, major_version, minor_version, platform):
+    if branch == '':
+        branch = 'ML' if major_version + minor_version == 0 else 'v{}_{}'.format(major_version, minor_version)
+    job_name = 'PKG_{}_{}'.format(branch, platform.capitalize())
+    return job_name
+
+
+# Get package names by looking up S3 bucket
+def get_package_names(branch, major_version, minor_version, include_filter, build_number):
+    package_names = []
+    prefix = include_filter[:include_filter.find('*')]
+    pattern = '.*{}.*{}..*'.format(prefix, build_number)
+    if branch == '':
+        bucket_name = 'ly-packages-mainline' if major_version + minor_version == 0 else 'ly-packages-release-candidate'
+        folder = 'lumberyard-packages'
+    else:
+        bucket_name = 'ly-packages-feature-branches'
+        folder = 'lumberyard-packages/{}'.format(branch)
+    s3 = boto3.resource('s3')
+    bucket = s3.Bucket(bucket_name)
+    for obj in bucket.objects.filter(Prefix='{}/{}'.format(folder, prefix)):
+        package_name = obj.key
+        if re.match(pattern, package_name):
+            package_names.append(package_name.replace('{}/'.format(folder), ''))
+    return package_names
+
+
+if __name__ == "__main__":
+    branch, major_version, minor_version, changelist_number, platform, build_number = parse_include_filter(INCLUDE_FILTER)
+    packaging_job_name = get_packaging_job_name(branch, major_version, minor_version, platform)
+    changelists = get_changelists(packaging_job_name, build_number)
+    package_names = get_package_names(branch, major_version, minor_version, INCLUDE_FILTER, build_number)
+    with open(os.path.join(WORKSPACE, EMAIL_TEMPLATE_FILE), 'w+') as output:
+        if len(package_names) > 0:
+            package_list_str = '\n'.join(package_names)
+            changelists_str = ''
+            for item in changelists:
+                changelists_str += '---------------------------------------------------------------------------------------------\n'
+                try:
+                    changelists_str += 'CL{} by {} on {}\n{}\n'.format(item['changeNumber'], item['author']['fullName'], item['changeTime'], item['msg'].encode('utf-8', 'ignore'))
+                except KeyError:
+                    error('Internal error, check the output of Jenkins API.')
+            output.write(EMAIL_TEMPLATE.format(S3_TARGET, package_list_str, changelists_str))
+
+

scripts/build/tools/upload_to_s3.py

@@ -0,0 +1,112 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+'''
+Usage:
+Use EC2 role to upload all .zip and .MD5 files in %WORKSPACE% folder to bucket ly-packages-mainline:
+python upload_to_s3.py --base_dir %WORKSPACE% --file_regex "(.*zip$|.*MD5$)" --bucket ly-packages-mainline
+
+Use profile to upload all .zip and .MD5 files in %WORKSPACE% folder to bucket ly-packages-mainline:
+python upload_to_s3.py --base_dir %WORKSPACE% --profile profile --file_regex "(.*zip$|.*MD5$)" --bucket ly-packages-mainline
+
+'''
+
+
+import os
+import re
+import json
+import boto3
+from optparse import OptionParser
+
+
+def parse_args():
+    parser = OptionParser()
+    parser.add_option("--base_dir", dest="base_dir", default=os.getcwd(), help="Base directory to upload files, If not given, then current directory is used.")
+    parser.add_option("--file_regex", dest="file_regex", default=None, help="Regular expression that used to match file names to upload.")
+    parser.add_option("--profile", dest="profile", default=None, help="The name of a profile to use. If not given, then the default profile is used.")
+    parser.add_option("--bucket", dest="bucket", default=None, help="S3 bucket the files are uploaded to.")
+    parser.add_option("--key_prefix", dest="key_prefix", default='', help="Object key prefix.")
+    '''
+    ExtraArgs used to call s3.upload_file(), should be in json format. extra_args key must be one of: ACL, CacheControl, ContentDisposition, ContentEncoding, ContentLanguage, ContentType, Expires,
+    GrantFullControl, GrantRead, GrantReadACP, GrantWriteACP, Metadata, RequestPayer, ServerSideEncryption, StorageClass,
+    SSECustomerAlgorithm, SSECustomerKey, SSECustomerKeyMD5, SSEKMSKeyId, WebsiteRedirectLocation
+    '''
+    parser.add_option("--extra_args", dest="extra_args", default=None, help="Additional parameters used to upload file.")
+    parser.add_option("--max_retry", dest="max_retry", default=1, help="Maximum retry times to upload file.")
+    (options, args) = parser.parse_args()
+    if not os.path.isdir(options.base_dir):
+        error('{} is not a valid directory'.format(options.base_dir))
+    if not options.file_regex:
+        error('Use --file_regex to specify regular expression that used to match file names to upload.')
+    if not options.bucket:
+        error('Use --bucket to specify bucket that the files are uploaded to.')
+    return options
+
+
+def error(message):
+    print(f'Error: {message}')
+    exit(1)
+
+
+def get_client(service_name, profile_name):
+    session = boto3.session.Session(profile_name=profile_name)
+    client = session.client(service_name)
+    return client
+
+
+def get_files_to_upload(base_dir, regex):
+    # Get all file names in base directory
+    files = [x for x in os.listdir(base_dir) if os.path.isfile(os.path.join(base_dir, x))]
+    # Get all file names matching the regular expression, those file will be uploaded to S3
+    files_to_upload = [x for x in files if re.match(regex, x)]
+    return files_to_upload
+
+
+def s3_upload_file(client, base_dir, file, bucket, key_prefix=None, extra_args=None, max_retry=1):
+    print(('Uploading file {} to bucket {}.'.format(file, bucket)))
+    key = file if key_prefix is None else '{}/{}'.format(key_prefix, file)
+    for x in range(max_retry):
+        try:
+            client.upload_file(
+                os.path.join(base_dir, file), bucket, key,
+                ExtraArgs=extra_args
+            )
+            print('Upload succeeded')
+            return True
+        except Exception as err:
+            print(('exception while uploading: {}'.format(err)))
+            print('Retrying upload...')
+    print('Upload failed')
+    return False
+
+
+if __name__ == "__main__":
+    options = parse_args()
+    client = get_client('s3', options.profile)
+    files_to_upload = get_files_to_upload(options.base_dir, options.file_regex)
+    extra_args = json.loads(options.extra_args) if options.extra_args else None
+
+    print(('Uploading {} files to bucket {}.'.format(len(files_to_upload), options.bucket)))
+    failure = []
+    success = []
+    for file in files_to_upload:
+        if not s3_upload_file(client, options.base_dir, file, options.bucket, options.key_prefix, extra_args, 2):
+            failure.append(file)
+        else:
+            success.append(file)
+    print('Upload finished.')
+    print(('{} files are uploaded successfully:'.format(len(success))))
+    print(('\n'.join(success)))
+    if len(failure) > 0:
+        print(('{} files failed to upload:'.format(len(failure))))
+        print(('\n'.join(failure)))
+        # Exit with error code 1 if any file is failed to upload
+        exit(1)