Initial commit

scripts/CMakeLists.txt

@@ -0,0 +1,13 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+add_subdirectory(detect_file_changes)
+add_subdirectory(commit_validation)

scripts/build/bootstrap/incremental_build_util.py

@@ -0,0 +1,525 @@
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+
+import argparse
+import ast
+import boto3
+import datetime
+import urllib2
+import os
+import psutil
+import time
+import requests
+import subprocess
+import sys
+import tempfile
+import traceback
+
+IAM_ROLE_NAME = 'ec2-jenkins-node'
+DEFAULT_REGION = 'us-west-2'
+DEFAULT_DISK_SIZE = 300
+DEFAULT_DISK_TYPE = 'gp2'
+DEFAULT_TIMEOUT = 300
+MAX_EBS_MOUNTING_ATTEMPT = 3
+LOW_EBS_DISK_SPACE_LIMIT = 10240
+MAX_EBS_DISK_SIZE = DEFAULT_DISK_SIZE * 2
+
+if os.name == 'nt':
+    MOUNT_PATH = 'D:\\'
+else:
+    MOUNT_PATH = '/data'
+
+if os.name == 'nt':
+    import ctypes
+    import win32api
+    import collections
+    import locale
+
+    locale.setlocale(locale.LC_ALL, '')  # set locale to default to get thousands separators
+
+    PULARGE_INTEGER = ctypes.POINTER(ctypes.c_ulonglong)  # Pointer to large unsigned integer
+    kernel32 = ctypes.WinDLL('kernel32', use_last_error=True)
+    kernel32.GetDiskFreeSpaceExW.argtypes = (ctypes.c_wchar_p,) + (PULARGE_INTEGER,) * 3
+
+    class UsageTuple(collections.namedtuple('UsageTuple', 'total, used, free')):
+        def __str__(self):
+            # Add thousands separator to numbers displayed
+            return self.__class__.__name__ + '(total={:n}, used={:n}, free={:n})'.format(*self)
+
+    def is_dir_symlink(path):
+        FILE_ATTRIBUTE_REPARSE_POINT = 0x0400
+        return os.path.isdir(path) and (ctypes.windll.kernel32.GetFileAttributesW(unicode(path)) & FILE_ATTRIBUTE_REPARSE_POINT)
+
+    def get_free_space_mb(path):
+        if sys.version_info < (3,):  # Python 2?
+            saved_conversion_mode = ctypes.set_conversion_mode('mbcs', 'strict')
+        else:
+            try:
+                path = os.fsdecode(path)  # allows str or bytes (or os.PathLike in Python 3.6+)
+            except AttributeError:  # fsdecode() not added until Python 3.2
+                pass
+
+        # Define variables to receive results when passed as "by reference" arguments
+        _, total, free = ctypes.c_ulonglong(), ctypes.c_ulonglong(), ctypes.c_ulonglong()
+
+        success = kernel32.GetDiskFreeSpaceExW(
+            path, ctypes.byref(_), ctypes.byref(total), ctypes.byref(free))
+        if not success:
+            error_code = ctypes.get_last_error()
+
+        if sys.version_info < (3,):  # Python 2?
+            ctypes.set_conversion_mode(*saved_conversion_mode)  # restore conversion mode
+
+        if not success:
+            windows_error_message = ctypes.FormatError(error_code)
+            raise ctypes.WinError(error_code, '{} {!r}'.format(windows_error_message, path))
+
+        used = total.value - free.value
+
+        return free.value / 1024 / 1024#for now
+else:
+    def get_free_space_mb(dirname):
+        st = os.statvfs(dirname)
+        return st.f_bavail * st.f_frsize / 1024 / 1024
+
+def error(message):
+    print message
+    exit(1)
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-a', '--action', dest="action", help="Action (mount|unmount|delete)")
+    parser.add_argument('-pipe', '--pipeline', dest="pipeline", help="Pipeline")
+    parser.add_argument('-b', '--branch', dest="branch", help="Branch")
+    parser.add_argument('-plat', '--platform', dest="platform", help="Platform")
+    parser.add_argument('-c', '--build_type', dest="build_type", help="Build type")
+    parser.add_argument('-ds', '--disk_size', dest="disk_size", help="Disk size in Gigabytes (defaults to {})".format(DEFAULT_DISK_SIZE), default=DEFAULT_DISK_SIZE)
+    parser.add_argument('-dt', '--disk_type', dest="disk_type", help="Disk type (defaults to {})".format(DEFAULT_DISK_TYPE), default=DEFAULT_DISK_TYPE)
+    args = parser.parse_args()
+
+    # Input validation
+    if args.action is None:
+        error('No action specified')
+    args.action = args.action.lower()
+    if args.action != 'unmount':
+        if args.pipeline is None:
+            error('No pipeline specified')
+        if args.branch is None:
+            error('No branch specified')
+        if args.platform is None:
+            error('No platform specified')
+        if args.build_type is None:
+            error('No build_type specified')
+    
+    return args
+
+def get_mount_name(pipeline, branch, platform, build_type):
+    mount_name = "{}_{}_{}_{}".format(pipeline, branch, platform, build_type)
+    mount_name = mount_name.replace('/','_').replace('\\','_')
+    return mount_name
+
+def get_pipeline_and_branch(pipeline, branch):
+    pipeline_and_branch = "{}_{}".format(pipeline, branch)
+    pipeline_and_branch = pipeline_and_branch.replace('/','_').replace('\\','_')
+    return pipeline_and_branch
+
+def get_iam_role_credentials(role_name):
+    security_metadata = None
+    try:
+        response = urllib2.urlopen(
+            'http://169.254.169.254/latest/meta-data/iam/security-credentials/{0}'.format(role_name)).read()
+        security_metadata = ast.literal_eval(response)
+    except:
+        print 'Unable to get iam role credentials'
+        print traceback.print_exc()
+
+    return security_metadata
+
+
+def get_ec2_client(region):
+    credentials = None
+    try:
+        response = urllib2.urlopen(
+            'http://169.254.169.254/latest/meta-data/iam/security-credentials/{0}'.format(IAM_ROLE_NAME)).read()
+        credentials = ast.literal_eval(response)
+    except Exception as e:
+        print e.message
+        error('Error: Unable to get IAM rols credentials, please contact ly-build@ for help.')
+
+    keys = ['AccessKeyId', 'SecretAccessKey', 'Token']
+    for key in keys:
+        if key not in credentials:
+            error('Error: Unable to find {0} in get_iam_role_credentials response {1}'.format(key, credentials))
+
+    aws_access_key_id = credentials['AccessKeyId']
+    aws_secret_access_key = credentials['SecretAccessKey']
+    aws_session_token = credentials['Token']
+
+    client = boto3.client('ec2', region_name=region, aws_access_key_id=aws_access_key_id,
+                          aws_secret_access_key=aws_secret_access_key,
+                          aws_session_token=aws_session_token)
+    return client
+
+
+def get_ec2_instance_id():
+    try:
+        instance_id = urllib2.urlopen('http://169.254.169.254/latest/meta-data/instance-id').read()
+        return instance_id
+    except Exception as e:
+        print e.message
+        error('No EC2 metadata! Check if you are running this script on an EC2 instance.')
+
+
+def get_availability_zone():
+    try:
+        availability_zone = urllib2.urlopen('http://169.254.169.254/latest/meta-data/placement/availability-zone').read()
+        return availability_zone
+    except Exception as e:
+        print e.message
+        error('No EC2 metadata! Check if you are running this script on an EC2 instance.')
+
+
+def kill_processes(workspace='/dev/'):
+    '''
+    Kills all processes that have open file paths associated with the workspace.
+    Uses PSUtil for cross-platform compatibility
+    '''
+    print 'Checking for any stuck processes...'
+    for proc in psutil.process_iter():
+        try:
+            if workspace in str(proc.open_files()):
+                print "{} has open files in {}. Terminating".format(proc.name(), proc.open_files())
+                proc.kill()
+                time.sleep(1) # Just to make sure a parent process has time to close
+        except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
+            continue
+
+
+def delete_volume(ec2_client, volume_id):
+    response = ec2_client.delete_volume(VolumeId=volume_id)
+    print 'Volume {} deleted'.format(volume_id)
+
+def find_snapshot_id(ec2_client, pipeline, platform, build_type, disk_size):
+    mount_name = get_mount_name(pipeline, 'main', platform, build_type) # we take snapshots out of main
+    response = ec2_client.describe_snapshots(Filters= [{
+        'Name': 'tag:Name', 'Values': [mount_name]
+    }])
+
+    snapshot_id = None
+    if 'Snapshots' in response and len(response['Snapshots']) > 0:
+        snapshot_start_time_max = None # find the latest snapshot
+        for snapshot in response['Snapshots']:
+            if snapshot['State'] == 'completed' and snapshot['VolumeSize'] == disk_size:
+                snapshot_start_time = snapshot['StartTime']
+                if not snapshot_start_time_max or snapshot_start_time > snapshot_start_time_max:
+                    snapshot_start_time_max = snapshot_start_time
+                    snapshot_id = snapshot['SnapshotId']
+    return snapshot_id
+
+def create_volume(ec2_client, availability_zone, pipeline, branch, platform, build_type, disk_size, disk_type):   
+    # The actual EBS default calculation for IOps is a floating point number, the closest approxmiation is 4x of the disk size for simplicity
+    mount_name = get_mount_name(pipeline, branch, platform, build_type)
+    pipeline_and_branch = get_pipeline_and_branch(pipeline, branch) 
+    parameters = dict(
+        AvailabilityZone = availability_zone,
+        VolumeType=disk_type,
+        TagSpecifications= [{
+            'ResourceType': 'volume',
+            'Tags': [
+                { 'Key': 'Name', 'Value': mount_name },
+                { 'Key': 'Pipeline', 'Value': pipeline },
+                { 'Key': 'BranchName', 'Value': branch },
+                { 'Key': 'Platform', 'Value': platform },
+                { 'Key': 'BuildType', 'Value': build_type }, 
+                { 'Key': 'PipelineAndBranch', 'Value': pipeline_and_branch }, # used so the snapshoting easily identifies which volumes to snapshot
+            ]
+        }]
+    )
+    if 'io1' in disk_type.lower(): 
+        parameters['Iops'] = (4 * disk_size)
+
+    snapshot_id = find_snapshot_id(ec2_client, pipeline, platform, build_type, disk_size)
+    if snapshot_id:
+        parameters['SnapshotId'] = snapshot_id
+        created = False
+    else:
+        # If no snapshot id, we need to specify the size
+        parameters['Size'] = disk_size
+        created = True
+
+    response = ec2_client.create_volume(**parameters)
+    volume_id = response['VolumeId']
+
+    # give some time for the creation call to complete
+    time.sleep(1)
+
+    response = ec2_client.describe_volumes(VolumeIds=[volume_id, ])
+    while (response['Volumes'][0]['State'] != 'available'):
+        time.sleep(1)
+        response = ec2_client.describe_volumes(VolumeIds=[volume_id, ])
+
+    print("Volume {} created\n\tSnapshot: {}\n\tPipeline {}\n\tBranch {}\n\tPlatform: {}\n\tBuild type: {}"
+        .format(volume_id, snapshot_id, pipeline, branch, platform, build_type))
+    return volume_id, created
+
+
+def mount_volume(created):
+    print 'Mounting volume...'
+    if os.name == 'nt':
+        f = tempfile.NamedTemporaryFile(delete=False)
+        f.write("""
+      select disk 1
+      online disk
+      attribute disk clear readonly
+      """) # assume disk # for now
+
+        if created:
+            print 'Creating filesystem on new volume'
+            f.write("""create partition primary
+          select partition 1
+          format quick fs=ntfs
+          assign
+          active
+          """)
+
+        f.close()
+        
+        subprocess.call(['diskpart', '/s', f.name])
+
+        time.sleep(5)
+
+        drives_after = win32api.GetLogicalDriveStrings()
+        drives_after = drives_after.split('\000')[:-1]
+
+        print drives_after
+
+        #drive_letter = next(item for item in drives_after if item not in drives_before)
+        drive_letter = MOUNT_PATH
+
+        os.unlink(f.name)
+
+        time.sleep(1)
+
+    else:
+        subprocess.call(['file', '-s', '/dev/xvdf'])
+        if created:
+            subprocess.call(['mkfs', '-t', 'ext4', '/dev/xvdf'])
+        subprocess.call(['mount', '/dev/xvdf', MOUNT_PATH])
+
+
+def attach_volume(volume, volume_id, instance_id, timeout=DEFAULT_TIMEOUT):
+    print 'Attaching volume {} to instance {}'.format(volume_id, instance_id)
+    volume.attach_to_instance(Device='xvdf',
+                              InstanceId=instance_id,
+                              VolumeId=volume_id)
+    # give a little bit of time for the aws call to process
+    time.sleep(2)
+    # reload the volume just in case
+    volume.load()
+    timeout_init = time.clock()
+    while (len(volume.attachments) and volume.attachments[0]['State'] != 'attached'):
+        time.sleep(1)
+        volume.load()
+        if (time.clock() - timeout_init) > timeout:
+            print 'ERROR: Timeout reached trying to mount EBS'
+            exit(1)
+
+    print 'Volume {} has been attached to instance {}'.format(volume_id, instance_id)
+
+
+def unmount_volume():
+    print 'Umounting volume...'
+    if os.name == 'nt':
+        kill_processes(MOUNT_PATH + 'workspace')
+        f = tempfile.NamedTemporaryFile(delete=False)
+        f.write("""
+          select disk 1
+          offline disk
+          """)
+        f.close()
+        subprocess.call('diskpart /s %s' % f.name)
+        os.unlink(f.name)
+    else:
+        kill_processes(MOUNT_PATH)
+        subprocess.call(['umount', '-f', MOUNT_PATH])
+
+
+def detach_volume(volume, ec2_instance_id, force, timeout=DEFAULT_TIMEOUT):
+    print 'Detaching volume {} from instance {}'.format(volume.volume_id, ec2_instance_id)
+    volume.detach_from_instance(Device='xvdf',
+                                Force=force,
+                                InstanceId=ec2_instance_id,
+                                VolumeId=volume.volume_id)
+    timeout_init = time.clock()
+    while len(volume.attachments) and volume.attachments[0]['State'] != 'detached':
+        time.sleep(1)
+        volume.load()
+        if (time.clock() - timeout_init) > timeout:
+            print 'ERROR: Timeout reached trying to unmount EBS.'
+            volume.detach_from_instance(Device='xvdf',Force=True,InstanceId=ec2_instance_id,VolumeId=volume.volume_id)
+            exit(1)
+            
+    print 'Volume {} has been detached from instance {}'.format(volume.volume_id, ec2_instance_id)
+    volume.load()
+    if len(volume.attachments):
+        print 'Volume still has attachments'
+        for attachment in volume.attachments:
+            print 'Volume {} {} to instance {}'.format(attachment['VolumeId'], attachment['State'], attachment['InstanceId'])
+
+
+def attach_ebs_and_create_partition_with_retry(volume, volume_id, ec2_instance_id, created):
+    attach_volume(volume, volume_id, ec2_instance_id)
+    mount_volume(created)
+    attempt = 1
+    while attempt <= MAX_EBS_MOUNTING_ATTEMPT:
+        if os.name == 'nt':
+            drives_after = win32api.GetLogicalDriveStrings()
+            drives_after = drives_after.split('\000')[:-1]
+            if MOUNT_PATH not in drives_after:
+                print('Disk partitioning failed, retrying...')
+                unmount_volume()
+                detach_volume(volume, ec2_instance_id, False)
+                attach_volume(volume, volume_id, ec2_instance_id)
+                mount_volume(created)
+        attempt += 1
+
+def mount_ebs(pipeline, branch, platform, build_type, disk_size, disk_type):
+    session = boto3.session.Session()
+    region = session.region_name
+    if region is None:
+        region = DEFAULT_REGION
+    ec2_client = get_ec2_client(region)
+    ec2_instance_id = get_ec2_instance_id()
+    ec2_availability_zone = get_availability_zone()
+    ec2_resource = boto3.resource('ec2', region_name=region)
+    ec2_instance = ec2_resource.Instance(ec2_instance_id)
+
+    for volume in ec2_instance.volumes.all():
+        for attachment in volume.attachments:
+            print 'attachment device: {}'.format(attachment['Device'])
+            if 'xvdf' in attachment['Device'] and attachment['State'] != 'detached':
+                print 'A device is already attached to xvdf. This likely means a previous build failed to detach its ' \
+                      'build volume. This volume is considered orphaned and will be detached from this instance.'
+                unmount_volume()
+                detach_volume(volume, ec2_instance_id, False) # Force unmounts should not be used, as that will cause the EBS block device driver to fail the remount
+
+    mount_name = get_mount_name(pipeline, branch, platform, build_type)
+    response = ec2_client.describe_volumes(Filters=[{
+        'Name': 'tag:Name', 'Values': [mount_name]
+        }])
+
+    created = False
+    if 'Volumes' in response and not len(response['Volumes']):
+        print 'Volume for {} doesn\'t exist creating it...'.format(mount_name)
+        # volume doesn't exist, create it
+        volume_id, created = create_volume(ec2_client, ec2_availability_zone, pipeline, branch, platform, build_type, disk_size, disk_type)
+    else:
+        volume = response['Volumes'][0]
+        volume_id = volume['VolumeId']
+        print 'Current volume {} is a {} GB {}'.format(volume_id, volume['Size'], volume['VolumeType'])
+        if (volume['Size'] != disk_size or volume['VolumeType'] != disk_type):
+            print 'Override disk attributes does not match the existing volume, deleting {} and replacing the volume'.format(volume_id)
+            delete_volume(ec2_client, volume_id)
+            volume_id, created = create_volume(ec2_client, ec2_availability_zone, pipeline, branch, platform, build_type, disk_size, disk_type)
+        if len(volume['Attachments']):
+            # this is bad we shouldn't be attached, we should have detached at the end of a build
+            attachment = volume['Attachments'][0]
+            print ('Volume already has attachment {}, detaching...'.format(attachment))
+            detach_volume(ec2_resource.Volume(volume_id), attachment['InstanceId'], True)
+
+    volume = ec2_resource.Volume(volume_id)
+
+    if os.name == 'nt':
+        drives_before = win32api.GetLogicalDriveStrings()
+        drives_before = drives_before.split('\000')[:-1]
+
+        print drives_before
+
+    attach_ebs_and_create_partition_with_retry(volume, volume_id, ec2_instance_id, created)
+
+    free_space_mb = get_free_space_mb(MOUNT_PATH)
+    print 'Free disk space {}MB'.format(free_space_mb)
+    
+    if free_space_mb < LOW_EBS_DISK_SPACE_LIMIT:
+        print 'Volume is running below EBS free disk space treshhold {}MB. Recreating volume and running clean build.'.format(LOW_EBS_DISK_SPACE_LIMIT)
+        unmount_volume()
+        detach_volume(volume, ec2_instance_id, False)
+        delete_volume(ec2_client, volume_id)
+        new_disk_size = int(volume.size * 1.25)
+        if new_disk_size > MAX_EBS_DISK_SIZE:
+            print 'Error: EBS disk size reached to the allowed maximum disk size {}MB, please contact ly-infra@ and ly-build@ to investigate.'.format(MAX_EBS_DISK_SIZE)
+            exit(1)
+        print 'Recreating the EBS with disk size {}'.format(new_disk_size)
+        volume_id, created = create_volume(ec2_client, ec2_availability_zone, pipeline, branch, platform, build_type, new_disk_size, disk_type)
+        volume = ec2_resource.Volume(volume_id)
+        attach_ebs_and_create_partition_with_retry(volume, volume_id, ec2_instance_id, created)
+
+def unmount_ebs():
+    session = boto3.session.Session()
+    region = session.region_name
+    if region is None:
+        region = DEFAULT_REGION
+    ec2_client = get_ec2_client(region)
+    ec2_instance_id = get_ec2_instance_id()
+    ec2_resource = boto3.resource('ec2', region_name=region)
+    ec2_instance = ec2_resource.Instance(ec2_instance_id)
+
+    if os.path.isfile('envinject.properties'):
+        os.remove('envinject.properties')
+
+    volume = None
+
+    for attached_volume in ec2_instance.volumes.all():
+        for attachment in attached_volume.attachments:
+            print 'attachment device: {}'.format(attachment['Device'])
+            if attachment['Device'] == 'xvdf':
+                volume = attached_volume
+
+    if not volume:
+        # volume is not mounted
+        print 'Volume is not mounted'
+    else:
+        unmount_volume()
+        detach_volume(volume, ec2_instance_id, False)
+
+def delete_ebs(pipeline, branch, platform, build_type):
+    unmount_ebs()
+
+    session = boto3.session.Session()
+    region = session.region_name
+    if region is None:
+        region = DEFAULT_REGION
+    ec2_client = get_ec2_client(region)
+    ec2_instance_id = get_ec2_instance_id()
+    ec2_resource = boto3.resource('ec2', region_name=region)
+    ec2_instance = ec2_resource.Instance(ec2_instance_id)
+
+    mount_name = get_mount_name(pipeline, branch, platform, build_type)
+    response = ec2_client.describe_volumes(Filters=[
+        { 'Name': 'tag:Name', 'Values': [mount_name] }
+    ])
+
+    if 'Volumes' in response and len(response['Volumes']):
+        volume = response['Volumes'][0]
+        volume_id = volume['VolumeId']
+        delete_volume(ec2_client, volume_id)
+
+
+def main(action, pipeline, branch, platform, build_type, disk_size, disk_type):
+    if action == 'mount':
+        mount_ebs(pipeline, branch, platform, build_type, disk_size, disk_type)
+    elif action == 'unmount':
+        unmount_ebs()
+    elif action == 'delete':
+        delete_ebs(pipeline, branch, platform, build_type)
+
+if __name__ == "__main__":
+    args = parse_args()
+    ret = main(args.action, args.pipeline, args.branch, args.platform, args.build_type, args.disk_size, args.disk_type)
+    sys.exit(ret)

scripts/build/build_node/Platform/Common/requirements.txt

@@ -0,0 +1,185 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --generate-hashes 'requirements.txt'
+#
+
+# These packages are used as part of the pre-SCM sync environment and not actually used in the build
+# The requirements file for build can be found here: dev\Tools\Python\<python version>\requirements.txt
+
+boto3==1.16.18 \
+    --hash=sha256:51c419d890ae216b9b031be31f3182739dc3deb5b64351f286bffca2818ddb35 \
+    --hash=sha256:d70d21ea137d786e84124639a62be42f92f4b09472ebfb761156057c92dc5366 \
+    # via -r requirements.txt
+botocore==1.19.18 \
+    --hash=sha256:288d43e85f12e3c1d6a0535a585a182ca04e8c6e742ebaaf15357a0e3b37ca7a \
+    --hash=sha256:bba18b5c4eef3eb2dc39b1b1f8959ba01ac27e7e12e413e281b0fb242990c0f5 \
+    # via -r requirements.txt, boto3, s3transfer
+certifi==2020.11.8 \
+    --hash=sha256:1f422849db327d534e3d0c5f02a263458c3955ec0aae4ff09b95f195c59f4edd \
+    --hash=sha256:f05def092c44fbf25834a51509ef6e631dc19765ab8a57b4e7ab85531f0a9cf4 \
+    # via requests
+chardet==3.0.4 \
+    --hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \
+    --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 \
+    # via requests
+colorama==0.4.4 \
+    --hash=sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b \
+    --hash=sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2 \
+    # via -r requirements.txt
+docutils==0.16 \
+    --hash=sha256:0c5b78adfbf7762415433f5515cd5c9e762339e23369dbe8000d84a4bf4ab3af \
+    --hash=sha256:c2de3a60e9e7d07be26b7f2b00ca0309c207e06c100f9cc2a94931fc75a478fc \
+    # via -r requirements.txt
+futures==3.3.0 \
+    --hash=sha256:49b3f5b064b6e3afc3316421a3f25f66c137ae88f068abbf72830170033c5e16 \
+    --hash=sha256:7e033af76a5e35f58e56da7a91e687706faf4e7bdfb2cbc3f2cca6b9bcda9794 \
+    # via -r requirements.txt, s3transfer
+idna==2.10 \
+    --hash=sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6 \
+    --hash=sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0 \
+    # via requests
+jinja2==2.11.2 \
+    --hash=sha256:89aab215427ef59c34ad58735269eb58b1a5808103067f7bb9d5836c651b3bb0 \
+    --hash=sha256:f0a4641d3cf955324a89c04f3d94663aa4d638abe8f733ecd3582848e1c37035 \
+    # via -r requirements.txt
+jmespath==0.10.0 \
+    --hash=sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9 \
+    --hash=sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f \
+    # via -r requirements.txt, boto3, botocore
+markupsafe==1.1.1 \
+    --hash=sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473 \
+    --hash=sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161 \
+    --hash=sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235 \
+    --hash=sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5 \
+    --hash=sha256:13d3144e1e340870b25e7b10b98d779608c02016d5184cfb9927a9f10c689f42 \
+    --hash=sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff \
+    --hash=sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b \
+    --hash=sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1 \
+    --hash=sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e \
+    --hash=sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183 \
+    --hash=sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66 \
+    --hash=sha256:596510de112c685489095da617b5bcbbac7dd6384aeebeda4df6025d0256a81b \
+    --hash=sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1 \
+    --hash=sha256:6788b695d50a51edb699cb55e35487e430fa21f1ed838122d722e0ff0ac5ba15 \
+    --hash=sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1 \
+    --hash=sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e \
+    --hash=sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b \
+    --hash=sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905 \
+    --hash=sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735 \
+    --hash=sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d \
+    --hash=sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e \
+    --hash=sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d \
+    --hash=sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c \
+    --hash=sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21 \
+    --hash=sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2 \
+    --hash=sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5 \
+    --hash=sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b \
+    --hash=sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6 \
+    --hash=sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f \
+    --hash=sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f \
+    --hash=sha256:cdb132fc825c38e1aeec2c8aa9338310d29d337bebbd7baa06889d09a60a1fa2 \
+    --hash=sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7 \
+    --hash=sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be \
+    # via -r requirements.txt, jinja2
+p4python==2017.2.1615960 \
+    --hash=sha256:0e6f7a1d654e32a2001a0b39ce360aabdc853e72d4f2ab59e9d1a3b1d1a0f9ed \
+    --hash=sha256:0fec222b4f1b625398630e20b5b63442432d0a029f456169aa8f2e3ba3ea0eff \
+    --hash=sha256:119a9c4c8262a072e8eaa6b70ab00813ff35b8b5a84b50af8308b71847af2166 \
+    --hash=sha256:316499580172a5ed5c873cc361cdd0ecda70bb36cffc2d84c189e53268a06312 \
+    --hash=sha256:31a22054e9a0342932da233c234ae6de54eee4d7797caf79912387a00457e13f \
+    --hash=sha256:3445ef1a1109d887a197aa2f137385747bb75d837fcb5991acaece14567444c2 \
+    --hash=sha256:3df3ebe4fb3bc47f338538dea86f1e45c59e13cb837b9bed29def3f5cfd5327d \
+    --hash=sha256:4e0303a4d482189033fcca0d05702dc91c3c62c3f35acfd7829e750b1905d95f \
+    --hash=sha256:549d5f6d2b094b86e63097650e217c65f65d82b87fdcc3013cf390aa552bb601 \
+    --hash=sha256:55c7c996a5c2897ea5a8fe4c8c9e65dfc6d6f65a09b1f499105d2b9a8a018498 \
+    --hash=sha256:85a083ab4749e3e980286de7eb06e9eae225549c5e5834d28fd2e9109372efaf \
+    --hash=sha256:8d446900216d9d1e42b7faa5f65d5c18d1e15b517f08a7e288631aee7d3cedd0 \
+    --hash=sha256:ae901d370d34097df145b91e754bf5463206c08ef3b3879888d37800d155c1c5 \
+    --hash=sha256:b5c61edda30e58ff1724771e83e03be047905d0708fbab4ef0733e6b1618c8c4 \
+    --hash=sha256:ba8bb0552f8e037a3fed74c6bc64d91dc3c3491015d066218bc91f23bfbcf76f \
+    --hash=sha256:bc3301a0ecf2161128b77bb1b4b23fa37059274a6be2f3ff7a8eb8b4bbf7c15d \
+    --hash=sha256:c5e3c145a2418a4e188f76f8eeb706fb04005887bca04146c202182bba21d2a4 \
+    --hash=sha256:d8917ed4e2ad23b57421cccf7cdfdff0a932cddbf45c53410ee390770beb7270 \
+    --hash=sha256:df99c2dc7e518a22f98fa5a9239bfbfae062c3736c362ca7cc6f9c9c1d3553d6 \
+    --hash=sha256:e539bc34c616a62aaae1d4c50b0f7311eb97c2f0daf5a1eedd4d181249a22d2b \
+    --hash=sha256:f39a7f4aee228f10437876f06a3f013453b04606c8e9c899632dbfeca690bcd4 \
+    # via -r requirements.txt
+psutil==5.7.3 \
+    --hash=sha256:01bc82813fbc3ea304914581954979e637bcc7084e59ac904d870d6eb8bb2bc7 \
+    --hash=sha256:1cd6a0c9fb35ece2ccf2d1dd733c1e165b342604c67454fd56a4c12e0a106787 \
+    --hash=sha256:2cb55ef9591b03ef0104bedf67cc4edb38a3edf015cf8cf24007b99cb8497542 \
+    --hash=sha256:56c85120fa173a5d2ad1d15a0c6e0ae62b388bfb956bb036ac231fbdaf9e4c22 \
+    --hash=sha256:5d9106ff5ec2712e2f659ebbd112967f44e7d33f40ba40530c485cc5904360b8 \
+    --hash=sha256:6a3e1fd2800ca45083d976b5478a2402dd62afdfb719b30ca46cd28bb25a2eb4 \
+    --hash=sha256:ade6af32eb80a536eff162d799e31b7ef92ddcda707c27bbd077238065018df4 \
+    --hash=sha256:af73f7bcebdc538eda9cc81d19db1db7bf26f103f91081d780bbacfcb620dee2 \
+    --hash=sha256:e02c31b2990dcd2431f4524b93491941df39f99619b0d312dfe1d4d530b08b4b \
+    --hash=sha256:fa38ac15dbf161ab1e941ff4ce39abd64b53fec5ddf60c23290daed2bc7d1157 \
+    --hash=sha256:fbcac492cb082fa38d88587d75feb90785d05d7e12d4565cbf1ecc727aff71b7 \
+    # via -r requirements.txt
+pyasn1==0.4.8 \
+    --hash=sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d \
+    --hash=sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba \
+    # via -r requirements.txt, rsa
+dateutil==2.8.1 \
+    --hash=sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c \
+    --hash=sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a \
+    # via -r requirements.txt, botocore
+pytz==2020.4 \
+    --hash=sha256:3e6b7dd2d1e0a59084bcee14a17af60c5c562cdc16d828e8eba2e683d3a7e268 \
+    --hash=sha256:5c55e189b682d420be27c6995ba6edce0c0a77dd67bfbe2ae6607134d5851ffd \
+    # via -r requirements.txt
+pywin32==228 \
+    --hash=sha256:00eaf43dbd05ba6a9b0080c77e161e0b7a601f9a3f660727a952e40140537de7 \
+    --hash=sha256:11cb6610efc2f078c9e6d8f5d0f957620c333f4b23466931a247fb945ed35e89 \
+    --hash=sha256:1f45db18af5d36195447b2cffacd182fe2d296849ba0aecdab24d3852fbf3f80 \
+    --hash=sha256:37dc9935f6a383cc744315ae0c2882ba1768d9b06700a70f35dc1ce73cd4ba9c \
+    --hash=sha256:6e38c44097a834a4707c1b63efa9c2435f5a42afabff634a17f563bc478dfcc8 \
+    --hash=sha256:8319bafdcd90b7202c50d6014efdfe4fde9311b3ff15fd6f893a45c0868de203 \
+    --hash=sha256:9b3466083f8271e1a5eb0329f4e0d61925d46b40b195a33413e0905dccb285e8 \
+    --hash=sha256:a60d795c6590a5b6baeacd16c583d91cce8038f959bd80c53bd9a68f40130f2d \
+    --hash=sha256:af40887b6fc200eafe4d7742c48417529a8702dcc1a60bf89eee152d1d11209f \
+    --hash=sha256:ec16d44b49b5f34e99eb97cf270806fdc560dff6f84d281eb2fcb89a014a56a9 \
+    --hash=sha256:ed74b72d8059a6606f64842e7917aeee99159ebd6b8d6261c518d002837be298 \
+    --hash=sha256:fa6ba028909cfc64ce9e24bcf22f588b14871980d9787f1e2002c99af8f1850c \
+    # via -r requirements.txt
+pyxb==1.2.6 \
+    --hash=sha256:2a00f38dd1d87b88f92d79bc5a09718d730419b88e814545f472bbd5a3bf27b4 \
+    # via -r requirements.txt
+pyyaml==5.3.1 \
+    --hash=sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97 \
+    --hash=sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76 \
+    --hash=sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2 \
+    --hash=sha256:6034f55dab5fea9e53f436aa68fa3ace2634918e8b5994d82f3621c04ff5ed2e \
+    --hash=sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648 \
+    --hash=sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf \
+    --hash=sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f \
+    --hash=sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2 \
+    --hash=sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee \
+    --hash=sha256:ad9c67312c84def58f3c04504727ca879cb0013b2517c85a9a253f0cb6380c0a \
+    --hash=sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d \
+    --hash=sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c \
+    --hash=sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a \
+    # via -r requirements.txt
+requests==2.25.0 \
+    --hash=sha256:7f1a0b932f4a60a1a65caa4263921bb7d9ee911957e0ae4a23a6dd08185ad5f8 \
+    --hash=sha256:e786fa28d8c9154e6a4de5d46a1d921b8749f8b74e28bde23768e5e16eece998 \
+    # via -r requirements.txt
+rsa==4.5 \
+    --hash=sha256:35c5b5f6675ac02120036d97cf96f1fde4d49670543db2822ba5015e21a18032 \
+    --hash=sha256:4d409f5a7d78530a4a2062574c7bd80311bc3af29b364e293aa9b03eea77714f \
+    # via -r requirements.txt
+s3transfer==0.3.3 \
+    --hash=sha256:2482b4259524933a022d59da830f51bd746db62f047d6eb213f2f8855dcb8a13 \
+    --hash=sha256:921a37e2aefc64145e7b73d50c71bb4f26f46e4c9f414dc648c6245ff92cf7db \
+    # via -r requirements.txt, boto3
+six==1.15.0 \
+    --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \
+    --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced \
+    # via -r requirements.txt, dateutil
+urllib3==1.26.2 \
+    --hash=sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08 \
+    --hash=sha256:d8ff90d979214d7b4f8ce956e80f4028fc6860e4431f731ea4a8c08f23f99473 \
+    # via -r requirements.txt, botocore, requests

scripts/build/build_node/Platform/Windows/init_setup.ps1

@@ -0,0 +1,42 @@
+<#
+All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+its licensors.
+
+For complete copyright and license terms please see the LICENSE at the root of this
+distribution (the "License"). All use of this software is governed by the License,
+or, if provided, by the license below or the license accompanying this file. Do not
+remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#>
+
+# Disable Windows Defender
+Set-MpPreference -DisableRealtimeMonitoring $true
+
+# Disable Admin prompts
+New-ItemProperty -Path HKLM:Software\Microsoft\Windows\CurrentVersion\policies\system -Name EnableLUA -PropertyType DWord -Value 0 -Force
+New-ItemProperty -Path HKLM:Software\Microsoft\Windows\CurrentVersion\policies\system -Name ConsentPromptBehaviorAdmin -PropertyType DWord -Value 0 -Force
+
+# Increase port number availablity (for P4) 
+netsh int ipv4 set dynamicport tcp start=1025 num=64511
+
+# Install Chocolatey and Carbon (local user util)
+[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))
+$env:ChocolateyInstall = Convert-Path "$((Get-Command choco).Path)\..\.."   
+Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
+choco install Carbon -y
+choco install awscli -y
+refreshenv
+
+# Grab credentials from parameter store (assumes AWS cli is installed and correct IAM policy is setup)
+$az = curl http://169.254.169.254/latest/meta-data/placement/availability-zone -UseBasicParsing
+$region = $az.Content -replace ".$"
+$username = aws ssm get-parameters --names "shared.builderuser" --region $region --with-decryption | ConvertFrom-Json
+$username = $username.Parameters.Value.ToString()
+$password = aws ssm get-parameters --names "shared.builderpass" --region $region --with-decryption | ConvertFrom-Json
+$password = ConvertTo-SecureString $password.Parameters.Value.ToString() -AsPlainText -Force
+$credential = New-Object System.Management.Automation.PSCredential -ArgumentList $username, $password
+
+Write-Host "Adding builder user"
+Import-Module 'Carbon'
+Install-User -Credential $credential -FullName "$($username)" -Description "Builder account for LY"
+Add-GroupMember -Name "Administrators" -Member "$($username)"

scripts/build/build_node/Platform/Windows/install_android.ps1

@@ -0,0 +1,43 @@
+<#
+All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+its licensors.
+
+For complete copyright and license terms please see the LICENSE at the root of this
+distribution (the "License"). All use of this software is governed by the License,
+or, if provided, by the license below or the license accompanying this file. Do not
+remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#>
+
+choco install -y android-sdk
+
+$Env:ANDROID_HOME = "C:\Android\android-sdk"
+setx ANDROID_HOME "C:\Android\android-sdk"
+
+$apis_packages = '"add-ons;addon-google_apis-google-19" "add-ons;addon-google_apis-google-21" "add-ons;addon-google_apis-google-22" "add-ons;addon-google_apis-google-23" "add-ons;addon-google_apis-google-24"'
+$android_packages = '"platforms;android-19" "platforms;android-21" "platforms;android-22" "platforms;android-23" "platforms;android-24" "platforms;android-25" "platforms;android-26" "platforms;android-27" "platforms;android-28" "platforms;android-29" "platforms;android-30"'
+$googleplay_packages = '"extras;google;market_apk_expansion" "extras;google;market_licensing"'
+$build_tools = '"build-tools;30.0.2" "tools"'
+
+$sdkmanager = "C:\Android\android-sdk\tools\bin\sdkmanager.bat"
+Start-Process -FilePath $sdkmanager -ArgumentList $apis_packages -NoNewWindow -Wait
+Start-Process -FilePath $sdkmanager -ArgumentList $android_packages -NoNewWindow -Wait
+Start-Process -FilePath $sdkmanager -ArgumentList $googleplay_packages -NoNewWindow -Wait
+Start-Process -FilePath $sdkmanager -ArgumentList $build_tools -NoNewWindow -Wait
+
+Write-Host "Installing Gradle and Ninja"
+Import-Module C:\ProgramData\chocolatey\helpers\chocolateyInstaller.psm1 #Grade needs a custom installer due to being hardcoded to C:\Programdata in Chocolatey
+$packageName = 'gradle'
+$version = '5.6.4'
+$checksum = 'ABC10BCEDB58806E8654210F96031DB541BCD2D6FC3161E81CB0572D6A15E821'
+$url = "https://services.gradle.org/distributions/gradle-$version-all.zip"
+$installDir = "C:\Gradle"
+
+Install-ChocolateyZipPackage $packageName $url $installDir -Checksum $checksum -ChecksumType 'sha256'
+
+$gradle_home = Join-Path $installDir "$packageName-$version"
+$gradle_bat = Join-Path $gradle_home 'bin/gradle.bat'
+
+Install-ChocolateyEnvironmentVariable "GRADLE_HOME" $gradle_home 'Machine'
+
+choco install -y ninja --version=1.10.0 --package-parameters="/installDir:C:\Ninja"

scripts/build/build_node/Platform/Windows/install_cygwin.ps1

@@ -0,0 +1,81 @@
+<#
+All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+its licensors.
+
+For complete copyright and license terms please see the LICENSE at the root of this
+distribution (the "License"). All use of this software is governed by the License,
+or, if provided, by the license below or the license accompanying this file. Do not
+remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#>
+
+# Grab credentials from parameter store (assumes AWS cli is installed and correct IAM policy is setup)
+$az = curl http://169.254.169.254/latest/meta-data/placement/availability-zone -UseBasicParsing
+$region = $az.Content -replace ".$"
+
+$username = aws ssm get-parameters --names "shared.builderuser" --region $region --with-decryption | ConvertFrom-Json
+$username = $username.Parameters.Value.ToString()
+$password = aws ssm get-parameters --names "shared.builderpass" --region $region --with-decryption | ConvertFrom-Json
+$password = ConvertTo-SecureString $password.Parameters.Value.ToString() -AsPlainText -Force
+$credential = New-Object System.Management.Automation.PSCredential -ArgumentList $username, $password
+
+$cygwin_packages = "openssh,vim,curl,tar,wget,zip,unzip,diffutils,bzr,nc,procps,ncdu"
+
+# Download cygwin
+New-Item C:\tools\cygwin -Force
+wget https://cygwin.com/setup-x86_64.exe -o C:\tools\cygwin\setup-x86_64.exe
+
+# Install cygwin (this is a specific version not subject to LGPLv3)
+# complete package list: https://cygwin.com/packages/package_list.html
+Write-Host "  * Starting Cygwin install"
+Start-Process "C:\tools\cygwin\setup-x86_64.exe" -ArgumentList ("--quiet-mode " +
+"--wait --root C:\cygwin --site http://cygwin.osuosl.org " +
+"--packages $cygwin_packages") -wait `
+-NoNewWindow -PassThru -RedirectStandardOutput "C:\cygwin_install.log" `
+-RedirectStandardError "C:\cygwin_install.err"
+
+# Open up firewall for ssh daemon
+Write-Host "  * Opening port 22 on firewall"
+New-NetFirewallRule -DisplayName "Allow SSH inbound" -Direction Inbound -LocalPort 22 -Protocol TCP -Action Allow
+
+# Workaround for https://www.cygwin.com/ml/cygwin/2015-10/msg00036.html
+# see:
+#   1) https://www.cygwin.com/ml/cygwin/2015-10/msg00038.html
+#   2) https://goo.gl/EWzeVV
+$env:LOGONSERVER = "\\" + $env:COMPUTERNAME
+
+# Configure sshd
+Write-Host "  * Configuring SSHD"
+Start-Process "C:\cygwin\bin\bash.exe" -ArgumentList "--login
+-c `"ssh-host-config -y -c 'ntsec mintty' -u '$($username)' -w '$($Credential.GetNetworkCredential().Password)'`"" `
+-wait -NoNewWindow -PassThru `
+-RedirectStandardOutput "C:\logs\cygrunsrv.log" -RedirectStandardError "C:\logs\cygrunsrv.err"
+Start-Process "C:\cygwin\bin\bash.exe" -ArgumentList "--login -c 'echo ""KexAlgorithms curve25519-sha256@libssh.org,ecdh-sha2-nistp256,ecdh-sha2-nistp384,ecdh-sha2-nistp521,diffie-hellman-group-exchange-sha256,diffie-hellman-group14-sha1,diffie-hellman-group-exchange-sha1,diffie-hellman-group1-sha1"" >> /etc/sshd_config'"
+
+# Copy bash script to add special permissions to builder account
+aws s3 cp s3://ly-jenkins-node-config/windows/setup.sh c:\cygwin\home\$username\
+
+# Run bash setup script
+echo "  * Configuring Bash"
+Start-Process "C:\cygwin\bin\bash.exe" -ArgumentList "--login -c 'chmod a+x ~/setup.sh; ~/setup.sh $username'" `
+-wait -NoNewWindow -PassThru `
+-RedirectStandardOutput "C:\logs\Administrator_cygwin_setup.log" `
+-RedirectStandardError "C:\logs\Administrator_cygwin_setup.err"
+
+# Start sshd
+echo "  * Starting SSHD"
+Start-Process "net" -ArgumentList "start cygsshd" `
+-wait -NoNewWindow -PassThru `
+-RedirectStandardOutput "C:\logs\net_start_sshd.log" -RedirectStandardError "C:\logs\net_start_sshd.err"
+
+# Add SSH key
+echo "  * Getting SSH key"
+$sshkey = aws ssm get-parameters --names "shared.buildersshkey" --region $region --with-decryption | ConvertFrom-Json
+New-Item "C:\cygwin\home\$username\.ssh" -Force 
+Add-Content "C:\cygwin\home\$username\.ssh\authorized_keys" "$($sshkey.Parameters.Value)"
+Start-Process "C:\cygwin\bin\bash.exe" -ArgumentList "--login -c 'chown -R $username /home/$username; chmod 600 /home/$username/.ssh/authorized_keys; sed -i 's/\r$//' /home/$username/.ssh/authorized_keys'" `
+
+# Clean up secure variables
+Remove-Variable password
+Remove-Variable credential
+Remove-Variable sshkey

scripts/build/build_node/Platform/Windows/install_python.ps1

@@ -0,0 +1,23 @@
+<#
+All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+its licensors.
+
+For complete copyright and license terms please see the LICENSE at the root of this
+distribution (the "License"). All use of this software is governed by the License,
+or, if provided, by the license below or the license accompanying this file. Do not
+remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#>
+
+choco install -y python2 --version=2.7.15
+choco install -y python3 --version=3.7.5
+
+# Ensure Python paths are set
+[Environment]::SetEnvironmentVariable(
+    "Path",
+    [Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarget]::Machine) + ";C:\Python27;C:\Python27\Scripts",
+    [EnvironmentVariableTarget]::Machine)
+
+Write-Host "Installing packages" # requirements.txt hould be in the "Platforms\Common" folder
+pip install -r ..\Common\requirements.txt
+pip3 install -r ..\Common\requirements.txt

scripts/build/build_node/Platform/Windows/install_utiltools.ps1

@@ -0,0 +1,28 @@
+<#
+All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+its licensors.
+
+For complete copyright and license terms please see the LICENSE at the root of this
+distribution (the "License"). All use of this software is governed by the License,
+or, if provided, by the license below or the license accompanying this file. Do not
+remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#>
+
+# Install dependancies
+choco install -y 7zip
+choco install -y procexp
+choco install -y windirstat
+choco install -y sysinternals
+
+# Install source control apps 
+choco install -y git
+choco install -y git-lfs
+choco install -y p4
+
+Write-Host "Configuring Git"
+git config --global "credential.helper" "!aws codecommit credential-helper $@"
+git config --global "credential.UseHttpPath" "true"
+
+# Install Java (for Jenkins)
+choco install corretto8jdk -y --ia INSTALLDIR="c:\jdk8" # Custom directory to handle cases where whitespace in the path is not quote wrapped

scripts/build/build_node/Platform/Windows/install_vsbuildtools.ps1

@@ -0,0 +1,30 @@
+<#
+All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+its licensors.
+
+For complete copyright and license terms please see the LICENSE at the root of this
+distribution (the "License"). All use of this software is governed by the License,
+or, if provided, by the license below or the license accompanying this file. Do not
+remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#>
+
+choco install visualstudio2019buildtools --version=16.8.2 --package-parameters "--config .\vs2019bt.vsconfig" -y
+choco install visualstudio2017buildtools --version=15.9.29 --package-parameters "--config .\vs2017bt.vsconfig" -y
+
+Write-Host "Installing downstream dependancies"
+choco install -y dotnet3.5 
+
+# This is a custom install of the Win10 SDK debugger which was not included in VS2019/2017. This resolves an issue where the older DbgHelp library was being referenced from C:\Windows\System32\
+# More details here: https://docs.microsoft.com/en-us/windows/win32/debug/calling-the-dbghelp-library
+
+Import-Module C:\ProgramData\chocolatey\helpers\chocolateyInstaller.psm1 
+$packageName = 'windows-sdk-10.1'
+$featureName = 'OptionId.WindowsDesktopDebuggers'
+$installerType = 'EXE'
+$url = 'https://download.microsoft.com/download/4/2/2/42245968-6A79-4DA7-A5FB-08C0AD0AE661/windowssdk/winsdksetup.exe'
+$checksum = '2E28117E82B4D02FE30D564B835ACE9976612609271265872F20F2256A9C506B'
+$checksumType = 'sha256'
+$silentArgs = "/Quiet /NoRestart /features $featureName /Log ""$env:temp\${packageName}_$([Guid]::NewGuid().ToString('D')).log"""
+$validExitCodes = @(0,3010)
+Install-ChocolateyPackage $packageName $installerType $silentArgs $url -checksum $checksum -checksumType $checksumType -validExitCodes $validExitCodes

scripts/build/build_node/Platform/Windows/setup.sh

@@ -0,0 +1,70 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+#!/bin/bash
+
+echo "Running setup.sh script..."
+echo
+
+# setup standard groups and user account db for ssh
+mkpasswd -c > /etc/passwd
+mkgroup -c > /etc/group
+
+# give special permissions to the builder user
+editrights -a SeAssignPrimaryTokenPrivilege -u $1
+editrights -a SeCreateTokenPrivilege -u $1
+editrights -a SeTcbPrivilege -u $1
+editrights -a SeServiceLogonRight -u $1
+
+set -e
+
+function add_if_missing {
+    LINE="${1}"
+    FILE="${2}"
+    grep -Fx "${LINE}" "${FILE}" >/dev/null 2>&1 || echo "${LINE}" >> "${FILE}"
+}
+
+echo "  * Setting PS1 environment variable in .bashrc"
+add_if_missing 'export PS1='"'"'\[\033[01;35m\]\u\[\033[34m\]@\[\033[36m\]\h\[\033[00m\]:\[\033[01;33m\]\w\[\033[31m\] \$\[\033[00m\] '"'" ~/.bashrc
+echo "  * Making sure PATH gets sourced in .bashrc"
+add_if_missing 'export PATH="/cygdrive/c/Windows/system32:${PATH}"' ~/.bashrc
+echo "  * Making sure there is a pair of public/private keys"
+[ ! -e ~/.ssh/id_rsa ] || [ ! -e ~/.ssh/id_rsa.pub ] && ssh-keygen -b 1024 -t rsa -N '' -f ~/.ssh/id_rsa
+echo "  * Setting syntax highlighting in vi"
+add_if_missing 'syntax on' ~/.vimrc
+echo "  * Setting colour scheme in vi"
+add_if_missing 'colorscheme murphy' ~/.vimrc
+echo "  * Setting tab to four spaces in vi"
+add_if_missing 'set tabstop=4' ~/.vimrc
+[ ! -e /c ] && ln -s /cygdrive/c /c
+if [ ! -e ~/bin/sudo ]; then
+echo "  * Creating ""Sudo"" command"
+cat > /bin/sudo << 'EOF'
+#!/usr/bin/bash
+cygstart --action=runas "$@"
+EOF
+chmod a+x /bin/sudo
+fi
+echo "  * Installing apt-cyg"
+cd ~
+wget rawgit.com/transcode-open/apt-cyg/master/apt-cyg
+mv apt-cyg.* apt-get
+install apt-get /bin
+apt-get mirror http://mirrors.kernel.org/sourceware/cygwin/
+apt-get update
+
+echo " * Configuring the Cygwin git client to use the AWS codecommit helper"
+git config --global credential.helper "!aws codecommit credential-helper $@"
+git config --global credential.UseHttpPath true
+git config --global filter.lfs.required true
+git config --global filter.lfs.clean "git-lfs clean -- %f"
+git config --global filter.lfs.smudge "git-lfs smudge -- %f"
+git config --global filter.lfs.process "git-lfs filter-process"

scripts/build/build_node/Platform/Windows/vs2017bt.vsconfig

@@ -0,0 +1,28 @@
+{
+  "version": "1.0",
+  "components": [
+    "Microsoft.Component.MSBuild",
+    "Microsoft.Component.NetFX.Native",
+    "Microsoft.Component.VC.Runtime.OSSupport",
+    "Microsoft.Component.VC.Runtime.UCRTSDK",
+    "Microsoft.Net.Component.4.7.1.SDK",
+    "Microsoft.NetCore.Component.SDK",
+    "Microsoft.VisualStudio.Component.CoreBuildTools",
+    "Microsoft.VisualStudio.Component.NuGet.BuildTools",
+    "Microsoft.VisualStudio.Component.Roslyn.Compiler",
+    "Microsoft.VisualStudio.Component.Static.Analysis.Tools",
+    "Microsoft.VisualStudio.Component.TestTools.BuildTools",
+    "Microsoft.VisualStudio.Component.VC.ATL",
+    "Microsoft.VisualStudio.Component.VC.CMake.Project",
+    "Microsoft.VisualStudio.Component.VC.CoreBuildTools",
+    "Microsoft.VisualStudio.Component.VC.Redist.14.Latest",
+    "Microsoft.VisualStudio.Component.VC.Tools.x86.x64",
+    "Microsoft.VisualStudio.Component.Windows10SDK",
+    "Microsoft.VisualStudio.Component.Windows81SDK",
+    "Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Win81",
+    "Microsoft.VisualStudio.ComponentGroup.UWP.BuildTools",
+    "Microsoft.VisualStudio.Workload.MSBuildTools",
+    "Microsoft.VisualStudio.Workload.UniversalBuildTools",
+    "Microsoft.VisualStudio.Workload.VCTools"
+  ]
+}

scripts/build/build_node/Platform/Windows/vs2019bt.vsconfig

@@ -0,0 +1,30 @@
+{
+  "version": "1.0",
+  "components": [
+    "Microsoft.Component.MSBuild",
+    "Microsoft.Component.NetFX.Native",
+    "Microsoft.Net.Component.4.8.SDK",
+    "Microsoft.NetCore.Component.Runtime.3.1",
+    "Microsoft.NetCore.Component.Runtime.5.0",
+    "Microsoft.NetCore.Component.SDK",
+    "Microsoft.VisualStudio.Component.CoreBuildTools",
+    "Microsoft.VisualStudio.Component.NuGet.BuildTools",
+    "Microsoft.VisualStudio.Component.Roslyn.Compiler",
+    "Microsoft.VisualStudio.Component.TestTools.BuildTools",
+    "Microsoft.VisualStudio.Component.TextTemplating",
+    "Microsoft.VisualStudio.Component.VC.ASAN",
+    "Microsoft.VisualStudio.Component.VC.ATL",
+    "Microsoft.VisualStudio.Component.VC.CMake.Project",
+    "Microsoft.VisualStudio.Component.VC.CoreBuildTools",
+    "Microsoft.VisualStudio.Component.VC.CoreIde",
+    "Microsoft.VisualStudio.Component.VC.Redist.14.Latest",
+    "Microsoft.VisualStudio.Component.VC.Tools.x86.x64",
+    "Microsoft.VisualStudio.Component.Windows10SDK",
+    "Microsoft.VisualStudio.Component.Windows10SDK.18362",
+    "Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Core",
+    "Microsoft.VisualStudio.ComponentGroup.UWP.BuildTools",
+    "Microsoft.VisualStudio.Workload.MSBuildTools",
+    "Microsoft.VisualStudio.Workload.UniversalBuildTools",
+    "Microsoft.VisualStudio.Workload.VCTools"
+  ]
+}

scripts/build/lambda/auto_delete_ebs.py

@@ -0,0 +1,57 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import boto3
+import time
+import logging
+
+TIMEOUT = 300
+
+
+def lambda_handler(event, context):
+    log = logging.getLogger(__name__)
+    log.setLevel(logging.INFO)
+    branch_name = event['detail']['referenceName']
+    ec2_client = boto3.resource('ec2')
+    response = ec2_client.volumes.filter(Filters=[
+        {
+            'Name': 'tag:BranchName',
+            'Values': [branch_name]
+        }
+    ])
+    log.info(f'Deleting EBS volumes for remote-branch {branch_name}.')
+    for volume in response:
+        if volume.attachments:
+            ec2_instance_id = volume.attachments[0]['InstanceId']
+            try:
+                log.info(f'Detaching volume {volume.volume_id} from ec2 {ec2_instance_id}.')
+                volume.detach_from_instance(Device='xvdf',
+                                            Force=True,
+                                            InstanceId=ec2_instance_id,
+                                            VolumeId=volume.volume_id)
+            except Exception as e:
+                log.error(f'Failed to detach volume {volume.volume_id} from {ec2_instance_id}.')
+                log.error(e)
+            timeout_init = time.clock()
+            while len(volume.attachments) and volume.attachments[0]['State'] != 'detached':
+                time.sleep(1)
+                volume.load()
+                if (time.clock() - timeout_init) > TIMEOUT:
+                    log.error('Timeout reached trying to detach EBS.')
+        try:
+            log.info(f'Deleting volume {volume.volume_id}')
+            volume.delete()
+        except Exception as e:
+            log.error(f'Failed to delete volume {volume.volume_id}.')
+            log.error(e)
+
+
+lambda_handler(event, context)

scripts/build/lambda/trigger_first_build.py

@@ -0,0 +1,74 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import boto3
+import logging
+import json
+import os
+import requests
+
+from botocore.exceptions import ClientError
+from requests.adapters import HTTPAdapter
+from requests.packages.urllib3.util.retry import Retry
+from requests.exceptions import RetryError
+
+log = logging.getLogger(__name__)
+log.setLevel(logging.INFO)
+
+JENKINS_CREDENTIAL_STORE = os.environ['JENKINS_CREDENTIAL_STORE']
+JENKINS_ENDPOINT = os.environ['JENKINS_ENDPOINT']
+REGION = os.environ['REGION']
+
+
+def get_secret(secret_name):
+    secrets = boto3.client(service_name='secretsmanager', region_name=REGION)
+    try:
+        response = secrets.get_secret_value(SecretId=secret_name)
+        return json.loads(response['SecretString'])
+    except ClientError as e:
+        log.error(f'Unable to get secret value: {e}')
+        return None
+
+
+def lambda_handler(event, context):
+    event_record = event['Records'][0]
+    log.info(f'Received Event: {event_record}')
+
+    ref_prefix = "refs/heads/"
+    branch_name = event_record['codecommit']['references'][0]['ref'][len(ref_prefix):]
+    repo_name = event_record['eventSourceARN'].split(":")[-1]
+
+    credentials = get_secret(JENKINS_CREDENTIAL_STORE)
+
+    if credentials:
+        retries = 3
+        backoff = 30
+        status_list = [404]  # Retry if the branch doesn't exist yet and provide time for Jenkins to discover it.
+        method_list = ['POST']
+        retry_config = Retry(total=retries, backoff_factor=backoff, status_forcelist=status_list, method_whitelist=method_list)
+
+        session = requests.Session()
+        session.mount('https://', HTTPAdapter(max_retries=retry_config))
+
+        encoded_branch_name = requests.utils.quote(branch_name, safe='')
+        build_path = f'/job/{repo_name}/job/{encoded_branch_name}/build'
+        jenkins_url = requests.compat.urljoin(JENKINS_ENDPOINT, build_path)
+
+        try:
+            response = session.post(jenkins_url, auth=(credentials['username'], credentials['apitoken']))
+            if response.status_code == 201:
+                log.info(f'Successfully triggered build on {repo_name}/{branch_name}')
+            elif response.status_code == 400:
+                log.info(f'Initial build already started for {repo_name}/{branch_name}. Parameters are already available.')
+            else:
+                log.error(f'Failed to start build on {repo_name}/{branch_name}. Status code: {response.status_code}')
+        except RetryError as e:
+            log.error(f'Pipeline for {repo_name}/{branch_name} does not exist in Jenkins: {e}')

scripts/build/package/PackageEnv.py

@@ -0,0 +1,194 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+from Params import Params
+from util import *
+
+
+class PackageEnv(Params):
+    def __init__(self, platform, type, json_file):
+        super(PackageEnv, self).__init__()
+        self.__cur_dir = os.path.dirname(os.path.abspath(__file__))
+        global_env_file = os.path.join(self.__cur_dir, json_file)
+        with open(global_env_file, 'r') as source:
+            data = json.load(source)
+            self.__global_env = data.get('global_env')
+        platform_env_file = os.path.join(self.__cur_dir, 'Platform', platform, json_file)
+        if not os.path.exists(platform_env_file):
+            print(f'{platform_env_file} is not found.')
+            # Search restricted platform folders
+            engine_root = self.get('ENGINE_ROOT')
+            # Use real path in case engine root is a symlink path
+            if os.name == 'posix' and os.path.islink(engine_root):
+                engine_root = os.readlink(engine_root)
+            rel_path = os.path.relpath(self.__cur_dir, engine_root)
+            platform_env_file = os.path.join(engine_root, 'restricted', platform, rel_path, json_file)
+            if not os.path.exists(platform_env_file):
+                ly_build_error(f'{platform_env_file} is not found.')
+        with open(platform_env_file, 'r') as source:
+            data = json.load(source)
+            types = data.get('types')
+            if type not in types:
+                ly_build_error(f'Package type {type} is not supported')
+            self.__platform = platform
+            self.__platform_env = data.get('local_env')
+            self.__platform_env.update(self.__global_env)
+            self.__type = type
+            self.__type_env = types.get(type)
+
+    def get_platform(self):
+        return self.__platform
+
+    def get_type(self):
+        return self.__type
+
+    def get_platform_env(self):
+        return self.__platform_env
+
+    def get_type_env(self):
+        return self.__type_env
+
+    def __get_platform_value(self, key):
+        key = key.upper()
+        value = self.__platform_env.get(key)
+        if value is None:
+            ly_build_error(f'{key} is not defined in global env nor in local env')
+        return value
+
+    def __get_type_value(self, key):
+        key = key.upper()
+        value = self.__type_env.get(key)
+        if value is None:
+            ly_build_error(f'{key} is not defined in package type {self.__type} for platform {self.__platform}')
+        return value
+
+    def __evaluate_boolean(self, v):
+        return str(v).lower() in ['1', 'true']
+
+    def __get_engine_root(self):
+        def validate_engine_root(engine_root):
+            if not os.path.isdir(engine_root):
+                return False
+            return os.path.exists(os.path.join(engine_root, 'engineroot.txt'))
+
+        workspace = os.getenv('WORKSPACE')
+        if workspace is not None:
+            print(f'Environment variable WORKSPACE={workspace} detected')
+            if validate_engine_root(workspace):
+                print(f'Setting ENGINE_ROOT to {workspace}')
+                return workspace
+            print('Cannot locate ENGINE_ROOT with Environment variable WORKSPACE')
+
+        engine_root = os.getenv('ENGINE_ROOT', '')
+        if validate_engine_root(engine_root):
+            return engine_root
+
+        print('Environment variable ENGINE_ROOT is not set or invalid, checking ENGINE_ROOT in env json file')
+        engine_root = self.__global_env.get('ENGINE_ROOT')
+        if validate_engine_root(engine_root):
+            return engine_root
+
+        # Set engine_root based on script location
+        engine_root = os.path.dirname(os.path.dirname(os.path.dirname(self.__cur_dir)))
+        print(f'ENGINE_ROOT from env json file is invalid, defaulting to {engine_root}')
+        if validate_engine_root(engine_root):
+            return engine_root
+        else:
+            error('Cannot Locate ENGINE_ROOT')
+
+    def __get_thirdparty_home(self):
+        third_party_home = os.getenv('LY_3RDPARTY_PATH', '')
+        if os.path.exists(third_party_home):
+            print(f'LY_3RDPARTY_PATH found, using {third_party_home} as 3rdParty path.')
+            return third_party_home
+        third_party_home = self.__get_platform_value('THIRDPARTY_HOME')
+        if os.path.isdir(third_party_home):
+            return third_party_home
+
+        # Set engine_root based on script location
+        print('THIRDPARTY_HOME is not valid, looking for THIRD_PARTY_HOME')
+
+        # Finding THIRD_PARTY_HOME
+        cur_dir = self.__get_engine_root()
+        last_dir = None
+        while last_dir != cur_dir:
+            third_party_home = os.path.join(cur_dir, '3rdParty')
+            print(f'Cheking THIRDPARTY_HOME {third_party_home}')
+            if os.path.exists(os.path.join(third_party_home, '3rdParty.txt')):
+                print(f'Setting THIRDPARTY_HOME to {third_party_home}')
+                return third_party_home
+            last_dir = cur_dir
+            cur_dir = os.path.dirname(cur_dir)
+        error('Cannot locate THIRDPARTY_HOME')
+
+    def __get_package_name_pattern(self):
+        package_name_pattern = self.__get_platform_value('PACKAGE_NAME_PATTERN')
+        if os.getenv('PACKAGE_NAME_PATTERN') is not None:
+            package_name_pattern = os.getenv('PACKAGE_NAME_PATTERN')
+        return package_name_pattern
+
+    def __get_branch_name(self):
+        branch_name = self.__get_platform_value('BRANCH_NAME')
+        if os.getenv('BRANCH_NAME') is not None:
+            branch_name = os.getenv('BRANCH_NAME')
+        branch_name = branch_name.replace('/', '_').replace('\\', '_')
+        return branch_name
+
+    def __get_build_number(self):
+        build_number = self.__get_platform_value('BUILD_NUMBER')
+        if os.getenv('BUILD_NUMBER') is not None:
+            build_number = os.getenv('BUILD_NUMBER')
+        return build_number
+
+    def __get_scrub_params(self):
+        return self.__get_type_value('SCRUB_PARAMS')
+
+    def __get_validator_platforms(self):
+        return self.__get_type_value('VALIDATOR_PLATFORMS')
+
+    def __get_package_targets(self):
+        return self.__get_type_value('PACKAGE_TARGETS')
+
+    def __get_build_targets(self):
+        return self.__get_type_value('BUILD_TARGETS')
+
+    def __get_asset_processor_path(self):
+        return self.__get_type_value('ASSET_PROCESSOR_PATH')
+
+    def __get_asset_game_folders(self):
+        return self.__get_type_value('ASSET_GAME_FOLDERS')
+
+    def __get_asset_platform(self):
+        return self.__get_type_value('ASSET_PLATFORM')
+
+    def __get_bootstrap_cfg_game_folder(self):
+        return self.__get_type_value('BOOTSTRAP_CFG_GAME_FOLDER')
+
+    def __get_skip_build(self):
+        skip_build = os.getenv('SKIP_BUILD')
+        if skip_build is None:
+            skip_build = self.__get_type_value('SKIP_BUILD')
+        return self.__evaluate_boolean(skip_build)
+
+    def __get_skip_scrubbing(self):
+        skip_scrubbing = os.getenv('SKIP_SCRUBBING')
+        if skip_scrubbing is None:
+            skip_scrubbing = self.__type_env.get('SKIP_SCRUBBING', 'False')
+        return self.__evaluate_boolean(skip_scrubbing)
+
+    def __get_internal_s3_bucket(self):
+        return self.__get_platform_value('INTERNAL_S3_BUCKET')
+
+    def __get_qa_s3_bucket(self):
+        return self.__get_platform_value('QA_S3_BUCKET')
+
+    def __get_s3_prefix(self):
+        return self.__get_platform_value('S3_PREFIX')

scripts/build/package/Params.py

@@ -0,0 +1,83 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+import os
+import sys
+import re
+from util import ly_build_error
+
+
+class Params(object):
+    def __init__(self):
+        # Cache params
+        self.__params = {}
+
+    def get(self, param_name):
+        param_value = self.__params.get(param_name)
+        if param_value is not None:
+            return param_value
+        # Call __get_${param_name} function
+        func = getattr(self, '_{}__get_{}'.format(self.__class__.__name__, param_name.lower()), None)
+        if func is not None:
+            param_value = func()
+            # Replace all ${env} in value
+            if isinstance(param_value, str):
+                param_value = self.__process_string(param_name, param_value)
+            elif isinstance(param_value, list):
+                param_value = self.__process_list(param_name, param_value)
+            elif isinstance(param_value, dict):
+                param_value = self.__process_dict(param_name, param_value)
+            # Cache param
+            self.__params[param_name] = param_value
+            return param_value
+        ly_build_error('method __get_{} is not defined in class {}'.format(param_name.lower(), self.__class__.__name__))
+
+    def set(self, param_name, param_value):
+        self.__params[param_name] = param_value
+
+    def exists(self, param_name):
+        try:
+            self.get(param_name)
+        except LyBuildError:
+            return False
+        return True
+
+    def __process_string(self, param_name, param_value):
+        # Find all param with format ${param}
+        params = re.findall('\${(\w+)}', param_value)
+        # Avoid using the same param name in value, like 'WORKSPACE': '${WORKSPACE} some string'
+        if param_name in params:
+            ly_build_error('The use of same parameter name({}) in value is not allowed'.format(param_name))
+        # Replace ${param} with actual value
+        for param in params:
+            param_value = param_value.replace('${' + param + '}', self.get(param))
+        return param_value
+
+    def __process_list(self, param_name, param_value):
+        processed_list = []
+        for entry in param_value:
+            if isinstance(entry, str):
+                entry = self.__process_string(param_name, entry)
+            elif isinstance(entry, list):
+                entry = self.__process_list(param_name, entry)
+            elif isinstance(entry, dict):
+                entry = self.__process_dict(param_name, entry)
+            processed_list.append(entry)
+        return processed_list
+
+    def __process_dict(self, param_name, param_value):
+        for key in param_value:
+            if isinstance(param_value[key], str):
+                param_value[key] = self.__process_string(param_name, param_value[key])
+            elif isinstance(param_value[key], list):
+                param_value[key] = self.__process_list(param_name, param_value[key])
+            elif isinstance(param_value[key], dict):
+                param_value[key] = self.__process_dict(param_name, param_value[key])
+        return param_value

scripts/build/package/Platform/3rdParty/package_env.json

@@ -0,0 +1,19 @@
+{
+    "local_env": {
+        "S3_PREFIX": "${BRANCH_NAME}/3rdParty"
+    },
+    "types": {
+        "3rdParty_all": {
+            "PACKAGE_TARGETS":[
+                {
+                    "FILE_LIST": "3rdParty.json",
+                    "FILE_LIST_TYPE": "3rdParty",
+                    "PACKAGE_NAME": "${PACKAGE_NAME_PATTERN}-3rdParty-all-${BUILD_NUMBER}.zip"
+                }
+            ],
+            "BOOTSTRAP_CFG_GAME_FOLDER":"CMakeTestbed",
+            "SKIP_BUILD": 1,
+            "SKIP_SCRUBBING": 1
+        }
+    }
+}

scripts/build/package/Platform/3rdParty/package_filelists/3rdParty.json

@@ -0,0 +1,45 @@
+{
+    "@3rdParty": {
+        "3rdParty.txt": "#include",
+        "AWS/AWSNativeSDK/1.7.167-az.2/**": "#include",
+        "AWS/GameLift/3.4.0/**": "#include",
+        "Blast/1.1.6-az.1/**": "#include",
+        "benchmark/1.5.0/**": "#include",
+        "cityhash/1.1-az.1/**": "#include",
+        "civetweb/civetweb-20160922-az.2/**": "#include",
+        "Clang/6.0.1-az/**": "#include",
+        "DirectXShaderCompiler/1.0.1-az.1/**": "#include",
+        "dyad/0.2.0-17-amazon/**": "#include",
+        "etc2comp/2017_04_24-az.2/**": "#include",
+        "expat/2.1.0-pkg.3/**": "#include",
+        "FbxSdk/2016.1.2-az.1/**": "#include",
+        "glad/2.0.0-beta/**": "#include",
+        "googletest/1.8.1-az.3/**": "#include",
+        "jsmn/78b1dca/**": "#include",
+        "libav/11.7/**": "#include",
+        "LibTomCrypt/1.17-az.2/**": "#include",
+        "LibTomMath/0.42.0-az.2/**": "#include",
+        "lz4/r128-pkg.3/**": "#include",
+        "Lzma/unknown-pkg.3/**": "#include",
+        "LZSS/unknown-pkg.3/**": "#include",
+        "md5/2.0-pkg.3/**": "#include",
+        "mikkelsen/1.0.0-az.2/**": "#include",
+        "nvapi/R361-developer_V2/**": "#include",
+        "NvCloth/1.1.6-az.4/**": "#include",
+        "OpenSSL/1.1.1b-noasm-az/**": "#include",
+        "p4api/2019.1/**": "#include",
+        "PhysX/4.1.0.25992954-az.1/**": "#include",
+        "poly2tri/0.3.3-az.2/**": "#include",
+        "PVRTexTool/2016_r1.1/**": "#include",
+        "Qt/5.15.1.2-az/**": "#include",
+        "RadTelemetry/3.5.0.17/**": "#include",
+        "rapidjson/rapidjson-1.1.0/**": "#include",
+        "rapidxml/1.13-modified.1/**": "#include",
+        "SQLite/v3.32.2/**": "#include",
+        "tiff/3.9.5-az.3/**": "#include",
+        "unwind/1.2.1/**": "#include",
+        "Wwise/2019.2.8.7432/**": "#include",
+        "xxhash/0.7.4/**": "#include",
+        "zstd/1.35-pkg.1/**": "#include"
+    }
+}

scripts/build/package/Platform/Mac/package_env.json

@@ -0,0 +1,28 @@
+{
+    "local_env": {},
+    "types":{
+        "all":{
+            "PACKAGE_TARGETS":[
+                {
+                    "FILE_LIST": "all.json",
+                    "FILE_LIST_TYPE": "All",
+                    "PACKAGE_NAME": "${PACKAGE_NAME_PATTERN}-mac-all-${BUILD_NUMBER}.zip"
+                }
+            ],
+            "BOOTSTRAP_CFG_GAME_FOLDER":"CMakeTestbed",
+            "SKIP_BUILD": 0,
+            "BUILD_TARGETS":[
+                {
+                    "BUILD_CONFIG_FILENAME": "build_config.json",
+                    "PLATFORM": "Mac",
+                    "TYPE": "profile"
+                },
+                {
+                    "BUILD_CONFIG_FILENAME": "build_config.json",
+                    "PLATFORM": "iOS",
+                    "TYPE": "profile"
+                }
+            ]
+        }
+    }
+}

scripts/build/package/Platform/Windows/package_env.json

@@ -0,0 +1,63 @@
+{
+    "local_env": {
+        "S3_PREFIX": "${BRANCH_NAME}/Windows"
+    },
+    "types":{
+        "all":{
+            "PACKAGE_TARGETS":[
+                {
+                    "FILE_LIST": "all.json",
+                    "FILE_LIST_TYPE": "All",
+                    "PACKAGE_NAME": "${PACKAGE_NAME_PATTERN}-windows-all-${BUILD_NUMBER}.zip"
+                },
+                {
+                    "FILE_LIST": "symbols.json",
+                    "FILE_LIST_TYPE": "All",
+                    "PACKAGE_NAME": "${PACKAGE_NAME_PATTERN}-windows-all-symbols-${BUILD_NUMBER}.zip"
+                }
+            ],
+            "BOOTSTRAP_CFG_GAME_FOLDER":"CMakeTestbed",
+            "SKIP_BUILD": 0,
+            "BUILD_TARGETS":[
+                {
+                    "BUILD_CONFIG_FILENAME": "build_config.json",
+                    "PLATFORM": "Windows",
+                    "TYPE": "profile_vs2017"
+                },
+                {
+                    "BUILD_CONFIG_FILENAME": "build_config.json",
+                    "PLATFORM": "Windows",
+                    "TYPE": "profile_vs2019"
+                }
+            ]
+        },
+        "atom":{
+            "PACKAGE_TARGETS":[
+                {
+                    "FILE_LIST": "atom.json",
+                    "FILE_LIST_TYPE": "Windows",
+                    "PACKAGE_NAME": "${PACKAGE_NAME_PATTERN}-windows-atom-${BUILD_NUMBER}.zip"
+                },
+                {
+                    "FILE_LIST": "symbols.json",
+                    "FILE_LIST_TYPE": "All",
+                    "PACKAGE_NAME": "${PACKAGE_NAME_PATTERN}-windows-atom-symbols-${BUILD_NUMBER}.zip"
+                }
+            ],
+            "BOOTSTRAP_CFG_GAME_FOLDER":"AtomSampleViewer;AtomTest",
+            "SKIP_BUILD": 1,
+            "BUILD_TARGETS":[
+                {
+                    "BUILD_CONFIG_FILENAME": "build_config.json",
+                    "PLATFORM": "Windows",
+                    "TYPE": "profile_vs2017_atom"
+                },
+                {
+                    "BUILD_CONFIG_FILENAME": "build_config.json",
+                    "PLATFORM": "Windows",
+                    "TYPE": "profile_vs2019_atom"
+                }
+            ]
+        }
+    }
+}

scripts/build/package/Platform/Windows/package_filelist/atom.json

@@ -0,0 +1,198 @@
+{
+    "@3rdParty": {
+        "**/.owner": "#exclude",
+        "3rdParty.txt": "#move:3rdParty",
+        "OpenEXR/**": "#move:3rdParty",
+        "CMake/3.19.1/**": "#move:3rdParty",
+        "Redistributables":{
+            "WwiseLTX": {
+                "LTX_2018.1.2.6762": {
+                    "**": "#move:dev/Tools/Redistributables/WwiseLTX/LTX_2018.1.2.6762",
+                    "*.app.zip": "#exclude",
+                    "WwiseLauncher.pkg": "#exclude"
+                }
+            },
+            "FbxSdk": {
+                "2016.1.2": {
+                    "*win*": "#move:dev/Tools/Redistributables/FbxSdk/2016.1.2-az.1",
+                    "*vs2013*": "#exclude"
+                }
+            }
+        }
+    },
+    "@lyengine": {
+        "**/*.pyc": "#exclude",
+        "*": "#include",
+        "AtomTest":
+        {
+            "**":"#include",
+            "**/*.ma":"#exclude",
+            "**/*.max":"#exclude",
+            "**/*.mb":"#exclude",
+            "**/*.psd":"#exclude"
+        },
+        "AtomSampleViewer":
+        {
+            "**":"#include",
+            "**/*.ma":"#exclude",
+            "**/*.max":"#exclude",
+            "**/*.mb":"#exclude",
+            "**/*.psd":"#exclude"
+        },
+        "cmake/**": "#include",
+        "Code": {
+            "CryEngine/**": "#include",
+            "Framework/**": "#include",
+            "LauncherUnified/**": "#include",
+            "Sandbox/**": "#include",
+            "Tools": {
+                "Android/**": "#include",
+                "AWSNativeSDKInit/**": "#include",
+                "AssetProcessor*/**": "#include",
+                "AssetBundler/**": "#include",
+                "AzTestRunner/**": "#include",
+                "CrashHandler/**": "#include",
+                "CryCommonTools/**": "#include",
+                "CrySCompileServer/**": "#include",
+                "CryXML/**": "#include",
+                "DeltaCataloger/**": "#include",
+                "GemRegistry/**": "#include",
+                "GridHub/**": "#include",
+                "HLSLCrossCompiler/**": "#include",
+                "HLSLCrossCompilerMETAL/**": "#include",
+                "LyIdentity/**": "#include",
+                "LyMetrics/**": "#include",
+                "News/**": "#include",
+                "PythonBindingsExample/**": "#include",
+                "RC/**": "#include",
+                "RemoteConsole/**": "#include",
+                "SceneAPI/**": "#include",
+                "SerializeContextTools/**": "#include",
+                "ShaderCacheGen/**": "#include",
+                "SharedQMLResource/**": "#include",
+                "Woodpecker/**": "#include",
+                "CMakeLists.txt": "#include"
+            },
+            "CMakeLists.txt": "#include"
+        },
+        "ctest_scripts/**": "#include",
+        "Editor/**": "#include",
+        "Engine/**": "#include",
+        "Gems": {
+            "Achievements": "#include",
+            "AssetMemoryAnalyzer": "#include",
+            "AssetValidation": "#include",
+            "Atom": "#include",
+            "AtomLyIntegration": "#include",
+            "AudioEngineWwise": "#include",
+            "AudioSystem": "#include",
+            "AutomatedLauncherTesting": "#include",
+            "Blast": "#include",
+            "Camera": "#include",
+            "CameraFramework": "#include",
+            "CertificateManager": "#include",
+            "ChatPlay": "#include",
+            "Clouds": "#include",
+            "CrashReporting": "#include",
+            "CustomAssetExample": "#include",
+            "DebugDraw": "#include",
+            "EditorPythonBindings": "#include",
+            "EMotionFX": "#include",
+            "ExpressionEvaluation": "#include",
+            "FastNoise": "#include",
+            "GameEffectSystem": "#include",
+            "GameLift": "#include",
+            "GameState": "#include",
+            "GameStateSamples": "#include",
+            "Gestures": "#include",
+            "GradientSignal": "#include",
+            "GraphCanvas": "#include",
+            "GraphModel": "#include",
+            "HttpRequestor": "#include",
+            "ImageProcessing": "#include",
+            "ImGui": "#include",
+            "InAppPurchases": "#include",
+            "LandscapeCanvas": "#include",
+            "LegacyTerrain": "#include",
+            "LmbrCentral": "#include",
+            "LocalUser": "#include",
+            "LyShine": "#include",
+            "LyShineExamples": "#include",
+            "Maestro": "#include",
+            "MessagePopup": "#include",
+            "Metastream": "#include",
+            "Microphone": "#include",
+            "Multiplayer": "#include",
+            "MultiplayerImGui": "#include",
+            "NvCloth": "#include",
+            "PhysX": "#include",
+            "PhysXDebug": "#include",
+            "Presence": "#include",
+            "QtForPython": "#include",
+            "RADTelemetry": "#include",
+            "RenderToTexture": "#include",
+            "SaveData": "#include",
+            "SceneLoggingExample": "#include",
+            "SceneProcessing": "#include",
+            "ScriptCanvas": "#include",
+            "ScriptCanvasDeveloper": "#include",
+            "ScriptCanvasDiagnosticLibrary": "#include",
+            "ScriptCanvasPhysics": "#include",
+            "ScriptCanvasTesting": "#include",
+            "ScriptedEntityTweener": "#include",
+            "ScriptEvents": "#include",
+            "SliceFavorites": "#include",
+            "StartingPointCamera": "#include",
+            "StartingPointInput": "#include",
+            "StartingPointMovement": "#include",
+            "Substance": "#include",
+            "SurfaceData": "#include",
+            "SVOGI": "#include",
+            "TestAssetBuilder": "#include",
+            "TextureAtlas": "#include",
+            "TickBusOrderViewer": "#include",
+            "TouchBending": "#include",
+            "Twitch": "#include",
+            "Vegetation": "#include",
+            "VideoPlayback": "#include",
+            "VideoPlaybackBink": "#include",
+            "VideoPlaybackFramework": "#include",
+            "VirtualGamepad": "#include",
+            "Visibility": "#include",
+            "Water": "#include",
+            "WhiteBox": "#include",
+            "CMakeLists.txt": "#include"
+        },
+        "Tools": {
+            "3dsmax/**": "#include",
+            "7za.exe": "#include",
+            "7za_legal_notice.txt": "#include",
+            "AWSNativeSDK": {
+                "**": "#include",
+                "Upgrader/restricted_platforms.py": "#exclude"
+            },
+            "AWSPythonSDK/**": "#include",
+            "Crashpad/**": "#include",
+            "CrySCompileServer/**": "#include",
+            "PakShaders/**": "#include",
+            "Python/**": "#include",
+            "Redistributables": {
+                "**": "#include",
+                "ANGLE/**": "#exclude",
+                "D3DCompiler/**": "#exclude",
+                "DbgHelp/**": "#exclude",
+                "FFMpeg/**": "#exclude",
+                "LuaCompiler/**": "#exclude",
+                "MSVC90/**": "#exclude",
+                "OpenGL32/**": "#exclude",
+                "SSLEAY/**": "#exclude"
+            },
+            "RemoteConsole/**": "#include",
+            "__init__.py": "#include",
+            "lmbr_aws/**": "#include",
+            "maxscript/**": "#include",
+            "maya/**": "#include",
+            "photoshop/**": "#include"
+        }
+    }
+}

scripts/build/package/glob3.py

@@ -0,0 +1,166 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import os
+import re
+import fnmatch
+
+__all__ = ["glob", "iglob", "escape"]
+
+def glob(pathname, recursive=False):
+    """Return a list of paths matching a pathname pattern.
+    The pattern may contain simple shell-style wildcards a la
+    fnmatch. However, unlike fnmatch, filenames starting with a
+    dot are special cases that are not matched by '*' and '?'
+    patterns.
+    If recursive is true, the pattern '**' will match any files and
+    zero or more directories and subdirectories.
+    """
+    return list(iglob(pathname, recursive=recursive))
+
+def iglob(pathname, recursive=False):
+    """Return an iterator which yields the paths matching a pathname pattern.
+    The pattern may contain simple shell-style wildcards a la
+    fnmatch. However, unlike fnmatch, filenames starting with a
+    dot are special cases that are not matched by '*' and '?'
+    patterns.
+    If recursive is true, the pattern '**' will match any files and
+    zero or more directories and subdirectories.
+    """
+    it = _iglob(pathname, recursive, False)
+    if recursive and _isrecursive(pathname):
+        s = next(it)  # skip empty string
+        assert not s
+    return it
+
+def _iglob(pathname, recursive, dironly):
+    dirname, basename = os.path.split(pathname)
+    if not has_magic(pathname):
+        assert not dironly
+        if basename:
+            if os.path.lexists(pathname):
+                yield pathname
+        else:
+            # Patterns ending with a slash should match only directories
+            if os.path.isdir(dirname):
+                yield pathname
+        return
+    if not dirname:
+        if recursive and _isrecursive(basename):
+            yield _glob2(dirname, basename, dironly)
+        else:
+            yield _glob1(dirname, basename, dironly)
+        return
+    # `os.path.split()` returns the argument itself as a dirname if it is a
+    # drive or UNC path.  Prevent an infinite recursion if a drive or UNC path
+    # contains magic characters (i.e. r'\\?\C:').
+    if dirname != pathname and has_magic(dirname):
+        dirs = _iglob(dirname, recursive, True)
+    else:
+        dirs = [dirname]
+    if has_magic(basename):
+        if recursive and _isrecursive(basename):
+            glob_in_dir = _glob2
+        else:
+            glob_in_dir = _glob1
+    else:
+        glob_in_dir = _glob0
+    for dirname in dirs:
+        for name in glob_in_dir(dirname, basename, dironly):
+            yield os.path.join(dirname, name)
+
+# These 2 helper functions non-recursively glob inside a literal directory.
+# They return a list of basenames.  _glob1 accepts a pattern while _glob0
+# takes a literal basename (so it only has to check for its existence).
+
+def _glob1(dirname, pattern, dironly):
+    names = list(_iterdir(dirname, dironly))
+    return fnmatch.filter(names, pattern)
+
+def _glob0(dirname, basename, dironly):
+    if not basename:
+        # `os.path.split()` returns an empty basename for paths ending with a
+        # directory separator.  'q*x/' should match only directories.
+        if os.path.isdir(dirname):
+            return [basename]
+    else:
+        if os.path.lexists(os.path.join(dirname, basename)):
+            return [basename]
+    return []
+
+# Following functions are not public but can be used by third-party code.
+
+def glob0(dirname, pattern):
+    return _glob0(dirname, pattern, False)
+
+def glob1(dirname, pattern):
+    return _glob1(dirname, pattern, False)
+
+# This helper function recursively yields relative pathnames inside a literal
+# directory.
+
+def _glob2(dirname, pattern, dironly):
+    assert _isrecursive(pattern)
+    return [pattern[:0]] + list(_rlistdir(dirname, dironly))
+
+# If dironly is false, yields all file names inside a directory.
+# If dironly is true, yields only directory names.
+def _iterdir(dirname, dironly):
+    if not dirname:
+        if isinstance(dirname, bytes):
+            dirname = bytes(os.curdir, 'ASCII')
+        else:
+            dirname = os.curdir
+    try:
+        for entry in os.listdir(dirname):
+            yield entry
+    except OSError:
+        return
+
+# Recursively yields relative pathnames inside a literal directory.
+def _rlistdir(dirname, dironly):
+    if not os.path.islink(dirname):
+        names = list(_iterdir(dirname, dironly))
+        for x in names:
+            yield x
+            path = os.path.join(dirname, x) if dirname else x
+            for y in _rlistdir(path, dironly):
+                yield os.path.join(x, y)
+magic_check = re.compile('([*?[])')
+magic_check_bytes = re.compile(b'([*?[])')
+
+def has_magic(s):
+    if isinstance(s, bytes):
+        match = magic_check_bytes.search(s)
+    else:
+        match = magic_check.search(s)
+    return match is not None
+
+def _ishidden(path):
+    return path[0] in ('.', b'.'[0])
+
+def _isrecursive(pattern):
+    if isinstance(pattern, bytes):
+        return pattern == b'**'
+    else:
+        return pattern == '**'
+
+def escape(pathname):
+    """Escape all special characters.
+    """
+    # Escaping is done by wrapping any of "*?[" between square brackets.
+    # Metacharacters do not work in the drive part and shouldn't be escaped.
+    drive, pathname = os.path.splitdrive(pathname)
+    if isinstance(pathname, bytes):
+        pathname = magic_check_bytes.sub(br'[\1]', pathname)
+    else:
+        pathname = magic_check.sub(r'[\1]', pathname)
+    return drive + pathname

scripts/build/package/glob_to_regex.py

@@ -0,0 +1,127 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+from __future__ import absolute_import
+import os
+import re
+import json
+import sys
+try:
+    import six
+except ImportError:
+    import pip
+    pip.main(['install', 'six', '--ignore-installed', '-q'])
+    import six
+from pathlib import Path
+
+this_file_path = os.path.dirname(os.path.realpath(__file__))
+    
+def convert_glob_pattern_to_regex_pattern(glob_pattern):
+    # switch to forward slashes because way easier to pattern match against
+    pattern = re.sub(r'\\', r'/', glob_pattern)
+
+    # Replace the dots and question marks
+    pattern = re.sub(r'\.', r'\\.', pattern)
+    pattern = re.sub(r'\?', r'.', pattern)
+
+    # Handle the * vs ** expansions
+    pattern = re.sub(r'([^*])\*($|[^*])', r'\1[^/\\\\]*\2', pattern)
+    pattern = re.sub(r'\*\*/', r'(.*/)?', pattern)
+    pattern = re.sub(r'\*\*', r'.*', pattern)
+
+    # replace the forward slashes with [/\\] so it works on PC/unix
+    pattern = re.sub(r'([^^])/', r'\1[/\\\\]', pattern)
+    return pattern
+
+# Convert the package json into a pair of regexes we can use to look for includes and excludes
+def convert_glob_list_to_regex_list(filelist, prefix):
+    includes = []
+    excludes = []
+    for key, value in six.iteritems(filelist):
+        glob_pattern = os.path.join(prefix, key)
+        if isinstance(value, dict):
+            (sub_includes, sub_excludes) = convert_glob_list_to_regex_list(value, glob_pattern)
+            includes.extend(sub_includes)
+            excludes.extend(sub_excludes)
+        else:
+            # Simulate what glob would do with file walking to scope the * within a directory
+            # and ** across directories
+            regex_pattern = convert_glob_pattern_to_regex_pattern(os.path.normpath(glob_pattern))
+
+            # Deal with the commands. include/exclude are straight forward. Moves/renames are to be considered
+            # includes, and we will stick with validating the original contents for now
+            if value == "#include":
+                includes.append(regex_pattern)
+            elif value == "#exclude":
+                excludes.append(regex_pattern)
+            elif value.startswith('#move:'):
+                includes.append(regex_pattern)
+            elif value.startswith('#rename:'):
+                includes.append(regex_pattern)
+            else:
+                pass
+    return (includes, excludes)
+
+def generate_excludes_for_platform(root, platform):
+    if platform == 'all':
+        platform_exclusions_filename = os.path.join(this_file_path, 'platform_exclusions.json')
+        with open(platform_exclusions_filename, 'r') as platform_exclusions_file:
+            platform_exclusions = json.load(platform_exclusions_file)
+    else:
+        # Use real path in case engine root is a symlink path
+        if os.name == 'posix' and os.path.islink(root):
+            root = os.readlink(root)
+        cur_dir = os.path.dirname(os.path.abspath(__file__))
+        relative_folder = os.path.relpath(cur_dir, root)
+        platform_exclusions_filename = os.path.join(root, 'restricted', platform, relative_folder, platform.lower() + '_exclusions.json')
+        with open(platform_exclusions_filename, 'r') as platform_exclusions_file:
+            platform_exclusions = json.load(platform_exclusions_file)
+        
+    if platform not in platform_exclusions:
+        raise KeyError('No {} found in {}'.format(platform, platform_exclusions_filename))
+    if '@lyengine' not in platform_exclusions[platform]:
+        raise KeyError('No {}/@lyengine found in {}'.format(platform, package_file_list))
+    (_, excludes) = convert_glob_list_to_regex_list(platform_exclusions[platform]['@lyengine'], root)
+    del _
+    return excludes
+
+def generate_include_exclude_regexes(package_platform, package_type, root, prohibited_platforms):
+    # The general contents will be indicated by the package file
+    if package_type == 'all':
+        package_file_list = os.path.join(this_file_path, 'package_filelists', 'all.json')
+    else:
+        # Search non-restricted platform first
+        package_file_list = os.path.join(this_file_path, 'Platform', package_platform, 'package_filelists', f'{package_type}.json')
+        if not os.path.exists(filelist):
+            # Use real path in case engine root is a symlink path
+            if os.name == 'posix' and os.path.islink(root):
+                root = os.readlink(root)
+            rel_path = os.path.relpath(cur_dir, root)
+            package_file_list = os.path.join(root, 'restricted', package_platform, rel_path, 'package_filelists',
+                                    f'{package_type}.json')
+    with open(package_file_list, 'r') as package_file:
+        package = json.load(package_file)
+
+    if '@lyengine' not in package:
+        raise KeyError('No @lyengine found in {}'.format(package_file_list))
+
+    (includes_list, excludes_list) = convert_glob_list_to_regex_list(package['@lyengine'], root)
+    prohibited_platforms.append('all')
+
+    # Add the exclusions of each prohibited platform
+    for p in prohibited_platforms:
+        excludes_list.extend(generate_excludes_for_platform(root, p))
+
+    includes = re.compile('|'.join(includes_list), re.IGNORECASE)
+    excludes = re.compile('|'.join(excludes_list), re.IGNORECASE)
+    return (includes, excludes)
+
+def generate_exclude_regexes_for_platform(root, platform):
+    return re.compile('|'.join(generate_excludes_for_platform(root, platform)), re.IGNORECASE)

scripts/build/package/package.py

@@ -0,0 +1,302 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+import os
+import sys
+import glob_to_regex
+import zipfile
+import timeit
+import stat
+import progressbar
+from optparse import OptionParser
+from PackageEnv import PackageEnv
+cur_dir = cur_dir = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, f'{cur_dir}/../../../Tools/build/JenkinsScripts/build')
+from ci_build import build
+from util import *
+from glob3 import glob
+
+
+def package(options):
+    package_env = PackageEnv(options.platform, options.type, options.package_env)
+    engine_root = package_env.get('ENGINE_ROOT')
+
+    if not package_env.get('SKIP_SCRUBBING'):
+        # Ask the validator code to tell us which files need to be removed from the package
+        prohibited_file_mask = get_prohibited_file_mask(options.platform, engine_root)
+
+        # Scrub files. This is destructive, but is necessary to allow the current file existance checks to work properly. Better to copy and then build, or to
+        # mask on sync, but this is what we have for now
+        scrub_files(package_env, prohibited_file_mask)
+
+        # validate files
+        validate_restricted_files(options.platform, options.type, package_env)
+
+    # Override values in bootstrap.cfg for PC package
+    override_bootstrap_cfg(package_env)
+
+    if not package_env.get('SKIP_BUILD'):
+        print(package_env.get('SKIP_BUILD'))
+        print('SKIP_BUILD is False, running CMake build...')
+        cmake_build(package_env)
+
+    # TODO Compile Assets
+    #if package_env.exists('ASSET_PROCESSOR_PATH'):
+    #    compile_assets(package_env)
+
+    #create packages
+    package_targets = package_env.get('PACKAGE_TARGETS')
+    for package_target in package_targets:
+        create_package(package_env, package_target)
+        upload_package(package_env, package_target)
+
+
+def get_python_path(package_env):
+    if sys.platform == 'win32':
+        return os.path.join(package_env.get('ENGINE_ROOT'), 'python', 'python.cmd')
+    else:
+        return os.path.join(package_env.get('ENGINE_ROOT'), 'python', 'python.sh')
+
+
+def override_bootstrap_cfg(package_env):
+    print('Override values in bootstrap.cfg')
+    engine_root = package_env.get('ENGINE_ROOT')
+    bootstrap_path = os.path.join(engine_root, 'bootstrap.cfg')
+    replace_values = {'sys_game_folder':'{}'.format(package_env.get('BOOTSTRAP_CFG_GAME_FOLDER'))}
+    try:
+        with open(bootstrap_path, 'r') as bootstrap_cfg:
+            content = bootstrap_cfg.read()
+    except:
+        error('Cannot read file {}'.format(bootstrap_path))
+    content = content.split('\n')
+    new_content = []
+    for line in content:
+        if not line.startswith('--'):
+            strs = line.split('=')
+            if len(strs):
+                key = strs[0].strip(' ')
+                if key in replace_values:
+                    line = '{}={}'.format(key, replace_values[key])
+        new_content.append(line)
+    try:
+        with open(bootstrap_path, 'w') as out:
+            out.write('\n'.join(new_content))
+    except:
+        error('Cannot write to file {}'.format(bootstrap_path))
+    print('{} updated with value {}'.format(bootstrap_path, replace_values))
+
+
+def get_prohibited_file_mask(platform, engine_root):
+    sys.path.append(os.path.join(engine_root, 'Tools', 'build', 'JenkinsScripts', 'distribution', 'scrubbing'))
+    from validator_data_LEGAL_REVIEW_REQUIRED import get_prohibited_platforms_for_package
+
+    # The list of prohibited platforms is controlled by the validator on a per-package basis
+    prohibited_platforms = get_prohibited_platforms_for_package(platform)
+    prohibited_platforms.append('all')
+    excludes_list = []
+    for p in prohibited_platforms:
+        platform_excludes = glob_to_regex.generate_excludes_for_platform(engine_root, p)
+        excludes_list.extend(platform_excludes)
+    prohibited_file_mask = re.compile('|'.join(excludes_list), re.IGNORECASE)
+    return prohibited_file_mask
+
+
+def scrub_files(package_env, prohibited_file_mask):
+    print('Perform the Code Scrubbing')
+    engine_root = package_env.get('ENGINE_ROOT')
+
+    success = True
+    for dirname, subFolders, files in os.walk(engine_root):
+        for filename in files:
+            full_path = os.path.join(dirname, filename)
+            if prohibited_file_mask.match(full_path):
+                try:
+                    print('Deleting: {}'.format(full_path))
+                    os.chmod(full_path, stat.S_IWRITE)
+                    os.unlink(full_path)
+                except:
+                    e = sys.exc_info()[0]
+                    sys.stderr.write('Error: could not delete {} ... aborting.\n'.format(full_path))
+                    sys.stderr.write('{}\n'.format(str(e)))
+                    success = False
+    if not success:
+        sys.stderr.write('ERROR: scrub_files failed\n')
+        sys.exit(1)
+
+
+def validate_restricted_files(package_platform, package_type, package_env):
+    print('Perform the Code Scrubbing')
+    engine_root = package_env.get('ENGINE_ROOT')
+
+    # Run validator
+    success = True
+    validator_path = os.path.join(engine_root, 'Tools/build/JenkinsScripts/distribution/scrubbing/validator.py')
+    python = get_python_path(package_env)
+    args = [python, validator_path, '--package_platform', package_platform, '--package_type', package_type, engine_root]
+    return_code = safe_execute_system_call(args)
+    if return_code != 0:
+        success = False
+    if not success:
+        error('Restricted file validator failed.')
+    print('Restricted file validator completed successfully.')
+
+
+def cmake_build(package_env):
+    build_targets = package_env.get('BUILD_TARGETS')
+    for build_target in build_targets:
+        build(build_target['BUILD_CONFIG_FILENAME'], build_target['PLATFORM'], build_target['TYPE'])
+
+
+def create_package(package_env, package_target):
+    print('Creating zipfile for package target {}'.format(package_target))
+    cur_dir = os.path.dirname(os.path.abspath(__file__))
+    file_list_type = package_target['FILE_LIST_TYPE']
+    if file_list_type == 'All':
+        filelist = os.path.join(cur_dir, 'package_filelists', package_target['FILE_LIST'])
+    else:
+        # Search non-restricted platform first
+        filelist = os.path.join(cur_dir, 'Platform', file_list_type, 'package_filelists', package_target['FILE_LIST'])
+        if not os.path.exists(filelist):
+            engine_root = package_env.get('ENGINE_ROOT')
+            # Use real path in case engine root is a symlink path
+            if os.name == 'posix' and os.path.islink(engine_root):
+                engine_root = os.readlink(engine_root)
+            rel_path = os.path.relpath(cur_dir, engine_root)
+            filelist = os.path.join(engine_root, 'restricted', file_list_type, rel_path, 'package_filelists', package_target['FILE_LIST'])
+    with open(filelist, 'r') as source:
+        data = json.load(source)
+    lyengine = package_env.get('ENGINE_ROOT')
+    print('Calculating filelists...')
+    files = {}
+
+    if '@lyengine' in data:
+        files.update(filter_files(data['@lyengine'], lyengine))
+    if '@3rdParty' in data:
+        files.update(filter_files(data['@3rdParty'], package_env.get('THIRDPARTY_HOME')))
+    package_path = os.path.join(lyengine, package_target['PACKAGE_NAME'])
+    print('Creating zipfile at {}'.format(package_path))
+    start = timeit.default_timer()
+    with progressbar.ProgressBar(max_value=len(files), redirect_stderr=True) as bar:
+        with zipfile.ZipFile(package_path, 'w', compression=zipfile.ZIP_DEFLATED, allowZip64=True) as myzip:
+            i = 0
+            bar.update(i)
+            last_bar_update = timeit.default_timer()
+            for f in files:
+                if os.path.islink(f):
+                    zipInfo = zipfile.ZipInfo(files[f])
+                    zipInfo.create_system = 3
+                    # long type of hex val of '0xA1ED0000L',
+                    # say, symlink attr magic...
+                    zipInfo.external_attr |= 0xA0000000
+                    myzip.writestr(zipInfo, os.readlink(f))
+                else:
+                    myzip.write(f, files[f])
+                i += 1
+                # Update progress bar every 2 minutes
+                if int(timeit.default_timer() - last_bar_update) > 120:
+                    last_bar_update = timeit.default_timer()
+                    bar.update(i)
+            bar.update(i)
+
+    stop = timeit.default_timer()
+    total_time = int(stop - start)
+    print('{} is created. Total time: {} seconds.'.format(package_path, total_time))
+
+    def get_MD5(file_path):
+        from hashlib import md5
+        chunk_size = 200 * 1024
+        h = md5()
+        with open(file_path, 'rb') as f:
+            while True:
+                chunk = f.read(chunk_size)
+                if len(chunk):
+                    h.update(chunk)
+                else:
+                    break
+        return h.hexdigest()
+
+    md5_file = '{}.MD5'.format(package_path)
+    print('Creating MD5 file at {}'.format(md5_file))
+    start = timeit.default_timer()
+    with open(md5_file, 'w') as output:
+        output.write(get_MD5(package_path))
+    stop = timeit.default_timer()
+    total_time = int(stop - start)
+    print('{} is created. Total time: {} seconds.'.format(md5_file, total_time))
+
+
+def upload_package(package_env, package_target):
+    package_name = package_target['PACKAGE_NAME']
+    engine_root = package_env.get('ENGINE_ROOT')
+    internal_s3_bucket = package_env.get('INTERNAL_S3_BUCKET')
+    qa_s3_bucket = package_env.get('QA_S3_BUCKET')
+    s3_prefix = package_env.get('S3_PREFIX')
+    print(f'Uploading {package_name} to S3://{internal_s3_bucket}/{s3_prefix}/{package_name}')
+    cmd = ['aws', 's3', 'cp', os.path.join(engine_root, package_name), f's3://{internal_s3_bucket}/{s3_prefix}/{package_name}']
+    execute_system_call(cmd, stdout=subprocess.DEVNULL)
+    print(f'Uploading {package_name} to S3://{qa_s3_bucket}/{s3_prefix}/{package_name}')
+    cmd = ['aws', 's3', 'cp', os.path.join(engine_root, package_name), f's3://{qa_s3_bucket}/{s3_prefix}/{package_name}', '--acl', 'bucket-owner-full-control']
+    execute_system_call(cmd, stdout=subprocess.DEVNULL)
+
+
+def filter_files(data, base, prefix='', support_symlinks=True):
+    includes = {}
+    excludes = set()
+    for key, value in data.items():
+        pattern = os.path.join(base, prefix, key)
+        if not isinstance(value, dict):
+            pattern = os.path.normpath(pattern)
+            result = glob(pattern, recursive=True)
+            files = [x for x in result if os.path.isfile(x) or (support_symlinks and os.path.islink(x))]
+            if value == "#exclude":
+                excludes.update(files)
+            elif value == "#include":
+                for file in files:
+                    includes[file] = os.path.relpath(file, base)
+            else:
+                if value.startswith('#move:'):
+                    for file in files:
+                        file_name = os.path.relpath(file, os.path.join(base, prefix))
+                        dst_dir = value.replace('#move:', '').strip(' ')
+                        includes[file] = os.path.join(dst_dir, file_name)
+                elif value.startswith('#rename:'):
+                    for file in files:
+                        dst_file = value.replace('#rename:', '').strip(' ')
+                        includes[file] = dst_file
+                else:
+                    warn('Unknown directive {} for pattern {}'.format(value, pattern))
+        else:
+            includes.update(filter_files(value, base, os.path.join(prefix, key), support_symlinks))
+
+    for exclude in excludes:
+        try:
+            includes.pop(exclude)
+        except KeyError:
+            pass
+    return includes
+
+
+def parse_args():
+    parser = OptionParser()
+    parser.add_option("--platform", dest="platform", default='consoles', help="Target platform to package")
+    parser.add_option("--type", dest="type", default='consoles', help="Package type")
+    parser.add_option("--package_env", dest="package_env", default="package_env.json",
+                      help="JSON file that defines package environment variables")
+    (options, args) = parser.parse_args()
+    return options, args
+
+
+if __name__ == "__main__":
+    (options, args) = parse_args()
+    package(options)
+
+
+
+

scripts/build/package/package_env.json

@@ -0,0 +1,11 @@
+{
+    "global_env":{
+        "ENGINE_ROOT":"",
+        "THIRDPARTY_HOME":"",
+        "BRANCH_NAME":"",
+        "PACKAGE_NAME_PATTERN":"${BRANCH_NAME}-spectra",
+        "BUILD_NUMBER":"0",
+        "INTERNAL_S3_BUCKET": "ly-spectra-packages",
+        "QA_S3_BUCKET": "amazon.ly.lionbridgeshare/ly-spectra-packages"
+    }
+}

scripts/build/package/package_filelists/all.json

@@ -0,0 +1,7 @@
+{
+    "@lyengine": {
+        "**": "#include",
+        "**/*.pyc": "#exclude",
+        "**/*.pdb": "#exclude"
+    }
+}

scripts/build/package/package_filelists/symbols.json

@@ -0,0 +1,6 @@
+{
+    "@lyengine": {
+        "**/*.pdb": "#include",
+        "Tools/Crashpad/**": "#exclude"
+    }
+}

scripts/build/package/platform_exclusions.json

@@ -0,0 +1,12 @@
+{ 
+    "all": {
+        "@lyengine": {
+            "**/Gems/Atom/RHI/DX12/External/pix/**": "#exclude",
+            "**/.idea/**": "#exclude",
+            "**/*.csproj*": "#exclude",
+            "**/.owner": "#exclude",
+            "**/WinPixEventRuntime.dll": "#exclude",
+            "**/XenonConsole.exe": "#exclude"
+        }
+    }
+}

scripts/build/package/util.py

@@ -0,0 +1,62 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import json
+import os
+import re
+import sys
+import subprocess
+
+
+class LyBuildError(Exception):
+    def __init__(self, message):
+        super(LyBuildError, self).__init__(message)
+
+
+def ly_build_error(message):
+    raise LyBuildError(message)
+
+
+def error(message):
+    print(('Error: {}'.format(message)))
+    exit(1)
+
+
+# Exit with status code 0 means it won't fail the whole build process
+def safe_exit_with_error(message):
+    print(('Error: {}'.format(message)))
+    exit(0)
+
+
+def warn(message):
+    print(('Warning: {}'.format(message)))
+
+
+def execute_system_call(command, **kwargs):
+    print(('Executing subprocess.check_call({})'.format(command)))
+    try:
+        subprocess.check_call(command, **kwargs)
+    except subprocess.CalledProcessError as e:
+        print((e.output))
+        error('Executing subprocess.check_call({}) failed with error {}'.format(command, e))
+    except FileNotFoundError as e:
+        error("File Not Found - Failed to call {} with error {}".format(command, e))
+
+
+def safe_execute_system_call(command, **kwargs):
+    print(('Executing subprocess.check_call({})'.format(command)))
+    try:
+        subprocess.check_call(command, **kwargs)
+    except subprocess.CalledProcessError as e:
+        print((e.output))
+        warn('Executing subprocess.check_call({}) failed'.format(command))
+        return e.returncode
+    return 0

scripts/build/tools/alert_build_failures.py

@@ -0,0 +1,76 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import os
+from incremental_build_util import get_iam_role_credentials
+try:
+    import requests
+except ImportError:
+    import pip
+    pip.main(['install', 'requests', '--ignore-installed', '-q'])
+    import requests
+try:
+    from requests_aws4auth import AWS4Auth
+except ImportError:
+    import pip
+    pip.main(['install', 'requests_aws4auth', '--ignore-installed', '-q'])
+    from requests_aws4auth import AWS4Auth
+
+IAM_ROLE_NAME = 'ec2-jenkins-node'
+TEAM = 'lumberyard-build'
+CHIME_ROOM_WEB_HOOK = "https://hooks.chime.aws/incomingwebhooks/2a6018c9-3bf5-4e03-851c-32ba82c7c4e2?token=YWhTVXZsWVJ8MXxaLTg5RkVZNlA5Q1NiMVdfQndGeS1TSHNnYW5VREVha3pjX1pUUEd5b1JF"
+
+
+def find_curr_oncalls_for_team(team):
+    host = "who-is-oncall-pdx.corp.amazon.com"
+    service_name = "who-is-oncall"
+    aws_region = "us-west-2"
+    headers = {"content-type": "application/json", "host": host}
+
+    credentials = get_iam_role_credentials(IAM_ROLE_NAME)
+    try:
+        aws_access_key_id = credentials['AccessKeyId']
+        aws_secret_access_key = credentials['SecretAccessKey']
+        aws_session_token = credentials['Token']
+    except Exception as e:
+        print(f'ERROR: Cannot get AWS credentials.\n{e}')
+        return ['All']
+
+    auth = AWS4Auth(aws_access_key_id, aws_secret_access_key, aws_region, service_name, session_token=aws_session_token)
+    r = requests.get(f"https://who-is-oncall-pdx.corp.amazon.com/teams/{team}", headers=headers, auth=auth, verify=False)
+    if r.ok:
+        res = r.json()
+        try:
+            return res['currOncalls']
+        except KeyError:
+            return ['All']
+    return ['All']
+
+
+def send_alert_to_chime_room(web_hook, content):
+    data = '{"Content":"' + content + '"}'
+    headers = {'Content-Type': 'application/json'}
+    requests.post(web_hook, headers=headers, data=data)
+
+
+def create_content():
+    content = ''
+    oncalls = find_curr_oncalls_for_team(TEAM)
+    for oncall in oncalls:
+        content += f'@{oncall} '
+    job_name = os.environ['JOB_NAME']
+    build_url = os.environ['BUILD_URL']
+    content += fr'\nJob {job_name} failed\nBuild URL: {build_url}\n'
+    return content
+
+
+send_alert_to_chime_room(CHIME_ROOM_WEB_HOOK, create_content())
+

scripts/build/tools/download_from_s3.py

@@ -0,0 +1,101 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+'''
+Usage:
+Use EC2 role to download files to %WORKSPACE% folder from bucket bucket_name:
+python download_from_s3.py --base_dir %WORKSPACE% --files_to_download "file1,file2" --bucket bucket_name
+
+Use profile to download files to %WORKSPACE% folder from bucket bucket_name:
+python download_from_s3.py --base_dir %WORKSPACE% --profile profile --files_to_download "file1,file2" --bucket bucket_name
+'''
+
+
+import os
+import json
+import boto3
+from optparse import OptionParser
+from util import error
+
+
+def parse_args():
+    parser = OptionParser()
+    parser.add_option("--base_dir", dest="base_dir", default=os.getcwd(), help="Base directory to download files, If not given, then current directory is used.")
+    parser.add_option("--files_to_download", dest="files_to_download", default=None, help="Files to download, separated by comma.")
+    parser.add_option("--profile", dest="profile", default=None, help="The name of a profile to use. If not given, then the default profile is used.")
+    parser.add_option("--bucket", dest="bucket", default=None, help="S3 bucket the files are downloaded from.")
+    parser.add_option("--key_prefix", dest="key_prefix", default='', help="Object key prefix.")
+    '''
+    ExtraArgs used to call s3.download_file(), should be in json format. extra_args key must be one of: ACL, CacheControl, ContentDisposition, ContentEncoding, ContentLanguage, ContentType, Expires,
+    GrantFullControl, GrantRead, GrantReadACP, GrantWriteACP, Metadata, RequestPayer, ServerSideEncryption, StorageClass,
+    SSECustomerAlgorithm, SSECustomerKey, SSECustomerKeyMD5, SSEKMSKeyId, WebsiteRedirectLocation
+    '''
+    parser.add_option("--extra_args", dest="extra_args", default=None, help="Additional parameters used to download file.")
+    parser.add_option("--max_retry", dest="max_retry", default=1, help="Maximum retry times to download file.")
+    (options, args) = parser.parse_args()
+    if not os.path.isdir(options.base_dir):
+        error('{} is not a valid directory'.format(options.base_dir))
+    if not options.files_to_download:
+        error('Use --files_to_download to specify files to download, separated by comma.')
+    if not options.bucket:
+        error('Use --bucket to specify bucket that the files are downloaded from.')
+    return options
+
+
+def get_client(service_name, profile_name=None):
+    session = boto3.session.Session(profile_name=profile_name)
+    client = session.client(service_name)
+    return client
+
+
+def s3_download_file(client, base_dir, file, bucket, key_prefix=None, extra_args=None, max_retry=1):
+    print('Downloading file {} from bucket {}.'.format(file, bucket))
+    key = file if key_prefix is None else '{}/{}'.format(key_prefix, file)
+    for x in range(max_retry):
+        try:
+            client.download_file(
+                bucket, key, os.path.join(base_dir, file),
+                ExtraArgs=extra_args
+            )
+            print('Download succeeded')
+            return True
+        except:
+            print('Retrying download...')
+    print('Download failed')
+    return False
+
+
+def download_files(base_dir, files_to_download, bucket, key_prefix=None, profile=None, extra_args=None, max_retry=1):
+    client = get_client('s3', profile)
+    files_to_download = files_to_download.split(',')
+    extra_args = json.loads(extra_args) if extra_args else None
+
+    print('Downloading {} files from bucket {}.'.format(len(files_to_download), bucket))
+    failure = []
+    success = []
+    for file in files_to_download:
+        if not s3_download_file(client, base_dir, file, bucket, key_prefix, extra_args, max_retry):
+            failure.append(file)
+        else:
+            success.append(file)
+    print('{} files are downloaded successfully:'.format(len(success)))
+    print('\n'.join(success))
+    print('{} files failed to download:'.format(len(failure)))
+    print('\n'.join(failure))
+    # Exit with error code 1 if any file is failed to download
+    if len(failure) > 0:
+        return False
+    return True
+
+
+if __name__ == "__main__":
+    options = parse_args()
+    download_files(options.base_dir, options.files_to_download, options.bucket, options.key_prefix, options.profile, options.extra_args, options.max_retry)

scripts/build/tools/email_to_lionbridge.py

@@ -0,0 +1,128 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+"""
+This script will be used in https://jenkins.agscollab.com/view/%7ESandbox/job/PACKAGE_COPY_S3/
+PACKAGE_COPY_S3 is a downstream job of nightly packaging job, it copies the nightly packages from Infra S3 bucket to Lionbridge S3 bucket based on the INCLUDE_FILTER passed from packaging job
+"""
+import os
+import re
+import json
+import requests
+from requests.auth import HTTPBasicAuth
+import boto3
+from util import error, warn
+
+
+# Write EMAIL_TEMPLATE to a file and inject it into the email sent to Lionbridge
+EMAIL_TEMPLATE = '''Packages are uploaded to S3 bucket {}
+Package List:
+{}
+
+
+Changelists:
+{}
+'''
+
+
+def get_jenkins_env(key):
+    try:
+        return os.environ[key]
+    except KeyError:
+        print('Error: Jenkins parameters {} is not set.'.format(key))
+    return None
+
+
+JENKINS_USERNAME = get_jenkins_env('JENKINS_USERNAME')
+JENKINS_API_TOKEN = get_jenkins_env('JENKINS_API_TOKEN')
+JENKINS_URL = get_jenkins_env('JENKINS_URL')
+WORKSPACE = get_jenkins_env('WORKSPACE')
+S3_TARGET = get_jenkins_env('S3_TARGET')
+INCLUDE_FILTER = get_jenkins_env('INCLUDE_FILTER')
+EMAIL_TEMPLATE_FILE = get_jenkins_env('EMAIL_TEMPLATE_FILE')
+if None in [JENKINS_USERNAME, JENKINS_API_TOKEN, JENKINS_URL, WORKSPACE, S3_TARGET, INCLUDE_FILTER, EMAIL_TEMPLATE_FILE]:
+    error('Please make sure all Jenkins parameters are set correctly.')
+
+
+def parse_include_filter(include_filter):
+    try:
+        res = re.search('^(\w*)-*lumberyard-(\d+)\.(\d+)-(\d+)-(\w+).*\*(\d+)\.\*', include_filter)
+        branch = res.group(1)
+        major_version = int(res.group(2))
+        minor_version = int(res.group(3))
+        changelist_number = res.group(4)
+        platform = res.group(5)
+        build_number = res.group(6)
+        return branch, major_version, minor_version, changelist_number, platform, build_number
+    except (AttributeError, IndexError):
+        error('Unable to parse INCLUDE_FILTER, please make sure the INCLUDE_FILTER is set correctly')
+
+
+# Get the changelists that trigger the build
+def get_changelists(job_name, build_number):
+    changelists = []
+    headers = {'Content-type': 'application/json', 'Accept': 'application/json'}
+    try:
+        res = requests.get('{}/job/{}/{}/api/json'.format(JENKINS_URL, job_name, build_number),
+                           auth=HTTPBasicAuth(JENKINS_USERNAME, JENKINS_API_TOKEN), headers=headers, verify=False)
+        res = json.loads(res.content)
+        changelists = res.get('changeSet').get('items')
+        return changelists
+    except:
+        warn('Error: Failed to get changes from build {} in job {}'.format(build_number, job_name))
+        return []
+
+
+def get_packaging_job_name(branch, major_version, minor_version, platform):
+    if branch == '':
+        branch = 'ML' if major_version + minor_version == 0 else 'v{}_{}'.format(major_version, minor_version)
+    job_name = 'PKG_{}_{}'.format(branch, platform.capitalize())
+    return job_name
+
+
+# Get package names by looking up S3 bucket
+def get_package_names(branch, major_version, minor_version, include_filter, build_number):
+    package_names = []
+    prefix = include_filter[:include_filter.find('*')]
+    pattern = '.*{}.*{}..*'.format(prefix, build_number)
+    if branch == '':
+        bucket_name = 'ly-packages-mainline' if major_version + minor_version == 0 else 'ly-packages-release-candidate'
+        folder = 'lumberyard-packages'
+    else:
+        bucket_name = 'ly-packages-feature-branches'
+        folder = 'lumberyard-packages/{}'.format(branch)
+    s3 = boto3.resource('s3')
+    bucket = s3.Bucket(bucket_name)
+    for obj in bucket.objects.filter(Prefix='{}/{}'.format(folder, prefix)):
+        package_name = obj.key
+        if re.match(pattern, package_name):
+            package_names.append(package_name.replace('{}/'.format(folder), ''))
+    return package_names
+
+
+if __name__ == "__main__":
+    branch, major_version, minor_version, changelist_number, platform, build_number = parse_include_filter(INCLUDE_FILTER)
+    packaging_job_name = get_packaging_job_name(branch, major_version, minor_version, platform)
+    changelists = get_changelists(packaging_job_name, build_number)
+    package_names = get_package_names(branch, major_version, minor_version, INCLUDE_FILTER, build_number)
+    with open(os.path.join(WORKSPACE, EMAIL_TEMPLATE_FILE), 'w+') as output:
+        if len(package_names) > 0:
+            package_list_str = '\n'.join(package_names)
+            changelists_str = ''
+            for item in changelists:
+                changelists_str += '---------------------------------------------------------------------------------------------\n'
+                try:
+                    changelists_str += 'CL{} by {} on {}\n{}\n'.format(item['changeNumber'], item['author']['fullName'], item['changeTime'], item['msg'].encode('utf-8', 'ignore'))
+                except KeyError:
+                    error('Internal error, check the output of Jenkins API.')
+            output.write(EMAIL_TEMPLATE.format(S3_TARGET, package_list_str, changelists_str))
+
+

scripts/build/tools/upload_to_s3.py

@@ -0,0 +1,112 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+'''
+Usage:
+Use EC2 role to upload all .zip and .MD5 files in %WORKSPACE% folder to bucket ly-packages-mainline:
+python upload_to_s3.py --base_dir %WORKSPACE% --file_regex "(.*zip$|.*MD5$)" --bucket ly-packages-mainline
+
+Use profile to upload all .zip and .MD5 files in %WORKSPACE% folder to bucket ly-packages-mainline:
+python upload_to_s3.py --base_dir %WORKSPACE% --profile profile --file_regex "(.*zip$|.*MD5$)" --bucket ly-packages-mainline
+
+'''
+
+
+import os
+import re
+import json
+import boto3
+from optparse import OptionParser
+
+
+def parse_args():
+    parser = OptionParser()
+    parser.add_option("--base_dir", dest="base_dir", default=os.getcwd(), help="Base directory to upload files, If not given, then current directory is used.")
+    parser.add_option("--file_regex", dest="file_regex", default=None, help="Regular expression that used to match file names to upload.")
+    parser.add_option("--profile", dest="profile", default=None, help="The name of a profile to use. If not given, then the default profile is used.")
+    parser.add_option("--bucket", dest="bucket", default=None, help="S3 bucket the files are uploaded to.")
+    parser.add_option("--key_prefix", dest="key_prefix", default='', help="Object key prefix.")
+    '''
+    ExtraArgs used to call s3.upload_file(), should be in json format. extra_args key must be one of: ACL, CacheControl, ContentDisposition, ContentEncoding, ContentLanguage, ContentType, Expires,
+    GrantFullControl, GrantRead, GrantReadACP, GrantWriteACP, Metadata, RequestPayer, ServerSideEncryption, StorageClass,
+    SSECustomerAlgorithm, SSECustomerKey, SSECustomerKeyMD5, SSEKMSKeyId, WebsiteRedirectLocation
+    '''
+    parser.add_option("--extra_args", dest="extra_args", default=None, help="Additional parameters used to upload file.")
+    parser.add_option("--max_retry", dest="max_retry", default=1, help="Maximum retry times to upload file.")
+    (options, args) = parser.parse_args()
+    if not os.path.isdir(options.base_dir):
+        error('{} is not a valid directory'.format(options.base_dir))
+    if not options.file_regex:
+        error('Use --file_regex to specify regular expression that used to match file names to upload.')
+    if not options.bucket:
+        error('Use --bucket to specify bucket that the files are uploaded to.')
+    return options
+
+
+def error(message):
+    print(f'Error: {message}')
+    exit(1)
+
+
+def get_client(service_name, profile_name):
+    session = boto3.session.Session(profile_name=profile_name)
+    client = session.client(service_name)
+    return client
+
+
+def get_files_to_upload(base_dir, regex):
+    # Get all file names in base directory
+    files = [x for x in os.listdir(base_dir) if os.path.isfile(os.path.join(base_dir, x))]
+    # Get all file names matching the regular expression, those file will be uploaded to S3
+    files_to_upload = [x for x in files if re.match(regex, x)]
+    return files_to_upload
+
+
+def s3_upload_file(client, base_dir, file, bucket, key_prefix=None, extra_args=None, max_retry=1):
+    print(('Uploading file {} to bucket {}.'.format(file, bucket)))
+    key = file if key_prefix is None else '{}/{}'.format(key_prefix, file)
+    for x in range(max_retry):
+        try:
+            client.upload_file(
+                os.path.join(base_dir, file), bucket, key,
+                ExtraArgs=extra_args
+            )
+            print('Upload succeeded')
+            return True
+        except Exception as err:
+            print(('exception while uploading: {}'.format(err)))
+            print('Retrying upload...')
+    print('Upload failed')
+    return False
+
+
+if __name__ == "__main__":
+    options = parse_args()
+    client = get_client('s3', options.profile)
+    files_to_upload = get_files_to_upload(options.base_dir, options.file_regex)
+    extra_args = json.loads(options.extra_args) if options.extra_args else None
+
+    print(('Uploading {} files to bucket {}.'.format(len(files_to_upload), options.bucket)))
+    failure = []
+    success = []
+    for file in files_to_upload:
+        if not s3_upload_file(client, options.base_dir, file, options.bucket, options.key_prefix, extra_args, 2):
+            failure.append(file)
+        else:
+            success.append(file)
+    print('Upload finished.')
+    print(('{} files are uploaded successfully:'.format(len(success))))
+    print(('\n'.join(success)))
+    if len(failure) > 0:
+        print(('{} files failed to upload:'.format(len(failure))))
+        print(('\n'.join(failure)))
+        # Exit with error code 1 if any file is failed to upload
+        exit(1)

scripts/commit_validation/CMakeLists.txt

@@ -0,0 +1,18 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+# this ctest makes sure that the commit validation function
+# also runs its tests during commit validation!
+ly_add_pytest(
+    NAME test_commit_validation
+    PATH ${CMAKE_CURRENT_LIST_DIR}
+)
+

scripts/commit_validation/commit_validation/__init__.py

@@ -0,0 +1,10 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#

scripts/commit_validation/commit_validation/commit_validation.py

@@ -0,0 +1,191 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import abc
+import importlib
+import os
+import pkgutil
+import re
+import time
+from typing import Dict, List, Tuple
+
+VERBOSE = False
+
+class Commit(abc.ABC):
+    """An interface for accessing details about a commit"""
+
+    @abc.abstractmethod
+    def get_files(self) -> List[str]:
+        """Returns a list of local files added/modified by the commit"""
+        pass
+
+    @abc.abstractmethod
+    def get_removed_files(self) -> List[str]:
+        """Returns a list of local files removed by the commit"""
+        pass
+
+    @abc.abstractmethod
+    def get_file_diff(self, str) -> str:
+        """
+        Given a file name, returns a string in unified diff format
+        that represents the changes made to that file for this commit.
+        Most validators will only pay attention to added lines (with + in front)
+        """
+        pass
+
+    @abc.abstractmethod
+    def get_description(self) -> str:
+        """Returns the description of the commit"""
+        pass
+
+    @abc.abstractmethod
+    def get_author(self) -> str:
+        """Returns the author of the commit"""
+        pass
+
+
+def validate_commit(commit: Commit, out_errors: List[str] = None, ignore_validators: List[str] = None) -> bool:
+    """Validates a commit against all validators
+
+    :param commit: The commit to validate
+    :param out_errors: if not None, will populate with the list of errors given by the validators
+    :param ignore_validators: Optional list of CommitValidator classes to ignore, by class name
+    :return: True if there are no validation errors, and False otherwise
+    """
+    failed_count = 0
+    passed_count = 0
+    start_time = time.time()
+
+    # Find all the validators in the validators package (recursively)
+    validator_classes = []
+    validators_dir = os.path.join(os.path.dirname(__file__), 'validators')
+    for _, module_name, is_package in pkgutil.iter_modules([validators_dir]):
+        if not is_package:
+            module = importlib.import_module('commit_validation.validators.' + module_name)
+            validator = module.get_validator()
+            if ignore_validators and validator.__name__ in ignore_validators:
+                print(f"Disabled validation for '{validator.__name__}'")
+            else:
+                validator_classes.append(validator)
+
+    error_summary = {}
+
+    # Process validators
+    for validator_class in validator_classes:
+        validator = validator_class()
+        validator_name = validator.__class__.__name__
+
+        error_list = []
+        passed = validator.run(commit, errors = error_list)
+        if passed:
+            passed_count += 1
+            print(f'{validator.__class__.__name__} PASSED')
+        else:
+            failed_count += 1
+            print(f'{validator.__class__.__name__} FAILED')
+        error_summary[validator_name] = error_list
+        
+    end_time = time.time()
+
+    if failed_count:
+        print("VALIDATION FAILURE SUMMARY")
+        for val_name in error_summary.keys():
+            errors = error_summary[val_name]
+            if errors:
+                for error_message in errors:
+                    first_line = True
+                    for line in error_message.splitlines():
+                        if first_line:
+                            first_line = False
+                            print(f'VALIDATOR_FAILED: {val_name} {line}')
+                        else:
+                            print(f'    {line}') # extra detail lines do not need machine parsing
+
+    stats_strs = []
+    if failed_count > 0:
+        stats_strs.append(f'{failed_count} failed')
+    if passed_count > 0:
+        stats_strs.append(f'{passed_count} passed')
+    stats_str = ', '.join(stats_strs) + f' in {end_time - start_time:.2f}s'
+
+    print()
+    print(stats_str)
+
+    return failed_count == 0
+
+def IsFileSkipped(file_name) -> bool:
+    if os.path.splitext(file_name)[1].lower() not in SOURCE_AND_SCRIPT_FILE_EXTENSIONS:
+        skipped = True
+        for pattern in SOURCE_AND_SCRIPT_FILE_PATTERNS:
+            if pattern.match(file_name):
+                skipped = False
+                break
+        return skipped
+    return False
+
+class CommitValidator(abc.ABC):
+    """A commit validator"""
+
+    @abc.abstractmethod
+    def run(self, commit: Commit, errors: List[str]) -> bool:
+        """Validates a commit
+
+        :param commit: The commit to validate
+        :param errors: List of errors generated, append them to this list
+        :return: True if the commit is valid, and False otherwise
+        """
+        pass
+
+
+SOURCE_FILE_EXTENSIONS: Tuple[str, ...] = (
+    '.c', '.cc', '.cpp', '.cxx', '.h', '.hpp', '.hxx', '.inl', '.m', '.mm', '.cs', '.java'
+)
+"""File extensions for compiled source code"""
+
+SCRIPT_FILE_EXTENSIONS: Tuple[str, ...] = (
+    '.py', '.lua', '.bat', '.cmd', '.sh', '.js'
+)
+"""File extensions for interpreted code"""
+
+BUILD_FILE_EXTENSIONS: Tuple[str, ...] = (
+    '.cmake',
+)
+"""File extensions for build files"""
+
+SOURCE_AND_SCRIPT_FILE_EXTENSIONS: Tuple[str, ...] = SOURCE_FILE_EXTENSIONS + SCRIPT_FILE_EXTENSIONS + BUILD_FILE_EXTENSIONS
+"""File extensions for both compiled and interpreted code"""
+
+BUILD_FILE_PATTERNS: Tuple[re.Pattern, ...] = (
+    re.compile(r'.*CMakeLists\.txt'),
+)
+"""File patterns for build files"""
+
+SOURCE_AND_SCRIPT_FILE_PATTERNS: Tuple[re.Pattern, ...] = BUILD_FILE_PATTERNS
+
+EXCLUDED_VALIDATION_PATTERNS = [
+    '.git/*',
+    '*/3rdParty/*',
+    '*/__pycache__/*',
+    '*/External/*',
+    'build',
+    'Cache',
+    'Code/Tools/CryFXC',
+    'Code/Tools/HLSLCrossCompiler',
+    'Code/Tools/HLSLCrossCompilerMETAL',
+    'Code/Tools/ProfVis',
+    'Code/Tools/UniversalRemoteConsole',
+    'Docs',
+    'python/runtime',
+    'restricted/*/Tools/*RemoteControl',
+    'Tools/3dsmax',
+    'Tools/AWSPythonSDK',
+    'Tools/Crashpad',
+]

scripts/commit_validation/commit_validation/pal_allowedlist.py

@@ -0,0 +1,46 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import fnmatch
+import os
+from typing import List
+
+DEFAULT_ALLOWEDLIST_FILE = os.path.join(os.path.dirname(__file__), 'pal_allowedlist.txt')
+"""The path to the default allowed-list file"""
+
+
+class PALAllowedlist:
+    """A utility class used for determining if PAL rules should apply to a particular file path. If a path matches the
+    allowed-list, then PAL rules should not apply to that path."""
+
+    def __init__(self, patterns: List[str]) -> None:
+        """Creates a new instance of :class:`PALAllowedlist` from a list of glob patterns
+
+        :param patterns: a list of glob patterns
+        """
+        self.patterns: List[str] = patterns
+
+    def is_match(self, path: str) -> bool:
+        """Determines if a path matches the allowed-list
+
+        :param path: the path to match
+        :return: True if the path matches the allowed-list, and False otherwise
+        """
+        for pattern in self.patterns:
+            if fnmatch.fnmatch(path, pattern):
+                return True
+        return False
+
+
+def load() -> PALAllowedlist:
+    """Returns an instance of :class:`PALAllowedlist` created from the glob patterns in :const:`DEFAULT_ALLOWEDLIST_FILE`"""
+    with open(DEFAULT_ALLOWEDLIST_FILE) as fh:
+        return PALAllowedlist(fh.read().splitlines())

scripts/commit_validation/commit_validation/pal_allowedlist.txt

@@ -0,0 +1,68 @@
+*/Code/CryEngine/*
+*/Code/Deprecated/Sandbox/*
+*/Code/Framework/AzCore/AzCore/Math/Aabb.h
+*/Code/Framework/AzCore/AzCore/Math/Color.h
+*/Code/Framework/AzCore/AzCore/Math/Crc.h
+*/Code/Framework/AzCore/AzCore/Math/Matrix3x3.h
+*/Code/Framework/AzCore/AzCore/Math/Matrix4x4.h
+*/Code/Framework/AzCore/AzCore/Math/Obb.h
+*/Code/Framework/AzCore/AzCore/Math/PackedVector3.h
+*/Code/Framework/AzCore/AzCore/Math/Plane.h
+*/Code/Framework/AzCore/AzCore/Math/Quaternion.h
+*/Code/Framework/AzCore/AzCore/Math/Transform.h
+*/Code/Framework/AzCore/AzCore/Math/Uuid.cpp
+*/Code/Framework/AzCore/AzCore/Math/Vector2.h
+*/Code/Framework/AzCore/AzCore/Math/Vector3.h
+*/Code/Framework/AzCore/AzCore/Math/Vector4.h
+*/Code/Framework/AzCore/AzCore/Math/VectorFloat.h
+*/Code/Framework/AzCore/AzCore/Memory/dlmalloc.inl
+*/Code/Framework/AzCore/AzCore/Memory/nedmalloc.inl
+*/Code/Framework/AzCore/AzCore/PlatformDef.h
+*/Code/Framework/AzCore/AzCore/std/containers/compressed_pair.h
+*/Code/Framework/AzCore/AzCore/std/containers/variant.h
+*/Code/Framework/AzCore/AzCore/std/parallel/binary_semaphore.h
+*/Code/Framework/AzCore/AzCore/std/parallel/semaphore.h
+*/Code/Framework/AzCore/Platform/Android/AzCore/AzCore_Traits_Android.h
+*/Code/Framework/AzCore/Platform/AppleTV/AzCore/AzCore_Traits_AppleTV.h
+*/Code/Framework/AzCore/Platform/iOS/AzCore/AzCore_Traits_iOS.h
+*/Code/Framework/AzCore/Platform/Jasper/AzCore/AzCore_Traits_Jasper.h
+*/Code/Framework/AzCore/Platform/Linux/AzCore/AzCore_Traits_Linux.h
+*/Code/Framework/AzCore/Platform/Mac/AzCore/AzCore_Traits_Mac.h
+*/Code/Framework/AzCore/Platform/Provo/AzCore/AzCore_Traits_Provo.h
+*/Code/Framework/AzCore/Platform/Salem/AzCore/AzCore_Traits_Salem.h
+*/Code/Framework/AzCore/Platform/Windows/AzCore/AzCore_Traits_Windows.h
+*/Code/Framework/AzCore/Platform/Xenia/AzCore/AzCore_Traits_Xenia.h
+*/Code/Framework/AzCore/Tests/AZStd/Examples.cpp
+*/Code/Framework/AzCore/Tests/Memory.cpp
+*/Code/Framework/AzFramework/AzFramework/Asset/AssetSystemComponentHelper.cpp
+*/Code/Framework/AzQtComponents/AzQtComponents/*
+*/Code/Framework/AzToolsFramework/AzToolsFramework/AzToolsFramework_precompiled.h
+*/Code/Framework/AzToolsFramework/AzToolsFramework/API/EditorAssetSystemAPI.h
+*/Code/Framework/AzToolsFramework/AzToolsFramework/Application/ToolsApplication.cpp
+*/Code/Framework/AzToolsFramework/AzToolsFramework/Asset/AssetSystemComponent.cpp
+*/Code/Framework/AzToolsFramework/AzToolsFramework/Process/internal/ProcessCommon_Win.h
+*/Code/Framework/AzToolsFramework/AzToolsFramework/Process/internal/ProcessCommunicator_Win.cpp
+*/Code/Framework/AzToolsFramework/AzToolsFramework/Process/internal/ProcessWatcher_Win.cpp
+*/Code/Framework/AzToolsFramework/AzToolsFramework/Process/ProcessCommunicator.h
+*/Code/Framework/AzToolsFramework/AzToolsFramework/UI/LegacyFramework/Core/EditorFrameworkAPI.h
+*/Code/Framework/AzToolsFramework/AzToolsFramework/UI/LegacyFramework/Core/EditorFrameworkApplication.cpp
+*/Code/Framework/AzToolsFramework/AzToolsFramework/UI/LegacyFramework/UIFramework.cpp
+*/Code/Sandbox/*
+*/Code/Tools/*
+*/Gems/*/3rdParty/*
+*/Gems/*/External/*
+*/Gems/CryLegacy*
+*/Gems/EMotionFX/Code/EMotionFX/Rendering/OpenGL2/Source/GLInclude.h
+*/Gems/EMotionFX/Code/EMotionFX/Tools/EMotionStudio/EMStudioSDK/Source/MainWindow.cpp
+*/Gems/EMotionFX/Code/EMotionFX/Tools/EMotionStudio/EMStudioSDK/Source/PluginManager.cpp
+*/Gems/EMotionFX/Code/EMotionFX/Tools/EMotionStudio/EMStudioSDK/Source/PluginManager.h
+*/Gems/EMotionFX/Code/EMotionFX/Tools/EMotionStudio/Plugins/StandardPlugins/Source/StandardPluginsConfig.h
+*/Gems/EMotionFX/Code/MCore/Source/Config.h
+*/Gems/GameStateSamples/Code/Include/GameStateSamples/GameStateLocalUserLobby.inl
+*/Gems/ImageProcessing/Code/Tests/AtlasBuilderTest.cpp
+*/Gems/PhysX/Code/Source/System/PhysXSystem.cpp
+*/Gems/SaveData/Code/Tests/SaveDataTest.cpp
+*/Gems/WhiteBox/Code/Source/Rendering/Legacy/WhiteBoxLegacyRenderMesh.cpp
+*/restricted/*/Code/Framework/AzCore/AzCore/AzCore_Traits_*.h
+*/SamplesProject/Gem/Code/Source/MetastreamTest/MetastreamTest.cpp
+*/Tools/CryDeprecation/precompile_check_defines.h

scripts/commit_validation/commit_validation/tests/__init__.py

@@ -0,0 +1,10 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#

scripts/commit_validation/commit_validation/tests/mocks/__init__.py

@@ -0,0 +1,10 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#

scripts/commit_validation/commit_validation/tests/mocks/mock_commit.py

@@ -0,0 +1,38 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+from typing import Dict, List
+
+from commit_validation.commit_validation import Commit
+
+class MockCommit(Commit):
+
+    def __init__(self, files=None, removed_files=None, file_diffs=None, description: str = '', author: str = ''):
+        self.files = [] if files is None else files
+        self.removed_files = [] if removed_files is None else removed_files
+        self.file_diffs = {} if file_diffs is None else file_diffs
+        self.description = description
+        self.author = author
+
+    def get_files(self) -> List[str]:
+        return self.files
+
+    def get_removed_files(self) -> List[str]:
+        return self.removed_files
+
+    def get_file_diff(self, file) -> Dict[str, str]:
+        return  self.file_diffs[file]
+
+    def get_description(self) -> str:
+        return self.description
+
+    def get_author(self) -> str:
+        return self.author

scripts/commit_validation/commit_validation/tests/test_pal_allowedlist.py

@@ -0,0 +1,40 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import unittest
+from unittest.mock import patch, mock_open
+
+import commit_validation.pal_allowedlist as pal_allowedlist
+
+
+class PALAllowedTests(unittest.TestCase):
+    @patch('builtins.open', mock_open(read_data='*/some/*\n'
+                                                '*/set/*\n'
+                                                '*/of/*\n'
+                                                '*/patterns/*'))
+    def setUp(self):
+        self.allowedlist = pal_allowedlist.load()
+
+    def test_load_allowedlistFile_patternsLoaded(self):
+        self.assertEqual(self.allowedlist.patterns[0], '*/some/*')
+        self.assertEqual(self.allowedlist.patterns[1], '*/set/*')
+        self.assertEqual(self.allowedlist.patterns[2], '*/of/*')
+        self.assertEqual(self.allowedlist.patterns[3], '*/patterns/*')
+
+    def test_isMatch_pathMatchesPattern_returnsTrue(self):
+        self.assertTrue(self.allowedlist.is_match('/path/to/some/file.cpp'))
+
+    def test_isMatch_pathDoesNotMatchPattern_returnsFalse(self):
+        self.assertFalse(self.allowedlist.is_match('/path/to/another/file.cpp'))
+
+
+if __name__ == '__main__':
+    unittest.main()

scripts/commit_validation/commit_validation/tests/validators/__init__.py

@@ -0,0 +1,12 @@
+"""
+
+ All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+ its licensors.
+
+ For complete copyright and license terms please see the LICENSE at the root of this
+ distribution (the "License"). All use of this software is governed by the License,
+ or, if provided, by the license below or the license accompanying this file. Do not
+ remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+"""
+

scripts/commit_validation/commit_validation/tests/validators/test_az_platform_validator.py

@@ -0,0 +1,72 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import unittest
+from unittest.mock import patch, mock_open
+
+from commit_validation import pal_allowedlist
+from commit_validation.tests.mocks.mock_commit import MockCommit
+from commit_validation.validators.az_platform_validator import AzPlatformValidator
+
+class AzPlatformValidatorTests(unittest.TestCase):
+    def test_fileWithNoAzPlatformMacro_passes(self):
+        commit = MockCommit(
+            files=['someCppFile.cpp'],
+            file_diffs = {
+                'someCppFile.cpp' : '+This file does not contain\n+AZ_PLATFORM_MACRO\n'
+            }
+        )
+        error_list = []
+        self.assertTrue(AzPlatformValidator().run(commit, error_list))
+        self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
+
+    # make sure it only examines parts that have actually been changed by this commit
+    # note that the line does not start with +
+    def test_fileWithMacroInNonChangedSection_passes(self):
+        commit = MockCommit(
+            files=['someCppFile.cpp', 'otherfile.cpp'],
+            file_diffs = { # one file has no diff marker, one file indicates its removed.
+                'someCppFile.cpp' : 'Stuff\n#if defined(AZ_PLATFORM_MACRO\n)',
+                'otherfile.cpp'  : 'Stuff\n-#if defined(AZ_PLATFORM_MACRO\n)' 
+            }
+        )
+        error_list = []
+        self.assertTrue(AzPlatformValidator().run(commit, error_list))
+        self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
+
+    def test_fileWithAzPlatformMacro_fails(self):
+        commit = MockCommit(
+            files=['someCppFile.cpp', 'otherfile.cpp'],
+            file_diffs = {
+                'someCppFile.cpp' : '+This file does contain\n'
+                                    '+#if defined(AZ_PLATFORM_MACRO)\n',
+                'otherfile.cpp' : 'This File has a different form\n'
+                                  '+#     if defined(AZ_PLATFORM_MACRO)\n'
+            })
+        error_list = []
+        self.assertFalse(AzPlatformValidator().run(commit, error_list))
+        self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
+
+    def test_fileExtensionIgnored_passes(self):
+        commit = MockCommit(files=['someCppFile.waf_files'])
+        error_list = []
+        self.assertTrue(AzPlatformValidator().run(commit, error_list))
+        self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
+
+    @patch('commit_validation.pal_allowedlist.load', return_value=pal_allowedlist.PALAllowedlist(['*/some/path/*']))
+    def test_fileAllowedlisted_passes(self, mocked_load):
+        commit = MockCommit(files=['/path/to/some/path/someCppFile.cpp'])
+        error_list = []
+        self.assertTrue(AzPlatformValidator().run(commit, error_list))
+        self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
+
+if __name__ == '__main__':
+    unittest.main()

scripts/commit_validation/commit_validation/tests/validators/test_az_trait_validator.py

@@ -0,0 +1,95 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import unittest
+from unittest.mock import patch, mock_open
+
+from commit_validation import pal_allowedlist
+from commit_validation.tests.mocks.mock_commit import MockCommit
+from commit_validation.validators.az_trait_validator import AzTraitValidator
+
+import pytest
+
+
+class Test_AzTraitValidatorTests():
+    def test_fileDoesntCheckAzTraitIsDefined_passes(self):
+        commit = MockCommit(
+                files=['someCppFile.cpp'], 
+                file_diffs={ 'someCppFile.cpp' : ''}
+        )
+        error_list = []
+        assert AzTraitValidator().run(commit, error_list)
+        assert len(error_list) == 0, f"Unexpected errors: {error_list}"
+
+    @pytest.mark.parametrize(
+        'file_diffs,expect_success', [
+            pytest.param('+This file does contain\n'
+                         '+a trait existence check\n'
+                         '+#ifdef AZ_TRAIT_USED_INCORRECTLY\n',
+                         False,
+                         id="AZ_TRAIT_inside_ifdef_fails" ), # gives the test a friendly name!
+            
+            pytest.param('+This file does contain\n'
+                         '+a trait existence check\n'
+                         '+#if defined(AZ_TRAIT_USED_INCORRECTLY)\n',
+                         False,
+                         id="AZ_TRAIT_inside_if_defined_fails" ),
+            
+            pytest.param('+This file does contain\n'
+                         '+a trait existence check\n'
+                         '+#ifndef AZ_TRAIT_USED_INCORRECTLY\n',
+                        False,
+                        id="AZ_TRAIT_inside_ifndef_fails" ),
+            
+            pytest.param('+This file contains a diff which REMOVES an incorrect usage\n'
+                         '-#ifndef AZ_TRAIT_USED_INCORRECTLY\n',
+                         True,
+                         id="AZ_TRAIT_removed_in_diff_passes" ),
+            
+            pytest.param('+This file contains a diff which has an old already okayed usage\n'
+                         '+which is not actually part of the diff.\n'
+                         '#ifndef AZ_TRAIT_USED_INCORRECTLY\n',
+                         True,
+                         id="AZ_TRAIT_in_unmodified_section_passes"),
+            
+            pytest.param('+This file contains the correct usage\n'
+                         '+#if AZ_TRAIT_USED_CORRECTLY\n',
+                         True,
+                         id="AZ_TRAIT_correct_usage_passes"),
+        ])
+    def test_fileChecksAzTraitIsDefined(self, file_diffs, expect_success):
+        commit = MockCommit(
+            files=['someCppFile.cpp'],
+            file_diffs={ 'someCppFile.cpp' : file_diffs })
+
+        error_list = []
+        if expect_success:
+            assert AzTraitValidator().run(commit, error_list)
+            assert len(error_list) == 0, f"Unexpected errors: {error_list}"
+        else:
+            assert not AzTraitValidator().run(commit, error_list)
+            assert len(error_list) != 0, f"Errors were expected but none were returned."
+    
+    def test_fileExtensionIgnored_passes(self):
+        commit = MockCommit(files=['someCppFile.waf_files'])
+        error_list = []
+        assert AzTraitValidator().run(commit, error_list)
+        assert len(error_list) == 0, f"Unexpected errors: {error_list}"
+
+    @patch('commit_validation.pal_allowedlist.load', return_value=pal_allowedlist.PALAllowedlist(['*/some/path/*']))
+    def test_fileAllowedlisted_passes(self, mocked_load):
+        commit = MockCommit(files=['/path/to/some/path/someCppFile.cpp'])
+        error_list = []
+        assert AzTraitValidator().run(commit, error_list)
+        assert len(error_list) == 0, f"Unexpected errors: {error_list}"
+
+if __name__ == '__main__':
+    unittest.main()

scripts/commit_validation/commit_validation/tests/validators/test_copyright_header_validator.py

@@ -0,0 +1,75 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import unittest
+from unittest.mock import patch, mock_open
+
+from commit_validation.tests.mocks.mock_commit import MockCommit
+from commit_validation.validators.copyright_header_validator import CopyrightHeaderValidator
+
+
+class CopyrightHeaderValidatorTests(unittest.TestCase):
+    @patch('builtins.open', mock_open(read_data='This file does contain\n'
+                                                'Copyright (c) Amazon.com, so it should pass\n'))
+    def test_fileWithCopyrightHeader_passes(self):
+        commit = MockCommit(files=['/someCppFile.cpp'])
+        files = [
+            'This file does contain\n'
+            'Copyright (c) Amazon.com, so it should pass\n',
+
+            'This file does contain\n'
+            'Copyright(c) Amazon.com, so it should pass\n'
+            'and there\'s no space between "Copyright" and "(c)"',
+
+            'This file has a upper-case C between the parenthesis\n'
+            '// Copyright (C) Amazon.com, Inc. or its affiliates.',
+        ]
+        for file in files:
+            with patch('builtins.open', mock_open(read_data=file)):
+                error_list = []
+                self.assertTrue(CopyrightHeaderValidator().run(commit, error_list))
+                self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
+
+    def test_fileWithCopyrightHeaderExternalDir_passes(self):
+        commit = MockCommit(files=['/External/someCppFile.cpp'])
+        error_list = []
+        self.assertTrue(CopyrightHeaderValidator().run(commit, error_list))
+        self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
+
+    @patch('builtins.open', mock_open(read_data='This file does not contain\n'
+                                                'the copyright header\n'))
+    def test_fileWithNoCopyrightHeader_fails(self):
+        commit = MockCommit(files=['/someCppFile.cpp'])
+        error_list = []
+        self.assertFalse(CopyrightHeaderValidator().run(commit, error_list))
+        self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
+
+    def test_fileExtensionIgnored_passes(self):
+        commit = MockCommit(files=['/someCppFile.waf_files'])
+        error_list = []
+        self.assertTrue(CopyrightHeaderValidator().run(commit, error_list))
+        self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
+
+    def test_fileWith3rdPartyPath_passes(self):
+        commit = MockCommit(files=['/3rdParty/someCppFile.cpp'])
+        error_list = []
+        self.assertTrue(CopyrightHeaderValidator().run(commit, error_list))
+        self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
+
+    def test_fileWithExternal_passes(self):
+        commit = MockCommit(files=['/External/someCppFile.cpp'])
+        error_list = []
+        self.assertTrue(CopyrightHeaderValidator().run(commit, error_list))
+        self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
+
+
+if __name__ == '__main__':
+    unittest.main()

scripts/commit_validation/commit_validation/tests/validators/test_diff_whitespace_validator.py

@@ -0,0 +1,102 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import unittest
+
+from commit_validation.tests.mocks.mock_commit import MockCommit
+from commit_validation.validators.diff_whitespace_validator import WhitespaceValidator
+
+
+class WhitespaceValidatorTests(unittest.TestCase):
+
+    def test_DiffWhitespace_WhitespaceLinesContiguousWithCodeChange_pass(self):
+        commit = MockCommit(
+            files = ['/someCppFile.cpp'],
+            file_diffs={'/someCppFile.cpp':
+                            'This file\n'
+                            '-was edited'
+                            '+  and has a diff including plus and minus symbols\n'
+                            '+  \n'
+                            'and should pass\n'})
+        error_list = []
+        self.assertTrue(WhitespaceValidator().run(commit, error_list))
+        self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
+
+    def test_DiffWhitespace_WhitespaceLinesSeparateAtEOF_fail(self):
+        commit = MockCommit(
+            files = ['/someCppFile.cpp'],
+            file_diffs={'/someCppFile.cpp':
+                            'This file\n'
+                            '-was edited'
+                            '+  and has an implementation diff\n'
+                            'but should not pass because of a whitespace only change:\n'
+                            '-    \n'})
+        error_list = []
+        self.assertFalse(WhitespaceValidator().run(commit, error_list))
+        self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
+
+    def test_DiffWhitespace_WhitespaceLinesSeparateAtEOF_fail_More(self):
+        commit = MockCommit(
+            files = ['/someCppFile.cpp'],
+            file_diffs={'/someCppFile.cpp':
+                            '+    \n'
+                            'This file\n'
+                            '-was edited'
+                            '+  and has an implementation diff\n'
+                            'but should not pass because of a whitespace only change\n'
+                            'on the first line\n'})
+        error_list = []
+        self.assertFalse(WhitespaceValidator().run(commit, error_list))
+        self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
+
+    def test_DiffWhitespace_WhitespaceLinesSeparateFromCode_fail(self):
+        commit = MockCommit(
+            files = ['/someCppFile.cpp'],
+            file_diffs={'/someCppFile.cpp':
+                            'This file\n'
+                            'should not pass because of a whitespace only change:\n'
+                            '-    \n'
+                            'since was edited'
+                            '+  and has an implementation diff\n'
+                            '-  later on'})
+        error_list = []
+        self.assertFalse(WhitespaceValidator().run(commit, error_list))
+        self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
+
+    def test_DiffWhitespace_WhitespaceOnlyChange_pass(self):
+        commit = MockCommit(
+            files = ['/someCppFile.cpp'],
+            file_diffs={'/someCppFile.cpp':
+                            'This file has whitespace only changes\n'
+                            '+  \n'
+                            '+  \n'
+                            'and should pass\n'
+                            '-  \n'})
+        error_list = []
+        self.assertTrue(WhitespaceValidator().run(commit, error_list))
+        self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
+
+    def test_DiffWhitespace_CodeChangeOnly_pass(self):
+        commit = MockCommit(
+            files = ['/someCppFile.cpp'],
+            file_diffs={'/someCppFile.cpp':
+                            'This file has'
+                            '+  no whitespace only lines changed\n'
+                            '  \n'
+                            '-and should pass\n'
+                            '  \n'})
+        error_list = []
+        self.assertTrue(WhitespaceValidator().run(commit, error_list))
+        self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
+
+
+if __name__ == '__main__':
+    unittest.main()

scripts/commit_validation/commit_validation/tests/validators/test_generated_files_validator.py

@@ -0,0 +1,96 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import unittest
+from unittest.mock import patch, mock_open
+
+from commit_validation.tests.mocks.mock_commit import MockCommit
+from commit_validation.validators.generated_files_validator import GeneratedFilesValidator
+
+
+@patch('builtins.open', mock_open(read_data='unused\n'))
+class GeneratedFilesValidatorTests(unittest.TestCase):
+
+    def test_IsGenerated_NormalFile_pass(self):
+        commit = MockCommit(files=['/someFile.cpp'])
+        error_list = []
+        self.assertTrue(GeneratedFilesValidator().run(commit, error_list))
+        self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
+
+    def test_IsGenerated_DotGeneratedFile_fail(self):
+        commit = MockCommit(files=['/someFile.generated.cpp'])
+        error_list = []
+        self.assertFalse(GeneratedFilesValidator().run(commit, error_list))
+        self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
+
+    def test_IsGenerated_QtMOC_fail(self):
+        commit = MockCommit(files=['/moc_someFile.cpp'])
+        error_list = []
+        self.assertFalse(GeneratedFilesValidator().run(commit, error_list))
+        self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
+
+    def test_IsGenerated_QtQRC_fail(self):
+        commit = MockCommit(files=['/qrc_someFile.cpp'])
+        error_list = []
+        self.assertFalse(GeneratedFilesValidator().run(commit, error_list))
+        self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
+
+    def test_IsGenerated_QtUiDotH_fail(self):
+        commit = MockCommit(files=['/ui_someFile.h'])
+        error_list = []
+        self.assertFalse(GeneratedFilesValidator().run(commit, error_list))
+        self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
+
+    def test_IsGenerated_CMakeCacheFile_fail(self):
+        commit = MockCommit(files=['/CMakeCache.txt'])
+        error_list = []
+        self.assertFalse(GeneratedFilesValidator().run(commit, error_list))
+        self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
+
+    def test_IsGenerated_CMakeCacheExtension_fail(self):
+        commit = MockCommit(files=['/AzCore.Benchmarks.rule'])
+        error_list = []
+        self.assertFalse(GeneratedFilesValidator().run(commit, error_list))
+        self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
+
+    def test_IsGenerated_Tempfile_fail(self):
+        commit = MockCommit(files=['/someFile.tmp'])
+        error_list = []
+        self.assertFalse(GeneratedFilesValidator().run(commit, error_list))
+        self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
+
+    def test_IsGenerated_UnixObject_fail(self):
+        commit = MockCommit(files=['/someFile.o'])
+        error_list = []
+        self.assertFalse(GeneratedFilesValidator().run(commit, error_list))
+        self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
+
+    def test_IsGenerated_WindowsObject_fail(self):
+        commit = MockCommit(files=['/someFile.obj'])
+        error_list = []
+        self.assertFalse(GeneratedFilesValidator().run(commit, error_list))
+        self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
+
+    def test_IsGenerated_MicrosoftSolution_fail(self):
+        commit = MockCommit(files=['/someFile.sln'])
+        error_list = []
+        self.assertFalse(GeneratedFilesValidator().run(commit, error_list))
+        self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
+
+    def test_IsGenerated_Logfile_fail(self):
+        commit = MockCommit(files=['/someFile.log'])
+        error_list = []
+        self.assertFalse(GeneratedFilesValidator().run(commit, error_list))
+        self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
+
+
+if __name__ == '__main__':
+    unittest.main()

scripts/commit_validation/commit_validation/tests/validators/test_git_conflict_validator.py

@@ -0,0 +1,99 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import unittest
+from unittest.mock import patch, mock_open
+
+from commit_validation.tests.mocks.mock_commit import MockCommit
+from commit_validation.validators.git_conflict_validator import GitConflictValidator
+
+
+class NewlineValidatorTests(unittest.TestCase):
+    @patch('builtins.open', mock_open(read_data='This file is completely normal\n'
+                                                'and should pass\n'))
+    def test_HasConflictMarkers_NoMarkers_Pass(self):
+        commit = MockCommit(files=['/someFile.cpp'])
+        error_list = []
+        self.assertTrue(GitConflictValidator().run(commit, error_list))
+        self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
+
+    @patch('builtins.open', mock_open(read_data='This file has a start marker\n'
+                                                '<<<<<<< ours\n'
+                                                'and should fail\n'))
+    def test_HasConflictMarkers_StartMarker_Fail(self):
+        commit = MockCommit(files=['/someFile.cpp'])
+        error_list = []
+        self.assertFalse(GitConflictValidator().run(commit, error_list))
+        self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
+
+    @patch('builtins.open', mock_open(read_data='This file has a diff3 marker from using --conflict=diff3\n'
+                                                '||||||| base\n'
+                                                'and should fail\n'))
+    def test_HasConflictMarkers_BaseMarker_Fail(self):
+        commit = MockCommit(files=['/someFile.cpp'])
+        error_list = []
+        self.assertFalse(GitConflictValidator().run(commit, error_list))
+        self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
+
+    @patch('builtins.open', mock_open(read_data='This file has a diff marker\n'
+                                                '=======\n'
+                                                'and should fail\n'))
+    def test_HasConflictMarkers_DiffMarker_Fail(self):
+        commit = MockCommit(files=['/someFile.cpp'])
+        error_list = []
+        self.assertFalse(GitConflictValidator().run(commit, error_list))
+        self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
+
+    @patch('builtins.open', mock_open(read_data='This file has and end marker\n'
+                                                '>>>>>>> theirs\n'
+                                                'and should fail\n'))
+    def test_HasConflictMarkers_EndMarker_Fail(self):
+        commit = MockCommit(files=['/someFile.cpp'])
+        error_list = []
+        self.assertFalse(GitConflictValidator().run(commit, error_list))
+        self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
+
+    @patch('builtins.open', mock_open(read_data='This file has a equals sign divider of length seven, but is indented\n'
+                                                '    /*\n'
+                                                '    =======\n'
+                                                '    */'
+                                                'and should pass\n'))
+    def test_HasConflictMarkers_IndentedCommentDivider_Pass(self):
+        commit = MockCommit(files=['/someFile.cpp'])
+        error_list = []
+        self.assertTrue(GitConflictValidator().run(commit, error_list))
+        self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
+
+    @patch('builtins.open', mock_open(read_data='This file has an unindented equals sign divider, of length eight\n'
+                                                '/*\n'
+                                                '========\n'
+                                                '*/'
+                                                'and should pass\n'))
+    def test_HasConflictMarkers_LongCommentDivider_Pass(self):
+        commit = MockCommit(files=['/someFile.cpp'])
+        error_list = []
+        self.assertTrue(GitConflictValidator().run(commit, error_list))
+        self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
+
+    @patch('builtins.open', mock_open(read_data='This file has an unindented equals sign divider, of length six\n'
+                                                '/*\n'
+                                                '======\n'
+                                                '*/'
+                                                'and should pass\n'))
+    def test_HasConflictMarkers_ShortCommentDivider_Pass(self):
+        commit = MockCommit(files=['/someFile.cpp'])
+        error_list = []
+        self.assertTrue(GitConflictValidator().run(commit, error_list))
+        self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
+
+
+if __name__ == '__main__':
+    unittest.main()

scripts/commit_validation/commit_validation/tests/validators/test_newline_validator.py

@@ -0,0 +1,88 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import unittest
+from unittest.mock import patch, mock_open
+
+from commit_validation.tests.mocks.mock_commit import MockCommit
+from commit_validation.validators.newline_validator import NewlineValidator
+
+class NewlineValidatorTests(unittest.TestCase):
+    @patch('builtins.open', mock_open(read_data='This file uses carriage returns with line feeds\r\n'
+                                                'and should fail since only line feeds are allowed\r\n'
+                                                'despite appropriately ending with an extra line\r\n'))
+    def test_LineEndings_CRLF_fail(self):
+        commit = MockCommit(files=['/someFile.cpp'])
+        error_list = []
+        self.assertFalse(NewlineValidator().run(commit, error_list))
+        self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
+
+    @patch('builtins.open', mock_open(read_data='This file uses line feeds\n'
+                                                'and ends with an extra line\n'))
+    def test_LineEndings_LF_pass(self):
+        commit = MockCommit(files=['/someFile.cpp'])
+        error_list = []
+        self.assertTrue(NewlineValidator().run(commit, error_list))
+        self.assertTrue(not error_list)
+
+    @patch('builtins.open', mock_open(read_data='This file is full of garbage\n'
+                                                'which would normally fail\r\n'
+                                                'though it should not due to its file extension\n\r'
+                                                'which should skip validation'))
+    def test_LineEndings_RandomFileExtension_skip(self):
+        commit = MockCommit(files=['/someFile.garbage'])
+        error_list = []
+        self.assertTrue(NewlineValidator().run(commit, error_list))
+        self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
+
+    @patch('builtins.open', mock_open(read_data='This file mixes line feed endings\n'
+                                                'with carriage return and line feed endings\r\n'))
+    def test_LineEndings_Mixed_fail(self):
+        commit = MockCommit(files=['/someFile.cpp'])
+        error_list = []
+        self.assertFalse(NewlineValidator().run(commit, error_list))
+        self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
+
+    @patch('builtins.open', mock_open(read_data='This file ends\n'
+                                                'with no newline'))
+    def test_LineEndings_NoFinalNewline_fail(self):
+        commit = MockCommit(files=['/someFile.cpp'])
+        error_list = []
+        self.assertFalse(NewlineValidator().run(commit, error_list))
+        self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
+
+    @patch('builtins.open', mock_open(read_data='This file ends with CRLF newlines\r\n'
+                                                'but not at the end'))
+    def test_LineEndings_NoFinalCRLF_fail(self):
+        commit = MockCommit(files=['/someFile.cpp'])
+        error_list = []
+        self.assertFalse(NewlineValidator().run(commit, error_list))
+        self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
+
+    @patch('builtins.open', mock_open(read_data='This file ends with extra newlines\n'
+                                                '\n'))
+    def test_LineEndings_ExtraFinalNewline_fail(self):
+        commit = MockCommit(files=['/someFile.cpp'])
+        error_list = []
+        self.assertFalse(NewlineValidator().run(commit, error_list))
+        self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
+
+    @patch('builtins.open', mock_open(read_data='This file ends with CRLF extra newlines\r\n'
+                                                '\r\n'))
+    def test_LineEndings_ExtraFinalCRLF_fail(self):
+        commit = MockCommit(files=['/someFile.cpp'])
+        error_list = []
+        self.assertFalse(NewlineValidator().run(commit, error_list))
+        self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
+
+
+if __name__ == '__main__':
+    unittest.main()

scripts/commit_validation/commit_validation/tests/validators/test_platform_macro_validator.py

@@ -0,0 +1,77 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import unittest
+from unittest.mock import patch, mock_open
+
+from commit_validation import pal_allowedlist
+from commit_validation.tests.mocks.mock_commit import MockCommit
+from commit_validation.validators.platform_macro_validator import PlatformMacroValidator
+
+
+class PlatformMacroValidatorTests(unittest.TestCase):
+    def test_fileWithNoPlatformMacro_passes(self):
+        commit = MockCommit(
+            files=['someCppFile.cpp'],
+            file_diffs={ 'someCppFile.cpp' : '+This file does not contain\n'
+                                             '+a platform macro\n'
+                                             '+#define ACCEPTABLE_MACRO\n'
+                        })
+        error_list = []
+        self.assertTrue(PlatformMacroValidator().run(commit, error_list))
+        self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
+
+    def test_fileWithPlatformMacro_errors(self):
+        commit = MockCommit(
+            files=['someCppFile.cpp'],
+            file_diffs={ 'someCppFile.cpp' : '+This file does contain\n'
+                                             '+a platform macro\n'
+                                             '+#if defined(APPLE)\n'
+                    })
+        error_list = []
+        self.assertFalse(PlatformMacroValidator().run(commit, error_list))
+        self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
+    
+    def test_fileWithPlatformMacro_ButNotInChangedPart_passes(self):
+        commit = MockCommit(
+            files=['someCppFile.cpp'],
+            file_diffs={ 'someCppFile.cpp' : '+This file does contain\n'
+                                             '+a platform macro\n'
+                                             '-#if defined(APPLE)\n'
+                                             'But not in a part thats relevant to the diff'
+                                             '#if defined(APPLE)\n'
+                    })
+        error_list = []
+        self.assertTrue(PlatformMacroValidator().run(commit, error_list))
+        self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
+
+    def test_fileExtensionIgnored_passes(self):
+        commit = MockCommit(files=['someCppFile.waf_files'])
+        error_list = []
+        self.assertTrue(PlatformMacroValidator().run(commit, error_list))
+        self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
+
+    def test_platformFolderIgnored_passes(self):
+        commit = MockCommit(files=['/path/to/Platform/SomePlatform/someCppFile.cpp'])
+        error_list = []
+        self.assertTrue(PlatformMacroValidator().run(commit, error_list))
+        self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
+
+    @patch('commit_validation.pal_allowedlist.load', return_value=pal_allowedlist.PALAllowedlist(['*/some/path/*']))
+    def test_fileAllowedlisted_passes(self, mocked_load):
+        commit = MockCommit(files=['/path/to/some/path/someCppFile.cpp'])
+        error_list = []
+        self.assertTrue(PlatformMacroValidator().run(commit, error_list))
+        self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
+
+
+if __name__ == '__main__':
+    unittest.main()

scripts/commit_validation/commit_validation/tests/validators/test_pragma_optimize_validator.py

@@ -0,0 +1,89 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import unittest
+from unittest.mock import patch, mock_open
+
+from commit_validation.tests.mocks.mock_commit import MockCommit
+from commit_validation.validators.pragma_optimize_validator import PragmaOptimizeValidator
+
+
+class PragmaOptimizeValidatorTests(unittest.TestCase):
+    def test_fileWithNoPragmaOptimize_passes(self):
+        commit = MockCommit(
+                files=['/someCppFile.cpp'],
+                file_diffs={ 
+                    '/someCppFile.cpp' : '+// This file does not contain\n'
+                                         '+// the p r a g m a o p t i m i z e'
+                })
+        error_list = []
+        self.assertTrue(PragmaOptimizeValidator().run(commit, error_list))
+        self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
+
+    def test_fileWithNoPragma3rdPartyPath_passes(self):
+        commit = MockCommit(files=['/3rdParty/someCppFile.cpp'])
+        error_list = []
+        self.assertTrue(PragmaOptimizeValidator().run(commit, error_list))
+        self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
+
+    def test_fileWithNoPragmaExternal_passes(self):
+        commit = MockCommit(files=['/External/someCppFile.cpp'])
+        error_list = []
+        self.assertTrue(PragmaOptimizeValidator().run(commit, error_list))
+        self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
+
+    def test_fileWithPragmaOptimize_fails(self):
+        commit = MockCommit(
+            files=['/someCppFile.cpp'],
+            file_diffs={ 
+                    '/someCppFile.cpp' : '+// This file contains\n'
+                                         '+// a line\n'
+                                         '+// with #pragma optimize("", off)'
+                })
+        error_list = []
+        self.assertFalse(PragmaOptimizeValidator().run(commit, error_list))
+        self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
+
+    def test_fileWithPragmaOptimize_in_unchanged_part_passes(self):
+        commit = MockCommit(
+            files=['/someCppFile.cpp'],
+            file_diffs={ 
+                    '/someCppFile.cpp' : '+// This file contains\n'
+                                         '+// a line\n'
+                                         '-#pragma optimize("", off)\n'
+                                         'but its in a deleted line or a line that is not part of the diff\n'
+                                         '#pragma optimize("", off)\n'
+                })
+        error_list = []
+        self.assertTrue(PragmaOptimizeValidator().run(commit, error_list))
+        self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
+
+    def test_fileExtensionIgnored_passes(self):
+        commit = MockCommit(files=['/someCppFile.waf_files'])
+        error_list = []
+        self.assertTrue(PragmaOptimizeValidator().run(commit, error_list))
+        self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
+        
+
+    def test_fileWithPragma3rdPartyPath_passes(self):
+        commit = MockCommit(files=['/3rdParty/someCppFile.cpp'])
+        error_list = []
+        self.assertTrue(PragmaOptimizeValidator().run(commit, error_list))
+        self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
+
+    def test_fileWithPragmaExternal_passes(self):
+        commit = MockCommit(files=['/External/someCppFile.cpp'])
+        error_list = []
+        self.assertTrue(PragmaOptimizeValidator().run(commit, error_list))
+        self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
+
+if __name__ == '__main__':
+    unittest.main()

scripts/commit_validation/commit_validation/tests/validators/test_tabs_validator.py

@@ -0,0 +1,78 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import unittest
+from unittest.mock import patch, mock_open
+
+from commit_validation.tests.mocks.mock_commit import MockCommit
+from commit_validation.validators.tabs_validator import TabsValidator
+
+
+class TabsValidatorTests(unittest.TestCase):
+    def test_fileWithNoTabs_passes(self):
+        commit = MockCommit(
+            files=['/someCppFile.cpp'],
+            file_diffs={ '/someCppFile.cpp' : '+This file contains\n'
+                                              '+    no tabs\n'
+                                              '+    but does have spaces\n'})
+        error_list = []
+        self.assertTrue(TabsValidator().run(commit, error_list))
+        self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
+
+    def test_fileWithTabs_fails(self):
+        commit = MockCommit(
+            files=['/someCppFile.cpp'],
+            file_diffs={ '/someCppFile.cpp' : '+This file contains\n'
+                                              '+\tTHE DREADED TAB CHARACTER\n'})
+        error_list = []
+        self.assertFalse(TabsValidator().run(commit, error_list))
+        self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
+
+    def test_fileWithTabs_butNotInDiffArea_passes(self):
+        commit = MockCommit(
+            files=['/someCppFile.cpp'],
+            file_diffs={ '/someCppFile.cpp' : '+This file contains\n'
+                                              '\tTHE DREADED TAB CHARACTER\n'
+                                              'But since its not in a diff line\n'
+                                              'It does not matter not even if in \n'
+                                              '-\tA REMOVED LINE with the tab character\n'})
+        error_list = []
+        self.assertTrue(TabsValidator().run(commit, error_list))
+        self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
+
+
+    def test_fileExtensionIgnored_passes(self):
+        commit = MockCommit(files=['/someCppFile.waf_files'])
+        error_list = []
+        self.assertTrue(TabsValidator().run(commit, error_list))
+        self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
+
+    def test_fileWithNoTabsbutAnEmoji_passes(self):
+        commit = MockCommit(
+            files=['/someCppFile_trap.cpp'],
+            file_diffs={ '/someCppFile_trap.cpp' : '+This file contains\n'
+                                                   '+    no tabs\n'
+                                                   '+    but has an emoji \U0001F4A9\n'})
+        error_list = []
+        self.assertTrue(TabsValidator().run(commit, error_list))
+        self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
+
+    def test_fileWithTabsAndEmoji_fails(self):
+        commit = MockCommit(
+            files=['/another_trap.cpp'],
+            file_diffs={ '/another_trap.cpp' : '+This file contains an emoji \U0001F4A9 \n'
+                                               '+\tbut also a tab!\n'})
+        error_list = []
+        self.assertFalse(TabsValidator().run(commit, error_list))
+        self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
+
+if __name__ == '__main__':
+    unittest.main()

scripts/commit_validation/commit_validation/validators/__init__.py

@@ -0,0 +1,10 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#

scripts/commit_validation/commit_validation/validators/az_platform_validator.py

@@ -0,0 +1,59 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import os
+import re
+from typing import Type, List
+
+import commit_validation.pal_allowedlist as pal_allowedlist
+from commit_validation.commit_validation import Commit, CommitValidator, SOURCE_FILE_EXTENSIONS, VERBOSE
+
+az_platform_regex = re.compile(r'^\+\s*#.*\bAZ_PLATFORM_')
+
+
+class AzPlatformValidator(CommitValidator):
+    """A file-level validator that makes sure a file does not contain AZ_PLATFORM macros"""
+
+    def __init__(self) -> None:
+        self.pal_allowedlist = pal_allowedlist.load()
+
+    def run(self, commit: Commit, errors: List[str]) -> bool:
+        for file_name in commit.get_files():
+            if os.path.splitext(file_name)[1].lower() not in SOURCE_FILE_EXTENSIONS:
+                if VERBOSE: print(f'{file_name}::{self.__class__.__name__} SKIPPED - File excluded based on extension.')
+                continue
+            if self.pal_allowedlist.is_match(file_name):
+                if VERBOSE: print(f'{file_name}::{self.__class__.__name__} SKIPPED - File excluded based on PAL allowedlist.')
+                continue
+            
+            file_diff = commit.get_file_diff(file_name)
+            previous_line_context = ""
+
+            line_number = 1
+            for line in file_diff.splitlines():
+                # we only care about lines that start with +
+                if line.startswith('+'):
+                    if az_platform_regex.search(line):
+                        error_message = str(f'{file_name}:{line_number}::{self.__class__.__name__} FAILED - Source file contains an AZ_PLATFORM '
+                                f'macro in code:\n'
+                                f'     {previous_line_context}\n'
+                                f'---> {line}\n')
+                        if VERBOSE: print(error_message)
+                        errors.append(error_message)
+                previous_line_context = line
+                line_number += 1
+                
+        return (not errors)
+
+
+def get_validator() -> Type[AzPlatformValidator]:
+    """Returns the validator class for this module"""
+    return AzPlatformValidator

scripts/commit_validation/commit_validation/validators/az_trait_validator.py

@@ -0,0 +1,60 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import os
+import re
+from typing import Type, List
+
+import commit_validation.pal_allowedlist as pal_allowedlist
+from commit_validation.commit_validation import Commit, CommitValidator, SOURCE_FILE_EXTENSIONS, VERBOSE
+
+ifdef_regex = re.compile(r'^\+\s*#\s*ifn?def\s+AZ_TRAIT_')
+defined_regex = re.compile(r'\sdefined\s*\(\s*AZ_TRAIT_')
+
+
+class AzTraitValidator(CommitValidator):
+    """A file-level validator that makes sure a file does not contain existence checks for AZ_TRAIT macros"""
+
+    def __init__(self) -> None:
+        self.pal_allowedlist = pal_allowedlist.load()
+
+    def run(self, commit: Commit, errors: List[str]) -> bool:
+        for file_name in commit.get_files():
+            if os.path.splitext(file_name)[1].lower() not in SOURCE_FILE_EXTENSIONS:
+                if VERBOSE: print(f'{file_name}::{self.__class__.__name__} SKIPPED - File excluded based on extension.')
+                continue
+            if self.pal_allowedlist.is_match(file_name):
+                if VERBOSE: print(f'{file_name}::{self.__class__.__name__} SKIPPED - File excluded based on PAL allowedlist.')
+                continue
+
+            file_diff = commit.get_file_diff(file_name)
+            previous_line_context = ""
+
+            for line in file_diff.splitlines():
+                # we only care about added lines.
+                if line.startswith('+'):
+                    if ifdef_regex.search(line) or defined_regex.search(line):
+                        error_message = str(
+                                f'{file_name}::{self.__class__.__name__} FAILED - Source file contains an existence '
+                                f'check for an AZ_TRAIT macro in this code: \n'
+                                f'       {previous_line_context}\n'
+                                f' ----> {line}\n'
+                                f'Traits should be tested for true/false, since they are guaranteed to exist on all platforms.')
+                        if VERBOSE: print(error_message)
+                        errors.append(error_message)
+                previous_line_context = line
+
+        return (not errors)
+
+
+def get_validator() -> Type[AzTraitValidator]:
+    """Returns the validator class for this module"""
+    return AzTraitValidator

scripts/commit_validation/commit_validation/validators/copyright_header_validator.py

@@ -0,0 +1,52 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import fnmatch
+import os
+import re
+from typing import Type, List
+
+from commit_validation.commit_validation import Commit, CommitValidator, IsFileSkipped, SOURCE_AND_SCRIPT_FILE_EXTENSIONS, EXCLUDED_VALIDATION_PATTERNS, VERBOSE
+
+
+class CopyrightHeaderValidator(CommitValidator):
+    """A file-level validator that makes sure a file contains the standard copyright header"""
+
+    def run(self, commit: Commit, errors: List[str]) -> bool:
+        for file_name in commit.get_files():
+            for pattern in EXCLUDED_VALIDATION_PATTERNS:
+                if fnmatch.fnmatch(file_name, pattern):
+                    if VERBOSE: print(f'{file_name}::{self.__class__.__name__} SKIPPED - Validation pattern excluded on path.')
+                    break
+            else:
+                if IsFileSkipped(file_name):
+                    if VERBOSE: print(f'{file_name}::{self.__class__.__name__} SKIPPED - File excluded based on extension.')
+                    continue
+
+                copyright_regex = re.compile(r'copyright[\s]*(?:\(c\))?[\s]*amazon\.com', re.IGNORECASE)
+
+                # copyright header validator does not use the diff, as it needs to check the front
+                # of the file for the header.
+                with open(file_name, 'rt', encoding='utf8', errors='replace') as fh:
+                    for line in fh:
+                        if copyright_regex.search(line):
+                            break
+                    else:
+                        error_message = str(f'{file_name}::{self.__class__.__name__} FAILED - Source file missing copyright headers.')
+                        errors.append(error_message)
+                        if VERBOSE: print(error_message)
+
+        return (not errors)
+
+
+def get_validator() -> Type[CopyrightHeaderValidator]:
+    """Returns the validator class for this module"""
+    return CopyrightHeaderValidator

scripts/commit_validation/commit_validation/validators/diff_whitespace_validator.py

@@ -0,0 +1,58 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import fnmatch
+import os
+import re
+from typing import Type, List
+
+from commit_validation.commit_validation import Commit, CommitValidator, IsFileSkipped, SOURCE_AND_SCRIPT_FILE_EXTENSIONS, EXCLUDED_VALIDATION_PATTERNS, VERBOSE
+
+__STARTS_WITH_UNCHANGED_LINE = r'(\A|^(?=[^+-]).*\n)'
+__NONZERO_CHANGED_WHITESPACE_LINES = r'(^[+-][\r\t\f\v ]*\n)+'
+__ENDS_WITH_UNCHANGED_LINE = r'([^+-]|\Z)'
+__NONADJACENT_WHITESPACE =\
+    __STARTS_WITH_UNCHANGED_LINE + __NONZERO_CHANGED_WHITESPACE_LINES + __ENDS_WITH_UNCHANGED_LINE
+_NONADJACENT_WHITESPACE_DIFF_REGEX = re.compile(__NONADJACENT_WHITESPACE, re.MULTILINE)
+
+_NONWHITESPACE_DIFF_REGEX = re.compile(r'^[+-][\t ]*\S', re.MULTILINE)
+
+
+class WhitespaceValidator(CommitValidator):
+    """A diff-level validator that makes sure a change does not mix whitespace-only changes with code changes"""
+
+    def run(self, commit: Commit, errors: List[str]) -> bool:
+        for file_name in commit.get_files():
+            file_identifier = f"{file_name}::{self.__class__.__name__}"
+            for pattern in EXCLUDED_VALIDATION_PATTERNS:
+                if fnmatch.fnmatch(file_name, pattern):
+                    if VERBOSE: print(f'{file_identifier} SKIPPED - Validation pattern excluded on path.')
+                    break
+            else:
+                if IsFileSkipped(file_name):
+                    if VERBOSE: print(f'{file_identifier} SKIPPED - File excluded based on extension.')
+                    continue
+
+                diff = commit.get_file_diff(file_name)
+                
+                if _NONADJACENT_WHITESPACE_DIFF_REGEX.search(diff) and _NONWHITESPACE_DIFF_REGEX.search(diff):
+                    error_message = str(f'{file_identifier} FAILED - Source file contains whitespace-only changes which are '
+                          f'non-contiguous with other non-whitespace changes.  Make whitespace-only changes as a '
+                          f'separate change.')
+                    errors.append(error_message)
+                    if VERBOSE: print(error_message)
+
+        return (not errors)
+
+
+def get_validator() -> Type[WhitespaceValidator]:
+    """Returns the validator class for this module"""
+    return WhitespaceValidator

scripts/commit_validation/commit_validation/validators/generated_files_validator.py

@@ -0,0 +1,139 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import fnmatch
+import os
+import pathlib
+from typing import Type, List
+
+from commit_validation.commit_validation import Commit, CommitValidator, EXCLUDED_VALIDATION_PATTERNS, VERBOSE
+
+# Disallowed File Name Patterns
+# these must be LOWER CASE since they will be compared after a 'lower'
+AZ_CODEGEN_FILENAME_PATTERN = "*.generated*"
+QT_COMPILER_PATTERNS = ["moc_*", "qrc_*", "ui_*.h"]
+CMAKE_CACHE_FILENAME_PATTERNS = ["cmakecache.txt", "cmake_install.cmake", "ctesttestfile.cmake", "ctestcostdata.txt"]
+OTHER_GENERATED_FILES = ["appxmanifest.xml"]
+
+# Disallowed File Extensions
+CMAKE_CACHE_EXTENSIONS = [".rule", ".stamp", ".depend"]
+TEMPFILE_EXTENSIONS = [".backup", ".bak", ".bkup", ".temp", ".tempfile", ".temporary", ".tmp"]
+EXECUTION_ARTIFACT_EXTENSIONS = [".cache", ".dmp", ".log", ".pyc"]
+CLANG_FILE_EXTENSIONS = [".o", ".s"]
+MSVS_FILE_EXTENSIONS = [
+    ".aps",
+    ".csproj",
+    ".exp",
+    ".filters",
+    ".idb",
+    ".ilk",
+    ".lastbuildstate",
+    ".meta",
+    ".ncb",
+    ".obj",
+    ".opensdf",
+    ".pch",
+    ".pdb",
+    ".pgc",
+    ".pgd",
+    ".psess",
+    ".rsp",
+    ".sbr",
+    ".sdf",
+    ".sln",
+    ".suo",
+    ".tlb",
+    ".tlh",
+    ".tli",
+    ".tlog",
+    ".vap",
+    ".vbg",
+    ".vbproj",
+    ".vcxitems",
+    ".vcxproj",
+    ".vdproj",
+    ".vmx",
+    ".vsp",
+    ".vspscc",
+    ".vspx",
+    ".vssscc",
+    ".vup",
+]
+
+
+class GeneratedFilesValidator(CommitValidator):
+    """A file-level validator to check if a file is a true source file, and not generated during build or execution"""
+
+    def run(self, commit: Commit, errors: List[str]) -> bool:
+        for file_name in commit.get_files():
+            file_identifier = f"{file_name}::{self.__class__.__name__}"
+            for pattern in EXCLUDED_VALIDATION_PATTERNS:
+                if fnmatch.fnmatch(file_name, pattern):
+                    if VERBOSE: print(f'{file_identifier} SKIPPED - Validation pattern excluded on path.')
+                    break
+            else:
+                file_path_lower = pathlib.Path(file_name.lower())
+                extension = file_path_lower.suffix
+                if extension in CMAKE_CACHE_EXTENSIONS:
+                    error_message = str(f'{file_identifier} FAILED - Autogenerated CMake file detected by extension.')
+                    if VERBOSE: print(error_message)
+                    errors.append(error_message)
+                    continue
+                if extension in MSVS_FILE_EXTENSIONS:
+                    error_message = str(f'{file_identifier} FAILED - Autogenerated Visual Studio file detected by extension.')
+                    if VERBOSE: print(error_message)
+                    errors.append(error_message)
+                    continue
+                if extension in CLANG_FILE_EXTENSIONS:
+                    error_message = str(f'{file_identifier} FAILED - Autogenerated Clang file detected by extension.')
+                    if VERBOSE: print(error_message)
+                    errors.append(error_message)
+                    continue
+                if extension in TEMPFILE_EXTENSIONS:
+                    error_message = str(f'{file_identifier} FAILED - Autogenerated temporary file detected by extension.')
+                    if VERBOSE: print(error_message)
+                    errors.append(error_message)
+                    continue
+                if extension in EXECUTION_ARTIFACT_EXTENSIONS:
+                    error_message = str(f'{file_identifier} FAILED - Execution artifact detected by extension.')
+                    if VERBOSE: print(error_message)
+                    errors.append(error_message)
+                    continue
+
+                for cmake_pattern in QT_COMPILER_PATTERNS:
+                    if fnmatch.fnmatch(file_path_lower.name, cmake_pattern):
+                        error_message = str(f'{file_identifier} FAILED - Autogenerated QT file detected by pattern.')
+                        if VERBOSE: print(error_message)
+                        errors.append(error_message)
+                        break
+                for cmake_pattern in CMAKE_CACHE_FILENAME_PATTERNS:
+                    if fnmatch.fnmatch(file_path_lower.name, cmake_pattern):
+                        error_message = str(f'{file_identifier} FAILED - Autogenerated CMake file detected by name.')
+                        if VERBOSE: print(error_message)
+                        errors.append(error_message)
+                        break
+                if fnmatch.fnmatch(file_path_lower.name, AZ_CODEGEN_FILENAME_PATTERN):
+                    error_message = str(f'{file_identifier} FAILED - Autogenerated AzCodeGen file detected by name.')
+                    if VERBOSE: print(error_message)
+                    errors.append(error_message)
+                    continue
+                if file_path_lower.name in OTHER_GENERATED_FILES:
+                    error_message = str(f'{file_identifier} FAILED - Generated file (others).')
+                    if VERBOSE: print(error_message)
+                    errors.append(error_message)
+                    continue
+
+        return (not errors)
+
+
+def get_validator() -> Type[GeneratedFilesValidator]:
+    """Returns the validator class for this module"""
+    return GeneratedFilesValidator

scripts/commit_validation/commit_validation/validators/git_conflict_validator.py

@@ -0,0 +1,80 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import os
+import re
+from typing import Type, List
+
+import commit_validation.pal_allowedlist as pal_allowedlist
+from commit_validation.commit_validation import Commit, CommitValidator, SOURCE_FILE_EXTENSIONS, VERBOSE
+
+MERGE_TO_MARKER_REGEX = re.compile(r'^<<<<<<<')
+MERGE_BASE_REGEX = re.compile(r'^\|\|\|\|\|\|\|')
+MERGE_DIFF_REGEX = re.compile(r'^=======\n')  # include newline as equals sign is a common visual separator
+MERGE_FROM_MARKER_REGEX = re.compile(r'^>>>>>>>')
+
+
+class GitConflictValidator(CommitValidator):
+    """A file-level validator that for conflict markers"""
+
+    def __init__(self) -> None:
+        self.pal_allowedlist = pal_allowedlist.load()
+
+    def run(self, commit: Commit, errors: List[str]) -> bool:
+        for file_name in commit.get_files():
+            if os.path.splitext(file_name)[1].lower() not in SOURCE_FILE_EXTENSIONS:
+                if VERBOSE: print(f'{file_name}::{self.__class__.__name__} SKIPPED - File excluded based on extension.')
+                continue
+            if self.pal_allowedlist.is_match(file_name):
+                if VERBOSE: print(f'{file_name}::{self.__class__.__name__} SKIPPED - File excluded based on PAL allowedlist.')
+                continue
+
+            # we never want conflict markers to be added to our repository
+            # so we don't look at the file diffs, but the file contents.
+            with open(file_name, 'rt', encoding='utf8', errors='replace') as fh:
+                previous_line_context = ""
+                for line_number, line in enumerate(fh):
+                    if MERGE_TO_MARKER_REGEX.search(line):
+                        error_message = str(f'{file_name}::{self.__class__.__name__} FAILED - Source file contains git merge '
+                              f'conflict start-marker on line {line_number + 1}:\n'
+                              f'      {previous_line_context}'
+                              f'----> {line}')
+                        if VERBOSE: print(error_message)
+                        errors.append(error_message)
+                    if MERGE_BASE_REGEX.search(line):
+                        error_message = str(f'{file_name}::{self.__class__.__name__} FAILED - Source file contains git merge '
+                              f'conflict diff3-marker on line {line_number + 1}:\n'
+                              f'      {previous_line_context}'
+                              f'----> {line}')
+                        if VERBOSE: print(error_message)
+                        errors.append(error_message)
+                    if MERGE_DIFF_REGEX.search(line):
+                        error_message = str(f'{file_name}::{self.__class__.__name__} FAILED - Source file contains git merge '
+                              f'conflict diff-marker on line {line_number + 1}:\n'
+                              f'      {previous_line_context}'
+                              f'----> {line}')
+                        if VERBOSE: print(error_message)
+                        errors.append(error_message)
+                    if MERGE_FROM_MARKER_REGEX.search(line):
+                        error_message = str(f'{file_name}::{self.__class__.__name__} FAILED - Source file contains git merge '
+                              f'conflict end-marker on line {line_number + 1}:\n'
+                              f'      {previous_line_context}'
+                              f'----> {line}')
+                        if VERBOSE: print(error_message)
+                        errors.append(error_message)
+                    previous_line_context = line
+
+        return (not errors)
+
+
+def get_validator() -> Type[GitConflictValidator]:
+    """Returns the validator class for this module"""
+    return GitConflictValidator

scripts/commit_validation/commit_validation/validators/newline_validator.py

@@ -0,0 +1,74 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import fnmatch
+import os
+import re
+from typing import Type, List
+
+from commit_validation.commit_validation import Commit, CommitValidator, IsFileSkipped, SOURCE_AND_SCRIPT_FILE_EXTENSIONS, EXCLUDED_VALIDATION_PATTERNS, VERBOSE
+
+_SINGLE_NEWLINE_ENDING_REGEX = re.compile(r'\n\Z', re.MULTILINE)
+_MULTI_NEWLINE_ENDING_REGEX = re.compile(r'\n\s*\n\Z', re.MULTILINE)
+_CRLF_REGEX = re.compile(r'^.*\r\n', re.MULTILINE)
+_ONLY_LF_REGEX = re.compile(r'^[^\r]*\n', re.MULTILINE)
+
+
+class NewlineValidator(CommitValidator):
+    """A file-level validator that makes sure a file contains valid line-endings"""
+
+    def run(self, commit: Commit, errors: List[str]) -> bool:
+        for file_name in commit.get_files():
+            file_identifier = f"{file_name}::{self.__class__.__name__}"
+            for pattern in EXCLUDED_VALIDATION_PATTERNS:
+                if fnmatch.fnmatch(file_name, pattern):
+                    if VERBOSE: print(f'{file_identifier} SKIPPED - Validation pattern excluded on path.')
+                    break
+            else:
+                file_extension = os.path.splitext(file_name)[1].lower()
+                if IsFileSkipped(file_name):
+                    if VERBOSE: print(f'{file_identifier} SKIPPED - File excluded based on extension.')
+                    continue
+
+                # since this validator focuses on newlines throughout the file, not just in diffs
+                # we use the real file data instead of a diff
+                with open(file_name, 'rt', encoding='utf8', errors='replace') as fh:
+                    lines = fh.read()
+                    if not _SINGLE_NEWLINE_ENDING_REGEX.search(lines):
+                        error_message = str(f'{file_identifier} FAILED - Source file does not end with a trailing newline.')
+                        if VERBOSE: print(error_message)
+                        errors.append(error_message)
+
+                    if _MULTI_NEWLINE_ENDING_REGEX.search(lines):
+                        error_message = str(f'{file_identifier} FAILED - Source file ends in multiple trailing newlines.')
+                        if VERBOSE: print(error_message)
+                        errors.append(error_message)
+
+                    crlf_result = _CRLF_REGEX.search(lines)
+                    only_lf_result = _ONLY_LF_REGEX.search(lines)
+                    if crlf_result and only_lf_result:
+                        error_message = str(f'{file_identifier} FAILED - Source file contains mixed line endings (\\r\\n and \\n)')
+                        if VERBOSE: print(error_message)
+                        errors.append(error_message)
+
+                    if crlf_result:
+                        error_message = str(f'{file_identifier} FAILED - Source file incorrectly contains Windows-style line endings'
+                              f' (\\r\\n), when Unix-style line endings (\\n) were expected.  Enable git option '
+                              f'"core.autocrlf" to avoid this error.')
+                        if VERBOSE: print(error_message)
+                        errors.append(error_message)
+
+        return (not errors)
+
+
+def get_validator() -> Type[NewlineValidator]:
+    """Returns the validator class for this module"""
+    return NewlineValidator

scripts/commit_validation/commit_validation/validators/platform_macro_validator.py

@@ -0,0 +1,86 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import fnmatch
+import os
+import re
+from typing import Type, List
+
+import commit_validation.pal_allowedlist as pal_allowedlist
+from commit_validation.commit_validation import Commit, CommitValidator, SOURCE_FILE_EXTENSIONS, VERBOSE
+
+platform_macros = [
+    'ANDROID',
+    'APPLE',
+    'APPLETV',
+    'DARWIN',
+    'IOS',
+    'LINUX',
+    'LINUX64',
+    'MAC',
+    'PROVO',
+    'SALEM',
+    'WIN32',
+    'WIN32_LEAN_AND_MEAN',
+    'WIN64',
+    'XENIA',
+    '_WIN32',
+    '_WIN32_WINDOWS',
+    '_WIN32_WINNT',
+    'linux',
+]
+
+platform_macro_regex = re.compile(r'^\+\s*#.*?\b(' + str.join('|', platform_macros) + r')\b')
+
+
+class PlatformMacroValidator(CommitValidator):
+    """A file-level validator that makes sure a file does not contain certain platform macros"""
+
+    def __init__(self) -> None:
+        self.pal_allowedlist = pal_allowedlist.load()
+
+    def run(self, commit: Commit, errors: List[str]) -> bool:
+        for file_name in commit.get_files():
+            if os.path.splitext(file_name)[1].lower() not in SOURCE_FILE_EXTENSIONS:
+                if VERBOSE: print(f'{file_name}::{self.__class__.__name__} SKIPPED - File excluded based on extension.')
+                continue
+            if self.pal_allowedlist.is_match(file_name):
+                if VERBOSE: print(f'{file_name}::{self.__class__.__name__} SKIPPED - File excluded based on PAL allowedlist.')
+                continue
+            if fnmatch.fnmatch(file_name, '*/Platform/*'):
+                if VERBOSE: print(f'{file_name}::{self.__class__.__name__} SKIPPED - File excluded because it is in a "Platform" '
+                      f'folder.')
+                continue
+
+            file_diff = commit.get_file_diff(file_name)
+            previous_line_context = ""
+
+            line_number = 1
+            for line in file_diff.splitlines():
+                # we only care about added lines.
+                if line.startswith('+'):
+                    match = platform_macro_regex.search(line)
+                    if match:
+                        error_message = str(f'{file_name}:{line_number}::{self.__class__.__name__} FAILED - Source file contains a forbidden '
+                                f'platform macro "{match.group(1)}" here:\n'
+                                f'     {previous_line_context}\n'
+                                f'---> {line}')
+                        if VERBOSE: print(error_message)
+                        errors.append(error_message)
+                previous_line_context = line
+                line_number += 1
+
+        return (not errors)
+
+
+def get_validator() -> Type[PlatformMacroValidator]:
+    """Returns the validator class for this module"""
+    return PlatformMacroValidator

scripts/commit_validation/commit_validation/validators/pragma_optimize_validator.py

@@ -0,0 +1,51 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import fnmatch
+import os.path
+from typing import Type, List
+
+from commit_validation.commit_validation import Commit, CommitValidator, SOURCE_FILE_EXTENSIONS, EXCLUDED_VALIDATION_PATTERNS, VERBOSE
+
+
+class PragmaOptimizeValidator(CommitValidator):
+    """A file-level validator that makes sure a file does not contain #pragma optimize directives"""
+
+    def run(self, commit: Commit, errors: List[str]) -> bool:
+        for file_name in commit.get_files():
+            for pattern in EXCLUDED_VALIDATION_PATTERNS:
+                if fnmatch.fnmatch(file_name, pattern):
+                    if VERBOSE: print(f'{file_name}::{self.__class__.__name__} SKIPPED - Validation pattern excluded on path.')
+                    break
+            else:
+                if os.path.splitext(file_name)[1].lower() not in SOURCE_FILE_EXTENSIONS:
+                    if VERBOSE: print(f'{file_name}::{self.__class__.__name__} SKIPPED - File excluded based on extension.')
+                    continue
+
+                file_diff = commit.get_file_diff(file_name)
+                previous_line_for_context = ""
+                for line in file_diff.splitlines():
+                    # we only care about added lines in a diff
+                    if line.startswith('+'):
+                        if '#pragma optimize' in line:
+                            error_message = str(f'{file_name}::{self.__class__.__name__} FAILED - Source file contains #pragma optimize!\n'
+                                                f'     {previous_line_for_context}\n'
+                                                f'---> {line}')
+                            if VERBOSE: print(error_message)
+                            errors.append(error_message)
+                        previous_line_for_context = line
+
+        return (not errors)
+
+
+def get_validator() -> Type[PragmaOptimizeValidator]:
+    """Returns the validator class for this module"""
+    return PragmaOptimizeValidator

scripts/commit_validation/commit_validation/validators/tabs_validator.py

@@ -0,0 +1,65 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import fnmatch
+import os.path
+from typing import Type, List
+
+from commit_validation.commit_validation import Commit, CommitValidator, IsFileSkipped, SOURCE_AND_SCRIPT_FILE_EXTENSIONS, EXCLUDED_VALIDATION_PATTERNS, VERBOSE
+
+class TabsValidator(CommitValidator):
+    """A file-level validator that makes sure a file does not contain tabs"""
+
+    def run(self, commit: Commit, errors: List[str]) -> bool:
+        for file_name in commit.get_files():
+            if IsFileSkipped(file_name):
+                if VERBOSE: print(f'{file_name}::{self.__class__.__name__} SKIPPED TabsValidator - File excluded based on extension.')
+                continue
+
+            for pattern in EXCLUDED_VALIDATION_PATTERNS:
+                if fnmatch.fnmatch(file_name, pattern):
+                    if VERBOSE: print(f'{file_name} SKIPPED TabsValidator - Validation pattern excluded on path.')
+                    break
+            else:
+                tab_line_count = 0
+                file_diff = commit.get_file_diff(file_name)
+
+                # Usually, code either has a very small number of tabs in the file by accident,
+                # or the entire file is full of tabs.  
+                # So we count the tabs, but we only print the first one in full.
+                first_tab_line_found = None
+                  
+                for line in file_diff.splitlines():
+                    # we only care about added lines.
+                    if line.startswith('+'):
+                        if '\t' in line:
+                            line = line.replace('\t','\\t') # make it obvious!
+                            if not first_tab_line_found:
+                                first_tab_line_found = str(
+                                    f'     {previous_line_context}\n'
+                                    f'---> {line}\n')
+                            tab_line_count = tab_line_count + 1
+                            
+                    previous_line_context = line
+                if tab_line_count:
+                    error_message = str(
+                                f'{file_name}::{self.__class__.__name__} FAILED TabsValidator - {tab_line_count} tabs in this file\n'
+                                f'First instance of a tab: \n'
+                                f'{first_tab_line_found}')
+                    errors.append(error_message)
+                    if VERBOSE: print(error_message)
+
+        return (not errors)
+
+
+def get_validator() -> Type[TabsValidator]:
+    """Returns the validator class for this module"""
+    return TabsValidator

scripts/commit_validation/fix_copyright_headers.py

@@ -0,0 +1,148 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import argparse
+import os
+import re
+
+copyrightre = re.compile(r'All or portions of this file Copyright \(c\) Amazon\.com')
+copyright_text = """
+ All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+ its licensors.
+
+ For complete copyright and license terms please see the LICENSE at the root of this
+ distribution (the "License"). All use of this software is governed by the License,
+ or, if provided, by the license below or the license accompanying this file. Do not
+ remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+
+""".splitlines(keepends=True)
+
+class copyright_builder:
+
+    def get(self):
+        pass
+
+    def apply(self, file_contents):
+        new_file_contents = self.get()
+        new_file_contents += '\n' + file_contents
+        return new_file_contents
+
+class copyright_builder_first_last_line(copyright_builder):
+    def __init__(self, comment_str):
+        self.comment_str = comment_str
+
+    def get(self):
+        copyright_str = self.comment_str + copyright_text[0]
+        for i in range(1, len(copyright_text)-1):
+            copyright_str += copyright_text[i]
+        copyright_str += self.comment_str + copyright_text[len(copyright_text)-1]
+        return copyright_str
+
+class copyright_builder_each_line(copyright_builder):
+    def __init__(self, comment_str):
+        self.comment_str = comment_str
+
+    def get(self):
+        copyright_str = ''
+        for i in range(0, len(copyright_text)):
+            copyright_str += self.comment_str + copyright_text[i]
+        return copyright_str
+
+class copyright_builder_each_line_bat(copyright_builder_each_line):
+    def __init__(self, comment_str):
+        self.comment_str = comment_str
+        self.echo_off_re = re.compile('@echo off\n', re.IGNORECASE) 
+
+    def apply(self, file_contents):
+        re_result = self.echo_off_re.search(file_contents)
+        if re_result:
+            return self.echo_off_re.sub(re_result.group(0) + '\n' + self.get() + '\n', file_contents)
+        return super().apply(file_contents)
+
+class copyright_builder_each_line_sh(copyright_builder_each_line):
+    def __init__(self, comment_str):
+        self.comment_str = comment_str
+        self.bin_str = [
+            '#!/bin/sh\n', 
+            '#!/bin/bash\n'
+        ]
+
+    def apply(self, file_contents):
+        for bin_str in self.bin_str:
+            if bin_str in file_contents:
+                return file_contents.replace(bin_str, bin_str + '\n' + self.get() + '\n')
+        return super().apply(file_contents)
+
+class copyright_builder_c(copyright_builder):
+    def get(self):
+        copyright_str = '/*' + copyright_text[0]
+        for i in range(1, len(copyright_text)-1):
+            copyright_str += '*' + copyright_text[i]
+        copyright_str += '*\n'
+        copyright_str += '*/' + copyright_text[len(copyright_text)-1]
+        return copyright_str
+
+copyright_comment_by_extension = {
+    'lua': copyright_builder_each_line('-- '),
+    'py': copyright_builder_first_last_line('"""'),
+    'bat': copyright_builder_each_line_bat('REM '),
+    'cmd': copyright_builder_each_line_bat('REM '),
+    'sh': copyright_builder_each_line_sh('# '),
+    'cs': copyright_builder_c(),
+}
+
+def fixCopyright(input_file):
+    try:
+        extension = os.path.splitext(input_file)[1].replace('.','')
+        if not extension in copyright_comment_by_extension:
+            print(f'[WARN] Extension {extension} not handled')
+        else:
+            copyright_builder = copyright_comment_by_extension[extension]
+            with open(input_file, 'r') as source_file:
+                fileContents = source_file.read()
+
+            reResult = copyrightre.search(fileContents)
+            if reResult:
+                # Copyright found, skip
+                return
+
+            newFileContents = copyright_builder.apply(fileContents)
+            
+            with open(input_file, 'w') as destination_file:
+                destination_file.write(newFileContents)
+
+            print(f'[INFO] Patched {input_file}')
+
+    except (IOError, UnicodeDecodeError) as err:
+        print('[ERROR] reading {}: {}'.format(input_file, err))
+        return
+
+def main():
+    """script main function"""
+    parser = argparse.ArgumentParser(description='This script fixes copyright headers',
+        formatter_class=argparse.RawTextHelpFormatter)
+    parser.add_argument('file_or_dir', type=str, nargs='+',
+        help='list of files or directories to search within for files to fix up copyright headers')
+
+    args = parser.parse_args()
+
+    for input_file in args.file_or_dir:
+        if os.path.isdir(input_file):
+            for dp, dn, filenames in os.walk(input_file):
+                for f in filenames:
+                    fixCopyright(os.path.join(dp, f))
+        else:
+            fixCopyright(input_file)
+
+#entrypoint
+if __name__ == '__main__':
+    main()

scripts/commit_validation/fix_tabs.py

@@ -0,0 +1,58 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import argparse
+import fnmatch
+import os
+
+handled_file_patterns = [
+    '*.c', '*.cc', '*.cpp', '*.cxx', '*.h', '*.hpp', '*.hxx', '*.inl', '*.m', '*.mm', '*.cs', '*.java',
+    '*.py', '*.lua', '*.bat', '*.cmd', '*.sh', '*.js',
+    '*.cmake', 'CMakeLists.txt'
+]
+def fixTabs(input_file):
+    try:
+        basename = os.path.basename(input_file)
+        for pattern in handled_file_patterns:
+            if fnmatch.fnmatch(basename, pattern):
+                with open(input_file, 'r') as source_file:
+                    fileContents = source_file.read()
+                if '\t' in fileContents:
+                    newFileContents = fileContents.replace('\t', '    ')
+                    with open(input_file, 'w') as destination_file:
+                        destination_file.write(newFileContents)
+                    print(f'[INFO] Patched {input_file}')
+                break
+
+    except (IOError, UnicodeDecodeError) as err:
+        print('[ERROR] reading {}: {}'.format(input_file, err))
+        return
+
+def main():
+    """script main function"""
+    parser = argparse.ArgumentParser(description='This script replaces tabs with spaces',
+        formatter_class=argparse.RawTextHelpFormatter)
+    parser.add_argument('file_or_dir', type=str, nargs='+',
+        help='list of files or directories to search within for files to fix up tabs')
+
+    args = parser.parse_args()
+
+    for input_file in args.file_or_dir:
+        if os.path.isdir(input_file):
+            for dp, dn, filenames in os.walk(input_file):
+                for f in filenames:
+                    fixTabs(os.path.join(dp, f))
+        else:
+            fixTabs(input_file)
+
+#entrypoint
+if __name__ == '__main__':
+    main()

scripts/commit_validation/git_validate_branch.py

@@ -0,0 +1,117 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import argparse
+import os
+import sys
+from typing import Dict, List
+import difflib
+
+from commit_validation.commit_validation import Commit, validate_commit
+import git
+
+
+class GitChange(Commit):
+    """An implementation of the :class:`Commit` interface for accessing details about a git change"""
+
+    def __init__(self, source: str = None, target: str = 'origin/main') -> None:
+        """Creates a new instance of :class:`GitChange`
+
+        :param source: source of the change, e.g. commit hash or branch
+        :param target: target of the change, e.g. main branch
+        """
+        root_path = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))
+        self.repo = git.Repo()
+        self.git_root_path = self.repo.git.rev_parse("--show-toplevel")
+        if source:
+            git.cmd.Git().fetch('origin', source) # So we can compare it
+            self.source_commit = self.repo.commit(target)
+        else:
+            self.source_commit = self.repo.commit()
+
+        if 'origin' in target:
+            origin_target = target.replace('origin/','')
+            git.cmd.Git().fetch('origin', origin_target) # So we can compare it
+        self.target_commit = self.repo.commit(target)       
+
+        # We only want to run the verification on the changes introduced by the
+        # branch being merged in. Find the merge base (the common ancestor) of
+        # the two commits, and then get the diff from merge_base..target_commit
+        self.merge_base = self.repo.merge_base(self.source_commit, self.target_commit)
+        if not len(self.merge_base) == 1:
+            raise RuntimeError(f"Cannot find the merge base of {self.source_commit} and {self.target_commit}")
+        self.merge_base = self.merge_base[0]
+        self.diff_index = self.merge_base.diff(self.source_commit)
+
+        print(f"Running validation from '{source}' ({self.source_commit}) to '{target}' ({self.target_commit}) using baseline {self.merge_base}")
+
+        # Cache the file lists since they are requested by each validator
+        self.files_list: List[str] = []
+        self.removed_files_list: List[str] = []
+
+        for diff_item in self.diff_index:
+            # 'A' for added paths
+            # 'C' for changed paths
+            # 'R' for renamed paths
+            # 'M' for paths with modified data
+            # 'T' for changed in the type paths
+            # 'D' for deleted paths
+            # 'R' for renamed paths
+            if diff_item.change_type in ('A', 'C', 'R', 'M', 'T'):
+                self.files_list.append(os.path.abspath(os.path.join(self.git_root_path, diff_item.b_path)))
+            if diff_item.change_type in ('D', 'R'):
+                self.removed_files_list.append(os.path.abspath(os.path.join(self.git_root_path, diff_item.a_path))) 
+
+    def get_files(self) -> List[str]:
+        """Returns a list of local files added/modified by the commit"""
+        return self.files_list
+
+    def get_removed_files(self) -> List[str]:
+        """Returns a list of local files removed by the commit"""
+        return self.removed_files_list
+
+    def get_file_diff(self, str) -> str:
+        """
+        Given a file name, returns a string in unified diff format
+        that represents the changes made to that file for this commit.
+        Most validators will only pay attention to added lines (with + in front)
+        """
+        diff = self.repo.git.diff(self.merge_base, self.source_commit, str)
+        return diff
+
+    def get_description(self) -> str:
+        """Returns the description of the commit"""
+        return self.target_commit.message
+
+    def get_author(self) -> str:
+        """Returns the author of the commit"""
+        return self.target_commit.author
+
+def init_parser():
+    """Prepares the command line parser"""
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--source', default=None, help='Change source (e.g. commit hash or branch), defaults to active branch')
+    parser.add_argument('--target', default='origin/main', help='Change target, defaults to "origin/main"')
+    return parser
+
+def main():
+    parser = init_parser()
+    args = parser.parse_args()
+
+    change = GitChange(source=args.source, target=args.target)
+
+    if not validate_commit(commit=change, ignore_validators=["NewlineValidator", "WhitespaceValidator"]):
+        sys.exit(1)
+
+    sys.exit(0)
+
+if __name__ == '__main__':
+    main()

scripts/commit_validation/p4.py

@@ -0,0 +1,53 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+import marshal
+from subprocess import Popen, PIPE, check_output
+from typing import List
+
+
+def run_p4_command(command: str, client: str = None, raw_output: bool = False) -> List[dict]:
+    """Executes a Perforce command by calling p4 in a subprocess
+
+    :param command: The command to run (not including p4, e.g., not ['p4', 'opened'] but ['opened'])
+    :param client: The Perforce client to use when running the command
+    :param raw_output: if True, return the raw output from p4 instead of using the python dict.
+    :return: The list of results from the command where each result is a dict (refer to p4's global -G option)
+             if 'raw_output' is true, return will be verbatim string from p4.
+    """
+    results = []
+
+    base_command = ['p4']
+
+    if not raw_output:
+        base_command.append('-G')
+
+    if client is not None:
+        base_command += ['-c', client]
+
+    if raw_output:
+        return check_output(base_command + command.split()).decode('utf8')
+    
+    pipe = Popen(base_command + command.split(), stdout=PIPE).stdout
+    try:
+        while True:
+            result = marshal.load(pipe)
+            results.append(result)
+    except EOFError:
+        pass
+    finally:
+        pipe.close()
+
+    decoded_results = [{k.decode(): v.decode() if isinstance(v, bytes) else str(v) for k, v in r.items()} for r in results]
+    if decoded_results and decoded_results[0]['code'] == 'error':
+        command_was = ' '.join(base_command + command.split())
+        error_was = decoded_results[0]['data']
+        raise RuntimeError(f'P4 Command failed: "{command_was}"\nERROR FROM P4: {error_was}\n')
+    return decoded_results

scripts/commit_validation/p4_validate_changelist.py

@@ -0,0 +1,121 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import argparse
+import os
+import sys
+from typing import Dict, List
+import difflib
+
+from commit_validation.commit_validation import Commit, validate_commit
+from p4 import run_p4_command
+
+
+class P4Changelist(Commit):
+    """An implementation of the :class:`Commit` interface for accessing details about a Perforce changelist"""
+
+    def __init__(self, client: str = None, change: str = 'default') -> None:
+        """Creates a new instance of :class:`P4Changelist`
+
+        :param client: the Perforce client
+        :param change: the Perforce changelist
+        """
+        self.client = client
+        self.change = change
+        self.files: List[str] = []
+        self.removed_files: List[str] = []
+        self.file_diffs: Dict[str, str] = {}
+        self._load_files()
+
+    def _load_files(self):
+        self.files = []
+        self.removed_files = []
+        self.added_files = []
+
+        client_spec = run_p4_command(f'client -o', client=self.client)[0]
+        files = run_p4_command(f'opened -c {self.change}', client=self.client)
+        for f in files:
+            file_path = os.path.abspath(f['clientFile'].replace(f'//{client_spec["Client"]}', client_spec['Root']))
+            if 'delete' in f['action']:
+                self.removed_files.append(file_path)
+            elif 'add' in f['action']:
+                self.added_files.append(file_path)
+            elif 'branch' in f['action']:
+                self.added_files.append(file_path)
+            else:
+                self.files.append(file_path)
+
+    def get_file_diff(self, file) -> str:
+        if file in self.file_diffs:  # allow caching
+            return self.file_diffs[file]
+
+        if file in self.added_files: # added files return the entire file as a diff.
+            with open(file, "rt", encoding='utf8', errors='replace') as opened_file:
+                data = opened_file.readlines()
+                diffs = difflib.unified_diff([], data, fromfile=file, tofile=file)
+                diff_being_built = ''.join(diffs)
+                self.file_diffs[file] = diff_being_built
+                return diff_being_built
+            
+        if file not in self.files:
+            raise RuntimeError(f"Cannot calculate a diff for a file not in the changelist:  {file}")
+        
+        try:
+            result = run_p4_command(f'diff -du {file}', client=self.client)
+            if len(result) > 1:
+                diff = result[1]['data'] # p4 returns a normal code but with no result if theres no diff
+            else:
+                diff = ''
+                print(f'Warning: File being committed contains no changes {file}')
+            # note that the p4 command handles the data and errors internally, no need to check.
+            self.file_diffs[file] = diff
+            return diff
+        except RuntimeError as e:
+            print(f'error during p4 operation, unable to get a diff: {e}')
+        
+        return ''
+
+    def get_files(self) -> List[str]:
+        # this is just files relevant to the operation
+        return self.files + self.added_files
+
+    def get_removed_files(self) -> List[str]:
+        return self.removed_files
+
+    def get_description(self) -> str:
+        raise NotImplementedError
+
+    def get_author(self) -> str:
+        raise NotImplementedError
+
+
+def init_parser():
+    """Prepares the command line parser"""
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--client', help='Perforce client')
+    parser.add_argument('--change', default='default', help='Perforce changelist')
+    return parser
+
+
+def main():
+    parser = init_parser()
+    args = parser.parse_args()
+
+    change = P4Changelist(client=args.client, change=args.change)
+
+    if not validate_commit(commit=change, ignore_validators=["NewlineValidator", "WhitespaceValidator"]):
+        sys.exit(1)
+
+    sys.exit(0)
+
+
+if __name__ == '__main__':
+    main()

scripts/commit_validation/p4_validate_submitted_changelists.py

@@ -0,0 +1,120 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import argparse
+import os
+import sys
+from typing import Dict, List
+
+from commit_validation.commit_validation import Commit, validate_commit
+from p4 import run_p4_command
+
+
+class P4SubmittedChangelists(Commit):
+    """An implementation of the :class:`Commit` interface for accessing details about Perforce submitted changelists"""
+
+    def __init__(self, from_change: str, to_change: str, client: str = None) -> None:
+        """Creates a new instance of :class:`P4SubmittedChangelists`
+
+        :param from_change: The oldest Perforce changelist to include
+        :param to_change: The newest Perforce changelist to include
+        :param client: The Perforce client
+        """
+        self.from_change = from_change
+        self.to_change = to_change
+        self.client = client
+        self.files: List[str] = []
+        self.removed_files: List[str] = []
+        self.file_diffs: Dict[str, str] = {}
+        self._load_files()
+
+    def _load_files(self):
+        self.files = []
+        self.removed_files = []
+        depot_root = None
+
+        client_root = run_p4_command(f'client -o', client=self.client)[0]['Root']
+        files = run_p4_command(f'files {client_root}{os.sep}...@{self.from_change},{self.to_change}',
+                               client=self.client)
+
+        for f in files:
+            # The following code optimizes for the specific use case where all files are mapped the same way on the 
+            # client. The reason for this is to avoid calling 'p4 where' on thousands of files. So instead, it only calls
+            # 'p4 where' on the first file and applies the same mapping to the rest of the files.
+            if depot_root is None:
+                file_path = run_p4_command(f'where {f["depotFile"]}', client=self.client)[0]['path']
+                relative_path = file_path.replace(client_root, '')
+                relative_path = relative_path.replace('\\', '/')
+                depot_root = f['depotFile'].replace(relative_path, '')
+            else:
+                file_path = os.path.abspath(f['depotFile'].replace(depot_root, client_root))
+
+            if 'delete' in f['action']:
+                self.removed_files.append(file_path)
+            else:
+                self.files.append(file_path)
+        pass
+
+    def get_file_diff(self, file) -> str:
+        if file in self.file_diffs:
+            return self.file_diffs[file]
+        
+        if file not in self.files:
+            raise RuntimeError(f"Cannot compute diff for file not in change set: {file}")
+
+        # the diff2 command does not behave like normal diff command, in that it only
+        # returns the actual diff if you do not use '-G' mode.  So we ask it for the raw output:
+        diff = run_p4_command(f'diff2 -du {file}@{self.from_change} {file}@{self.to_change}', client=self.client, raw_output=True)
+        self.file_diffs[file] = diff
+        # note that if you feed the same changelist as the 'before' and 'after' on the command line
+        # you will get no diffs, because there is no difference between a changelist and itself.
+        # In addition, if you are diffing across changelists, but the file only existed
+        # in one changelist, it will also not indicate a diff (due to how the p4 diff2 command operates)
+        # This means that this validator is blind to branch integrates.  This is not the case
+        # for the 'live' or 'shelved' validator as those files show up as additions.
+        return diff
+
+    def get_files(self) -> List[str]:
+        return self.files
+
+    def get_removed_files(self) -> List[str]:
+        return self.removed_files
+
+    def get_description(self) -> str:
+        raise NotImplementedError
+
+    def get_author(self) -> str:
+        raise NotImplementedError
+
+
+def init_parser():
+    """Prepares the command line parser"""
+    parser = argparse.ArgumentParser()
+    parser.add_argument('from_change', help='The oldest changelist to include')
+    parser.add_argument('to_change', help='The newest changelist to include')
+    parser.add_argument('--client', help='The Perforce client')
+    return parser
+
+
+def main():
+    parser = init_parser()
+    args = parser.parse_args()
+
+    change = P4SubmittedChangelists(client=args.client, from_change=args.from_change, to_change=args.to_change)
+
+    if not validate_commit(commit=change, ignore_validators=["NewlineValidator", "WhitespaceValidator"]):
+        sys.exit(1)
+
+    sys.exit(0)
+
+
+if __name__ == '__main__':
+    main()

scripts/commit_validation/validate_file_or_folder.py

@@ -0,0 +1,111 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import argparse
+import fnmatch
+import os
+import sys
+from typing import Dict, List
+import difflib
+
+from commit_validation.commit_validation import Commit, validate_commit, IsFileSkipped, SOURCE_AND_SCRIPT_FILE_EXTENSIONS, EXCLUDED_VALIDATION_PATTERNS
+
+class FolderBased(Commit):
+    """An implementation of the :class:`Commit` which populates it from files and folders on disk"""
+
+    def __init__(self, item_path: str = '.') -> None:
+        """Creates a new instance of :class:`FolderBased`
+        :param item_path - The file name or the folder to search (recursively)
+        """
+        self.item_path = os.path.abspath(item_path)
+        self.files: List[str] = []
+        self.removed_files: List[str] = []
+        self.file_diffs: Dict[str, str] = {}
+        self._load_files()
+
+    def _load_files(self):
+        self.files = []
+        self.removed_files = []
+        # if its a file, we just add that file.
+        # if its a folder, we add it recursively
+        if os.path.isfile(self.item_path):
+            self.files.append(self.item_path)
+        else:
+            engine_root_path = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))
+            for root, _, file_names in os.walk(self.item_path):
+                for file_name in file_names:
+                    file_to_scan = os.path.abspath(os.path.join(root, file_name))
+                    skip = False
+                    for excluded_folder in EXCLUDED_VALIDATION_PATTERNS:
+                        matcher = os.path.abspath(os.path.join(engine_root_path, excluded_folder, '*'))
+                        if fnmatch.fnmatch(file_to_scan, matcher):
+                            skip = True
+                            break
+                    if not skip:
+                        self.files.append(file_to_scan)
+
+    def get_file_diff(self, file) -> str:
+        if file in self.file_diffs:
+            return self.file_diffs[file]
+        # we count the entire file as changed in this mode
+        # we do not include diffs for things that are not source files
+        # this also prevents us trying to decode binary files as utf8
+        if IsFileSkipped(file):
+            return ""
+        
+        diff_being_built = ""
+        # We simulate every line having been changed by making a blank file
+        # being the 'from' and the whole file being the 'to' part of the diff
+        with open(file, "rt", encoding='utf8', errors='replace') as opened_file:
+            data = opened_file.readlines()
+            diffs = difflib.unified_diff([], data, fromfile=file, tofile=file)
+            diff_being_built = ''.join(diffs)
+    
+        self.file_diffs[file] = diff_being_built
+        return diff_being_built
+
+    def get_files(self) -> List[str]:
+        return self.files
+
+    def get_removed_files(self) -> List[str]:
+        return self.removed_files
+
+    def get_description(self) -> str:
+        raise NotImplementedError
+
+    def get_author(self) -> str:
+        raise NotImplementedError
+
+
+def init_parser():
+    """Prepares the command line parser"""
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--path', default='.', help='Filename of single file, or a folder path to scan recursively')
+    return parser
+
+
+def main():
+    parser = init_parser()
+    args = parser.parse_args()
+
+    change = FolderBased(item_path = args.path)
+    validators_to_ignore = [
+                "NewlineValidator", 
+                "WhitespaceValidator"
+                ]
+                
+    if not validate_commit(commit=change, ignore_validators = validators_to_ignore):
+        sys.exit(1)
+
+    sys.exit(0)
+
+if __name__ == '__main__':
+    main()

scripts/detect_file_changes/CMakeLists.txt

@@ -0,0 +1,16 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+ly_add_pytest(
+    NAME test_detect_file_changes
+    PATH ${CMAKE_CURRENT_LIST_DIR}
+)
+

scripts/detect_file_changes/compare_snapshots.py

@@ -0,0 +1,59 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import argparse
+import pickle
+import sys
+
+"""This is a command-line entry point which, when given two snapshots created using make_snapshot.py
+compares them and reports on the differences.  It returns 0 if and only if there are no differences.
+
+To use it in a scripting environment instead of a CLI, invoke do_compare(filename1, filename2)
+Or do it manually using FolderSnapshot.CompareSnapshots and then enumerate_changes
+"""
+
+ignore_patterns = []
+
+from snapshot_folder.snapshot_folder import FolderSnapshot, SnapshotComparison
+
+def do_compare(filename1, filename2):
+    """Given two filenames, returns the diffs as a list of tuples [(type, file)]"""
+    snap1 = pickle.load(open(filename1, 'rb'))
+    snap2 = pickle.load(open(filename2, 'rb'))
+
+    comparison = FolderSnapshot.CompareSnapshots(snap1, snap2)
+
+    changes = [change_entry for change_entry in comparison.enumerate_changes()]
+    return changes
+
+def init_parser():
+    """Prepares the command line parser"""
+    parser = argparse.ArgumentParser()
+    parser.description = "Compares two snapshots previously saved, outputs the changes.  Exit code is 0 if no diff, 1 otherwise."
+    parser.add_argument('first', default='.', help='first file to use')
+    parser.add_argument('second', default='.', help='second file to use')
+    return parser
+
+def main():
+    parser = init_parser()
+    args = parser.parse_args()
+    changes = do_compare(args.first, args.second)
+    
+    for change in changes:
+        print(f"Change detected: {change}")
+    
+    if changes:
+        return 1
+    
+    return 0
+
+if __name__ == '__main__':
+    sys.exit(main())

scripts/detect_file_changes/make_snapshot.py

@@ -0,0 +1,50 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import argparse
+import pickle
+import sys
+
+"""This is a command line entry point that, given an out file name, and a folder to scan
+generates a file which contains the current snapshot of the files and folders in that folder.
+It is to be used later by compare_snapshot.py
+
+If you want to use this in a scripting environment instead of a CLI, use the dump_snapshot 
+function or use FolderSnapshot.CreateSnapshot directly.
+"""
+
+default_ignore_patterns = ['*.pyc', '__pycache__', '*.snapshot', 'Cache', 'build_*', 'build' ]
+
+from snapshot_folder.snapshot_folder import FolderSnapshot, SnapshotComparison
+
+def dump_snapshot(folder_to_scan, filename, ignore_patterns):
+    """Workhorse function of this module.  Saves the snapshot to the given file"""
+    snap = FolderSnapshot.CreateSnapshot(folder_to_scan, ignore_patterns=ignore_patterns)
+    pickle.dump(snap, open(filename, 'wb'))
+
+def init_parser():
+    """Prepares the command line parser"""
+    parser = argparse.ArgumentParser()
+    parser.description = "Takes a snapshot of the current files and folders into a given file for later comparison"
+    parser.add_argument('path_to_check', default='.', help='Path To start iterating at')
+    parser.add_argument('--out', required=True, help='Path to output to')
+    parser.add_argument('--ignore', action='append', nargs='+', default=default_ignore_patterns)
+    return parser
+
+def main():
+    """ Entry point to use if you want to supply args on the command line"""
+    parser = init_parser()
+    args = parser.parse_args()
+    print(f"Snapshotting: {args.path_to_check} into {args.out} with ignore {args.ignore}")
+    return dump_snapshot(args.path_to_check, args.out, args.ignore)
+
+if __name__ == '__main__':
+    main()

scripts/detect_file_changes/snapshot_folder/__init__.py

@@ -0,0 +1,10 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#

scripts/detect_file_changes/snapshot_folder/snapshot_folder.py

@@ -0,0 +1,118 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import os
+import fnmatch
+import pathlib
+
+"""This module contains FolderSnapshot, a class which can create and compare 'snapshots'
+of folders (The snapshots just store the modtimes / existence of files and folders), and
+also can compare two snapshots to return a SnapshotComparison which represents the diffs
+"""
+
+class SnapshotComparison:
+    """ This class just holds the diffs calculated between two folder trees."""
+    def __init__(self):
+        self.deleted_files = []
+        self.added_files = []
+        self.changed_files = []
+        self.dirs_added = []
+        self.dirs_removed = []
+    
+    def any_changed(self):
+        """Returns True if any changes were detected"""
+        return self.deleted_files or self.added_files or self.changed_files or self.dirs_added or self.dirs_removed
+
+    def enumerate_changes(self):
+        """Enumerates changes, yielding each as a pair of (string, string), that is, (type of change, filename)"""
+        for file_entry in self.deleted_files:
+            yield ("DELETED", file_entry)
+        for file_entry in self.added_files:
+            yield ("ADDED", file_entry)
+        for file_entry in self.changed_files:
+            yield ("CHANGED", file_entry)
+        for dir_entry in self.dirs_added:
+            yield ("FOLDER_ADDED", dir_entry)
+        for dir_entry in self.dirs_removed:
+            yield ("FOLDER_DELETED", dir_entry)
+
+class FolderSnapshot:
+    """ This class stores a snapshot of a folder state and has utility functions to compare snapshots"""
+    def __init__(self):
+        self.file_modtimes = {}
+        self.folder_paths = []
+        pass
+
+    @staticmethod
+    def _matches_ignore_pattern(in_string, ignore_patterns):
+        for pattern in ignore_patterns:
+            if fnmatch.fnmatch(in_string, pattern):
+                return True
+            # we also care if the last part is in the patterns
+            # this is to cover situatiosn where the pattern has 'build' in it as opposed to *build*
+            # and the name of the file is literally 'build'
+            _, filepart = os.path.split(in_string)
+            if fnmatch.fnmatch(filepart, pattern):
+                return True
+
+        return False
+    
+    @staticmethod
+    def CreateSnapshot(root_folder, ignore_patterns):
+        """Create a new FolderSnapshot based on a root folder and ignore patterns."""
+        folder_snap = FolderSnapshot()
+        ignored_folders = []
+        for root, dir_names, file_names in os.walk(root_folder, followlinks=False):
+            for dir_name in dir_names:
+                fullpath = os.path.normpath(os.path.join(root, dir_name)).replace('\\', '/')
+                if FolderSnapshot._matches_ignore_pattern(fullpath, ignore_patterns):
+                    ignored_folders.append(fullpath)
+                    continue
+                if os.path.dirname(fullpath) in ignored_folders:
+                    # we want to emulate not 'walking' down any folders themselves that have been omitted:
+                    ignored_folders.append(fullpath)
+                    continue
+                folder_snap.folder_paths.append(fullpath)
+            
+            for file_name in file_names:
+                fullpath = os.path.normpath(os.path.join(root, file_name)).replace('\\', '/')
+                if FolderSnapshot._matches_ignore_pattern(fullpath, ignore_patterns):
+                    continue
+                if os.path.dirname(fullpath) in ignored_folders:
+                    # we want to emulate not 'walking' down any folders themselves that have been omitted:
+                    continue
+                folder_snap.file_modtimes[fullpath] = os.stat(fullpath).st_mtime
+        
+        return folder_snap        
+
+    @staticmethod
+    def CompareSnapshots(before, after):
+        """Return a SnapshotComparison representing the difference between two FolderShapshot objects"""
+        comparison = SnapshotComparison()
+        for file_name in before.file_modtimes.keys():
+            if file_name not in after.file_modtimes:
+                comparison.deleted_files.append(file_name)
+            else:
+                if before.file_modtimes[file_name] != after.file_modtimes[file_name]:
+                    comparison.changed_files.append(file_name)
+        
+        for file_name in after.file_modtimes.keys():
+            if file_name not in before.file_modtimes:
+                comparison.added_files.append(file_name)
+
+        for folder_path in before.folder_paths:
+            if folder_path not in after.folder_paths:
+                comparison.dirs_removed.append(folder_path)
+        
+        for folder_path in after.folder_paths:
+            if folder_path not in before.folder_paths:
+                comparison.dirs_added.append(folder_path)
+        return comparison

scripts/detect_file_changes/snapshot_folder/tests/__init__.py

@@ -0,0 +1,10 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#

scripts/detect_file_changes/snapshot_folder/tests/test_snapshots.py

@@ -0,0 +1,189 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import unittest
+from unittest.mock import patch
+from snapshot_folder.snapshot_folder import FolderSnapshot
+
+class empty_object():
+    pass
+
+def fakestat(file_path):
+    f = empty_object()
+    f.st_mtime = 12345
+    return f
+
+@patch('os.stat', side_effect=fakestat)
+@patch('os.walk')
+def test_CreateSnapshot_sanity(mock_os_walk, mock_os_stat):
+    mock_os_walk.return_value = [
+        # this mocks os.walk, which always returns a tuple of (root name, folder names, file names)
+        ( '',  # root name
+          ['subfolder1', 'subfolder2', 'subfolder3'], # folders in here
+          ['file1.cpp'], # files in here
+        ),
+        ( 'subfolder1', 
+          [],
+          ['file1.cpp', 'file2.cpp'], # file1 is same name as above, but different folder name!
+        ),
+        ( 'subfolder2', 
+          [''],
+          [], # empty folders still get tracked
+        ),
+        ( 'subfolder3', 
+          ['subfolder4'], # folders only containing folders
+          [],
+        ),
+        ( 'subfolder3/subfolder4', 
+          [],
+          ['file4.cpp'],
+        )
+    ]
+    snap = FolderSnapshot.CreateSnapshot('.', ignore_patterns=[])
+    
+    assert 'subfolder1' in snap.folder_paths
+    assert 'subfolder2' in snap.folder_paths
+    assert 'subfolder3' in snap.folder_paths
+    assert 'subfolder3/subfolder4' in snap.folder_paths
+    assert 'file1.cpp' in snap.file_modtimes
+    assert 'subfolder1/file1.cpp' in snap.file_modtimes
+    assert 'subfolder1/file2.cpp' in snap.file_modtimes
+    assert 'subfolder3/subfolder4/file4.cpp' in snap.file_modtimes
+    
+@patch('os.stat', side_effect=fakestat)
+@patch('os.walk')
+def test_CreateSnapshot_obeys_exclusions(mock_os_walk, mock_os_stat):
+    mock_os_walk.return_value = [
+        (   '.', 
+            ['sub_buildfolder', 'build', 'build_mac', 'normal_subfolder'], # sneaky trap, sub_buildfolder should not be ignored
+            ['file.tif', 'file_not_a_tif.bmp'],
+        ),
+        (   'sub_buildfolder',  # should not be ignored, its not a match to build_*
+            [],
+            ['file2.cpp', 'file2.tif'],
+        ),
+        (   'build_mac', 
+            ['normalfolder'], # does not have build in its name but should still be ignored because parent is
+            ['file3.cpp'],  # even though it doesnt match a rule itself, it should be omitted since its in build
+        ),
+        (   'build_mac/normalfolder', 
+            [],
+            ['file4.cpp'],  # even though it doesnt match a rule itself, it should be omitted since its in build
+        ),
+        (   'build', 
+            [],
+            ['file4.cpp'],  # even though it doesnt match a rule itself, it should be omitted since its in build
+        ),
+        (   'normal_subfolder', 
+            ['build'], # a matching folder in a subfolder
+            [],
+        ),
+        (   'normal_subfolder/build', 
+            ['file.txt'], # a matching folder in a subfolder
+            [],
+        )
+    ]
+    snap = FolderSnapshot.CreateSnapshot('.', ignore_patterns=['*.tif', 'build', 'build_*'])
+    
+    assert 'sub_buildfolder' in snap.folder_paths
+    assert 'file_not_a_tif.bmp' in snap.file_modtimes
+    assert 'sub_buildfolder/file2.cpp' in snap.file_modtimes
+    assert 'normal_subfolder' in snap.folder_paths
+
+    assert 'build' not in snap.folder_paths
+    assert 'build_mac' not in snap.folder_paths
+    assert 'normal_subfolder/build' not in snap.folder_paths
+    assert 'build_mac/normalfolder' not in snap.folder_paths
+
+    assert 'file1.tif' not in snap.file_modtimes
+    assert 'sub_buildfolder/file2.tif' not in snap.file_modtimes
+    assert 'build/file3.cpp' not in snap.file_modtimes
+    assert 'build_mac/file4.cpp' not in snap.file_modtimes
+    assert 'build_mac/normalfolder/file4.cpp' not in snap.file_modtimes
+    assert 'normal_subfolder/build/file.txt' not in snap.file_modtimes
+    
+
+
+def test_CompareSnapshots_identical_snapshots_nodiffs():
+    # emulate identical snapshots
+    snap1 = FolderSnapshot()
+    snap1.folder_paths = ['myfolder1', 'myfolder2']
+    snap1.file_modtimes = {
+        'rootfile.txt' : 12345,
+        'myfolder1/file.txt' : 12345
+    }
+    
+    snap2 = FolderSnapshot()
+    snap2.folder_paths = ['myfolder1', 'myfolder2']
+    snap2.file_modtimes = {
+        'rootfile.txt' : 12345,
+        'myfolder1/file.txt' : 12345
+    }
+
+    changes = FolderSnapshot.CompareSnapshots(snap1, snap2)
+    assert not changes.any_changed()
+    changed_things = [c for c in changes.enumerate_changes()]
+    assert not changed_things
+
+def test_CompareSnapshots_two_of_each_kind_of_change():
+    # emulate identical snapshots
+    snap1 = FolderSnapshot()
+    snap1.folder_paths = ['myfolder1', 'myfolder2', 'myfolder3', 'myfolder4']
+    snap1.file_modtimes = {
+        'rootfile.txt' : 12345,
+        'rootfile2.txt' : 12345,
+        'rootfile3.txt' : 12345,
+        'myfolder1/file.txt' : 12345,
+        'myfolder2/file2.txt' : 12345,
+        'myfolder2/file3.txt' : 12345,
+    }
+    
+    snap2 = FolderSnapshot()
+    # myfolder1 deleted
+    # myfolder2 unchanged
+    # myfolder3 unchanged
+    # myfolder4 deleted
+    # myfolder5 as well as its subfolder, myfolder6 added
+    snap2.folder_paths = ['myfolder3', 'myfolder2', 'myfolder5', 'myfolder5/myfolder6']
+    snap2.file_modtimes = {
+        'rootfile2.txt' : 12345,    # unchanged, in root
+        'rootfile3.txt' : 33333,    # modified
+        'rootfile4.txt' : 12345,    # a new file
+        # myfolder1 is deleted, so myfolder1/file.txt should show up as deleted
+        'myfolder2/file2.txt' : 12345, # unchanged, but in subfolder
+        'myfolder2/file3.txt' : 33333, # modified in subfolder
+        'myfolder5/file4.txt' : 12345, # a new file in a new folder
+    }
+
+    changes = FolderSnapshot.CompareSnapshots(snap1, snap2)
+    assert changes.any_changed()
+    changed_things = [c for c in changes.enumerate_changes()]
+    # folders
+
+    for expected_element in [
+            ('FOLDER_ADDED', 'myfolder5'),
+            ('FOLDER_ADDED', 'myfolder5/myfolder6'),
+            ('FOLDER_DELETED', 'myfolder1'),
+            ('FOLDER_DELETED', 'myfolder4'),
+            ('DELETED', 'rootfile.txt'),
+            ('DELETED', 'myfolder1/file.txt'),
+            ('ADDED',   'rootfile4.txt'),
+            ('ADDED',   'myfolder5/file4.txt'),
+            ('CHANGED',   'rootfile3.txt'),
+            ('CHANGED',   'myfolder2/file3.txt')
+    ]:
+        assert expected_element in changed_things
+        changed_things.remove(expected_element)
+
+    # every time we did an assert above, we removed the matching element
+    # from the change list.  This means that the change list should now be empty 
+    # since everything that changed has been accounted for
+    assert not changed_things, f"Unexpected change {changed_things}"

scripts/lmbr.bat

@@ -0,0 +1,36 @@
+@ECHO OFF
+REM
+REM All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+REM its licensors.
+REM
+REM For complete copyright and license terms please see the LICENSE at the root of this
+REM distribution (the "License"). All use of this software is governed by the License,
+REM or, if provided, by the license below or the license accompanying this file. Do not
+REM remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+REM
+
+pushd %~dp0%
+CD %~dp0..
+SET BASE_PATH=%CD%
+CD %~dp0
+SET PYTHON_DIRECTORY=%BASE_PATH%\python
+IF EXIST "%PYTHON_DIRECTORY%" GOTO pythonPathAvailable
+GOTO pythonDirNotFound
+:pythonPathAvailable
+SET PYTHON_EXECUTABLE=%PYTHON_DIRECTORY%\python.cmd
+IF NOT EXIST "%PYTHON_EXECUTABLE%" GOTO pythonExeNotFound
+CALL "%PYTHON_EXECUTABLE%" %BASE_PATH%\scripts\lmbr.py %*
+GOTO end
+:pythonDirNotFound
+ECHO Python directory not found: %PYTHON_DIRECTORY%
+GOTO fail
+:pythonExeNotFound
+ECHO Python executable not found: %PYTHON_EXECUTABLE%
+GOTO fail
+:fail
+popd
+EXIT /b 1
+:end
+popd
+

scripts/lmbr.py

@@ -0,0 +1,54 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import argparse
+import sys
+import os
+
+# Resolve the common python module
+ROOT_DEV_PATH = os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))
+if ROOT_DEV_PATH not in sys.path:
+    sys.path.append(ROOT_DEV_PATH)
+
+from cmake.Tools import engine_template
+from cmake.Tools import current_project
+from cmake.Tools import add_remove_gem
+
+
+def add_args(parser, subparsers) -> None:
+    current_project.add_args(parser, subparsers)
+    engine_template.add_args(parser, subparsers)
+    add_remove_gem.add_args(parser, subparsers)
+
+
+if __name__ == "__main__":
+    # parse the command line args
+    the_parser = argparse.ArgumentParser()
+
+    # add subparsers
+    the_subparsers = the_parser.add_subparsers(help='sub-command help')
+
+    # add args to the parser
+    add_args(the_parser, the_subparsers)
+
+    # parse args
+    the_args = the_parser.parse_args()
+
+    # if empty print help
+    if len(sys.argv) == 1:
+        the_parser.print_help(sys.stderr)
+        sys.exit(1)
+
+    # run
+    ret = the_args.func(the_args)
+
+    # return
+    sys.exit(ret)

scripts/lmbr.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+CURRENT=$PWD
+cd "$PWD/.." || exit
+BASE_PATH=$PWD
+cd "$CURRENT" || exit
+PYTHON_DIRECTORY="${BASE_PATH}/python"
+if [ ! -d "$PYTHON_DIRECTORY" ]; then
+  echo "Python dir not found: $PYTHON_DIRECTORY"
+  exit 1
+fi
+PYTHON_EXECUTABLE="$PYTHON_DIRECTORY/python.sh"
+if [ ! -f "$PYTHON_EXECUTABLE" ]; then
+  echo "Python executable not found: $PYTHON_EXECUTABLE"
+  exit 1
+fi
+$PYTHON_EXECUTABLE "$BASE_PATH/scripts/lmbr.py" $*
+cd "$CURRENT" || exit
+