Integrating up through commit 90f050496

scripts/build/bootstrap/incremental_build_util.py

@@ -21,7 +21,6 @@ import sys
 import tempfile
 import traceback
 
-IAM_ROLE_NAME = 'ec2-jenkins-node'
 DEFAULT_REGION = 'us-west-2'
 DEFAULT_DISK_SIZE = 300
 DEFAULT_DISK_TYPE = 'gp2'
@@ -129,41 +128,8 @@ def get_pipeline_and_branch(pipeline, branch):
     pipeline_and_branch = pipeline_and_branch.replace('/','_').replace('\\','_')
     return pipeline_and_branch
 
-def get_iam_role_credentials(role_name):
-    security_metadata = None
-    try:
-        response = urllib2.urlopen(
-            'http://169.254.169.254/latest/meta-data/iam/security-credentials/{0}'.format(role_name)).read()
-        security_metadata = ast.literal_eval(response)
-    except:
-        print 'Unable to get iam role credentials'
-        print traceback.print_exc()
-
-    return security_metadata
-
-
 def get_ec2_client(region):
-    credentials = None
-    try:
-        response = urllib2.urlopen(
-            'http://169.254.169.254/latest/meta-data/iam/security-credentials/{0}'.format(IAM_ROLE_NAME)).read()
-        credentials = ast.literal_eval(response)
-    except Exception as e:
-        print e.message
-        error('Error: Unable to get IAM rols credentials, please contact ly-build@ for help.')
-
-    keys = ['AccessKeyId', 'SecretAccessKey', 'Token']
-    for key in keys:
-        if key not in credentials:
-            error('Error: Unable to find {0} in get_iam_role_credentials response {1}'.format(key, credentials))
-
-    aws_access_key_id = credentials['AccessKeyId']
-    aws_secret_access_key = credentials['SecretAccessKey']
-    aws_session_token = credentials['Token']
-
-    client = boto3.client('ec2', region_name=region, aws_access_key_id=aws_access_key_id,
-                          aws_secret_access_key=aws_secret_access_key,
-                          aws_session_token=aws_session_token)
+    client = boto3.client('ec2', region_name=region)
     return client
 
 

scripts/build/tools/delete_stale_ebs.py

@@ -0,0 +1,137 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import os
+import requests
+import traceback
+import boto3
+import json
+from datetime import datetime
+from requests.auth import HTTPBasicAuth
+from urllib.parse import unquote
+
+
+class JenkinsAPIClient:
+    def __init__(self, jenkins_base_url, jenkins_username, jenkins_api_token):
+        self.jenkins_base_url = jenkins_base_url.rstrip('/')
+        self.jenkins_username = jenkins_username
+        self.jenkins_api_token = jenkins_api_token
+
+    def get(self, url, retry=1):
+        for i in range(retry):
+            try:
+                response = requests.get(url, auth=HTTPBasicAuth(self.jenkins_username, self.jenkins_api_token))
+                if response.ok:
+                    return response.json()
+            except Exception:
+                traceback.print_exc()
+                print(f'WARN: Get request {url} failed, retying....')
+        print(f'WARN: Get request {url} failed, see exception for more details.')
+
+    def get_pipeline(self, pipeline_name):
+        url = f'{self.jenkins_base_url}/job/{pipeline_name}/api/json'
+        # Use retry because Jenkins API call sometimes may fail when Jenkins server is on high load
+        return self.get(url, retry=3)
+
+    def get_branch_job(self, pipeline_name, branch_name):
+        url = f'{self.jenkins_base_url}/blue/rest/organizations/jenkins/pipelines/{pipeline_name}/branches/{branch_name}'
+        # Use retry because Jenkins API call sometimes may fail when Jenkins server is on high load
+        return self.get(url, retry=3)
+
+
+def delete_branch_ebs_volumes(branch_name):
+    """
+    Make a fake branch deleted event and invoke the lambda function that we use to delete branch EBS volumes after a branch is deleted.
+    """
+    # Unescape branch name as it's URL encoded
+    branch_name = unquote(branch_name)
+    input = {
+        "detail": {
+            "event": "referenceDeleted",
+            "repositoryName": "Lumberyard",
+            "referenceName": branch_name
+        }
+    }
+    client = boto3.client('lambda')
+    # Invoke lambda function "AutoDeleteEBS-Lambda" asynchronously.
+    # This lambda function can have 1000 concurrent runs.
+    # we will setup a SQS/SNS queue to process the events if the event number exceeds the function capacity.
+    client.invoke(
+        FunctionName='AutoDeleteEBS-Lambda',
+        InvocationType='Event',
+        Payload=json.dumps(input),
+    )
+
+
+def delete_old_branch_ebs_volumes(env):
+    """
+    Check last run time of each branch build, if it exceeds the retention days, delete the EBS volumes that are tied to the branch.
+    """
+    branch_volumes_deleted = []
+    jenkins_client = JenkinsAPIClient(env['JENKINS_URL'], env['JENKINS_USERNAME'], env['JENKINS_API_TOKEN'])
+    today_date = datetime.today().date()
+    pipeline_name = env['PIPELINE_NAME']
+    pipeline_job = jenkins_client.get_pipeline(pipeline_name)
+    if not pipeline_job:
+        print(f'ERROR: Cannot get data of pipeline job {pipeline_name}.')
+        exit(1)
+    branch_jobs = pipeline_job.get('jobs', [])
+    retention_days = int(env['RETENTION_DAYS'])
+    for branch_job in branch_jobs:
+        branch_name = branch_job['name']
+        branch_job = jenkins_client.get_branch_job(pipeline_name, branch_name)
+        if not branch_job:
+            print(f'WARN: Cannot get data of {branch_name} job , skipping branch {pipeline_name}.')
+            continue
+        latest_run = branch_job.get('latestRun')
+        # If the job hasn't run, then there is no EBS volumes tied to that job
+        if latest_run:
+            latest_run_start_time = latest_run.get('startTime')
+            latest_run_datetime = datetime.strptime(latest_run_start_time, '%Y-%m-%dT%H:%M:%S.%f%z')
+            # Convert startTime to local timezone to compare, because Jenkins server may use a different timezone.
+            latest_run_date = latest_run_datetime.astimezone().date()
+            date_diff = today_date - latest_run_date
+            if date_diff.days > retention_days:
+                print(f'Branch {branch_name} job hasn\'t run for over {retention_days} days, deleting the EBS volumes of this branch...')
+                delete_branch_ebs_volumes(branch_name)
+                branch_volumes_deleted.append(branch_name)
+    print('Deleted EBS volumes for branches:')
+    print('\n'.join(branch_volumes_deleted))
+
+
+def get_required_env(env, keys):
+    success = True
+    for key in keys:
+        try:
+            env[key] = os.environ[key].strip()
+        except KeyError:
+            print(f'ERROR: {key} is not set in environment variable')
+            success = False
+    return success
+
+
+def main():
+    env = {}
+    required_env_list = [
+        'JENKINS_URL',
+        'JENKINS_USERNAME',
+        'JENKINS_API_TOKEN',
+        'PIPELINE_NAME',
+        'RETENTION_DAYS'
+    ]
+    if not get_required_env(env, required_env_list):
+        print('ERROR: Required environment variable is not set, see log for more details.')
+
+    delete_old_branch_ebs_volumes(env)
+
+
+if __name__ == "__main__":
+    main()

scripts/commit_validation/commit_validation/commit_validation.py

@@ -178,13 +178,15 @@ EXCLUDED_VALIDATION_PATTERNS = [
     '*/External/*',
     'build',
     'Cache',
+    '*/Code/Framework/AzCore/azgnmx/azgnmx/*',
     'Code/Tools/CryFXC',
     'Code/Tools/HLSLCrossCompiler',
     'Code/Tools/HLSLCrossCompilerMETAL',
-    'Code/Tools/UniversalRemoteConsole',
     'Docs',
     'python/runtime',
     'restricted/*/Tools/*RemoteControl',
     'Tools/3dsmax',
     'Tools/Crashpad',
+    '*/user/Cache/*',
+    '*/user/log/*',
 ]

scripts/commit_validation/commit_validation/pal_allowedlist.txt

@@ -63,5 +63,4 @@
 */Gems/SaveData/Code/Tests/SaveDataTest.cpp
 */Gems/WhiteBox/Code/Source/Rendering/Legacy/WhiteBoxLegacyRenderMesh.cpp
 */restricted/*/Code/Framework/AzCore/AzCore/AzCore_Traits_*.h
-*/SamplesProject/Gem/Code/Source/MetastreamTest/MetastreamTest.cpp
 */Tools/CryDeprecation/precompile_check_defines.h

scripts/commit_validation/commit_validation/tests/validators/test_unicode_validator.py

@@ -0,0 +1,50 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import unittest
+from unittest.mock import patch, mock_open
+
+from commit_validation.tests.mocks.mock_commit import MockCommit
+from commit_validation.validators.unicode_validator import UnicodeValidator
+
+
+class UnicodeValidatorTests(unittest.TestCase):
+
+    @patch('builtins.open', mock_open(read_data='This file contains no unicode characters'))
+    def test_fileWithNoUnicode_passes(self):
+        commit = MockCommit(files=['/someCppFile.cpp'])
+        error_list = []
+        self.assertTrue(UnicodeValidator().run(commit, error_list))
+        self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
+
+    @patch('builtins.open', mock_open(read_data='This file contains unicode character \u2318'))
+    def test_fileWithUnicode_fails(self):
+        commit = MockCommit(files=['/someCppFile.cpp'])
+        error_list = []
+        self.assertFalse(UnicodeValidator().run(commit, error_list))
+        self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
+
+    @patch('builtins.open', mock_open(read_data='This file contains unicode character \u2318'))
+    def test_fileExtensionIgnored_passes(self):
+        commit = MockCommit(files=['/someCppFile.somerandomextension'])
+        error_list = []
+        self.assertTrue(UnicodeValidator().run(commit, error_list))
+        self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
+
+    @patch('builtins.open', mock_open(read_data='This file contains unicode character: \\u2318'))
+    def test_fileWithEscapedUnicode_passes(self):
+        commit = MockCommit(files=['/someCppFile.cpp'])
+        error_list = []
+        self.assertTrue(UnicodeValidator().run(commit, error_list))
+        self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
+
+if __name__ == '__main__':
+    unittest.main()

scripts/commit_validation/commit_validation/validators/unicode_validator.py

@@ -0,0 +1,54 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import fnmatch
+import os.path
+from typing import Type, List
+
+from commit_validation.commit_validation import Commit, CommitValidator, IsFileSkipped, SOURCE_AND_SCRIPT_FILE_EXTENSIONS, EXCLUDED_VALIDATION_PATTERNS, VERBOSE
+
+allowed_chars = {  
+    0xAD, # '_'
+    0xAE, # '®'
+    0xB0, # '°'
+}
+class UnicodeValidator(CommitValidator):
+    """A file-level validator that makes sure a file does not contain unicode characters"""
+
+    def run(self, commit: Commit, errors: List[str]) -> bool:
+        for file_name in commit.get_files():
+            if IsFileSkipped(file_name):
+                if VERBOSE: print(f'{file_name}::{self.__class__.__name__} SKIPPED UnicodeValidator - File excluded based on extension.')
+                continue
+
+            for pattern in EXCLUDED_VALIDATION_PATTERNS:
+                if fnmatch.fnmatch(file_name, pattern):
+                    if VERBOSE: print(f'{file_name} SKIPPED UnicodeValidator - Validation pattern excluded on path.')
+                    break
+            else:
+                with open(file_name, 'r', encoding='utf-8', errors='strict') as fh:
+                    linecount = 1
+                    for line in fh:
+                        columncount = 0
+                        for ch in line:
+                            ord_ch = ord(ch)
+                            if ord_ch > 127 and ord_ch not in allowed_chars:
+                                error_message = str(f'{file_name}::{self.__class__.__name__}:{linecount},{columncount} FAILED - Source file contains unicode character, replace with \\u{ord_ch:X}.')
+                                errors.append(error_message)
+                                if VERBOSE: print(error_message)
+                            columncount += 1
+                        linecount += 1
+
+        return (not errors)
+
+def get_validator() -> Type[UnicodeValidator]:
+    """Returns the validator class for this module"""
+    return UnicodeValidator

scripts/commit_validation/fix_unicode.py

@@ -0,0 +1,75 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import argparse
+import fnmatch
+import os
+
+handled_file_patterns = [
+    '*.c', '*.cc', '*.cpp', '*.cxx', '*.h', '*.hpp', '*.hxx', '*.inl', '*.m', '*.mm', '*.cs', '*.java',
+    '*.py', '*.lua', '*.bat', '*.cmd', '*.sh', '*.js',
+    '*.cmake', 'CMakeLists.txt'
+]
+replacement_map = {
+    0xA0: ' ',
+    0xA6: '|',
+    0x2019: '\'',
+    0x2014: '-',
+    0x2191: '^',
+    0x2212: '-',
+    0x2217: '*',
+    0x2248: 'is close to',
+    0xFEFF: '',
+}
+
+def fixUnicode(input_file):
+    try:
+        basename = os.path.basename(input_file)
+        for pattern in handled_file_patterns:
+            if fnmatch.fnmatch(basename, pattern):
+                with open(input_file, 'r', encoding='utf-8', errors='replace') as fh:
+                    fileContents = fh.read()
+                modified = False
+                for uni, repl in replacement_map.items():
+                    uni_str = chr(uni)
+                    if uni_str in fileContents:
+                        fileContents = fileContents.replace(uni_str, repl)
+                        modified = True
+                if modified:
+                    with open(input_file, 'w') as destination_file:
+                        destination_file.writelines(fileContents)
+                    print(f'[INFO] Patched {input_file}')
+                break
+
+    except (IOError, UnicodeDecodeError) as err:
+        print('[ERROR] reading {}: {}'.format(input_file, err))
+        return
+
+def main():
+    """script main function"""
+    parser = argparse.ArgumentParser(description='This script replaces unicode characters, some of them are replaced for spaces (e.g. xA0), others are replaced with the escape sequence',
+        formatter_class=argparse.RawTextHelpFormatter)
+    parser.add_argument('file_or_dir', type=str, nargs='+',
+        help='list of files or directories to search within for files to fix up unicode characters')
+
+    args = parser.parse_args()
+
+    for input_file in args.file_or_dir:
+        if os.path.isdir(input_file):
+            for dp, dn, filenames in os.walk(input_file):
+                for f in filenames:
+                    fixUnicode(os.path.join(dp, f))
+        else:
+            fixUnicode(input_file)
+
+#entrypoint
+if __name__ == '__main__':
+    main()

scripts/project_manager/projects.py

@@ -384,11 +384,18 @@ class ProjectDialog(QObject):
         :return: None
         """
         remove_gems = self.get_selected_project_gems()
+
         for this_gem in remove_gems:
             gem_path = self.path_for_gem(this_gem)
-            add_remove_gem.add_remove_gem(False, engine_path, gem_path or os.path.join(engine_path, 'Gems', this_gem),
-                                          self.path_for_selection(), gem_name=this_gem)
-
+            add_remove_gem.add_remove_gem(add=False,
+                                          dev_root=engine_path,
+                                          gem_path=gem_path or os.path.join(engine_path, 'Gems', this_gem),
+                                          gem_target=this_gem,
+                                          project_path=self.path_for_selection(),
+                                          dependencies_file=None,
+                                          runtime_dependency=True,
+                                          tool_dependency=True,
+                                          server_dependency=True)
         self.update_gems()
 
     def manage_gems_handler(self):
@@ -455,10 +462,15 @@ class ProjectDialog(QObject):
             if not gem_info:
                 logger.error(f'Unknown gem {this_gem}!')
                 continue
-            add_remove_gem.add_remove_gem(True, engine_path, this_gem[1], self.path_for_selection(),
-                                          gem_name=gem_info.get('Name'),
+            add_remove_gem.add_remove_gem(add=True,
+                                          dev_root=engine_path,
+                                          gem_path=this_gem[1],
+                                          gem_target=gem_info.get('Name'),
+                                          project_path=self.path_for_selection(),
+                                          dependencies_file=None,
                                           runtime_dependency=gem_info.get('Runtime', False),
-                                          tool_dependency=gem_info.get('Tools', False))
+                                          tool_dependency=gem_info.get('Tools', False),
+                                          server_dependency=gem_info.get('Tools', False))
         self.update_gems()
 
     def create_project_handler(self):

scripts/project_manager/pyside.py

@@ -29,6 +29,7 @@ def add_pyside_environment(bin_path):
     old_env = os.environ.copy()
     binaries_path = Path(os.path.normpath(bin_path))
     platforms_path = binaries_path.joinpath("platforms")
+    logger.info(f'Adding binaries path {binaries_path}')
     os.environ["QT_QPA_PLATFORM_PLUGIN_PATH"] = str(platforms_path)
 
     path = os.environ['PATH']