Integrating up through commit 90f050496

scripts/commit_validation/commit_validation/commit_validation.py

@@ -178,13 +178,15 @@ EXCLUDED_VALIDATION_PATTERNS = [
     '*/External/*',
     'build',
     'Cache',
+    '*/Code/Framework/AzCore/azgnmx/azgnmx/*',
     'Code/Tools/CryFXC',
     'Code/Tools/HLSLCrossCompiler',
     'Code/Tools/HLSLCrossCompilerMETAL',
-    'Code/Tools/UniversalRemoteConsole',
     'Docs',
     'python/runtime',
     'restricted/*/Tools/*RemoteControl',
     'Tools/3dsmax',
     'Tools/Crashpad',
+    '*/user/Cache/*',
+    '*/user/log/*',
 ]

scripts/commit_validation/commit_validation/pal_allowedlist.txt

@@ -63,5 +63,4 @@
 */Gems/SaveData/Code/Tests/SaveDataTest.cpp
 */Gems/WhiteBox/Code/Source/Rendering/Legacy/WhiteBoxLegacyRenderMesh.cpp
 */restricted/*/Code/Framework/AzCore/AzCore/AzCore_Traits_*.h
-*/SamplesProject/Gem/Code/Source/MetastreamTest/MetastreamTest.cpp
 */Tools/CryDeprecation/precompile_check_defines.h

scripts/commit_validation/commit_validation/tests/validators/test_unicode_validator.py

@@ -0,0 +1,50 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import unittest
+from unittest.mock import patch, mock_open
+
+from commit_validation.tests.mocks.mock_commit import MockCommit
+from commit_validation.validators.unicode_validator import UnicodeValidator
+
+
+class UnicodeValidatorTests(unittest.TestCase):
+
+    @patch('builtins.open', mock_open(read_data='This file contains no unicode characters'))
+    def test_fileWithNoUnicode_passes(self):
+        commit = MockCommit(files=['/someCppFile.cpp'])
+        error_list = []
+        self.assertTrue(UnicodeValidator().run(commit, error_list))
+        self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
+
+    @patch('builtins.open', mock_open(read_data='This file contains unicode character \u2318'))
+    def test_fileWithUnicode_fails(self):
+        commit = MockCommit(files=['/someCppFile.cpp'])
+        error_list = []
+        self.assertFalse(UnicodeValidator().run(commit, error_list))
+        self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
+
+    @patch('builtins.open', mock_open(read_data='This file contains unicode character \u2318'))
+    def test_fileExtensionIgnored_passes(self):
+        commit = MockCommit(files=['/someCppFile.somerandomextension'])
+        error_list = []
+        self.assertTrue(UnicodeValidator().run(commit, error_list))
+        self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
+
+    @patch('builtins.open', mock_open(read_data='This file contains unicode character: \\u2318'))
+    def test_fileWithEscapedUnicode_passes(self):
+        commit = MockCommit(files=['/someCppFile.cpp'])
+        error_list = []
+        self.assertTrue(UnicodeValidator().run(commit, error_list))
+        self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
+
+if __name__ == '__main__':
+    unittest.main()

scripts/commit_validation/commit_validation/validators/unicode_validator.py

@@ -0,0 +1,54 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import fnmatch
+import os.path
+from typing import Type, List
+
+from commit_validation.commit_validation import Commit, CommitValidator, IsFileSkipped, SOURCE_AND_SCRIPT_FILE_EXTENSIONS, EXCLUDED_VALIDATION_PATTERNS, VERBOSE
+
+allowed_chars = {  
+    0xAD, # '_'
+    0xAE, # '®'
+    0xB0, # '°'
+}
+class UnicodeValidator(CommitValidator):
+    """A file-level validator that makes sure a file does not contain unicode characters"""
+
+    def run(self, commit: Commit, errors: List[str]) -> bool:
+        for file_name in commit.get_files():
+            if IsFileSkipped(file_name):
+                if VERBOSE: print(f'{file_name}::{self.__class__.__name__} SKIPPED UnicodeValidator - File excluded based on extension.')
+                continue
+
+            for pattern in EXCLUDED_VALIDATION_PATTERNS:
+                if fnmatch.fnmatch(file_name, pattern):
+                    if VERBOSE: print(f'{file_name} SKIPPED UnicodeValidator - Validation pattern excluded on path.')
+                    break
+            else:
+                with open(file_name, 'r', encoding='utf-8', errors='strict') as fh:
+                    linecount = 1
+                    for line in fh:
+                        columncount = 0
+                        for ch in line:
+                            ord_ch = ord(ch)
+                            if ord_ch > 127 and ord_ch not in allowed_chars:
+                                error_message = str(f'{file_name}::{self.__class__.__name__}:{linecount},{columncount} FAILED - Source file contains unicode character, replace with \\u{ord_ch:X}.')
+                                errors.append(error_message)
+                                if VERBOSE: print(error_message)
+                            columncount += 1
+                        linecount += 1
+
+        return (not errors)
+
+def get_validator() -> Type[UnicodeValidator]:
+    """Returns the validator class for this module"""
+    return UnicodeValidator