birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Integrating up through commit 90f050496

Browse Source
This commit is contained in:
alexpete
2021-04-07 14:03:29 -07:00
parent 8f2ed080a9
commit c2cbd430fe
2694 changed files with 285622 additions and 176874 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/commit_validation/commit_validation/commit_validation.py
+3 -1
View File
@@ -178,13 +178,15 @@ EXCLUDED_VALIDATION_PATTERNS = [
'*/External/*',
'build',
'Cache',
'*/Code/Framework/AzCore/azgnmx/azgnmx/*',
'Code/Tools/CryFXC',
'Code/Tools/HLSLCrossCompiler',
'Code/Tools/HLSLCrossCompilerMETAL',
'Code/Tools/UniversalRemoteConsole',
'Docs',
'python/runtime',
'restricted/*/Tools/*RemoteControl',
'Tools/3dsmax',
'Tools/Crashpad',
'*/user/Cache/*',
'*/user/log/*',
]
scripts/commit_validation/commit_validation/pal_allowedlist.txt
-1
View File
@@ -63,5 +63,4 @@
*/Gems/SaveData/Code/Tests/SaveDataTest.cpp
*/Gems/WhiteBox/Code/Source/Rendering/Legacy/WhiteBoxLegacyRenderMesh.cpp
*/restricted/*/Code/Framework/AzCore/AzCore/AzCore_Traits_*.h
*/SamplesProject/Gem/Code/Source/MetastreamTest/MetastreamTest.cpp
*/Tools/CryDeprecation/precompile_check_defines.h
scripts/commit_validation/commit_validation/tests/validators/test_unicode_validator.py
+50
View File
@@ -0,0 +1,50 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import unittest
from unittest.mock import patch, mock_open
from commit_validation.tests.mocks.mock_commit import MockCommit
from commit_validation.validators.unicode_validator import UnicodeValidator
class UnicodeValidatorTests(unittest.TestCase):
@patch('builtins.open', mock_open(read_data='This file contains no unicode characters'))
def test_fileWithNoUnicode_passes(self):
commit = MockCommit(files=['/someCppFile.cpp'])
error_list = []
self.assertTrue(UnicodeValidator().run(commit, error_list))
self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
@patch('builtins.open', mock_open(read_data='This file contains unicode character \u2318'))
def test_fileWithUnicode_fails(self):
commit = MockCommit(files=['/someCppFile.cpp'])
error_list = []
self.assertFalse(UnicodeValidator().run(commit, error_list))
self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
@patch('builtins.open', mock_open(read_data='This file contains unicode character \u2318'))
def test_fileExtensionIgnored_passes(self):
commit = MockCommit(files=['/someCppFile.somerandomextension'])
error_list = []
self.assertTrue(UnicodeValidator().run(commit, error_list))
self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
@patch('builtins.open', mock_open(read_data='This file contains unicode character: \\u2318'))
def test_fileWithEscapedUnicode_passes(self):
commit = MockCommit(files=['/someCppFile.cpp'])
error_list = []
self.assertTrue(UnicodeValidator().run(commit, error_list))
self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
if __name__ == '__main__':
unittest.main()
scripts/commit_validation/commit_validation/validators/unicode_validator.py
+54
View File
@@ -0,0 +1,54 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import fnmatch
import os.path
from typing import Type, List
from commit_validation.commit_validation import Commit, CommitValidator, IsFileSkipped, SOURCE_AND_SCRIPT_FILE_EXTENSIONS, EXCLUDED_VALIDATION_PATTERNS, VERBOSE
allowed_chars = {
0xAD, # '_'
0xAE, # '®'
0xB0, # '°'
}
class UnicodeValidator(CommitValidator):
"""A file-level validator that makes sure a file does not contain unicode characters"""
def run(self, commit: Commit, errors: List[str]) -> bool:
for file_name in commit.get_files():
if IsFileSkipped(file_name):
if VERBOSE: print(f'{file_name}::{self.__class__.__name__} SKIPPED UnicodeValidator - File excluded based on extension.')
continue
for pattern in EXCLUDED_VALIDATION_PATTERNS:
if fnmatch.fnmatch(file_name, pattern):
if VERBOSE: print(f'{file_name} SKIPPED UnicodeValidator - Validation pattern excluded on path.')
break
else:
with open(file_name, 'r', encoding='utf-8', errors='strict') as fh:
linecount = 1
for line in fh:
columncount = 0
for ch in line:
ord_ch = ord(ch)
if ord_ch > 127 and ord_ch not in allowed_chars:
error_message = str(f'{file_name}::{self.__class__.__name__}:{linecount},{columncount} FAILED - Source file contains unicode character, replace with \\u{ord_ch:X}.')
errors.append(error_message)
if VERBOSE: print(error_message)
columncount += 1
linecount += 1
return (not errors)
def get_validator() -> Type[UnicodeValidator]:
"""Returns the validator class for this module"""
return UnicodeValidator
scripts/commit_validation/fix_unicode.py
+75
View File
@@ -0,0 +1,75 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import argparse
import fnmatch
import os
handled_file_patterns = [
'*.c', '*.cc', '*.cpp', '*.cxx', '*.h', '*.hpp', '*.hxx', '*.inl', '*.m', '*.mm', '*.cs', '*.java',
'*.py', '*.lua', '*.bat', '*.cmd', '*.sh', '*.js',
'*.cmake', 'CMakeLists.txt'
]
replacement_map = {
0xA0: ' ',
0xA6: '|',
0x2019: '\'',
0x2014: '-',
0x2191: '^',
0x2212: '-',
0x2217: '*',
0x2248: 'is close to',
0xFEFF: '',
}
def fixUnicode(input_file):
try:
basename = os.path.basename(input_file)
for pattern in handled_file_patterns:
if fnmatch.fnmatch(basename, pattern):
with open(input_file, 'r', encoding='utf-8', errors='replace') as fh:
fileContents = fh.read()
modified = False
for uni, repl in replacement_map.items():
uni_str = chr(uni)
if uni_str in fileContents:
fileContents = fileContents.replace(uni_str, repl)
modified = True
if modified:
with open(input_file, 'w') as destination_file:
destination_file.writelines(fileContents)
print(f'[INFO] Patched {input_file}')
break
except (IOError, UnicodeDecodeError) as err:
print('[ERROR] reading {}: {}'.format(input_file, err))
return
def main():
"""script main function"""
parser = argparse.ArgumentParser(description='This script replaces unicode characters, some of them are replaced for spaces (e.g. xA0), others are replaced with the escape sequence',
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('file_or_dir', type=str, nargs='+',
help='list of files or directories to search within for files to fix up unicode characters')
args = parser.parse_args()
for input_file in args.file_or_dir:
if os.path.isdir(input_file):
for dp, dn, filenames in os.walk(input_file):
for f in filenames:
fixUnicode(os.path.join(dp, f))
else:
fixUnicode(input_file)
#entrypoint
if __name__ == '__main__':
main()