Initial commit

This commit is contained in:
alexpete
2021-03-05 11:26:34 -08:00
commit a10351f38d
27091 changed files with 5521199 additions and 0 deletions
+18
View File
@@ -0,0 +1,18 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
# this ctest makes sure that the commit validation function
# also runs its tests during commit validation!
ly_add_pytest(
NAME test_commit_validation
PATH ${CMAKE_CURRENT_LIST_DIR}
)
@@ -0,0 +1,10 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
@@ -0,0 +1,191 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import abc
import importlib
import os
import pkgutil
import re
import time
from typing import Dict, List, Tuple
VERBOSE = False
class Commit(abc.ABC):
"""An interface for accessing details about a commit"""
@abc.abstractmethod
def get_files(self) -> List[str]:
"""Returns a list of local files added/modified by the commit"""
pass
@abc.abstractmethod
def get_removed_files(self) -> List[str]:
"""Returns a list of local files removed by the commit"""
pass
@abc.abstractmethod
def get_file_diff(self, str) -> str:
"""
Given a file name, returns a string in unified diff format
that represents the changes made to that file for this commit.
Most validators will only pay attention to added lines (with + in front)
"""
pass
@abc.abstractmethod
def get_description(self) -> str:
"""Returns the description of the commit"""
pass
@abc.abstractmethod
def get_author(self) -> str:
"""Returns the author of the commit"""
pass
def validate_commit(commit: Commit, out_errors: List[str] = None, ignore_validators: List[str] = None) -> bool:
"""Validates a commit against all validators
:param commit: The commit to validate
:param out_errors: if not None, will populate with the list of errors given by the validators
:param ignore_validators: Optional list of CommitValidator classes to ignore, by class name
:return: True if there are no validation errors, and False otherwise
"""
failed_count = 0
passed_count = 0
start_time = time.time()
# Find all the validators in the validators package (recursively)
validator_classes = []
validators_dir = os.path.join(os.path.dirname(__file__), 'validators')
for _, module_name, is_package in pkgutil.iter_modules([validators_dir]):
if not is_package:
module = importlib.import_module('commit_validation.validators.' + module_name)
validator = module.get_validator()
if ignore_validators and validator.__name__ in ignore_validators:
print(f"Disabled validation for '{validator.__name__}'")
else:
validator_classes.append(validator)
error_summary = {}
# Process validators
for validator_class in validator_classes:
validator = validator_class()
validator_name = validator.__class__.__name__
error_list = []
passed = validator.run(commit, errors = error_list)
if passed:
passed_count += 1
print(f'{validator.__class__.__name__} PASSED')
else:
failed_count += 1
print(f'{validator.__class__.__name__} FAILED')
error_summary[validator_name] = error_list
end_time = time.time()
if failed_count:
print("VALIDATION FAILURE SUMMARY")
for val_name in error_summary.keys():
errors = error_summary[val_name]
if errors:
for error_message in errors:
first_line = True
for line in error_message.splitlines():
if first_line:
first_line = False
print(f'VALIDATOR_FAILED: {val_name} {line}')
else:
print(f' {line}') # extra detail lines do not need machine parsing
stats_strs = []
if failed_count > 0:
stats_strs.append(f'{failed_count} failed')
if passed_count > 0:
stats_strs.append(f'{passed_count} passed')
stats_str = ', '.join(stats_strs) + f' in {end_time - start_time:.2f}s'
print()
print(stats_str)
return failed_count == 0
def IsFileSkipped(file_name) -> bool:
if os.path.splitext(file_name)[1].lower() not in SOURCE_AND_SCRIPT_FILE_EXTENSIONS:
skipped = True
for pattern in SOURCE_AND_SCRIPT_FILE_PATTERNS:
if pattern.match(file_name):
skipped = False
break
return skipped
return False
class CommitValidator(abc.ABC):
"""A commit validator"""
@abc.abstractmethod
def run(self, commit: Commit, errors: List[str]) -> bool:
"""Validates a commit
:param commit: The commit to validate
:param errors: List of errors generated, append them to this list
:return: True if the commit is valid, and False otherwise
"""
pass
SOURCE_FILE_EXTENSIONS: Tuple[str, ...] = (
'.c', '.cc', '.cpp', '.cxx', '.h', '.hpp', '.hxx', '.inl', '.m', '.mm', '.cs', '.java'
)
"""File extensions for compiled source code"""
SCRIPT_FILE_EXTENSIONS: Tuple[str, ...] = (
'.py', '.lua', '.bat', '.cmd', '.sh', '.js'
)
"""File extensions for interpreted code"""
BUILD_FILE_EXTENSIONS: Tuple[str, ...] = (
'.cmake',
)
"""File extensions for build files"""
SOURCE_AND_SCRIPT_FILE_EXTENSIONS: Tuple[str, ...] = SOURCE_FILE_EXTENSIONS + SCRIPT_FILE_EXTENSIONS + BUILD_FILE_EXTENSIONS
"""File extensions for both compiled and interpreted code"""
BUILD_FILE_PATTERNS: Tuple[re.Pattern, ...] = (
re.compile(r'.*CMakeLists\.txt'),
)
"""File patterns for build files"""
SOURCE_AND_SCRIPT_FILE_PATTERNS: Tuple[re.Pattern, ...] = BUILD_FILE_PATTERNS
EXCLUDED_VALIDATION_PATTERNS = [
'.git/*',
'*/3rdParty/*',
'*/__pycache__/*',
'*/External/*',
'build',
'Cache',
'Code/Tools/CryFXC',
'Code/Tools/HLSLCrossCompiler',
'Code/Tools/HLSLCrossCompilerMETAL',
'Code/Tools/ProfVis',
'Code/Tools/UniversalRemoteConsole',
'Docs',
'python/runtime',
'restricted/*/Tools/*RemoteControl',
'Tools/3dsmax',
'Tools/AWSPythonSDK',
'Tools/Crashpad',
]
@@ -0,0 +1,46 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import fnmatch
import os
from typing import List
DEFAULT_ALLOWEDLIST_FILE = os.path.join(os.path.dirname(__file__), 'pal_allowedlist.txt')
"""The path to the default allowed-list file"""
class PALAllowedlist:
"""A utility class used for determining if PAL rules should apply to a particular file path. If a path matches the
allowed-list, then PAL rules should not apply to that path."""
def __init__(self, patterns: List[str]) -> None:
"""Creates a new instance of :class:`PALAllowedlist` from a list of glob patterns
:param patterns: a list of glob patterns
"""
self.patterns: List[str] = patterns
def is_match(self, path: str) -> bool:
"""Determines if a path matches the allowed-list
:param path: the path to match
:return: True if the path matches the allowed-list, and False otherwise
"""
for pattern in self.patterns:
if fnmatch.fnmatch(path, pattern):
return True
return False
def load() -> PALAllowedlist:
"""Returns an instance of :class:`PALAllowedlist` created from the glob patterns in :const:`DEFAULT_ALLOWEDLIST_FILE`"""
with open(DEFAULT_ALLOWEDLIST_FILE) as fh:
return PALAllowedlist(fh.read().splitlines())
@@ -0,0 +1,68 @@
*/Code/CryEngine/*
*/Code/Deprecated/Sandbox/*
*/Code/Framework/AzCore/AzCore/Math/Aabb.h
*/Code/Framework/AzCore/AzCore/Math/Color.h
*/Code/Framework/AzCore/AzCore/Math/Crc.h
*/Code/Framework/AzCore/AzCore/Math/Matrix3x3.h
*/Code/Framework/AzCore/AzCore/Math/Matrix4x4.h
*/Code/Framework/AzCore/AzCore/Math/Obb.h
*/Code/Framework/AzCore/AzCore/Math/PackedVector3.h
*/Code/Framework/AzCore/AzCore/Math/Plane.h
*/Code/Framework/AzCore/AzCore/Math/Quaternion.h
*/Code/Framework/AzCore/AzCore/Math/Transform.h
*/Code/Framework/AzCore/AzCore/Math/Uuid.cpp
*/Code/Framework/AzCore/AzCore/Math/Vector2.h
*/Code/Framework/AzCore/AzCore/Math/Vector3.h
*/Code/Framework/AzCore/AzCore/Math/Vector4.h
*/Code/Framework/AzCore/AzCore/Math/VectorFloat.h
*/Code/Framework/AzCore/AzCore/Memory/dlmalloc.inl
*/Code/Framework/AzCore/AzCore/Memory/nedmalloc.inl
*/Code/Framework/AzCore/AzCore/PlatformDef.h
*/Code/Framework/AzCore/AzCore/std/containers/compressed_pair.h
*/Code/Framework/AzCore/AzCore/std/containers/variant.h
*/Code/Framework/AzCore/AzCore/std/parallel/binary_semaphore.h
*/Code/Framework/AzCore/AzCore/std/parallel/semaphore.h
*/Code/Framework/AzCore/Platform/Android/AzCore/AzCore_Traits_Android.h
*/Code/Framework/AzCore/Platform/AppleTV/AzCore/AzCore_Traits_AppleTV.h
*/Code/Framework/AzCore/Platform/iOS/AzCore/AzCore_Traits_iOS.h
*/Code/Framework/AzCore/Platform/Jasper/AzCore/AzCore_Traits_Jasper.h
*/Code/Framework/AzCore/Platform/Linux/AzCore/AzCore_Traits_Linux.h
*/Code/Framework/AzCore/Platform/Mac/AzCore/AzCore_Traits_Mac.h
*/Code/Framework/AzCore/Platform/Provo/AzCore/AzCore_Traits_Provo.h
*/Code/Framework/AzCore/Platform/Salem/AzCore/AzCore_Traits_Salem.h
*/Code/Framework/AzCore/Platform/Windows/AzCore/AzCore_Traits_Windows.h
*/Code/Framework/AzCore/Platform/Xenia/AzCore/AzCore_Traits_Xenia.h
*/Code/Framework/AzCore/Tests/AZStd/Examples.cpp
*/Code/Framework/AzCore/Tests/Memory.cpp
*/Code/Framework/AzFramework/AzFramework/Asset/AssetSystemComponentHelper.cpp
*/Code/Framework/AzQtComponents/AzQtComponents/*
*/Code/Framework/AzToolsFramework/AzToolsFramework/AzToolsFramework_precompiled.h
*/Code/Framework/AzToolsFramework/AzToolsFramework/API/EditorAssetSystemAPI.h
*/Code/Framework/AzToolsFramework/AzToolsFramework/Application/ToolsApplication.cpp
*/Code/Framework/AzToolsFramework/AzToolsFramework/Asset/AssetSystemComponent.cpp
*/Code/Framework/AzToolsFramework/AzToolsFramework/Process/internal/ProcessCommon_Win.h
*/Code/Framework/AzToolsFramework/AzToolsFramework/Process/internal/ProcessCommunicator_Win.cpp
*/Code/Framework/AzToolsFramework/AzToolsFramework/Process/internal/ProcessWatcher_Win.cpp
*/Code/Framework/AzToolsFramework/AzToolsFramework/Process/ProcessCommunicator.h
*/Code/Framework/AzToolsFramework/AzToolsFramework/UI/LegacyFramework/Core/EditorFrameworkAPI.h
*/Code/Framework/AzToolsFramework/AzToolsFramework/UI/LegacyFramework/Core/EditorFrameworkApplication.cpp
*/Code/Framework/AzToolsFramework/AzToolsFramework/UI/LegacyFramework/UIFramework.cpp
*/Code/Sandbox/*
*/Code/Tools/*
*/Gems/*/3rdParty/*
*/Gems/*/External/*
*/Gems/CryLegacy*
*/Gems/EMotionFX/Code/EMotionFX/Rendering/OpenGL2/Source/GLInclude.h
*/Gems/EMotionFX/Code/EMotionFX/Tools/EMotionStudio/EMStudioSDK/Source/MainWindow.cpp
*/Gems/EMotionFX/Code/EMotionFX/Tools/EMotionStudio/EMStudioSDK/Source/PluginManager.cpp
*/Gems/EMotionFX/Code/EMotionFX/Tools/EMotionStudio/EMStudioSDK/Source/PluginManager.h
*/Gems/EMotionFX/Code/EMotionFX/Tools/EMotionStudio/Plugins/StandardPlugins/Source/StandardPluginsConfig.h
*/Gems/EMotionFX/Code/MCore/Source/Config.h
*/Gems/GameStateSamples/Code/Include/GameStateSamples/GameStateLocalUserLobby.inl
*/Gems/ImageProcessing/Code/Tests/AtlasBuilderTest.cpp
*/Gems/PhysX/Code/Source/System/PhysXSystem.cpp
*/Gems/SaveData/Code/Tests/SaveDataTest.cpp
*/Gems/WhiteBox/Code/Source/Rendering/Legacy/WhiteBoxLegacyRenderMesh.cpp
*/restricted/*/Code/Framework/AzCore/AzCore/AzCore_Traits_*.h
*/SamplesProject/Gem/Code/Source/MetastreamTest/MetastreamTest.cpp
*/Tools/CryDeprecation/precompile_check_defines.h
@@ -0,0 +1,10 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
@@ -0,0 +1,10 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
@@ -0,0 +1,38 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
from typing import Dict, List
from commit_validation.commit_validation import Commit
class MockCommit(Commit):
def __init__(self, files=None, removed_files=None, file_diffs=None, description: str = '', author: str = ''):
self.files = [] if files is None else files
self.removed_files = [] if removed_files is None else removed_files
self.file_diffs = {} if file_diffs is None else file_diffs
self.description = description
self.author = author
def get_files(self) -> List[str]:
return self.files
def get_removed_files(self) -> List[str]:
return self.removed_files
def get_file_diff(self, file) -> Dict[str, str]:
return self.file_diffs[file]
def get_description(self) -> str:
return self.description
def get_author(self) -> str:
return self.author
@@ -0,0 +1,40 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import unittest
from unittest.mock import patch, mock_open
import commit_validation.pal_allowedlist as pal_allowedlist
class PALAllowedTests(unittest.TestCase):
@patch('builtins.open', mock_open(read_data='*/some/*\n'
'*/set/*\n'
'*/of/*\n'
'*/patterns/*'))
def setUp(self):
self.allowedlist = pal_allowedlist.load()
def test_load_allowedlistFile_patternsLoaded(self):
self.assertEqual(self.allowedlist.patterns[0], '*/some/*')
self.assertEqual(self.allowedlist.patterns[1], '*/set/*')
self.assertEqual(self.allowedlist.patterns[2], '*/of/*')
self.assertEqual(self.allowedlist.patterns[3], '*/patterns/*')
def test_isMatch_pathMatchesPattern_returnsTrue(self):
self.assertTrue(self.allowedlist.is_match('/path/to/some/file.cpp'))
def test_isMatch_pathDoesNotMatchPattern_returnsFalse(self):
self.assertFalse(self.allowedlist.is_match('/path/to/another/file.cpp'))
if __name__ == '__main__':
unittest.main()
@@ -0,0 +1,12 @@
"""
All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
its licensors.
For complete copyright and license terms please see the LICENSE at the root of this
distribution (the "License"). All use of this software is governed by the License,
or, if provided, by the license below or the license accompanying this file. Do not
remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
"""
@@ -0,0 +1,72 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import unittest
from unittest.mock import patch, mock_open
from commit_validation import pal_allowedlist
from commit_validation.tests.mocks.mock_commit import MockCommit
from commit_validation.validators.az_platform_validator import AzPlatformValidator
class AzPlatformValidatorTests(unittest.TestCase):
def test_fileWithNoAzPlatformMacro_passes(self):
commit = MockCommit(
files=['someCppFile.cpp'],
file_diffs = {
'someCppFile.cpp' : '+This file does not contain\n+AZ_PLATFORM_MACRO\n'
}
)
error_list = []
self.assertTrue(AzPlatformValidator().run(commit, error_list))
self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
# make sure it only examines parts that have actually been changed by this commit
# note that the line does not start with +
def test_fileWithMacroInNonChangedSection_passes(self):
commit = MockCommit(
files=['someCppFile.cpp', 'otherfile.cpp'],
file_diffs = { # one file has no diff marker, one file indicates its removed.
'someCppFile.cpp' : 'Stuff\n#if defined(AZ_PLATFORM_MACRO\n)',
'otherfile.cpp' : 'Stuff\n-#if defined(AZ_PLATFORM_MACRO\n)'
}
)
error_list = []
self.assertTrue(AzPlatformValidator().run(commit, error_list))
self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
def test_fileWithAzPlatformMacro_fails(self):
commit = MockCommit(
files=['someCppFile.cpp', 'otherfile.cpp'],
file_diffs = {
'someCppFile.cpp' : '+This file does contain\n'
'+#if defined(AZ_PLATFORM_MACRO)\n',
'otherfile.cpp' : 'This File has a different form\n'
'+# if defined(AZ_PLATFORM_MACRO)\n'
})
error_list = []
self.assertFalse(AzPlatformValidator().run(commit, error_list))
self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
def test_fileExtensionIgnored_passes(self):
commit = MockCommit(files=['someCppFile.waf_files'])
error_list = []
self.assertTrue(AzPlatformValidator().run(commit, error_list))
self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
@patch('commit_validation.pal_allowedlist.load', return_value=pal_allowedlist.PALAllowedlist(['*/some/path/*']))
def test_fileAllowedlisted_passes(self, mocked_load):
commit = MockCommit(files=['/path/to/some/path/someCppFile.cpp'])
error_list = []
self.assertTrue(AzPlatformValidator().run(commit, error_list))
self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
if __name__ == '__main__':
unittest.main()
@@ -0,0 +1,95 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import unittest
from unittest.mock import patch, mock_open
from commit_validation import pal_allowedlist
from commit_validation.tests.mocks.mock_commit import MockCommit
from commit_validation.validators.az_trait_validator import AzTraitValidator
import pytest
class Test_AzTraitValidatorTests():
def test_fileDoesntCheckAzTraitIsDefined_passes(self):
commit = MockCommit(
files=['someCppFile.cpp'],
file_diffs={ 'someCppFile.cpp' : ''}
)
error_list = []
assert AzTraitValidator().run(commit, error_list)
assert len(error_list) == 0, f"Unexpected errors: {error_list}"
@pytest.mark.parametrize(
'file_diffs,expect_success', [
pytest.param('+This file does contain\n'
'+a trait existence check\n'
'+#ifdef AZ_TRAIT_USED_INCORRECTLY\n',
False,
id="AZ_TRAIT_inside_ifdef_fails" ), # gives the test a friendly name!
pytest.param('+This file does contain\n'
'+a trait existence check\n'
'+#if defined(AZ_TRAIT_USED_INCORRECTLY)\n',
False,
id="AZ_TRAIT_inside_if_defined_fails" ),
pytest.param('+This file does contain\n'
'+a trait existence check\n'
'+#ifndef AZ_TRAIT_USED_INCORRECTLY\n',
False,
id="AZ_TRAIT_inside_ifndef_fails" ),
pytest.param('+This file contains a diff which REMOVES an incorrect usage\n'
'-#ifndef AZ_TRAIT_USED_INCORRECTLY\n',
True,
id="AZ_TRAIT_removed_in_diff_passes" ),
pytest.param('+This file contains a diff which has an old already okayed usage\n'
'+which is not actually part of the diff.\n'
'#ifndef AZ_TRAIT_USED_INCORRECTLY\n',
True,
id="AZ_TRAIT_in_unmodified_section_passes"),
pytest.param('+This file contains the correct usage\n'
'+#if AZ_TRAIT_USED_CORRECTLY\n',
True,
id="AZ_TRAIT_correct_usage_passes"),
])
def test_fileChecksAzTraitIsDefined(self, file_diffs, expect_success):
commit = MockCommit(
files=['someCppFile.cpp'],
file_diffs={ 'someCppFile.cpp' : file_diffs })
error_list = []
if expect_success:
assert AzTraitValidator().run(commit, error_list)
assert len(error_list) == 0, f"Unexpected errors: {error_list}"
else:
assert not AzTraitValidator().run(commit, error_list)
assert len(error_list) != 0, f"Errors were expected but none were returned."
def test_fileExtensionIgnored_passes(self):
commit = MockCommit(files=['someCppFile.waf_files'])
error_list = []
assert AzTraitValidator().run(commit, error_list)
assert len(error_list) == 0, f"Unexpected errors: {error_list}"
@patch('commit_validation.pal_allowedlist.load', return_value=pal_allowedlist.PALAllowedlist(['*/some/path/*']))
def test_fileAllowedlisted_passes(self, mocked_load):
commit = MockCommit(files=['/path/to/some/path/someCppFile.cpp'])
error_list = []
assert AzTraitValidator().run(commit, error_list)
assert len(error_list) == 0, f"Unexpected errors: {error_list}"
if __name__ == '__main__':
unittest.main()
@@ -0,0 +1,75 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import unittest
from unittest.mock import patch, mock_open
from commit_validation.tests.mocks.mock_commit import MockCommit
from commit_validation.validators.copyright_header_validator import CopyrightHeaderValidator
class CopyrightHeaderValidatorTests(unittest.TestCase):
@patch('builtins.open', mock_open(read_data='This file does contain\n'
'Copyright (c) Amazon.com, so it should pass\n'))
def test_fileWithCopyrightHeader_passes(self):
commit = MockCommit(files=['/someCppFile.cpp'])
files = [
'This file does contain\n'
'Copyright (c) Amazon.com, so it should pass\n',
'This file does contain\n'
'Copyright(c) Amazon.com, so it should pass\n'
'and there\'s no space between "Copyright" and "(c)"',
'This file has a upper-case C between the parenthesis\n'
'// Copyright (C) Amazon.com, Inc. or its affiliates.',
]
for file in files:
with patch('builtins.open', mock_open(read_data=file)):
error_list = []
self.assertTrue(CopyrightHeaderValidator().run(commit, error_list))
self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
def test_fileWithCopyrightHeaderExternalDir_passes(self):
commit = MockCommit(files=['/External/someCppFile.cpp'])
error_list = []
self.assertTrue(CopyrightHeaderValidator().run(commit, error_list))
self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
@patch('builtins.open', mock_open(read_data='This file does not contain\n'
'the copyright header\n'))
def test_fileWithNoCopyrightHeader_fails(self):
commit = MockCommit(files=['/someCppFile.cpp'])
error_list = []
self.assertFalse(CopyrightHeaderValidator().run(commit, error_list))
self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
def test_fileExtensionIgnored_passes(self):
commit = MockCommit(files=['/someCppFile.waf_files'])
error_list = []
self.assertTrue(CopyrightHeaderValidator().run(commit, error_list))
self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
def test_fileWith3rdPartyPath_passes(self):
commit = MockCommit(files=['/3rdParty/someCppFile.cpp'])
error_list = []
self.assertTrue(CopyrightHeaderValidator().run(commit, error_list))
self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
def test_fileWithExternal_passes(self):
commit = MockCommit(files=['/External/someCppFile.cpp'])
error_list = []
self.assertTrue(CopyrightHeaderValidator().run(commit, error_list))
self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
if __name__ == '__main__':
unittest.main()
@@ -0,0 +1,102 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import unittest
from commit_validation.tests.mocks.mock_commit import MockCommit
from commit_validation.validators.diff_whitespace_validator import WhitespaceValidator
class WhitespaceValidatorTests(unittest.TestCase):
def test_DiffWhitespace_WhitespaceLinesContiguousWithCodeChange_pass(self):
commit = MockCommit(
files = ['/someCppFile.cpp'],
file_diffs={'/someCppFile.cpp':
'This file\n'
'-was edited'
'+ and has a diff including plus and minus symbols\n'
'+ \n'
'and should pass\n'})
error_list = []
self.assertTrue(WhitespaceValidator().run(commit, error_list))
self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
def test_DiffWhitespace_WhitespaceLinesSeparateAtEOF_fail(self):
commit = MockCommit(
files = ['/someCppFile.cpp'],
file_diffs={'/someCppFile.cpp':
'This file\n'
'-was edited'
'+ and has an implementation diff\n'
'but should not pass because of a whitespace only change:\n'
'- \n'})
error_list = []
self.assertFalse(WhitespaceValidator().run(commit, error_list))
self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
def test_DiffWhitespace_WhitespaceLinesSeparateAtEOF_fail_More(self):
commit = MockCommit(
files = ['/someCppFile.cpp'],
file_diffs={'/someCppFile.cpp':
'+ \n'
'This file\n'
'-was edited'
'+ and has an implementation diff\n'
'but should not pass because of a whitespace only change\n'
'on the first line\n'})
error_list = []
self.assertFalse(WhitespaceValidator().run(commit, error_list))
self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
def test_DiffWhitespace_WhitespaceLinesSeparateFromCode_fail(self):
commit = MockCommit(
files = ['/someCppFile.cpp'],
file_diffs={'/someCppFile.cpp':
'This file\n'
'should not pass because of a whitespace only change:\n'
'- \n'
'since was edited'
'+ and has an implementation diff\n'
'- later on'})
error_list = []
self.assertFalse(WhitespaceValidator().run(commit, error_list))
self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
def test_DiffWhitespace_WhitespaceOnlyChange_pass(self):
commit = MockCommit(
files = ['/someCppFile.cpp'],
file_diffs={'/someCppFile.cpp':
'This file has whitespace only changes\n'
'+ \n'
'+ \n'
'and should pass\n'
'- \n'})
error_list = []
self.assertTrue(WhitespaceValidator().run(commit, error_list))
self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
def test_DiffWhitespace_CodeChangeOnly_pass(self):
commit = MockCommit(
files = ['/someCppFile.cpp'],
file_diffs={'/someCppFile.cpp':
'This file has'
'+ no whitespace only lines changed\n'
' \n'
'-and should pass\n'
' \n'})
error_list = []
self.assertTrue(WhitespaceValidator().run(commit, error_list))
self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
if __name__ == '__main__':
unittest.main()
@@ -0,0 +1,96 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import unittest
from unittest.mock import patch, mock_open
from commit_validation.tests.mocks.mock_commit import MockCommit
from commit_validation.validators.generated_files_validator import GeneratedFilesValidator
@patch('builtins.open', mock_open(read_data='unused\n'))
class GeneratedFilesValidatorTests(unittest.TestCase):
def test_IsGenerated_NormalFile_pass(self):
commit = MockCommit(files=['/someFile.cpp'])
error_list = []
self.assertTrue(GeneratedFilesValidator().run(commit, error_list))
self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
def test_IsGenerated_DotGeneratedFile_fail(self):
commit = MockCommit(files=['/someFile.generated.cpp'])
error_list = []
self.assertFalse(GeneratedFilesValidator().run(commit, error_list))
self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
def test_IsGenerated_QtMOC_fail(self):
commit = MockCommit(files=['/moc_someFile.cpp'])
error_list = []
self.assertFalse(GeneratedFilesValidator().run(commit, error_list))
self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
def test_IsGenerated_QtQRC_fail(self):
commit = MockCommit(files=['/qrc_someFile.cpp'])
error_list = []
self.assertFalse(GeneratedFilesValidator().run(commit, error_list))
self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
def test_IsGenerated_QtUiDotH_fail(self):
commit = MockCommit(files=['/ui_someFile.h'])
error_list = []
self.assertFalse(GeneratedFilesValidator().run(commit, error_list))
self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
def test_IsGenerated_CMakeCacheFile_fail(self):
commit = MockCommit(files=['/CMakeCache.txt'])
error_list = []
self.assertFalse(GeneratedFilesValidator().run(commit, error_list))
self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
def test_IsGenerated_CMakeCacheExtension_fail(self):
commit = MockCommit(files=['/AzCore.Benchmarks.rule'])
error_list = []
self.assertFalse(GeneratedFilesValidator().run(commit, error_list))
self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
def test_IsGenerated_Tempfile_fail(self):
commit = MockCommit(files=['/someFile.tmp'])
error_list = []
self.assertFalse(GeneratedFilesValidator().run(commit, error_list))
self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
def test_IsGenerated_UnixObject_fail(self):
commit = MockCommit(files=['/someFile.o'])
error_list = []
self.assertFalse(GeneratedFilesValidator().run(commit, error_list))
self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
def test_IsGenerated_WindowsObject_fail(self):
commit = MockCommit(files=['/someFile.obj'])
error_list = []
self.assertFalse(GeneratedFilesValidator().run(commit, error_list))
self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
def test_IsGenerated_MicrosoftSolution_fail(self):
commit = MockCommit(files=['/someFile.sln'])
error_list = []
self.assertFalse(GeneratedFilesValidator().run(commit, error_list))
self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
def test_IsGenerated_Logfile_fail(self):
commit = MockCommit(files=['/someFile.log'])
error_list = []
self.assertFalse(GeneratedFilesValidator().run(commit, error_list))
self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
if __name__ == '__main__':
unittest.main()
@@ -0,0 +1,99 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import unittest
from unittest.mock import patch, mock_open
from commit_validation.tests.mocks.mock_commit import MockCommit
from commit_validation.validators.git_conflict_validator import GitConflictValidator
class NewlineValidatorTests(unittest.TestCase):
@patch('builtins.open', mock_open(read_data='This file is completely normal\n'
'and should pass\n'))
def test_HasConflictMarkers_NoMarkers_Pass(self):
commit = MockCommit(files=['/someFile.cpp'])
error_list = []
self.assertTrue(GitConflictValidator().run(commit, error_list))
self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
@patch('builtins.open', mock_open(read_data='This file has a start marker\n'
'<<<<<<< ours\n'
'and should fail\n'))
def test_HasConflictMarkers_StartMarker_Fail(self):
commit = MockCommit(files=['/someFile.cpp'])
error_list = []
self.assertFalse(GitConflictValidator().run(commit, error_list))
self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
@patch('builtins.open', mock_open(read_data='This file has a diff3 marker from using --conflict=diff3\n'
'||||||| base\n'
'and should fail\n'))
def test_HasConflictMarkers_BaseMarker_Fail(self):
commit = MockCommit(files=['/someFile.cpp'])
error_list = []
self.assertFalse(GitConflictValidator().run(commit, error_list))
self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
@patch('builtins.open', mock_open(read_data='This file has a diff marker\n'
'=======\n'
'and should fail\n'))
def test_HasConflictMarkers_DiffMarker_Fail(self):
commit = MockCommit(files=['/someFile.cpp'])
error_list = []
self.assertFalse(GitConflictValidator().run(commit, error_list))
self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
@patch('builtins.open', mock_open(read_data='This file has and end marker\n'
'>>>>>>> theirs\n'
'and should fail\n'))
def test_HasConflictMarkers_EndMarker_Fail(self):
commit = MockCommit(files=['/someFile.cpp'])
error_list = []
self.assertFalse(GitConflictValidator().run(commit, error_list))
self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
@patch('builtins.open', mock_open(read_data='This file has a equals sign divider of length seven, but is indented\n'
' /*\n'
' =======\n'
' */'
'and should pass\n'))
def test_HasConflictMarkers_IndentedCommentDivider_Pass(self):
commit = MockCommit(files=['/someFile.cpp'])
error_list = []
self.assertTrue(GitConflictValidator().run(commit, error_list))
self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
@patch('builtins.open', mock_open(read_data='This file has an unindented equals sign divider, of length eight\n'
'/*\n'
'========\n'
'*/'
'and should pass\n'))
def test_HasConflictMarkers_LongCommentDivider_Pass(self):
commit = MockCommit(files=['/someFile.cpp'])
error_list = []
self.assertTrue(GitConflictValidator().run(commit, error_list))
self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
@patch('builtins.open', mock_open(read_data='This file has an unindented equals sign divider, of length six\n'
'/*\n'
'======\n'
'*/'
'and should pass\n'))
def test_HasConflictMarkers_ShortCommentDivider_Pass(self):
commit = MockCommit(files=['/someFile.cpp'])
error_list = []
self.assertTrue(GitConflictValidator().run(commit, error_list))
self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
if __name__ == '__main__':
unittest.main()
@@ -0,0 +1,88 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import unittest
from unittest.mock import patch, mock_open
from commit_validation.tests.mocks.mock_commit import MockCommit
from commit_validation.validators.newline_validator import NewlineValidator
class NewlineValidatorTests(unittest.TestCase):
@patch('builtins.open', mock_open(read_data='This file uses carriage returns with line feeds\r\n'
'and should fail since only line feeds are allowed\r\n'
'despite appropriately ending with an extra line\r\n'))
def test_LineEndings_CRLF_fail(self):
commit = MockCommit(files=['/someFile.cpp'])
error_list = []
self.assertFalse(NewlineValidator().run(commit, error_list))
self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
@patch('builtins.open', mock_open(read_data='This file uses line feeds\n'
'and ends with an extra line\n'))
def test_LineEndings_LF_pass(self):
commit = MockCommit(files=['/someFile.cpp'])
error_list = []
self.assertTrue(NewlineValidator().run(commit, error_list))
self.assertTrue(not error_list)
@patch('builtins.open', mock_open(read_data='This file is full of garbage\n'
'which would normally fail\r\n'
'though it should not due to its file extension\n\r'
'which should skip validation'))
def test_LineEndings_RandomFileExtension_skip(self):
commit = MockCommit(files=['/someFile.garbage'])
error_list = []
self.assertTrue(NewlineValidator().run(commit, error_list))
self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
@patch('builtins.open', mock_open(read_data='This file mixes line feed endings\n'
'with carriage return and line feed endings\r\n'))
def test_LineEndings_Mixed_fail(self):
commit = MockCommit(files=['/someFile.cpp'])
error_list = []
self.assertFalse(NewlineValidator().run(commit, error_list))
self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
@patch('builtins.open', mock_open(read_data='This file ends\n'
'with no newline'))
def test_LineEndings_NoFinalNewline_fail(self):
commit = MockCommit(files=['/someFile.cpp'])
error_list = []
self.assertFalse(NewlineValidator().run(commit, error_list))
self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
@patch('builtins.open', mock_open(read_data='This file ends with CRLF newlines\r\n'
'but not at the end'))
def test_LineEndings_NoFinalCRLF_fail(self):
commit = MockCommit(files=['/someFile.cpp'])
error_list = []
self.assertFalse(NewlineValidator().run(commit, error_list))
self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
@patch('builtins.open', mock_open(read_data='This file ends with extra newlines\n'
'\n'))
def test_LineEndings_ExtraFinalNewline_fail(self):
commit = MockCommit(files=['/someFile.cpp'])
error_list = []
self.assertFalse(NewlineValidator().run(commit, error_list))
self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
@patch('builtins.open', mock_open(read_data='This file ends with CRLF extra newlines\r\n'
'\r\n'))
def test_LineEndings_ExtraFinalCRLF_fail(self):
commit = MockCommit(files=['/someFile.cpp'])
error_list = []
self.assertFalse(NewlineValidator().run(commit, error_list))
self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
if __name__ == '__main__':
unittest.main()
@@ -0,0 +1,77 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import unittest
from unittest.mock import patch, mock_open
from commit_validation import pal_allowedlist
from commit_validation.tests.mocks.mock_commit import MockCommit
from commit_validation.validators.platform_macro_validator import PlatformMacroValidator
class PlatformMacroValidatorTests(unittest.TestCase):
def test_fileWithNoPlatformMacro_passes(self):
commit = MockCommit(
files=['someCppFile.cpp'],
file_diffs={ 'someCppFile.cpp' : '+This file does not contain\n'
'+a platform macro\n'
'+#define ACCEPTABLE_MACRO\n'
})
error_list = []
self.assertTrue(PlatformMacroValidator().run(commit, error_list))
self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
def test_fileWithPlatformMacro_errors(self):
commit = MockCommit(
files=['someCppFile.cpp'],
file_diffs={ 'someCppFile.cpp' : '+This file does contain\n'
'+a platform macro\n'
'+#if defined(APPLE)\n'
})
error_list = []
self.assertFalse(PlatformMacroValidator().run(commit, error_list))
self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
def test_fileWithPlatformMacro_ButNotInChangedPart_passes(self):
commit = MockCommit(
files=['someCppFile.cpp'],
file_diffs={ 'someCppFile.cpp' : '+This file does contain\n'
'+a platform macro\n'
'-#if defined(APPLE)\n'
'But not in a part thats relevant to the diff'
'#if defined(APPLE)\n'
})
error_list = []
self.assertTrue(PlatformMacroValidator().run(commit, error_list))
self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
def test_fileExtensionIgnored_passes(self):
commit = MockCommit(files=['someCppFile.waf_files'])
error_list = []
self.assertTrue(PlatformMacroValidator().run(commit, error_list))
self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
def test_platformFolderIgnored_passes(self):
commit = MockCommit(files=['/path/to/Platform/SomePlatform/someCppFile.cpp'])
error_list = []
self.assertTrue(PlatformMacroValidator().run(commit, error_list))
self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
@patch('commit_validation.pal_allowedlist.load', return_value=pal_allowedlist.PALAllowedlist(['*/some/path/*']))
def test_fileAllowedlisted_passes(self, mocked_load):
commit = MockCommit(files=['/path/to/some/path/someCppFile.cpp'])
error_list = []
self.assertTrue(PlatformMacroValidator().run(commit, error_list))
self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
if __name__ == '__main__':
unittest.main()
@@ -0,0 +1,89 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import unittest
from unittest.mock import patch, mock_open
from commit_validation.tests.mocks.mock_commit import MockCommit
from commit_validation.validators.pragma_optimize_validator import PragmaOptimizeValidator
class PragmaOptimizeValidatorTests(unittest.TestCase):
def test_fileWithNoPragmaOptimize_passes(self):
commit = MockCommit(
files=['/someCppFile.cpp'],
file_diffs={
'/someCppFile.cpp' : '+// This file does not contain\n'
'+// the p r a g m a o p t i m i z e'
})
error_list = []
self.assertTrue(PragmaOptimizeValidator().run(commit, error_list))
self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
def test_fileWithNoPragma3rdPartyPath_passes(self):
commit = MockCommit(files=['/3rdParty/someCppFile.cpp'])
error_list = []
self.assertTrue(PragmaOptimizeValidator().run(commit, error_list))
self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
def test_fileWithNoPragmaExternal_passes(self):
commit = MockCommit(files=['/External/someCppFile.cpp'])
error_list = []
self.assertTrue(PragmaOptimizeValidator().run(commit, error_list))
self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
def test_fileWithPragmaOptimize_fails(self):
commit = MockCommit(
files=['/someCppFile.cpp'],
file_diffs={
'/someCppFile.cpp' : '+// This file contains\n'
'+// a line\n'
'+// with #pragma optimize("", off)'
})
error_list = []
self.assertFalse(PragmaOptimizeValidator().run(commit, error_list))
self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
def test_fileWithPragmaOptimize_in_unchanged_part_passes(self):
commit = MockCommit(
files=['/someCppFile.cpp'],
file_diffs={
'/someCppFile.cpp' : '+// This file contains\n'
'+// a line\n'
'-#pragma optimize("", off)\n'
'but its in a deleted line or a line that is not part of the diff\n'
'#pragma optimize("", off)\n'
})
error_list = []
self.assertTrue(PragmaOptimizeValidator().run(commit, error_list))
self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
def test_fileExtensionIgnored_passes(self):
commit = MockCommit(files=['/someCppFile.waf_files'])
error_list = []
self.assertTrue(PragmaOptimizeValidator().run(commit, error_list))
self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
def test_fileWithPragma3rdPartyPath_passes(self):
commit = MockCommit(files=['/3rdParty/someCppFile.cpp'])
error_list = []
self.assertTrue(PragmaOptimizeValidator().run(commit, error_list))
self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
def test_fileWithPragmaExternal_passes(self):
commit = MockCommit(files=['/External/someCppFile.cpp'])
error_list = []
self.assertTrue(PragmaOptimizeValidator().run(commit, error_list))
self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
if __name__ == '__main__':
unittest.main()
@@ -0,0 +1,78 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import unittest
from unittest.mock import patch, mock_open
from commit_validation.tests.mocks.mock_commit import MockCommit
from commit_validation.validators.tabs_validator import TabsValidator
class TabsValidatorTests(unittest.TestCase):
def test_fileWithNoTabs_passes(self):
commit = MockCommit(
files=['/someCppFile.cpp'],
file_diffs={ '/someCppFile.cpp' : '+This file contains\n'
'+ no tabs\n'
'+ but does have spaces\n'})
error_list = []
self.assertTrue(TabsValidator().run(commit, error_list))
self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
def test_fileWithTabs_fails(self):
commit = MockCommit(
files=['/someCppFile.cpp'],
file_diffs={ '/someCppFile.cpp' : '+This file contains\n'
'+\tTHE DREADED TAB CHARACTER\n'})
error_list = []
self.assertFalse(TabsValidator().run(commit, error_list))
self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
def test_fileWithTabs_butNotInDiffArea_passes(self):
commit = MockCommit(
files=['/someCppFile.cpp'],
file_diffs={ '/someCppFile.cpp' : '+This file contains\n'
'\tTHE DREADED TAB CHARACTER\n'
'But since its not in a diff line\n'
'It does not matter not even if in \n'
'-\tA REMOVED LINE with the tab character\n'})
error_list = []
self.assertTrue(TabsValidator().run(commit, error_list))
self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
def test_fileExtensionIgnored_passes(self):
commit = MockCommit(files=['/someCppFile.waf_files'])
error_list = []
self.assertTrue(TabsValidator().run(commit, error_list))
self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
def test_fileWithNoTabsbutAnEmoji_passes(self):
commit = MockCommit(
files=['/someCppFile_trap.cpp'],
file_diffs={ '/someCppFile_trap.cpp' : '+This file contains\n'
'+ no tabs\n'
'+ but has an emoji \U0001F4A9\n'})
error_list = []
self.assertTrue(TabsValidator().run(commit, error_list))
self.assertEqual(len(error_list), 0, f"Unexpected errors: {error_list}")
def test_fileWithTabsAndEmoji_fails(self):
commit = MockCommit(
files=['/another_trap.cpp'],
file_diffs={ '/another_trap.cpp' : '+This file contains an emoji \U0001F4A9 \n'
'+\tbut also a tab!\n'})
error_list = []
self.assertFalse(TabsValidator().run(commit, error_list))
self.assertNotEqual(len(error_list), 0, f"Errors were expected but none were returned.")
if __name__ == '__main__':
unittest.main()
@@ -0,0 +1,10 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
@@ -0,0 +1,59 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import os
import re
from typing import Type, List
import commit_validation.pal_allowedlist as pal_allowedlist
from commit_validation.commit_validation import Commit, CommitValidator, SOURCE_FILE_EXTENSIONS, VERBOSE
az_platform_regex = re.compile(r'^\+\s*#.*\bAZ_PLATFORM_')
class AzPlatformValidator(CommitValidator):
"""A file-level validator that makes sure a file does not contain AZ_PLATFORM macros"""
def __init__(self) -> None:
self.pal_allowedlist = pal_allowedlist.load()
def run(self, commit: Commit, errors: List[str]) -> bool:
for file_name in commit.get_files():
if os.path.splitext(file_name)[1].lower() not in SOURCE_FILE_EXTENSIONS:
if VERBOSE: print(f'{file_name}::{self.__class__.__name__} SKIPPED - File excluded based on extension.')
continue
if self.pal_allowedlist.is_match(file_name):
if VERBOSE: print(f'{file_name}::{self.__class__.__name__} SKIPPED - File excluded based on PAL allowedlist.')
continue
file_diff = commit.get_file_diff(file_name)
previous_line_context = ""
line_number = 1
for line in file_diff.splitlines():
# we only care about lines that start with +
if line.startswith('+'):
if az_platform_regex.search(line):
error_message = str(f'{file_name}:{line_number}::{self.__class__.__name__} FAILED - Source file contains an AZ_PLATFORM '
f'macro in code:\n'
f' {previous_line_context}\n'
f'---> {line}\n')
if VERBOSE: print(error_message)
errors.append(error_message)
previous_line_context = line
line_number += 1
return (not errors)
def get_validator() -> Type[AzPlatformValidator]:
"""Returns the validator class for this module"""
return AzPlatformValidator
@@ -0,0 +1,60 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import os
import re
from typing import Type, List
import commit_validation.pal_allowedlist as pal_allowedlist
from commit_validation.commit_validation import Commit, CommitValidator, SOURCE_FILE_EXTENSIONS, VERBOSE
ifdef_regex = re.compile(r'^\+\s*#\s*ifn?def\s+AZ_TRAIT_')
defined_regex = re.compile(r'\sdefined\s*\(\s*AZ_TRAIT_')
class AzTraitValidator(CommitValidator):
"""A file-level validator that makes sure a file does not contain existence checks for AZ_TRAIT macros"""
def __init__(self) -> None:
self.pal_allowedlist = pal_allowedlist.load()
def run(self, commit: Commit, errors: List[str]) -> bool:
for file_name in commit.get_files():
if os.path.splitext(file_name)[1].lower() not in SOURCE_FILE_EXTENSIONS:
if VERBOSE: print(f'{file_name}::{self.__class__.__name__} SKIPPED - File excluded based on extension.')
continue
if self.pal_allowedlist.is_match(file_name):
if VERBOSE: print(f'{file_name}::{self.__class__.__name__} SKIPPED - File excluded based on PAL allowedlist.')
continue
file_diff = commit.get_file_diff(file_name)
previous_line_context = ""
for line in file_diff.splitlines():
# we only care about added lines.
if line.startswith('+'):
if ifdef_regex.search(line) or defined_regex.search(line):
error_message = str(
f'{file_name}::{self.__class__.__name__} FAILED - Source file contains an existence '
f'check for an AZ_TRAIT macro in this code: \n'
f' {previous_line_context}\n'
f' ----> {line}\n'
f'Traits should be tested for true/false, since they are guaranteed to exist on all platforms.')
if VERBOSE: print(error_message)
errors.append(error_message)
previous_line_context = line
return (not errors)
def get_validator() -> Type[AzTraitValidator]:
"""Returns the validator class for this module"""
return AzTraitValidator
@@ -0,0 +1,52 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import fnmatch
import os
import re
from typing import Type, List
from commit_validation.commit_validation import Commit, CommitValidator, IsFileSkipped, SOURCE_AND_SCRIPT_FILE_EXTENSIONS, EXCLUDED_VALIDATION_PATTERNS, VERBOSE
class CopyrightHeaderValidator(CommitValidator):
"""A file-level validator that makes sure a file contains the standard copyright header"""
def run(self, commit: Commit, errors: List[str]) -> bool:
for file_name in commit.get_files():
for pattern in EXCLUDED_VALIDATION_PATTERNS:
if fnmatch.fnmatch(file_name, pattern):
if VERBOSE: print(f'{file_name}::{self.__class__.__name__} SKIPPED - Validation pattern excluded on path.')
break
else:
if IsFileSkipped(file_name):
if VERBOSE: print(f'{file_name}::{self.__class__.__name__} SKIPPED - File excluded based on extension.')
continue
copyright_regex = re.compile(r'copyright[\s]*(?:\(c\))?[\s]*amazon\.com', re.IGNORECASE)
# copyright header validator does not use the diff, as it needs to check the front
# of the file for the header.
with open(file_name, 'rt', encoding='utf8', errors='replace') as fh:
for line in fh:
if copyright_regex.search(line):
break
else:
error_message = str(f'{file_name}::{self.__class__.__name__} FAILED - Source file missing copyright headers.')
errors.append(error_message)
if VERBOSE: print(error_message)
return (not errors)
def get_validator() -> Type[CopyrightHeaderValidator]:
"""Returns the validator class for this module"""
return CopyrightHeaderValidator
@@ -0,0 +1,58 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import fnmatch
import os
import re
from typing import Type, List
from commit_validation.commit_validation import Commit, CommitValidator, IsFileSkipped, SOURCE_AND_SCRIPT_FILE_EXTENSIONS, EXCLUDED_VALIDATION_PATTERNS, VERBOSE
__STARTS_WITH_UNCHANGED_LINE = r'(\A|^(?=[^+-]).*\n)'
__NONZERO_CHANGED_WHITESPACE_LINES = r'(^[+-][\r\t\f\v ]*\n)+'
__ENDS_WITH_UNCHANGED_LINE = r'([^+-]|\Z)'
__NONADJACENT_WHITESPACE =\
__STARTS_WITH_UNCHANGED_LINE + __NONZERO_CHANGED_WHITESPACE_LINES + __ENDS_WITH_UNCHANGED_LINE
_NONADJACENT_WHITESPACE_DIFF_REGEX = re.compile(__NONADJACENT_WHITESPACE, re.MULTILINE)
_NONWHITESPACE_DIFF_REGEX = re.compile(r'^[+-][\t ]*\S', re.MULTILINE)
class WhitespaceValidator(CommitValidator):
"""A diff-level validator that makes sure a change does not mix whitespace-only changes with code changes"""
def run(self, commit: Commit, errors: List[str]) -> bool:
for file_name in commit.get_files():
file_identifier = f"{file_name}::{self.__class__.__name__}"
for pattern in EXCLUDED_VALIDATION_PATTERNS:
if fnmatch.fnmatch(file_name, pattern):
if VERBOSE: print(f'{file_identifier} SKIPPED - Validation pattern excluded on path.')
break
else:
if IsFileSkipped(file_name):
if VERBOSE: print(f'{file_identifier} SKIPPED - File excluded based on extension.')
continue
diff = commit.get_file_diff(file_name)
if _NONADJACENT_WHITESPACE_DIFF_REGEX.search(diff) and _NONWHITESPACE_DIFF_REGEX.search(diff):
error_message = str(f'{file_identifier} FAILED - Source file contains whitespace-only changes which are '
f'non-contiguous with other non-whitespace changes. Make whitespace-only changes as a '
f'separate change.')
errors.append(error_message)
if VERBOSE: print(error_message)
return (not errors)
def get_validator() -> Type[WhitespaceValidator]:
"""Returns the validator class for this module"""
return WhitespaceValidator
@@ -0,0 +1,139 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import fnmatch
import os
import pathlib
from typing import Type, List
from commit_validation.commit_validation import Commit, CommitValidator, EXCLUDED_VALIDATION_PATTERNS, VERBOSE
# Disallowed File Name Patterns
# these must be LOWER CASE since they will be compared after a 'lower'
AZ_CODEGEN_FILENAME_PATTERN = "*.generated*"
QT_COMPILER_PATTERNS = ["moc_*", "qrc_*", "ui_*.h"]
CMAKE_CACHE_FILENAME_PATTERNS = ["cmakecache.txt", "cmake_install.cmake", "ctesttestfile.cmake", "ctestcostdata.txt"]
OTHER_GENERATED_FILES = ["appxmanifest.xml"]
# Disallowed File Extensions
CMAKE_CACHE_EXTENSIONS = [".rule", ".stamp", ".depend"]
TEMPFILE_EXTENSIONS = [".backup", ".bak", ".bkup", ".temp", ".tempfile", ".temporary", ".tmp"]
EXECUTION_ARTIFACT_EXTENSIONS = [".cache", ".dmp", ".log", ".pyc"]
CLANG_FILE_EXTENSIONS = [".o", ".s"]
MSVS_FILE_EXTENSIONS = [
".aps",
".csproj",
".exp",
".filters",
".idb",
".ilk",
".lastbuildstate",
".meta",
".ncb",
".obj",
".opensdf",
".pch",
".pdb",
".pgc",
".pgd",
".psess",
".rsp",
".sbr",
".sdf",
".sln",
".suo",
".tlb",
".tlh",
".tli",
".tlog",
".vap",
".vbg",
".vbproj",
".vcxitems",
".vcxproj",
".vdproj",
".vmx",
".vsp",
".vspscc",
".vspx",
".vssscc",
".vup",
]
class GeneratedFilesValidator(CommitValidator):
"""A file-level validator to check if a file is a true source file, and not generated during build or execution"""
def run(self, commit: Commit, errors: List[str]) -> bool:
for file_name in commit.get_files():
file_identifier = f"{file_name}::{self.__class__.__name__}"
for pattern in EXCLUDED_VALIDATION_PATTERNS:
if fnmatch.fnmatch(file_name, pattern):
if VERBOSE: print(f'{file_identifier} SKIPPED - Validation pattern excluded on path.')
break
else:
file_path_lower = pathlib.Path(file_name.lower())
extension = file_path_lower.suffix
if extension in CMAKE_CACHE_EXTENSIONS:
error_message = str(f'{file_identifier} FAILED - Autogenerated CMake file detected by extension.')
if VERBOSE: print(error_message)
errors.append(error_message)
continue
if extension in MSVS_FILE_EXTENSIONS:
error_message = str(f'{file_identifier} FAILED - Autogenerated Visual Studio file detected by extension.')
if VERBOSE: print(error_message)
errors.append(error_message)
continue
if extension in CLANG_FILE_EXTENSIONS:
error_message = str(f'{file_identifier} FAILED - Autogenerated Clang file detected by extension.')
if VERBOSE: print(error_message)
errors.append(error_message)
continue
if extension in TEMPFILE_EXTENSIONS:
error_message = str(f'{file_identifier} FAILED - Autogenerated temporary file detected by extension.')
if VERBOSE: print(error_message)
errors.append(error_message)
continue
if extension in EXECUTION_ARTIFACT_EXTENSIONS:
error_message = str(f'{file_identifier} FAILED - Execution artifact detected by extension.')
if VERBOSE: print(error_message)
errors.append(error_message)
continue
for cmake_pattern in QT_COMPILER_PATTERNS:
if fnmatch.fnmatch(file_path_lower.name, cmake_pattern):
error_message = str(f'{file_identifier} FAILED - Autogenerated QT file detected by pattern.')
if VERBOSE: print(error_message)
errors.append(error_message)
break
for cmake_pattern in CMAKE_CACHE_FILENAME_PATTERNS:
if fnmatch.fnmatch(file_path_lower.name, cmake_pattern):
error_message = str(f'{file_identifier} FAILED - Autogenerated CMake file detected by name.')
if VERBOSE: print(error_message)
errors.append(error_message)
break
if fnmatch.fnmatch(file_path_lower.name, AZ_CODEGEN_FILENAME_PATTERN):
error_message = str(f'{file_identifier} FAILED - Autogenerated AzCodeGen file detected by name.')
if VERBOSE: print(error_message)
errors.append(error_message)
continue
if file_path_lower.name in OTHER_GENERATED_FILES:
error_message = str(f'{file_identifier} FAILED - Generated file (others).')
if VERBOSE: print(error_message)
errors.append(error_message)
continue
return (not errors)
def get_validator() -> Type[GeneratedFilesValidator]:
"""Returns the validator class for this module"""
return GeneratedFilesValidator
@@ -0,0 +1,80 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import os
import re
from typing import Type, List
import commit_validation.pal_allowedlist as pal_allowedlist
from commit_validation.commit_validation import Commit, CommitValidator, SOURCE_FILE_EXTENSIONS, VERBOSE
MERGE_TO_MARKER_REGEX = re.compile(r'^<<<<<<<')
MERGE_BASE_REGEX = re.compile(r'^\|\|\|\|\|\|\|')
MERGE_DIFF_REGEX = re.compile(r'^=======\n') # include newline as equals sign is a common visual separator
MERGE_FROM_MARKER_REGEX = re.compile(r'^>>>>>>>')
class GitConflictValidator(CommitValidator):
"""A file-level validator that for conflict markers"""
def __init__(self) -> None:
self.pal_allowedlist = pal_allowedlist.load()
def run(self, commit: Commit, errors: List[str]) -> bool:
for file_name in commit.get_files():
if os.path.splitext(file_name)[1].lower() not in SOURCE_FILE_EXTENSIONS:
if VERBOSE: print(f'{file_name}::{self.__class__.__name__} SKIPPED - File excluded based on extension.')
continue
if self.pal_allowedlist.is_match(file_name):
if VERBOSE: print(f'{file_name}::{self.__class__.__name__} SKIPPED - File excluded based on PAL allowedlist.')
continue
# we never want conflict markers to be added to our repository
# so we don't look at the file diffs, but the file contents.
with open(file_name, 'rt', encoding='utf8', errors='replace') as fh:
previous_line_context = ""
for line_number, line in enumerate(fh):
if MERGE_TO_MARKER_REGEX.search(line):
error_message = str(f'{file_name}::{self.__class__.__name__} FAILED - Source file contains git merge '
f'conflict start-marker on line {line_number + 1}:\n'
f' {previous_line_context}'
f'----> {line}')
if VERBOSE: print(error_message)
errors.append(error_message)
if MERGE_BASE_REGEX.search(line):
error_message = str(f'{file_name}::{self.__class__.__name__} FAILED - Source file contains git merge '
f'conflict diff3-marker on line {line_number + 1}:\n'
f' {previous_line_context}'
f'----> {line}')
if VERBOSE: print(error_message)
errors.append(error_message)
if MERGE_DIFF_REGEX.search(line):
error_message = str(f'{file_name}::{self.__class__.__name__} FAILED - Source file contains git merge '
f'conflict diff-marker on line {line_number + 1}:\n'
f' {previous_line_context}'
f'----> {line}')
if VERBOSE: print(error_message)
errors.append(error_message)
if MERGE_FROM_MARKER_REGEX.search(line):
error_message = str(f'{file_name}::{self.__class__.__name__} FAILED - Source file contains git merge '
f'conflict end-marker on line {line_number + 1}:\n'
f' {previous_line_context}'
f'----> {line}')
if VERBOSE: print(error_message)
errors.append(error_message)
previous_line_context = line
return (not errors)
def get_validator() -> Type[GitConflictValidator]:
"""Returns the validator class for this module"""
return GitConflictValidator
@@ -0,0 +1,74 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import fnmatch
import os
import re
from typing import Type, List
from commit_validation.commit_validation import Commit, CommitValidator, IsFileSkipped, SOURCE_AND_SCRIPT_FILE_EXTENSIONS, EXCLUDED_VALIDATION_PATTERNS, VERBOSE
_SINGLE_NEWLINE_ENDING_REGEX = re.compile(r'\n\Z', re.MULTILINE)
_MULTI_NEWLINE_ENDING_REGEX = re.compile(r'\n\s*\n\Z', re.MULTILINE)
_CRLF_REGEX = re.compile(r'^.*\r\n', re.MULTILINE)
_ONLY_LF_REGEX = re.compile(r'^[^\r]*\n', re.MULTILINE)
class NewlineValidator(CommitValidator):
"""A file-level validator that makes sure a file contains valid line-endings"""
def run(self, commit: Commit, errors: List[str]) -> bool:
for file_name in commit.get_files():
file_identifier = f"{file_name}::{self.__class__.__name__}"
for pattern in EXCLUDED_VALIDATION_PATTERNS:
if fnmatch.fnmatch(file_name, pattern):
if VERBOSE: print(f'{file_identifier} SKIPPED - Validation pattern excluded on path.')
break
else:
file_extension = os.path.splitext(file_name)[1].lower()
if IsFileSkipped(file_name):
if VERBOSE: print(f'{file_identifier} SKIPPED - File excluded based on extension.')
continue
# since this validator focuses on newlines throughout the file, not just in diffs
# we use the real file data instead of a diff
with open(file_name, 'rt', encoding='utf8', errors='replace') as fh:
lines = fh.read()
if not _SINGLE_NEWLINE_ENDING_REGEX.search(lines):
error_message = str(f'{file_identifier} FAILED - Source file does not end with a trailing newline.')
if VERBOSE: print(error_message)
errors.append(error_message)
if _MULTI_NEWLINE_ENDING_REGEX.search(lines):
error_message = str(f'{file_identifier} FAILED - Source file ends in multiple trailing newlines.')
if VERBOSE: print(error_message)
errors.append(error_message)
crlf_result = _CRLF_REGEX.search(lines)
only_lf_result = _ONLY_LF_REGEX.search(lines)
if crlf_result and only_lf_result:
error_message = str(f'{file_identifier} FAILED - Source file contains mixed line endings (\\r\\n and \\n)')
if VERBOSE: print(error_message)
errors.append(error_message)
if crlf_result:
error_message = str(f'{file_identifier} FAILED - Source file incorrectly contains Windows-style line endings'
f' (\\r\\n), when Unix-style line endings (\\n) were expected. Enable git option '
f'"core.autocrlf" to avoid this error.')
if VERBOSE: print(error_message)
errors.append(error_message)
return (not errors)
def get_validator() -> Type[NewlineValidator]:
"""Returns the validator class for this module"""
return NewlineValidator
@@ -0,0 +1,86 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import fnmatch
import os
import re
from typing import Type, List
import commit_validation.pal_allowedlist as pal_allowedlist
from commit_validation.commit_validation import Commit, CommitValidator, SOURCE_FILE_EXTENSIONS, VERBOSE
platform_macros = [
'ANDROID',
'APPLE',
'APPLETV',
'DARWIN',
'IOS',
'LINUX',
'LINUX64',
'MAC',
'PROVO',
'SALEM',
'WIN32',
'WIN32_LEAN_AND_MEAN',
'WIN64',
'XENIA',
'_WIN32',
'_WIN32_WINDOWS',
'_WIN32_WINNT',
'linux',
]
platform_macro_regex = re.compile(r'^\+\s*#.*?\b(' + str.join('|', platform_macros) + r')\b')
class PlatformMacroValidator(CommitValidator):
"""A file-level validator that makes sure a file does not contain certain platform macros"""
def __init__(self) -> None:
self.pal_allowedlist = pal_allowedlist.load()
def run(self, commit: Commit, errors: List[str]) -> bool:
for file_name in commit.get_files():
if os.path.splitext(file_name)[1].lower() not in SOURCE_FILE_EXTENSIONS:
if VERBOSE: print(f'{file_name}::{self.__class__.__name__} SKIPPED - File excluded based on extension.')
continue
if self.pal_allowedlist.is_match(file_name):
if VERBOSE: print(f'{file_name}::{self.__class__.__name__} SKIPPED - File excluded based on PAL allowedlist.')
continue
if fnmatch.fnmatch(file_name, '*/Platform/*'):
if VERBOSE: print(f'{file_name}::{self.__class__.__name__} SKIPPED - File excluded because it is in a "Platform" '
f'folder.')
continue
file_diff = commit.get_file_diff(file_name)
previous_line_context = ""
line_number = 1
for line in file_diff.splitlines():
# we only care about added lines.
if line.startswith('+'):
match = platform_macro_regex.search(line)
if match:
error_message = str(f'{file_name}:{line_number}::{self.__class__.__name__} FAILED - Source file contains a forbidden '
f'platform macro "{match.group(1)}" here:\n'
f' {previous_line_context}\n'
f'---> {line}')
if VERBOSE: print(error_message)
errors.append(error_message)
previous_line_context = line
line_number += 1
return (not errors)
def get_validator() -> Type[PlatformMacroValidator]:
"""Returns the validator class for this module"""
return PlatformMacroValidator
@@ -0,0 +1,51 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import fnmatch
import os.path
from typing import Type, List
from commit_validation.commit_validation import Commit, CommitValidator, SOURCE_FILE_EXTENSIONS, EXCLUDED_VALIDATION_PATTERNS, VERBOSE
class PragmaOptimizeValidator(CommitValidator):
"""A file-level validator that makes sure a file does not contain #pragma optimize directives"""
def run(self, commit: Commit, errors: List[str]) -> bool:
for file_name in commit.get_files():
for pattern in EXCLUDED_VALIDATION_PATTERNS:
if fnmatch.fnmatch(file_name, pattern):
if VERBOSE: print(f'{file_name}::{self.__class__.__name__} SKIPPED - Validation pattern excluded on path.')
break
else:
if os.path.splitext(file_name)[1].lower() not in SOURCE_FILE_EXTENSIONS:
if VERBOSE: print(f'{file_name}::{self.__class__.__name__} SKIPPED - File excluded based on extension.')
continue
file_diff = commit.get_file_diff(file_name)
previous_line_for_context = ""
for line in file_diff.splitlines():
# we only care about added lines in a diff
if line.startswith('+'):
if '#pragma optimize' in line:
error_message = str(f'{file_name}::{self.__class__.__name__} FAILED - Source file contains #pragma optimize!\n'
f' {previous_line_for_context}\n'
f'---> {line}')
if VERBOSE: print(error_message)
errors.append(error_message)
previous_line_for_context = line
return (not errors)
def get_validator() -> Type[PragmaOptimizeValidator]:
"""Returns the validator class for this module"""
return PragmaOptimizeValidator
@@ -0,0 +1,65 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import fnmatch
import os.path
from typing import Type, List
from commit_validation.commit_validation import Commit, CommitValidator, IsFileSkipped, SOURCE_AND_SCRIPT_FILE_EXTENSIONS, EXCLUDED_VALIDATION_PATTERNS, VERBOSE
class TabsValidator(CommitValidator):
"""A file-level validator that makes sure a file does not contain tabs"""
def run(self, commit: Commit, errors: List[str]) -> bool:
for file_name in commit.get_files():
if IsFileSkipped(file_name):
if VERBOSE: print(f'{file_name}::{self.__class__.__name__} SKIPPED TabsValidator - File excluded based on extension.')
continue
for pattern in EXCLUDED_VALIDATION_PATTERNS:
if fnmatch.fnmatch(file_name, pattern):
if VERBOSE: print(f'{file_name} SKIPPED TabsValidator - Validation pattern excluded on path.')
break
else:
tab_line_count = 0
file_diff = commit.get_file_diff(file_name)
# Usually, code either has a very small number of tabs in the file by accident,
# or the entire file is full of tabs.
# So we count the tabs, but we only print the first one in full.
first_tab_line_found = None
for line in file_diff.splitlines():
# we only care about added lines.
if line.startswith('+'):
if '\t' in line:
line = line.replace('\t','\\t') # make it obvious!
if not first_tab_line_found:
first_tab_line_found = str(
f' {previous_line_context}\n'
f'---> {line}\n')
tab_line_count = tab_line_count + 1
previous_line_context = line
if tab_line_count:
error_message = str(
f'{file_name}::{self.__class__.__name__} FAILED TabsValidator - {tab_line_count} tabs in this file\n'
f'First instance of a tab: \n'
f'{first_tab_line_found}')
errors.append(error_message)
if VERBOSE: print(error_message)
return (not errors)
def get_validator() -> Type[TabsValidator]:
"""Returns the validator class for this module"""
return TabsValidator
@@ -0,0 +1,148 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import argparse
import os
import re
copyrightre = re.compile(r'All or portions of this file Copyright \(c\) Amazon\.com')
copyright_text = """
All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
its licensors.
For complete copyright and license terms please see the LICENSE at the root of this
distribution (the "License"). All use of this software is governed by the License,
or, if provided, by the license below or the license accompanying this file. Do not
remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
""".splitlines(keepends=True)
class copyright_builder:
def get(self):
pass
def apply(self, file_contents):
new_file_contents = self.get()
new_file_contents += '\n' + file_contents
return new_file_contents
class copyright_builder_first_last_line(copyright_builder):
def __init__(self, comment_str):
self.comment_str = comment_str
def get(self):
copyright_str = self.comment_str + copyright_text[0]
for i in range(1, len(copyright_text)-1):
copyright_str += copyright_text[i]
copyright_str += self.comment_str + copyright_text[len(copyright_text)-1]
return copyright_str
class copyright_builder_each_line(copyright_builder):
def __init__(self, comment_str):
self.comment_str = comment_str
def get(self):
copyright_str = ''
for i in range(0, len(copyright_text)):
copyright_str += self.comment_str + copyright_text[i]
return copyright_str
class copyright_builder_each_line_bat(copyright_builder_each_line):
def __init__(self, comment_str):
self.comment_str = comment_str
self.echo_off_re = re.compile('@echo off\n', re.IGNORECASE)
def apply(self, file_contents):
re_result = self.echo_off_re.search(file_contents)
if re_result:
return self.echo_off_re.sub(re_result.group(0) + '\n' + self.get() + '\n', file_contents)
return super().apply(file_contents)
class copyright_builder_each_line_sh(copyright_builder_each_line):
def __init__(self, comment_str):
self.comment_str = comment_str
self.bin_str = [
'#!/bin/sh\n',
'#!/bin/bash\n'
]
def apply(self, file_contents):
for bin_str in self.bin_str:
if bin_str in file_contents:
return file_contents.replace(bin_str, bin_str + '\n' + self.get() + '\n')
return super().apply(file_contents)
class copyright_builder_c(copyright_builder):
def get(self):
copyright_str = '/*' + copyright_text[0]
for i in range(1, len(copyright_text)-1):
copyright_str += '*' + copyright_text[i]
copyright_str += '*\n'
copyright_str += '*/' + copyright_text[len(copyright_text)-1]
return copyright_str
copyright_comment_by_extension = {
'lua': copyright_builder_each_line('-- '),
'py': copyright_builder_first_last_line('"""'),
'bat': copyright_builder_each_line_bat('REM '),
'cmd': copyright_builder_each_line_bat('REM '),
'sh': copyright_builder_each_line_sh('# '),
'cs': copyright_builder_c(),
}
def fixCopyright(input_file):
try:
extension = os.path.splitext(input_file)[1].replace('.','')
if not extension in copyright_comment_by_extension:
print(f'[WARN] Extension {extension} not handled')
else:
copyright_builder = copyright_comment_by_extension[extension]
with open(input_file, 'r') as source_file:
fileContents = source_file.read()
reResult = copyrightre.search(fileContents)
if reResult:
# Copyright found, skip
return
newFileContents = copyright_builder.apply(fileContents)
with open(input_file, 'w') as destination_file:
destination_file.write(newFileContents)
print(f'[INFO] Patched {input_file}')
except (IOError, UnicodeDecodeError) as err:
print('[ERROR] reading {}: {}'.format(input_file, err))
return
def main():
"""script main function"""
parser = argparse.ArgumentParser(description='This script fixes copyright headers',
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('file_or_dir', type=str, nargs='+',
help='list of files or directories to search within for files to fix up copyright headers')
args = parser.parse_args()
for input_file in args.file_or_dir:
if os.path.isdir(input_file):
for dp, dn, filenames in os.walk(input_file):
for f in filenames:
fixCopyright(os.path.join(dp, f))
else:
fixCopyright(input_file)
#entrypoint
if __name__ == '__main__':
main()
+58
View File
@@ -0,0 +1,58 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import argparse
import fnmatch
import os
handled_file_patterns = [
'*.c', '*.cc', '*.cpp', '*.cxx', '*.h', '*.hpp', '*.hxx', '*.inl', '*.m', '*.mm', '*.cs', '*.java',
'*.py', '*.lua', '*.bat', '*.cmd', '*.sh', '*.js',
'*.cmake', 'CMakeLists.txt'
]
def fixTabs(input_file):
try:
basename = os.path.basename(input_file)
for pattern in handled_file_patterns:
if fnmatch.fnmatch(basename, pattern):
with open(input_file, 'r') as source_file:
fileContents = source_file.read()
if '\t' in fileContents:
newFileContents = fileContents.replace('\t', ' ')
with open(input_file, 'w') as destination_file:
destination_file.write(newFileContents)
print(f'[INFO] Patched {input_file}')
break
except (IOError, UnicodeDecodeError) as err:
print('[ERROR] reading {}: {}'.format(input_file, err))
return
def main():
"""script main function"""
parser = argparse.ArgumentParser(description='This script replaces tabs with spaces',
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('file_or_dir', type=str, nargs='+',
help='list of files or directories to search within for files to fix up tabs')
args = parser.parse_args()
for input_file in args.file_or_dir:
if os.path.isdir(input_file):
for dp, dn, filenames in os.walk(input_file):
for f in filenames:
fixTabs(os.path.join(dp, f))
else:
fixTabs(input_file)
#entrypoint
if __name__ == '__main__':
main()
@@ -0,0 +1,117 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import argparse
import os
import sys
from typing import Dict, List
import difflib
from commit_validation.commit_validation import Commit, validate_commit
import git
class GitChange(Commit):
"""An implementation of the :class:`Commit` interface for accessing details about a git change"""
def __init__(self, source: str = None, target: str = 'origin/main') -> None:
"""Creates a new instance of :class:`GitChange`
:param source: source of the change, e.g. commit hash or branch
:param target: target of the change, e.g. main branch
"""
root_path = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))
self.repo = git.Repo()
self.git_root_path = self.repo.git.rev_parse("--show-toplevel")
if source:
git.cmd.Git().fetch('origin', source) # So we can compare it
self.source_commit = self.repo.commit(target)
else:
self.source_commit = self.repo.commit()
if 'origin' in target:
origin_target = target.replace('origin/','')
git.cmd.Git().fetch('origin', origin_target) # So we can compare it
self.target_commit = self.repo.commit(target)
# We only want to run the verification on the changes introduced by the
# branch being merged in. Find the merge base (the common ancestor) of
# the two commits, and then get the diff from merge_base..target_commit
self.merge_base = self.repo.merge_base(self.source_commit, self.target_commit)
if not len(self.merge_base) == 1:
raise RuntimeError(f"Cannot find the merge base of {self.source_commit} and {self.target_commit}")
self.merge_base = self.merge_base[0]
self.diff_index = self.merge_base.diff(self.source_commit)
print(f"Running validation from '{source}' ({self.source_commit}) to '{target}' ({self.target_commit}) using baseline {self.merge_base}")
# Cache the file lists since they are requested by each validator
self.files_list: List[str] = []
self.removed_files_list: List[str] = []
for diff_item in self.diff_index:
# 'A' for added paths
# 'C' for changed paths
# 'R' for renamed paths
# 'M' for paths with modified data
# 'T' for changed in the type paths
# 'D' for deleted paths
# 'R' for renamed paths
if diff_item.change_type in ('A', 'C', 'R', 'M', 'T'):
self.files_list.append(os.path.abspath(os.path.join(self.git_root_path, diff_item.b_path)))
if diff_item.change_type in ('D', 'R'):
self.removed_files_list.append(os.path.abspath(os.path.join(self.git_root_path, diff_item.a_path)))
def get_files(self) -> List[str]:
"""Returns a list of local files added/modified by the commit"""
return self.files_list
def get_removed_files(self) -> List[str]:
"""Returns a list of local files removed by the commit"""
return self.removed_files_list
def get_file_diff(self, str) -> str:
"""
Given a file name, returns a string in unified diff format
that represents the changes made to that file for this commit.
Most validators will only pay attention to added lines (with + in front)
"""
diff = self.repo.git.diff(self.merge_base, self.source_commit, str)
return diff
def get_description(self) -> str:
"""Returns the description of the commit"""
return self.target_commit.message
def get_author(self) -> str:
"""Returns the author of the commit"""
return self.target_commit.author
def init_parser():
"""Prepares the command line parser"""
parser = argparse.ArgumentParser()
parser.add_argument('--source', default=None, help='Change source (e.g. commit hash or branch), defaults to active branch')
parser.add_argument('--target', default='origin/main', help='Change target, defaults to "origin/main"')
return parser
def main():
parser = init_parser()
args = parser.parse_args()
change = GitChange(source=args.source, target=args.target)
if not validate_commit(commit=change, ignore_validators=["NewlineValidator", "WhitespaceValidator"]):
sys.exit(1)
sys.exit(0)
if __name__ == '__main__':
main()
+53
View File
@@ -0,0 +1,53 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import marshal
from subprocess import Popen, PIPE, check_output
from typing import List
def run_p4_command(command: str, client: str = None, raw_output: bool = False) -> List[dict]:
"""Executes a Perforce command by calling p4 in a subprocess
:param command: The command to run (not including p4, e.g., not ['p4', 'opened'] but ['opened'])
:param client: The Perforce client to use when running the command
:param raw_output: if True, return the raw output from p4 instead of using the python dict.
:return: The list of results from the command where each result is a dict (refer to p4's global -G option)
if 'raw_output' is true, return will be verbatim string from p4.
"""
results = []
base_command = ['p4']
if not raw_output:
base_command.append('-G')
if client is not None:
base_command += ['-c', client]
if raw_output:
return check_output(base_command + command.split()).decode('utf8')
pipe = Popen(base_command + command.split(), stdout=PIPE).stdout
try:
while True:
result = marshal.load(pipe)
results.append(result)
except EOFError:
pass
finally:
pipe.close()
decoded_results = [{k.decode(): v.decode() if isinstance(v, bytes) else str(v) for k, v in r.items()} for r in results]
if decoded_results and decoded_results[0]['code'] == 'error':
command_was = ' '.join(base_command + command.split())
error_was = decoded_results[0]['data']
raise RuntimeError(f'P4 Command failed: "{command_was}"\nERROR FROM P4: {error_was}\n')
return decoded_results
@@ -0,0 +1,121 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import argparse
import os
import sys
from typing import Dict, List
import difflib
from commit_validation.commit_validation import Commit, validate_commit
from p4 import run_p4_command
class P4Changelist(Commit):
"""An implementation of the :class:`Commit` interface for accessing details about a Perforce changelist"""
def __init__(self, client: str = None, change: str = 'default') -> None:
"""Creates a new instance of :class:`P4Changelist`
:param client: the Perforce client
:param change: the Perforce changelist
"""
self.client = client
self.change = change
self.files: List[str] = []
self.removed_files: List[str] = []
self.file_diffs: Dict[str, str] = {}
self._load_files()
def _load_files(self):
self.files = []
self.removed_files = []
self.added_files = []
client_spec = run_p4_command(f'client -o', client=self.client)[0]
files = run_p4_command(f'opened -c {self.change}', client=self.client)
for f in files:
file_path = os.path.abspath(f['clientFile'].replace(f'//{client_spec["Client"]}', client_spec['Root']))
if 'delete' in f['action']:
self.removed_files.append(file_path)
elif 'add' in f['action']:
self.added_files.append(file_path)
elif 'branch' in f['action']:
self.added_files.append(file_path)
else:
self.files.append(file_path)
def get_file_diff(self, file) -> str:
if file in self.file_diffs: # allow caching
return self.file_diffs[file]
if file in self.added_files: # added files return the entire file as a diff.
with open(file, "rt", encoding='utf8', errors='replace') as opened_file:
data = opened_file.readlines()
diffs = difflib.unified_diff([], data, fromfile=file, tofile=file)
diff_being_built = ''.join(diffs)
self.file_diffs[file] = diff_being_built
return diff_being_built
if file not in self.files:
raise RuntimeError(f"Cannot calculate a diff for a file not in the changelist: {file}")
try:
result = run_p4_command(f'diff -du {file}', client=self.client)
if len(result) > 1:
diff = result[1]['data'] # p4 returns a normal code but with no result if theres no diff
else:
diff = ''
print(f'Warning: File being committed contains no changes {file}')
# note that the p4 command handles the data and errors internally, no need to check.
self.file_diffs[file] = diff
return diff
except RuntimeError as e:
print(f'error during p4 operation, unable to get a diff: {e}')
return ''
def get_files(self) -> List[str]:
# this is just files relevant to the operation
return self.files + self.added_files
def get_removed_files(self) -> List[str]:
return self.removed_files
def get_description(self) -> str:
raise NotImplementedError
def get_author(self) -> str:
raise NotImplementedError
def init_parser():
"""Prepares the command line parser"""
parser = argparse.ArgumentParser()
parser.add_argument('--client', help='Perforce client')
parser.add_argument('--change', default='default', help='Perforce changelist')
return parser
def main():
parser = init_parser()
args = parser.parse_args()
change = P4Changelist(client=args.client, change=args.change)
if not validate_commit(commit=change, ignore_validators=["NewlineValidator", "WhitespaceValidator"]):
sys.exit(1)
sys.exit(0)
if __name__ == '__main__':
main()
@@ -0,0 +1,120 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import argparse
import os
import sys
from typing import Dict, List
from commit_validation.commit_validation import Commit, validate_commit
from p4 import run_p4_command
class P4SubmittedChangelists(Commit):
"""An implementation of the :class:`Commit` interface for accessing details about Perforce submitted changelists"""
def __init__(self, from_change: str, to_change: str, client: str = None) -> None:
"""Creates a new instance of :class:`P4SubmittedChangelists`
:param from_change: The oldest Perforce changelist to include
:param to_change: The newest Perforce changelist to include
:param client: The Perforce client
"""
self.from_change = from_change
self.to_change = to_change
self.client = client
self.files: List[str] = []
self.removed_files: List[str] = []
self.file_diffs: Dict[str, str] = {}
self._load_files()
def _load_files(self):
self.files = []
self.removed_files = []
depot_root = None
client_root = run_p4_command(f'client -o', client=self.client)[0]['Root']
files = run_p4_command(f'files {client_root}{os.sep}...@{self.from_change},{self.to_change}',
client=self.client)
for f in files:
# The following code optimizes for the specific use case where all files are mapped the same way on the
# client. The reason for this is to avoid calling 'p4 where' on thousands of files. So instead, it only calls
# 'p4 where' on the first file and applies the same mapping to the rest of the files.
if depot_root is None:
file_path = run_p4_command(f'where {f["depotFile"]}', client=self.client)[0]['path']
relative_path = file_path.replace(client_root, '')
relative_path = relative_path.replace('\\', '/')
depot_root = f['depotFile'].replace(relative_path, '')
else:
file_path = os.path.abspath(f['depotFile'].replace(depot_root, client_root))
if 'delete' in f['action']:
self.removed_files.append(file_path)
else:
self.files.append(file_path)
pass
def get_file_diff(self, file) -> str:
if file in self.file_diffs:
return self.file_diffs[file]
if file not in self.files:
raise RuntimeError(f"Cannot compute diff for file not in change set: {file}")
# the diff2 command does not behave like normal diff command, in that it only
# returns the actual diff if you do not use '-G' mode. So we ask it for the raw output:
diff = run_p4_command(f'diff2 -du {file}@{self.from_change} {file}@{self.to_change}', client=self.client, raw_output=True)
self.file_diffs[file] = diff
# note that if you feed the same changelist as the 'before' and 'after' on the command line
# you will get no diffs, because there is no difference between a changelist and itself.
# In addition, if you are diffing across changelists, but the file only existed
# in one changelist, it will also not indicate a diff (due to how the p4 diff2 command operates)
# This means that this validator is blind to branch integrates. This is not the case
# for the 'live' or 'shelved' validator as those files show up as additions.
return diff
def get_files(self) -> List[str]:
return self.files
def get_removed_files(self) -> List[str]:
return self.removed_files
def get_description(self) -> str:
raise NotImplementedError
def get_author(self) -> str:
raise NotImplementedError
def init_parser():
"""Prepares the command line parser"""
parser = argparse.ArgumentParser()
parser.add_argument('from_change', help='The oldest changelist to include')
parser.add_argument('to_change', help='The newest changelist to include')
parser.add_argument('--client', help='The Perforce client')
return parser
def main():
parser = init_parser()
args = parser.parse_args()
change = P4SubmittedChangelists(client=args.client, from_change=args.from_change, to_change=args.to_change)
if not validate_commit(commit=change, ignore_validators=["NewlineValidator", "WhitespaceValidator"]):
sys.exit(1)
sys.exit(0)
if __name__ == '__main__':
main()
@@ -0,0 +1,111 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import argparse
import fnmatch
import os
import sys
from typing import Dict, List
import difflib
from commit_validation.commit_validation import Commit, validate_commit, IsFileSkipped, SOURCE_AND_SCRIPT_FILE_EXTENSIONS, EXCLUDED_VALIDATION_PATTERNS
class FolderBased(Commit):
"""An implementation of the :class:`Commit` which populates it from files and folders on disk"""
def __init__(self, item_path: str = '.') -> None:
"""Creates a new instance of :class:`FolderBased`
:param item_path - The file name or the folder to search (recursively)
"""
self.item_path = os.path.abspath(item_path)
self.files: List[str] = []
self.removed_files: List[str] = []
self.file_diffs: Dict[str, str] = {}
self._load_files()
def _load_files(self):
self.files = []
self.removed_files = []
# if its a file, we just add that file.
# if its a folder, we add it recursively
if os.path.isfile(self.item_path):
self.files.append(self.item_path)
else:
engine_root_path = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))
for root, _, file_names in os.walk(self.item_path):
for file_name in file_names:
file_to_scan = os.path.abspath(os.path.join(root, file_name))
skip = False
for excluded_folder in EXCLUDED_VALIDATION_PATTERNS:
matcher = os.path.abspath(os.path.join(engine_root_path, excluded_folder, '*'))
if fnmatch.fnmatch(file_to_scan, matcher):
skip = True
break
if not skip:
self.files.append(file_to_scan)
def get_file_diff(self, file) -> str:
if file in self.file_diffs:
return self.file_diffs[file]
# we count the entire file as changed in this mode
# we do not include diffs for things that are not source files
# this also prevents us trying to decode binary files as utf8
if IsFileSkipped(file):
return ""
diff_being_built = ""
# We simulate every line having been changed by making a blank file
# being the 'from' and the whole file being the 'to' part of the diff
with open(file, "rt", encoding='utf8', errors='replace') as opened_file:
data = opened_file.readlines()
diffs = difflib.unified_diff([], data, fromfile=file, tofile=file)
diff_being_built = ''.join(diffs)
self.file_diffs[file] = diff_being_built
return diff_being_built
def get_files(self) -> List[str]:
return self.files
def get_removed_files(self) -> List[str]:
return self.removed_files
def get_description(self) -> str:
raise NotImplementedError
def get_author(self) -> str:
raise NotImplementedError
def init_parser():
"""Prepares the command line parser"""
parser = argparse.ArgumentParser()
parser.add_argument('--path', default='.', help='Filename of single file, or a folder path to scan recursively')
return parser
def main():
parser = init_parser()
args = parser.parse_args()
change = FolderBased(item_path = args.path)
validators_to_ignore = [
"NewlineValidator",
"WhitespaceValidator"
]
if not validate_commit(commit=change, ignore_validators = validators_to_ignore):
sys.exit(1)
sys.exit(0)
if __name__ == '__main__':
main()