Initial commit

scripts/detect_file_changes/CMakeLists.txt

@@ -0,0 +1,16 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+ly_add_pytest(
+    NAME test_detect_file_changes
+    PATH ${CMAKE_CURRENT_LIST_DIR}
+)
+

scripts/detect_file_changes/compare_snapshots.py

@@ -0,0 +1,59 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import argparse
+import pickle
+import sys
+
+"""This is a command-line entry point which, when given two snapshots created using make_snapshot.py
+compares them and reports on the differences.  It returns 0 if and only if there are no differences.
+
+To use it in a scripting environment instead of a CLI, invoke do_compare(filename1, filename2)
+Or do it manually using FolderSnapshot.CompareSnapshots and then enumerate_changes
+"""
+
+ignore_patterns = []
+
+from snapshot_folder.snapshot_folder import FolderSnapshot, SnapshotComparison
+
+def do_compare(filename1, filename2):
+    """Given two filenames, returns the diffs as a list of tuples [(type, file)]"""
+    snap1 = pickle.load(open(filename1, 'rb'))
+    snap2 = pickle.load(open(filename2, 'rb'))
+
+    comparison = FolderSnapshot.CompareSnapshots(snap1, snap2)
+
+    changes = [change_entry for change_entry in comparison.enumerate_changes()]
+    return changes
+
+def init_parser():
+    """Prepares the command line parser"""
+    parser = argparse.ArgumentParser()
+    parser.description = "Compares two snapshots previously saved, outputs the changes.  Exit code is 0 if no diff, 1 otherwise."
+    parser.add_argument('first', default='.', help='first file to use')
+    parser.add_argument('second', default='.', help='second file to use')
+    return parser
+
+def main():
+    parser = init_parser()
+    args = parser.parse_args()
+    changes = do_compare(args.first, args.second)
+    
+    for change in changes:
+        print(f"Change detected: {change}")
+    
+    if changes:
+        return 1
+    
+    return 0
+
+if __name__ == '__main__':
+    sys.exit(main())

scripts/detect_file_changes/make_snapshot.py

@@ -0,0 +1,50 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import argparse
+import pickle
+import sys
+
+"""This is a command line entry point that, given an out file name, and a folder to scan
+generates a file which contains the current snapshot of the files and folders in that folder.
+It is to be used later by compare_snapshot.py
+
+If you want to use this in a scripting environment instead of a CLI, use the dump_snapshot 
+function or use FolderSnapshot.CreateSnapshot directly.
+"""
+
+default_ignore_patterns = ['*.pyc', '__pycache__', '*.snapshot', 'Cache', 'build_*', 'build' ]
+
+from snapshot_folder.snapshot_folder import FolderSnapshot, SnapshotComparison
+
+def dump_snapshot(folder_to_scan, filename, ignore_patterns):
+    """Workhorse function of this module.  Saves the snapshot to the given file"""
+    snap = FolderSnapshot.CreateSnapshot(folder_to_scan, ignore_patterns=ignore_patterns)
+    pickle.dump(snap, open(filename, 'wb'))
+
+def init_parser():
+    """Prepares the command line parser"""
+    parser = argparse.ArgumentParser()
+    parser.description = "Takes a snapshot of the current files and folders into a given file for later comparison"
+    parser.add_argument('path_to_check', default='.', help='Path To start iterating at')
+    parser.add_argument('--out', required=True, help='Path to output to')
+    parser.add_argument('--ignore', action='append', nargs='+', default=default_ignore_patterns)
+    return parser
+
+def main():
+    """ Entry point to use if you want to supply args on the command line"""
+    parser = init_parser()
+    args = parser.parse_args()
+    print(f"Snapshotting: {args.path_to_check} into {args.out} with ignore {args.ignore}")
+    return dump_snapshot(args.path_to_check, args.out, args.ignore)
+
+if __name__ == '__main__':
+    main()

scripts/detect_file_changes/snapshot_folder/__init__.py

@@ -0,0 +1,10 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#

scripts/detect_file_changes/snapshot_folder/snapshot_folder.py

@@ -0,0 +1,118 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import os
+import fnmatch
+import pathlib
+
+"""This module contains FolderSnapshot, a class which can create and compare 'snapshots'
+of folders (The snapshots just store the modtimes / existence of files and folders), and
+also can compare two snapshots to return a SnapshotComparison which represents the diffs
+"""
+
+class SnapshotComparison:
+    """ This class just holds the diffs calculated between two folder trees."""
+    def __init__(self):
+        self.deleted_files = []
+        self.added_files = []
+        self.changed_files = []
+        self.dirs_added = []
+        self.dirs_removed = []
+    
+    def any_changed(self):
+        """Returns True if any changes were detected"""
+        return self.deleted_files or self.added_files or self.changed_files or self.dirs_added or self.dirs_removed
+
+    def enumerate_changes(self):
+        """Enumerates changes, yielding each as a pair of (string, string), that is, (type of change, filename)"""
+        for file_entry in self.deleted_files:
+            yield ("DELETED", file_entry)
+        for file_entry in self.added_files:
+            yield ("ADDED", file_entry)
+        for file_entry in self.changed_files:
+            yield ("CHANGED", file_entry)
+        for dir_entry in self.dirs_added:
+            yield ("FOLDER_ADDED", dir_entry)
+        for dir_entry in self.dirs_removed:
+            yield ("FOLDER_DELETED", dir_entry)
+
+class FolderSnapshot:
+    """ This class stores a snapshot of a folder state and has utility functions to compare snapshots"""
+    def __init__(self):
+        self.file_modtimes = {}
+        self.folder_paths = []
+        pass
+
+    @staticmethod
+    def _matches_ignore_pattern(in_string, ignore_patterns):
+        for pattern in ignore_patterns:
+            if fnmatch.fnmatch(in_string, pattern):
+                return True
+            # we also care if the last part is in the patterns
+            # this is to cover situatiosn where the pattern has 'build' in it as opposed to *build*
+            # and the name of the file is literally 'build'
+            _, filepart = os.path.split(in_string)
+            if fnmatch.fnmatch(filepart, pattern):
+                return True
+
+        return False
+    
+    @staticmethod
+    def CreateSnapshot(root_folder, ignore_patterns):
+        """Create a new FolderSnapshot based on a root folder and ignore patterns."""
+        folder_snap = FolderSnapshot()
+        ignored_folders = []
+        for root, dir_names, file_names in os.walk(root_folder, followlinks=False):
+            for dir_name in dir_names:
+                fullpath = os.path.normpath(os.path.join(root, dir_name)).replace('\\', '/')
+                if FolderSnapshot._matches_ignore_pattern(fullpath, ignore_patterns):
+                    ignored_folders.append(fullpath)
+                    continue
+                if os.path.dirname(fullpath) in ignored_folders:
+                    # we want to emulate not 'walking' down any folders themselves that have been omitted:
+                    ignored_folders.append(fullpath)
+                    continue
+                folder_snap.folder_paths.append(fullpath)
+            
+            for file_name in file_names:
+                fullpath = os.path.normpath(os.path.join(root, file_name)).replace('\\', '/')
+                if FolderSnapshot._matches_ignore_pattern(fullpath, ignore_patterns):
+                    continue
+                if os.path.dirname(fullpath) in ignored_folders:
+                    # we want to emulate not 'walking' down any folders themselves that have been omitted:
+                    continue
+                folder_snap.file_modtimes[fullpath] = os.stat(fullpath).st_mtime
+        
+        return folder_snap        
+
+    @staticmethod
+    def CompareSnapshots(before, after):
+        """Return a SnapshotComparison representing the difference between two FolderShapshot objects"""
+        comparison = SnapshotComparison()
+        for file_name in before.file_modtimes.keys():
+            if file_name not in after.file_modtimes:
+                comparison.deleted_files.append(file_name)
+            else:
+                if before.file_modtimes[file_name] != after.file_modtimes[file_name]:
+                    comparison.changed_files.append(file_name)
+        
+        for file_name in after.file_modtimes.keys():
+            if file_name not in before.file_modtimes:
+                comparison.added_files.append(file_name)
+
+        for folder_path in before.folder_paths:
+            if folder_path not in after.folder_paths:
+                comparison.dirs_removed.append(folder_path)
+        
+        for folder_path in after.folder_paths:
+            if folder_path not in before.folder_paths:
+                comparison.dirs_added.append(folder_path)
+        return comparison

scripts/detect_file_changes/snapshot_folder/tests/__init__.py

@@ -0,0 +1,10 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#

scripts/detect_file_changes/snapshot_folder/tests/test_snapshots.py

@@ -0,0 +1,189 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import unittest
+from unittest.mock import patch
+from snapshot_folder.snapshot_folder import FolderSnapshot
+
+class empty_object():
+    pass
+
+def fakestat(file_path):
+    f = empty_object()
+    f.st_mtime = 12345
+    return f
+
+@patch('os.stat', side_effect=fakestat)
+@patch('os.walk')
+def test_CreateSnapshot_sanity(mock_os_walk, mock_os_stat):
+    mock_os_walk.return_value = [
+        # this mocks os.walk, which always returns a tuple of (root name, folder names, file names)
+        ( '',  # root name
+          ['subfolder1', 'subfolder2', 'subfolder3'], # folders in here
+          ['file1.cpp'], # files in here
+        ),
+        ( 'subfolder1', 
+          [],
+          ['file1.cpp', 'file2.cpp'], # file1 is same name as above, but different folder name!
+        ),
+        ( 'subfolder2', 
+          [''],
+          [], # empty folders still get tracked
+        ),
+        ( 'subfolder3', 
+          ['subfolder4'], # folders only containing folders
+          [],
+        ),
+        ( 'subfolder3/subfolder4', 
+          [],
+          ['file4.cpp'],
+        )
+    ]
+    snap = FolderSnapshot.CreateSnapshot('.', ignore_patterns=[])
+    
+    assert 'subfolder1' in snap.folder_paths
+    assert 'subfolder2' in snap.folder_paths
+    assert 'subfolder3' in snap.folder_paths
+    assert 'subfolder3/subfolder4' in snap.folder_paths
+    assert 'file1.cpp' in snap.file_modtimes
+    assert 'subfolder1/file1.cpp' in snap.file_modtimes
+    assert 'subfolder1/file2.cpp' in snap.file_modtimes
+    assert 'subfolder3/subfolder4/file4.cpp' in snap.file_modtimes
+    
+@patch('os.stat', side_effect=fakestat)
+@patch('os.walk')
+def test_CreateSnapshot_obeys_exclusions(mock_os_walk, mock_os_stat):
+    mock_os_walk.return_value = [
+        (   '.', 
+            ['sub_buildfolder', 'build', 'build_mac', 'normal_subfolder'], # sneaky trap, sub_buildfolder should not be ignored
+            ['file.tif', 'file_not_a_tif.bmp'],
+        ),
+        (   'sub_buildfolder',  # should not be ignored, its not a match to build_*
+            [],
+            ['file2.cpp', 'file2.tif'],
+        ),
+        (   'build_mac', 
+            ['normalfolder'], # does not have build in its name but should still be ignored because parent is
+            ['file3.cpp'],  # even though it doesnt match a rule itself, it should be omitted since its in build
+        ),
+        (   'build_mac/normalfolder', 
+            [],
+            ['file4.cpp'],  # even though it doesnt match a rule itself, it should be omitted since its in build
+        ),
+        (   'build', 
+            [],
+            ['file4.cpp'],  # even though it doesnt match a rule itself, it should be omitted since its in build
+        ),
+        (   'normal_subfolder', 
+            ['build'], # a matching folder in a subfolder
+            [],
+        ),
+        (   'normal_subfolder/build', 
+            ['file.txt'], # a matching folder in a subfolder
+            [],
+        )
+    ]
+    snap = FolderSnapshot.CreateSnapshot('.', ignore_patterns=['*.tif', 'build', 'build_*'])
+    
+    assert 'sub_buildfolder' in snap.folder_paths
+    assert 'file_not_a_tif.bmp' in snap.file_modtimes
+    assert 'sub_buildfolder/file2.cpp' in snap.file_modtimes
+    assert 'normal_subfolder' in snap.folder_paths
+
+    assert 'build' not in snap.folder_paths
+    assert 'build_mac' not in snap.folder_paths
+    assert 'normal_subfolder/build' not in snap.folder_paths
+    assert 'build_mac/normalfolder' not in snap.folder_paths
+
+    assert 'file1.tif' not in snap.file_modtimes
+    assert 'sub_buildfolder/file2.tif' not in snap.file_modtimes
+    assert 'build/file3.cpp' not in snap.file_modtimes
+    assert 'build_mac/file4.cpp' not in snap.file_modtimes
+    assert 'build_mac/normalfolder/file4.cpp' not in snap.file_modtimes
+    assert 'normal_subfolder/build/file.txt' not in snap.file_modtimes
+    
+
+
+def test_CompareSnapshots_identical_snapshots_nodiffs():
+    # emulate identical snapshots
+    snap1 = FolderSnapshot()
+    snap1.folder_paths = ['myfolder1', 'myfolder2']
+    snap1.file_modtimes = {
+        'rootfile.txt' : 12345,
+        'myfolder1/file.txt' : 12345
+    }
+    
+    snap2 = FolderSnapshot()
+    snap2.folder_paths = ['myfolder1', 'myfolder2']
+    snap2.file_modtimes = {
+        'rootfile.txt' : 12345,
+        'myfolder1/file.txt' : 12345
+    }
+
+    changes = FolderSnapshot.CompareSnapshots(snap1, snap2)
+    assert not changes.any_changed()
+    changed_things = [c for c in changes.enumerate_changes()]
+    assert not changed_things
+
+def test_CompareSnapshots_two_of_each_kind_of_change():
+    # emulate identical snapshots
+    snap1 = FolderSnapshot()
+    snap1.folder_paths = ['myfolder1', 'myfolder2', 'myfolder3', 'myfolder4']
+    snap1.file_modtimes = {
+        'rootfile.txt' : 12345,
+        'rootfile2.txt' : 12345,
+        'rootfile3.txt' : 12345,
+        'myfolder1/file.txt' : 12345,
+        'myfolder2/file2.txt' : 12345,
+        'myfolder2/file3.txt' : 12345,
+    }
+    
+    snap2 = FolderSnapshot()
+    # myfolder1 deleted
+    # myfolder2 unchanged
+    # myfolder3 unchanged
+    # myfolder4 deleted
+    # myfolder5 as well as its subfolder, myfolder6 added
+    snap2.folder_paths = ['myfolder3', 'myfolder2', 'myfolder5', 'myfolder5/myfolder6']
+    snap2.file_modtimes = {
+        'rootfile2.txt' : 12345,    # unchanged, in root
+        'rootfile3.txt' : 33333,    # modified
+        'rootfile4.txt' : 12345,    # a new file
+        # myfolder1 is deleted, so myfolder1/file.txt should show up as deleted
+        'myfolder2/file2.txt' : 12345, # unchanged, but in subfolder
+        'myfolder2/file3.txt' : 33333, # modified in subfolder
+        'myfolder5/file4.txt' : 12345, # a new file in a new folder
+    }
+
+    changes = FolderSnapshot.CompareSnapshots(snap1, snap2)
+    assert changes.any_changed()
+    changed_things = [c for c in changes.enumerate_changes()]
+    # folders
+
+    for expected_element in [
+            ('FOLDER_ADDED', 'myfolder5'),
+            ('FOLDER_ADDED', 'myfolder5/myfolder6'),
+            ('FOLDER_DELETED', 'myfolder1'),
+            ('FOLDER_DELETED', 'myfolder4'),
+            ('DELETED', 'rootfile.txt'),
+            ('DELETED', 'myfolder1/file.txt'),
+            ('ADDED',   'rootfile4.txt'),
+            ('ADDED',   'myfolder5/file4.txt'),
+            ('CHANGED',   'rootfile3.txt'),
+            ('CHANGED',   'myfolder2/file3.txt')
+    ]:
+        assert expected_element in changed_things
+        changed_things.remove(expected_element)
+
+    # every time we did an assert above, we removed the matching element
+    # from the change list.  This means that the change list should now be empty 
+    # since everything that changed has been accounted for
+    assert not changed_things, f"Unexpected change {changed_things}"