birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update to licenses/notices script (#5214)

Browse Source 
This updates the licenses script to pull in all PackageInfo.json files with a specific argument, then follows each license file defined and writes the contents to a file. In this mode, if a packageinfo file is found, it will only grab the license file path defined within.

Also has the following features:

* Generalizes the function and variable names for non-license specific references
* Sorts os.walk to maintain consistent ordering
* Uses an ordered dict for the output, also to maintain ordering if using Python below 3.7
* Adds an additional json config file to have specific exclusion rules for 3p packages
* Adds a package creation function and config file entry
* Allow multipath scans, optional use of gitignore, merged license file scan

Signed-off-by: Mike Chang <changml@amazon.com>
This commit is contained in:
Mike Chang
2021-11-05 14:04:05 -07:00
committed by GitHub
parent eecf6ab920
commit 46f4935ee4
2 changed files with 103 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/license_scanner/license_scanner.py
+100 -40
View File
@@ -6,6 +6,7 @@
#
import argparse
from collections import OrderedDict
import fnmatch
import json
import os
@@ -24,15 +25,19 @@ class LicenseScanner:
"""
DEFAULT_CONFIG_FILE = 'scanner_config.json'
DEFAULT_EXCLUDE_FILE = '.gitignore'
DEFAULT_PACKAGE_INFO_FILE = 'PackageInfo.json'
def __init__(self, config_file=None):
self.config_file = config_file
self.config_data = self._load_config()
self.license_regex = self._load_license_regex()
self.file_regex = self._load_file_regex(self.config_data['license_patterns'])
self.package_info = self._load_file_regex(self.config_data['package_patterns'])
self.excluded_directories = self._load_file_regex(self.config_data['excluded_directories'])
def _load_config(self):
"""Load config from the provided file. Sets default file if one is not provided."""
if self.config_file is None:
if not self.config_file:
script_directory = os.path.dirname(os.path.abspath(__file__)) # Default file expected in same dir as script
self.config_file = os.path.join(script_directory, self.DEFAULT_CONFIG_FILE)
@@ -43,45 +48,68 @@ class LicenseScanner:
print('Config file cannot be found')
raise
def _load_license_regex(self):
def _load_file_regex(self, patterns):
"""Returns regex object with case-insensitive matching from the list of filename patterns."""
regex_patterns = []
for pattern in self.config_data['license_patterns']:
for pattern in patterns:
regex_patterns.append(fnmatch.translate(pattern))
if not regex_patterns:
print(f'Warning: No patterns from {patterns} found')
return None
return re.compile('|'.join(regex_patterns), re.IGNORECASE)
def scan(self, path=os.curdir):
"""Scan directory tree for filenames matching license_regex.
def scan(self, paths=os.curdir):
"""Scan directory tree for filenames matching file_regex, package info, and exclusion files.
:param path: Path of the directory to run scanner
:return: Package paths and their corresponding license file contents
:rtype: dict
:param paths: Paths of the directory to run scanner
:return: Package paths and their corresponding file contents
:rtype: Ordered dict
"""
licenses = 0
license_files = {}
files = 0
matching_files = OrderedDict()
excluded_directories = None
for dirpath, dirnames, filenames in os.walk(path):
for file in filenames:
if self.license_regex.match(file):
license_file_content = self._get_license_file_contents(os.path.join(dirpath, file))
rel_dirpath = os.path.relpath(dirpath, path) # Limit path inside scanned directory
license_files[rel_dirpath] = license_file_content
licenses += 1
print(f'License file: {os.path.join(dirpath, file)}')
if not self.package_info:
self.package_info = self.DEFAULT_PACKAGE_INFO_FILE
# Remove directories that should not be scanned
for dir in self.config_data['excluded_directories']:
if dir in dirnames:
dirnames.remove(dir)
print(f'{licenses} license files found.')
return license_files
if not self.excluded_directories:
print(f'No excluded directory in config, looking for {self.DEFAULT_EXCLUDE_FILE} instead')
def _get_license_file_contents(self, filepath):
for path in paths:
for dirpath, dirnames, filenames in os.walk(path, topdown=True):
dirnames.sort(key=str.casefold) # Ensure that results are sorted
for file in filenames:
if self.file_regex.match(file) or self.package_info.match(file):
file_path = os.path.join(dirpath, file)
matching_file_content = self._get_file_contents(file_path)
matching_files[file_path] = matching_file_content
files += 1
print(f'Matching file: {file_path}')
if self.package_info.match(file):
dirnames[:] = [] # Stop scanning subdirectories if package info file found
if self.DEFAULT_EXCLUDE_FILE in file and not self.excluded_directories:
ignore_list = self._get_file_contents(os.path.join(dirpath, file)).splitlines()
ignore_list.append('.git') # .gitignore doesn't usually have .git in its exclusions
excluded_directories = self._load_file_regex(ignore_list)
# Remove directories that should not be scanned
if self.excluded_directories:
excluded_directories = self.excluded_directories
for dir in dirnames:
if excluded_directories.match(dir):
dirnames.remove(dir)
print(f'{files} files found.')
return matching_files
def _get_file_contents(self, filepath):
try:
with open(filepath, encoding='utf8') as f:
return f.read()
except UnicodeDecodeError:
print(f'Unable to read license file: {filepath}')
print(f'Unable to read file: {filepath}')
pass
def create_license_file(self, licenses, filepath='NOTICES.txt'):
@@ -89,18 +117,44 @@ class LicenseScanner:
:param licenses: Dict with package paths and their corresponding license file contents
:param filepath: Path to write the file
"""
package_separator = '------------------------------------'
with open(filepath, 'w', encoding='utf8') as f:
"""
license_separator = '------------------------------------'
with open(filepath, 'w', encoding='utf8') as lf:
for directory, license in licenses.items():
license_output = '\n\n'.join([
f'{package_separator}',
f'Package path: {directory}',
'License:',
f'{license}\n'
])
f.write(license_output)
if not self.package_info.match(os.path.basename(directory)):
license_output = '\n\n'.join([
f'{license_separator}',
f'Package path: {os.path.relpath(directory)}',
'License:',
f'{license}\n'
])
lf.write(license_output)
return None
def create_package_file(self, packages, filepath='SPDX-Licenses.json', get_contents=False):
"""Creates file with all the provided SPDX package info summaries in json.
Optional dirpath parameter will follow the license file path in the package info and return its contents in a dictionary
:param licenses: Dict with package info paths and their corresponding file contents
:param filepath: Path to write the file
:param dirpath: Root path for packages
:rtype: Ordered dict
"""
licenses = OrderedDict()
package_json = []
with open(filepath, 'w', encoding='utf8') as pf:
for directory, package in packages.items():
if self.package_info.match(os.path.basename(directory)):
package_obj = json.loads(package)
package_json.append(package_obj)
if get_contents:
license_path = os.path.join(os.path.dirname(directory), pathlib.Path(package_obj['LicenseFile']))
licenses[license_path] = self._get_file_contents(license_path)
else:
licenses[directory] = package
pf.write(json.dumps(package_json, indent=4))
return licenses
def parse_args():
@@ -108,7 +162,8 @@ def parse_args():
description='Script to run LicenseScanner and generate license file')
parser.add_argument('--config-file', '-c', type=pathlib.Path, help='Config file for LicenseScanner')
parser.add_argument('--license-file-path', '-l', type=pathlib.Path, help='Create license file in the provided path')
parser.add_argument('--scan-path', '-s', default=os.curdir, type=pathlib.Path, help='Path to scan')
parser.add_argument('--package-file-path', '-p', type=pathlib.Path, help='Create package summary file in the provided path')
parser.add_argument('--scan-path', '-s', default=os.curdir, type=pathlib.Path, nargs='+', help='Path to scan, multiple space separated paths can be used')
return parser.parse_args()
@@ -116,10 +171,15 @@ def main():
try:
args = parse_args()
ls = LicenseScanner(args.config_file)
licenses = ls.scan(args.scan_path)
scanned_path_data = ls.scan(args.scan_path)
if args.license_file_path:
ls.create_license_file(licenses, args.license_file_path)
ls.create_license_file(scanned_path_data, args.license_file_path)
if args.package_file_path:
ls.create_package_file(scanned_path_data, args.package_file_path)
if args.license_file_path and args.package_file_path:
license_files = ls.create_package_file(scanned_path_data, args.package_file_path, True)
ls.create_license_file(license_files, args.license_file_path)
except FileNotFoundError as e:
print(f'Type: {type(e).__name__}, Error: {e}')
return 1
scripts/license_scanner/scanner_config.json
+3
View File
@@ -8,5 +8,8 @@
"license_patterns": [
"LICENSE*",
"COPYING*"
],
"package_patterns": [
"PackageInfo.json"
]
}