#!/usr/bin/env python3
+"""
+crcchecker is a tool to used to enforce that .api messages do not change.
+API files with a semantic version < 1.0.0 are ignored.
+"""
+
import sys
import os
import json
import argparse
+import re
from subprocess import run, PIPE, check_output, CalledProcessError
-rootdir = os.path.dirname(os.path.realpath(__file__)) + '/../..'
+# pylint: disable=subprocess-run-check
+
+ROOTDIR = os.path.dirname(os.path.realpath(__file__)) + "/../.."
+APIGENBIN = f"{ROOTDIR}/src/tools/vppapigen/vppapigen.py"
+
def crc_from_apigen(revision, filename):
+ """Runs vppapigen with crc plugin returning a JSON object with CRCs for
+ all APIs in filename"""
if not revision and not os.path.isfile(filename):
- print(f'skipping: {filename}', file=sys.stderr)
- return {}
- apigen_bin = f'{rootdir}/src/tools/vppapigen/vppapigen.py'
+ print(f"skipping: {filename}", file=sys.stderr)
+ # Return <class 'set'> instead of <class 'dict'>
+ return {-1}
+
if revision:
- apigen = (f'{apigen_bin} --git-revision {revision} --includedir src '
- f'--input {filename} CRC')
+ apigen = (
+ f"{APIGENBIN} --git-revision {revision} --includedir src "
+ f"--input {filename} CRC"
+ )
else:
- apigen = (f'{apigen_bin} --includedir src --input {filename} CRC')
- rv = run(apigen.split(), stdout=PIPE, stderr=PIPE)
- if rv.returncode == 2: # No such file
- print(f'skipping: {revision}:{filename} {rv}', file=sys.stderr)
+ apigen = f"{APIGENBIN} --includedir src --input {filename} CRC"
+ returncode = run(apigen.split(), stdout=PIPE, stderr=PIPE)
+ if returncode.returncode == 2: # No such file
+ print(f"skipping: {revision}:{filename} {returncode}", file=sys.stderr)
return {}
- if rv.returncode != 0:
- print(f'vppapigen failed for {revision}:{filename} with command\n {apigen}\n error: {rv}',
- rv.stderr.decode('ascii'), file=sys.stderr)
+ if returncode.returncode != 0:
+ print(
+ f"vppapigen failed for {revision}:{filename} with "
+ "command\n {apigen}\n error: {rv}",
+ returncode.stderr.decode("ascii"),
+ file=sys.stderr,
+ )
sys.exit(-2)
- return json.loads(rv.stdout)
+ return json.loads(returncode.stdout)
-def dict_compare(d1, d2):
- d1_keys = set(d1.keys())
- d2_keys = set(d2.keys())
+def dict_compare(dict1, dict2):
+ """Compare two dictionaries returning added, removed, modified
+ and equal entries"""
+ d1_keys = set(dict1.keys())
+ d2_keys = set(dict2.keys())
intersect_keys = d1_keys.intersection(d2_keys)
added = d1_keys - d2_keys
removed = d2_keys - d1_keys
- modified = {o: (d1[o], d2[o]) for o in intersect_keys if d1[o]['crc'] != d2[o]['crc']}
- same = set(o for o in intersect_keys if d1[o] == d2[o])
+ modified = {
+ o: (dict1[o], dict2[o])
+ for o in intersect_keys
+ if dict1[o]["crc"] != dict2[o]["crc"]
+ }
+ same = set(o for o in intersect_keys if dict1[o] == dict2[o])
return added, removed, modified, same
def filelist_from_git_ls():
+ """Returns a list of all api files in the git repository"""
filelist = []
- git_ls = 'git ls-files *.api'
- rv = run(git_ls.split(), stdout=PIPE, stderr=PIPE)
- if rv.returncode != 0:
- sys.exit(rv.returncode)
-
- for l in rv.stdout.decode('ascii').split('\n'):
- if len(l):
- filelist.append(l)
+ git_ls = "git ls-files *.api"
+ returncode = run(git_ls.split(), stdout=PIPE, stderr=PIPE)
+ if returncode.returncode != 0:
+ sys.exit(returncode.returncode)
+
+ for line in returncode.stdout.decode("ascii").split("\n"):
+ if line:
+ filelist.append(line)
return filelist
def is_uncommitted_changes():
- git_status = 'git status --porcelain -uno'
- rv = run(git_status.split(), stdout=PIPE, stderr=PIPE)
- if rv.returncode != 0:
- sys.exit(rv.returncode)
+ """Returns true if there are uncommitted changes in the repo"""
+ git_status = "git status --porcelain -uno"
+ returncode = run(git_status.split(), stdout=PIPE, stderr=PIPE)
+ if returncode.returncode != 0:
+ sys.exit(returncode.returncode)
- if len(rv.stdout):
+ if returncode.stdout:
return True
return False
def filelist_from_git_grep(filename):
+ """Returns a list of api files that this <filename> api files imports."""
filelist = []
try:
- rv = check_output(f'git grep -e "import .*{filename}" -- *.api', shell=True)
- except CalledProcessError as err:
+ returncode = check_output(
+ f'git grep -e "import .*{filename}"' " -- *.api", shell=True
+ )
+ except CalledProcessError:
return []
- print('RV', err.returncode)
- for l in rv.decode('ascii').split('\n'):
- if l:
- f, p = l.split(':')
- filelist.append(f)
+ for line in returncode.decode("ascii").split("\n"):
+ if line:
+ filename, _ = line.split(":")
+ filelist.append(filename)
return filelist
-def filelist_from_patchset():
+def filelist_from_patchset(pattern):
+ """Returns list of api files in changeset and the list of api
+ files they import."""
filelist = []
- git_cmd = '((git diff HEAD~1.. --name-only;git ls-files -m) | sort -u)'
- rv = check_output(git_cmd, shell=True)
- for l in rv.decode('ascii').split('\n'):
- if len(l) and os.path.splitext(l)[1] == '.api':
- filelist.append(l)
+ git_cmd = (
+ "((git diff HEAD~1.. --name-only;git ls-files -m) | "
+ 'sort -u | grep "\\.api$")'
+ )
+ try:
+ res = check_output(git_cmd, shell=True)
+ except CalledProcessError:
+ return []
# Check for dependencies (imports)
imported_files = []
- for f in filelist:
- imported_files.extend(filelist_from_git_grep(os.path.basename(f)))
+ for line in res.decode("ascii").split("\n"):
+ if not line:
+ continue
+ if not re.search(pattern, line):
+ continue
+ filelist.append(line)
+ imported_files.extend(filelist_from_git_grep(os.path.basename(line)))
filelist.extend(imported_files)
return set(filelist)
-def is_deprecated(d, k):
- if 'options' in d[k] and 'deprecated' in d[k]['options']:
- return True
+
+def is_deprecated(message):
+ """Given a message, return True if message is deprecated"""
+ if "options" in message:
+ if "deprecated" in message["options"]:
+ return True
+ # recognize the deprecated format
+ if (
+ "status" in message["options"]
+ and message["options"]["status"] == "deprecated"
+ ):
+ print("WARNING: please use 'option deprecated;'")
+ return True
return False
-def is_in_progress(d, k):
- try:
- if d[k]['options']['status'] == 'in_progress':
+
+def is_in_progress(message):
+ """Given a message, return True if message is marked as in_progress"""
+ if "options" in message:
+ if "in_progress" in message["options"]:
+ return True
+ # recognize the deprecated format
+ if (
+ "status" in message["options"]
+ and message["options"]["status"] == "in_progress"
+ ):
+ print("WARNING: please use 'option in_progress;'")
return True
- except:
- return False
+ return False
+
def report(new, old):
- added, removed, modified, same = dict_compare(new, old)
+ """Given a dictionary of new crcs and old crcs, print all the
+ added, removed, modified, in-progress, deprecated messages.
+ Return the number of backwards incompatible changes made."""
+
+ # pylint: disable=too-many-branches
+
+ new.pop("_version", None)
+ old.pop("_version", None)
+ added, removed, modified, _ = dict_compare(new, old)
backwards_incompatible = 0
+
+ # print the full list of in-progress messages
+ # they should eventually either disappear of become supported
+ for k in new.keys():
+ newversion = int(new[k]["version"])
+ if newversion == 0 or is_in_progress(new[k]):
+ print(f"in-progress: {k}")
for k in added:
- print(f'added: {k}')
+ print(f"added: {k}")
for k in removed:
- oldversion = int(old[k]['version'])
- if oldversion > 0 and not is_deprecated(old, k) and not is_in_progress(old, k):
+ oldversion = int(old[k]["version"])
+ if oldversion > 0 and not is_deprecated(old[k]) and not is_in_progress(old[k]):
backwards_incompatible += 1
- print(f'removed: ** {k}')
+ print(f"removed: ** {k}")
else:
- print(f'removed: {k}')
+ print(f"removed: {k}")
for k in modified.keys():
- oldversion = int(old[k]['version'])
- newversion = int(new[k]['version'])
- if oldversion > 0 and not is_in_progress(old, k):
+ oldversion = int(old[k]["version"])
+ newversion = int(new[k]["version"])
+ if oldversion > 0 and not is_in_progress(old[k]):
backwards_incompatible += 1
- print(f'modified: ** {k}')
+ print(f"modified: ** {k}")
else:
- print(f'modified: {k}')
+ print(f"modified: {k}")
# check which messages are still there but were marked for deprecation
for k in new.keys():
- newversion = int(new[k]['version'])
- if newversion > 0 and is_deprecated(new, k):
+ newversion = int(new[k]["version"])
+ if newversion > 0 and is_deprecated(new[k]):
if k in old:
- if not is_deprecated(old, k):
- print(f'deprecated: {k}')
+ if not is_deprecated(old[k]):
+ print(f"deprecated: {k}")
else:
- print(f'added+deprecated: {k}')
+ print(f"added+deprecated: {k}")
return backwards_incompatible
+def check_patchset():
+ """Compare the changes to API messages in this changeset.
+ Ignores API files with version < 1.0.0.
+ Only considers API files located under the src directory in the repo.
+ """
+ files = filelist_from_patchset("^src/")
+ revision = "HEAD~1"
+
+ oldcrcs = {}
+ newcrcs = {}
+ for filename in files:
+ # Ignore files that have version < 1.0.0
+ _ = crc_from_apigen(None, filename)
+ # Ignore removed files
+ if isinstance(_, set) == 0:
+ if isinstance(_, set) == 0 and _["_version"]["major"] == "0":
+ continue
+ newcrcs.update(_)
+
+ oldcrcs.update(crc_from_apigen(revision, filename))
+
+ backwards_incompatible = report(newcrcs, oldcrcs)
+ if backwards_incompatible:
+ # alert on changing production API
+ print(
+ "crcchecker: Changing production APIs in an incompatible way",
+ file=sys.stderr,
+ )
+ sys.exit(-1)
+ else:
+ print("*" * 67)
+ print("* VPP CHECKAPI SUCCESSFULLY COMPLETED")
+ print("*" * 67)
+
+
def main():
- parser = argparse.ArgumentParser(description='VPP CRC checker.')
- parser.add_argument('--git-revision',
- help='Git revision to compare against')
- parser.add_argument('--dump-manifest', action='store_true',
- help='Dump CRC for all messages')
- parser.add_argument('--check-patchset', action='store_true',
- help='Dump CRC for all messages')
- parser.add_argument('files', nargs='*')
- parser.add_argument('--diff', help='Files to compare (on filesystem)', nargs=2)
+ """Main entry point."""
+ parser = argparse.ArgumentParser(description="VPP CRC checker.")
+ parser.add_argument("--git-revision", help="Git revision to compare against")
+ parser.add_argument(
+ "--dump-manifest", action="store_true", help="Dump CRC for all messages"
+ )
+ parser.add_argument(
+ "--check-patchset",
+ action="store_true",
+ help="Check patchset for backwards incompatbile changes",
+ )
+ parser.add_argument("files", nargs="*")
+ parser.add_argument("--diff", help="Files to compare (on filesystem)", nargs=2)
args = parser.parse_args()
if args.dump_manifest:
files = args.files if args.files else filelist_from_git_ls()
crcs = {}
- for f in files:
- crcs.update(crc_from_apigen(args.git_revision, f))
- for k, v in crcs.items():
- print(f'{k}: {v}')
+ for filename in files:
+ crcs.update(crc_from_apigen(args.git_revision, filename))
+ for k, value in crcs.items():
+ print(f"{k}: {value}")
sys.exit(0)
# Find changes between current patchset and given revision (previous)
if args.check_patchset:
if args.git_revision:
- print('Argument git-revision ignored', file=sys.stderr)
+ print("Argument git-revision ignored", file=sys.stderr)
# Check there are no uncomitted changes
if is_uncommitted_changes():
- print('Please stash or commit changes in workspace', file=sys.stderr)
+ print("Please stash or commit changes in workspace", file=sys.stderr)
sys.exit(-1)
- files = filelist_from_patchset()
- else:
- # Find changes between current workspace and revision
- # Find changes between a given file and a revision
- files = args.files if args.files else filelist_from_git_ls()
+ check_patchset()
+ sys.exit(0)
- revision = args.git_revision if args.git_revision else 'HEAD~1'
+ # Find changes between current workspace and revision
+ # Find changes between a given file and a revision
+ files = args.files if args.files else filelist_from_git_ls()
+
+ revision = args.git_revision if args.git_revision else "HEAD~1"
oldcrcs = {}
newcrcs = {}
- for f in files:
- newcrcs.update(crc_from_apigen(None, f))
- oldcrcs.update(crc_from_apigen(revision, f))
+ for file in files:
+ newcrcs.update(crc_from_apigen(None, file))
+ oldcrcs.update(crc_from_apigen(revision, file))
backwards_incompatible = report(newcrcs, oldcrcs)
if args.check_patchset:
if backwards_incompatible:
# alert on changing production API
- print("crcchecker: Changing production APIs in an incompatible way", file=sys.stderr)
+ print(
+ "crcchecker: Changing production APIs in an incompatible way",
+ file=sys.stderr,
+ )
sys.exit(-1)
else:
- print('*' * 67)
- print('* VPP CHECKAPI SUCCESSFULLY COMPLETED')
- print('*' * 67)
+ print("*" * 67)
+ print("* VPP CHECKAPI SUCCESSFULLY COMPLETED")
+ print("*" * 67)
+
-if __name__ == '__main__':
+if __name__ == "__main__":
main()
Code Review / vpp.git / blobdiff