4 crcchecker is a tool to used to enforce that .api messages do not change.
5 API files with a semantic version < 1.0.0 are ignored.
13 from subprocess import run, PIPE, check_output, CalledProcessError
15 # pylint: disable=subprocess-run-check
17 ROOTDIR = os.path.dirname(os.path.realpath(__file__)) + "/../.."
18 APIGENBIN = f"{ROOTDIR}/src/tools/vppapigen/vppapigen.py"
21 def crc_from_apigen(revision, filename):
22 """Runs vppapigen with crc plugin returning a JSON object with CRCs for
23 all APIs in filename"""
24 if not revision and not os.path.isfile(filename):
25 print(f"skipping: {filename}", file=sys.stderr)
26 # Return <class 'set'> instead of <class 'dict'>
31 f"{APIGENBIN} --git-revision {revision} --includedir src "
32 f"--input {filename} CRC"
35 apigen = f"{APIGENBIN} --includedir src --input {filename} CRC"
36 returncode = run(apigen.split(), stdout=PIPE, stderr=PIPE)
37 if returncode.returncode == 2: # No such file
38 print(f"skipping: {revision}:{filename} {returncode}", file=sys.stderr)
40 if returncode.returncode != 0:
42 f"vppapigen failed for {revision}:{filename} with "
43 "command\n {apigen}\n error: {rv}",
44 returncode.stderr.decode("ascii"),
49 return json.loads(returncode.stdout)
52 def dict_compare(dict1, dict2):
53 """Compare two dictionaries returning added, removed, modified
55 d1_keys = set(dict1.keys())
56 d2_keys = set(dict2.keys())
57 intersect_keys = d1_keys.intersection(d2_keys)
58 added = d1_keys - d2_keys
59 removed = d2_keys - d1_keys
61 o: (dict1[o], dict2[o])
62 for o in intersect_keys
63 if dict1[o]["crc"] != dict2[o]["crc"]
65 same = set(o for o in intersect_keys if dict1[o] == dict2[o])
66 return added, removed, modified, same
69 def filelist_from_git_ls():
70 """Returns a list of all api files in the git repository"""
72 git_ls = "git ls-files *.api"
73 returncode = run(git_ls.split(), stdout=PIPE, stderr=PIPE)
74 if returncode.returncode != 0:
75 sys.exit(returncode.returncode)
77 for line in returncode.stdout.decode("ascii").split("\n"):
83 def is_uncommitted_changes():
84 """Returns true if there are uncommitted changes in the repo"""
85 git_status = "git status --porcelain -uno"
86 returncode = run(git_status.split(), stdout=PIPE, stderr=PIPE)
87 if returncode.returncode != 0:
88 sys.exit(returncode.returncode)
95 def filelist_from_git_grep(filename):
96 """Returns a list of api files that this <filename> api files imports."""
99 returncode = check_output(
100 f'git grep -e "import .*{filename}"' " -- *.api", shell=True
102 except CalledProcessError:
104 for line in returncode.decode("ascii").split("\n"):
106 filename, _ = line.split(":")
107 filelist.append(filename)
111 def filelist_from_patchset(pattern):
112 """Returns list of api files in changeset and the list of api
113 files they import."""
116 "((git diff HEAD~1.. --name-only;git ls-files -m) | "
117 'sort -u | grep "\\.api$")'
120 res = check_output(git_cmd, shell=True)
121 except CalledProcessError:
124 # Check for dependencies (imports)
126 for line in res.decode("ascii").split("\n"):
129 if not re.search(pattern, line):
131 filelist.append(line)
132 imported_files.extend(filelist_from_git_grep(os.path.basename(line)))
134 filelist.extend(imported_files)
138 def is_deprecated(message):
139 """Given a message, return True if message is deprecated"""
140 if "options" in message:
141 if "deprecated" in message["options"]:
143 # recognize the deprecated format
145 "status" in message["options"]
146 and message["options"]["status"] == "deprecated"
148 print("WARNING: please use 'option deprecated;'")
153 def is_in_progress(message):
154 """Given a message, return True if message is marked as in_progress"""
155 if "options" in message:
156 if "in_progress" in message["options"]:
158 # recognize the deprecated format
160 "status" in message["options"]
161 and message["options"]["status"] == "in_progress"
163 print("WARNING: please use 'option in_progress;'")
168 def report(new, old):
169 """Given a dictionary of new crcs and old crcs, print all the
170 added, removed, modified, in-progress, deprecated messages.
171 Return the number of backwards incompatible changes made."""
173 # pylint: disable=too-many-branches
175 new.pop("_version", None)
176 old.pop("_version", None)
177 added, removed, modified, _ = dict_compare(new, old)
178 backwards_incompatible = 0
180 # print the full list of in-progress messages
181 # they should eventually either disappear of become supported
183 newversion = int(new[k]["version"])
184 if newversion == 0 or is_in_progress(new[k]):
185 print(f"in-progress: {k}")
189 oldversion = int(old[k]["version"])
190 if oldversion > 0 and not is_deprecated(old[k]) and not is_in_progress(old[k]):
191 backwards_incompatible += 1
192 print(f"removed: ** {k}")
194 print(f"removed: {k}")
195 for k in modified.keys():
196 oldversion = int(old[k]["version"])
197 newversion = int(new[k]["version"])
198 if oldversion > 0 and not is_in_progress(old[k]):
199 backwards_incompatible += 1
200 print(f"modified: ** {k}")
202 print(f"modified: {k}")
204 # check which messages are still there but were marked for deprecation
206 newversion = int(new[k]["version"])
207 if newversion > 0 and is_deprecated(new[k]):
209 if not is_deprecated(old[k]):
210 print(f"deprecated: {k}")
212 print(f"added+deprecated: {k}")
214 return backwards_incompatible
217 def check_patchset():
218 """Compare the changes to API messages in this changeset.
219 Ignores API files with version < 1.0.0.
220 Only considers API files located under the src directory in the repo.
222 files = filelist_from_patchset("^src/")
227 for filename in files:
228 # Ignore files that have version < 1.0.0
229 _ = crc_from_apigen(None, filename)
230 # Ignore removed files
231 if isinstance(_, set) == 0:
232 if isinstance(_, set) == 0 and _["_version"]["major"] == "0":
236 oldcrcs.update(crc_from_apigen(revision, filename))
238 backwards_incompatible = report(newcrcs, oldcrcs)
239 if backwards_incompatible:
240 # alert on changing production API
242 "crcchecker: Changing production APIs in an incompatible way",
248 print("* VPP CHECKAPI SUCCESSFULLY COMPLETED")
253 """Main entry point."""
254 parser = argparse.ArgumentParser(description="VPP CRC checker.")
255 parser.add_argument("--git-revision", help="Git revision to compare against")
257 "--dump-manifest", action="store_true", help="Dump CRC for all messages"
262 help="Check patchset for backwards incompatbile changes",
264 parser.add_argument("files", nargs="*")
265 parser.add_argument("--diff", help="Files to compare (on filesystem)", nargs=2)
267 args = parser.parse_args()
269 if args.diff and args.files:
275 oldcrcs = crc_from_apigen(None, args.diff[0])
276 newcrcs = crc_from_apigen(None, args.diff[1])
277 backwards_incompatible = report(newcrcs, oldcrcs)
280 # Dump CRC for messages in given files / revision
281 if args.dump_manifest:
282 files = args.files if args.files else filelist_from_git_ls()
284 for filename in files:
285 crcs.update(crc_from_apigen(args.git_revision, filename))
286 for k, value in crcs.items():
287 print(f"{k}: {value}")
290 # Find changes between current patchset and given revision (previous)
291 if args.check_patchset:
292 if args.git_revision:
293 print("Argument git-revision ignored", file=sys.stderr)
294 # Check there are no uncomitted changes
295 if is_uncommitted_changes():
296 print("Please stash or commit changes in workspace", file=sys.stderr)
301 # Find changes between current workspace and revision
302 # Find changes between a given file and a revision
303 files = args.files if args.files else filelist_from_git_ls()
305 revision = args.git_revision if args.git_revision else "HEAD~1"
310 newcrcs.update(crc_from_apigen(None, file))
311 oldcrcs.update(crc_from_apigen(revision, file))
313 backwards_incompatible = report(newcrcs, oldcrcs)
315 if args.check_patchset:
316 if backwards_incompatible:
317 # alert on changing production API
319 "crcchecker: Changing production APIs in an incompatible way",
325 print("* VPP CHECKAPI SUCCESSFULLY COMPLETED")
329 if __name__ == "__main__":