4 crcchecker is a tool to used to enforce that .api messages do not change.
5 API files with a semantic version < 1.0.0 are ignored.
13 from subprocess import run, PIPE, check_output, CalledProcessError
15 # pylint: disable=subprocess-run-check
17 ROOTDIR = os.path.dirname(os.path.realpath(__file__)) + '/../..'
18 APIGENBIN = f'{ROOTDIR}/src/tools/vppapigen/vppapigen.py'
21 def crc_from_apigen(revision, filename):
22 '''Runs vppapigen with crc plugin returning a JSON object with CRCs for
23 all APIs in filename'''
24 if not revision and not os.path.isfile(filename):
25 print(f'skipping: {filename}', file=sys.stderr)
26 # Return <class 'set'> instead of <class 'dict'>
30 apigen = (f'{APIGENBIN} --git-revision {revision} --includedir src '
31 f'--input {filename} CRC')
33 apigen = (f'{APIGENBIN} --includedir src --input {filename} CRC')
34 returncode = run(apigen.split(), stdout=PIPE, stderr=PIPE)
35 if returncode.returncode == 2: # No such file
36 print(f'skipping: {revision}:{filename} {returncode}', file=sys.stderr)
38 if returncode.returncode != 0:
39 print(f'vppapigen failed for {revision}:{filename} with '
40 'command\n {apigen}\n error: {rv}',
41 returncode.stderr.decode('ascii'), file=sys.stderr)
44 return json.loads(returncode.stdout)
47 def dict_compare(dict1, dict2):
48 '''Compare two dictionaries returning added, removed, modified
50 d1_keys = set(dict1.keys())
51 d2_keys = set(dict2.keys())
52 intersect_keys = d1_keys.intersection(d2_keys)
53 added = d1_keys - d2_keys
54 removed = d2_keys - d1_keys
55 modified = {o: (dict1[o], dict2[o]) for o in intersect_keys
56 if dict1[o]['crc'] != dict2[o]['crc']}
57 same = set(o for o in intersect_keys if dict1[o] == dict2[o])
58 return added, removed, modified, same
61 def filelist_from_git_ls():
62 '''Returns a list of all api files in the git repository'''
64 git_ls = 'git ls-files *.api'
65 returncode = run(git_ls.split(), stdout=PIPE, stderr=PIPE)
66 if returncode.returncode != 0:
67 sys.exit(returncode.returncode)
69 for line in returncode.stdout.decode('ascii').split('\n'):
75 def is_uncommitted_changes():
76 '''Returns true if there are uncommitted changes in the repo'''
77 git_status = 'git status --porcelain -uno'
78 returncode = run(git_status.split(), stdout=PIPE, stderr=PIPE)
79 if returncode.returncode != 0:
80 sys.exit(returncode.returncode)
87 def filelist_from_git_grep(filename):
88 '''Returns a list of api files that this <filename> api files imports.'''
91 returncode = check_output(f'git grep -e "import .*{filename}"'
94 except CalledProcessError:
96 for line in returncode.decode('ascii').split('\n'):
98 filename, _ = line.split(':')
99 filelist.append(filename)
103 def filelist_from_patchset(pattern):
104 '''Returns list of api files in changeset and the list of api
105 files they import.'''
107 git_cmd = ('((git diff HEAD~1.. --name-only;git ls-files -m) | '
108 'sort -u | grep "\\.api$")')
110 res = check_output(git_cmd, shell=True)
111 except CalledProcessError:
114 # Check for dependencies (imports)
116 for line in res.decode('ascii').split('\n'):
119 if not re.search(pattern, line):
121 filelist.append(line)
122 imported_files.extend(filelist_from_git_grep(os.path.basename(line)))
124 filelist.extend(imported_files)
128 def is_deprecated(message):
129 '''Given a message, return True if message is deprecated'''
130 if 'options' in message:
131 if 'deprecated' in message['options']:
133 # recognize the deprecated format
134 if 'status' in message['options'] and \
135 message['options']['status'] == 'deprecated':
136 print("WARNING: please use 'option deprecated;'")
141 def is_in_progress(message):
142 '''Given a message, return True if message is marked as in_progress'''
143 if 'options' in message:
144 if 'in_progress' in message['options']:
146 # recognize the deprecated format
147 if 'status' in message['options'] and \
148 message['options']['status'] == 'in_progress':
149 print("WARNING: please use 'option in_progress;'")
154 def report(new, old):
155 '''Given a dictionary of new crcs and old crcs, print all the
156 added, removed, modified, in-progress, deprecated messages.
157 Return the number of backwards incompatible changes made.'''
159 # pylint: disable=too-many-branches
161 new.pop('_version', None)
162 old.pop('_version', None)
163 added, removed, modified, _ = dict_compare(new, old)
164 backwards_incompatible = 0
166 # print the full list of in-progress messages
167 # they should eventually either disappear of become supported
169 newversion = int(new[k]['version'])
170 if newversion == 0 or is_in_progress(new[k]):
171 print(f'in-progress: {k}')
175 oldversion = int(old[k]['version'])
176 if oldversion > 0 and not is_deprecated(old[k]) and not \
177 is_in_progress(old[k]):
178 backwards_incompatible += 1
179 print(f'removed: ** {k}')
181 print(f'removed: {k}')
182 for k in modified.keys():
183 oldversion = int(old[k]['version'])
184 newversion = int(new[k]['version'])
185 if oldversion > 0 and not is_in_progress(old[k]):
186 backwards_incompatible += 1
187 print(f'modified: ** {k}')
189 print(f'modified: {k}')
191 # check which messages are still there but were marked for deprecation
193 newversion = int(new[k]['version'])
194 if newversion > 0 and is_deprecated(new[k]):
196 if not is_deprecated(old[k]):
197 print(f'deprecated: {k}')
199 print(f'added+deprecated: {k}')
201 return backwards_incompatible
204 def check_patchset():
205 '''Compare the changes to API messages in this changeset.
206 Ignores API files with version < 1.0.0.
207 Only considers API files located under the src directory in the repo.
209 files = filelist_from_patchset('^src/')
214 for filename in files:
215 # Ignore files that have version < 1.0.0
216 _ = crc_from_apigen(None, filename)
217 # Ignore removed files
218 if isinstance(_, set) == 0:
219 if isinstance(_, set) == 0 and _['_version']['major'] == '0':
223 oldcrcs.update(crc_from_apigen(revision, filename))
225 backwards_incompatible = report(newcrcs, oldcrcs)
226 if backwards_incompatible:
227 # alert on changing production API
228 print("crcchecker: Changing production APIs in an incompatible way",
233 print('* VPP CHECKAPI SUCCESSFULLY COMPLETED')
238 '''Main entry point.'''
239 parser = argparse.ArgumentParser(description='VPP CRC checker.')
240 parser.add_argument('--git-revision',
241 help='Git revision to compare against')
242 parser.add_argument('--dump-manifest', action='store_true',
243 help='Dump CRC for all messages')
244 parser.add_argument('--check-patchset', action='store_true',
245 help='Check patchset for backwards incompatbile changes')
246 parser.add_argument('files', nargs='*')
247 parser.add_argument('--diff', help='Files to compare (on filesystem)',
250 args = parser.parse_args()
252 if args.diff and args.files:
258 oldcrcs = crc_from_apigen(None, args.diff[0])
259 newcrcs = crc_from_apigen(None, args.diff[1])
260 backwards_incompatible = report(newcrcs, oldcrcs)
263 # Dump CRC for messages in given files / revision
264 if args.dump_manifest:
265 files = args.files if args.files else filelist_from_git_ls()
267 for filename in files:
268 crcs.update(crc_from_apigen(args.git_revision, filename))
269 for k, value in crcs.items():
270 print(f'{k}: {value}')
273 # Find changes between current patchset and given revision (previous)
274 if args.check_patchset:
275 if args.git_revision:
276 print('Argument git-revision ignored', file=sys.stderr)
277 # Check there are no uncomitted changes
278 if is_uncommitted_changes():
279 print('Please stash or commit changes in workspace',
285 # Find changes between current workspace and revision
286 # Find changes between a given file and a revision
287 files = args.files if args.files else filelist_from_git_ls()
289 revision = args.git_revision if args.git_revision else 'HEAD~1'
294 newcrcs.update(crc_from_apigen(None, file))
295 oldcrcs.update(crc_from_apigen(revision, file))
297 backwards_incompatible = report(newcrcs, oldcrcs)
299 if args.check_patchset:
300 if backwards_incompatible:
301 # alert on changing production API
302 print("crcchecker: Changing production APIs in an incompatible way", file=sys.stderr)
306 print('* VPP CHECKAPI SUCCESSFULLY COMPLETED')
310 if __name__ == '__main__':