blob: f5be51461e799f9c1f655f423c1b2e13d0bddd5d [file] [log] [blame]
Simon Glass3db916d2020-10-29 21:46:35 -06001# SPDX-License-Identifier: GPL-2.0+
2#
3# Copyright 2020 Google LLC
4#
5"""Talks to the patchwork service to figure out what patches have been reviewed
Simon Glass2112d072020-10-29 21:46:38 -06006and commented on. Provides a way to display review tags and comments.
7Allows creation of a new branch based on the old but with the review tags
8collected from patchwork.
Simon Glass3db916d2020-10-29 21:46:35 -06009"""
10
Simon Glass3729b8b2025-04-29 07:22:24 -060011import asyncio
Simon Glass21a23c22025-04-29 07:22:23 -060012from collections import defaultdict
Simon Glass3db916d2020-10-29 21:46:35 -060013import concurrent.futures
14from itertools import repeat
Simon Glassd0a0a582020-10-29 21:46:36 -060015
Simon Glass3729b8b2025-04-29 07:22:24 -060016import aiohttp
Simon Glassd0a0a582020-10-29 21:46:36 -060017import pygit2
Simon Glass3db916d2020-10-29 21:46:35 -060018
Simon Glass131444f2023-02-23 18:18:04 -070019from u_boot_pylib import terminal
20from u_boot_pylib import tout
Simon Glass232eefd2025-04-29 07:22:14 -060021from patman import patchstream
22from patman import patchwork
Simon Glass3db916d2020-10-29 21:46:35 -060023
Simon Glass3db916d2020-10-29 21:46:35 -060024
25def to_int(vals):
26 """Convert a list of strings into integers, using 0 if not an integer
27
28 Args:
29 vals (list): List of strings
30
31 Returns:
32 list: List of integers, one for each input string
33 """
34 out = [int(val) if val.isdigit() else 0 for val in vals]
35 return out
36
Simon Glass2112d072020-10-29 21:46:38 -060037
Simon Glass49b92292025-04-29 07:22:18 -060038def process_reviews(content, comment_data, base_rtags):
39 """Process and return review data
40
41 Args:
42 content (str): Content text of the patch itself - see pwork.get_patch()
43 comment_data (list of dict): Comments for the patch - see
44 pwork._get_patch_comments()
45 base_rtags (dict): base review tags (before any comments)
46 key: Response tag (e.g. 'Reviewed-by')
47 value: Set of people who gave that response, each a name/email
48 string
49
50 Return: tuple:
51 dict: new review tags (noticed since the base_rtags)
52 key: Response tag (e.g. 'Reviewed-by')
53 value: Set of people who gave that response, each a name/email
54 string
55 list of patchwork.Review: reviews received on the patch
56 """
57 pstrm = patchstream.PatchStream.process_text(content, True)
Simon Glass21a23c22025-04-29 07:22:23 -060058 rtags = defaultdict(set)
Simon Glass49b92292025-04-29 07:22:18 -060059 for response, people in pstrm.commit.rtags.items():
60 rtags[response].update(people)
61
62 reviews = []
63 for comment in comment_data:
64 pstrm = patchstream.PatchStream.process_text(comment['content'], True)
65 if pstrm.snippets:
66 submitter = comment['submitter']
67 person = f"{submitter['name']} <{submitter['email']}>"
68 reviews.append(patchwork.Review(person, pstrm.snippets))
69 for response, people in pstrm.commit.rtags.items():
70 rtags[response].update(people)
71
72 # Find the tags that are not in the commit
Simon Glass21a23c22025-04-29 07:22:23 -060073 new_rtags = defaultdict(set)
Simon Glass49b92292025-04-29 07:22:18 -060074 for tag, people in rtags.items():
75 for who in people:
76 is_new = (tag not in base_rtags or
77 who not in base_rtags[tag])
78 if is_new:
79 new_rtags[tag].add(who)
80 return new_rtags, reviews
81
82
Simon Glass3db916d2020-10-29 21:46:35 -060083def compare_with_series(series, patches):
84 """Compare a list of patches with a series it came from
85
86 This prints any problems as warnings
87
88 Args:
89 series (Series): Series to compare against
90 patches (:type: list of Patch): list of Patch objects to compare with
91
92 Returns:
93 tuple
94 dict:
95 key: Commit number (0...n-1)
96 value: Patch object for that commit
97 dict:
98 key: Patch number (0...n-1)
99 value: Commit object for that patch
100 """
101 # Check the names match
102 warnings = []
103 patch_for_commit = {}
104 all_patches = set(patches)
105 for seq, cmt in enumerate(series.commits):
106 pmatch = [p for p in all_patches if p.subject == cmt.subject]
107 if len(pmatch) == 1:
108 patch_for_commit[seq] = pmatch[0]
109 all_patches.remove(pmatch[0])
110 elif len(pmatch) > 1:
111 warnings.append("Multiple patches match commit %d ('%s'):\n %s" %
112 (seq + 1, cmt.subject,
113 '\n '.join([p.subject for p in pmatch])))
114 else:
115 warnings.append("Cannot find patch for commit %d ('%s')" %
116 (seq + 1, cmt.subject))
117
Simon Glass3db916d2020-10-29 21:46:35 -0600118 # Check the names match
119 commit_for_patch = {}
120 all_commits = set(series.commits)
121 for seq, patch in enumerate(patches):
122 cmatch = [c for c in all_commits if c.subject == patch.subject]
123 if len(cmatch) == 1:
124 commit_for_patch[seq] = cmatch[0]
125 all_commits.remove(cmatch[0])
126 elif len(cmatch) > 1:
127 warnings.append("Multiple commits match patch %d ('%s'):\n %s" %
128 (seq + 1, patch.subject,
129 '\n '.join([c.subject for c in cmatch])))
130 else:
131 warnings.append("Cannot find commit for patch %d ('%s')" %
132 (seq + 1, patch.subject))
133
134 return patch_for_commit, commit_for_patch, warnings
135
Simon Glass25b91c12025-04-29 07:22:19 -0600136def collect_patches(series_id, pwork):
Simon Glass3db916d2020-10-29 21:46:35 -0600137 """Collect patch information about a series from patchwork
138
139 Uses the Patchwork REST API to collect information provided by patchwork
140 about the status of each patch.
141
142 Args:
Simon Glass3db916d2020-10-29 21:46:35 -0600143 series_id (str): Patch series ID number
Simon Glass25b91c12025-04-29 07:22:19 -0600144 pwork (Patchwork): Patchwork object to use for reading
Simon Glass3db916d2020-10-29 21:46:35 -0600145
146 Returns:
Simon Glass27280f42025-04-29 07:22:17 -0600147 list of Patch: List of patches sorted by sequence number
Simon Glass3db916d2020-10-29 21:46:35 -0600148
149 Raises:
150 ValueError: if the URL could not be read or the web page does not follow
151 the expected structure
152 """
Simon Glass25b91c12025-04-29 07:22:19 -0600153 data = pwork.request('series/%s/' % series_id)
Simon Glass3db916d2020-10-29 21:46:35 -0600154
155 # Get all the rows, which are patches
156 patch_dict = data['patches']
157 count = len(patch_dict)
Simon Glass3db916d2020-10-29 21:46:35 -0600158
159 patches = []
160
161 # Work through each row (patch) one at a time, collecting the information
162 warn_count = 0
163 for pw_patch in patch_dict:
Simon Glass232eefd2025-04-29 07:22:14 -0600164 patch = patchwork.Patch(pw_patch['id'])
Simon Glass3db916d2020-10-29 21:46:35 -0600165 patch.parse_subject(pw_patch['name'])
166 patches.append(patch)
167 if warn_count > 1:
Simon Glass011f1b32022-01-29 14:14:15 -0700168 tout.warning(' (total of %d warnings)' % warn_count)
Simon Glass3db916d2020-10-29 21:46:35 -0600169
170 # Sort patches by patch number
171 patches = sorted(patches, key=lambda x: x.seq)
172 return patches
173
Simon Glass25b91c12025-04-29 07:22:19 -0600174def find_new_responses(new_rtag_list, review_list, seq, cmt, patch, pwork):
Simon Glass3db916d2020-10-29 21:46:35 -0600175 """Find new rtags collected by patchwork that we don't know about
176
177 This is designed to be run in parallel, once for each commit/patch
178
179 Args:
180 new_rtag_list (list): New rtags are written to new_rtag_list[seq]
181 list, each a dict:
182 key: Response tag (e.g. 'Reviewed-by')
183 value: Set of people who gave that response, each a name/email
184 string
Simon Glass2112d072020-10-29 21:46:38 -0600185 review_list (list): New reviews are written to review_list[seq]
186 list, each a
187 List of reviews for the patch, each a Review
Simon Glass3db916d2020-10-29 21:46:35 -0600188 seq (int): Position in new_rtag_list to update
189 cmt (Commit): Commit object for this commit
190 patch (Patch): Corresponding Patch object for this patch
Simon Glass25b91c12025-04-29 07:22:19 -0600191 pwork (Patchwork): Patchwork object to use for reading
Simon Glass3db916d2020-10-29 21:46:35 -0600192 """
193 if not patch:
194 return
195
196 # Get the content for the patch email itself as well as all comments
Simon Glass25b91c12025-04-29 07:22:19 -0600197 data = pwork.request('patches/%s/' % patch.id)
198 comment_data = pwork.request('patches/%s/comments/' % patch.id)
Simon Glass3db916d2020-10-29 21:46:35 -0600199
Simon Glass49b92292025-04-29 07:22:18 -0600200 new_rtags, reviews = process_reviews(data['content'], comment_data,
201 cmt.rtags)
Simon Glass3db916d2020-10-29 21:46:35 -0600202 new_rtag_list[seq] = new_rtags
Simon Glass2112d072020-10-29 21:46:38 -0600203 review_list[seq] = reviews
Simon Glass3db916d2020-10-29 21:46:35 -0600204
Simon Glassd4d3fb42025-04-29 07:22:21 -0600205def show_responses(col, rtags, indent, is_new):
Simon Glass3db916d2020-10-29 21:46:35 -0600206 """Show rtags collected
207
208 Args:
Simon Glassd4d3fb42025-04-29 07:22:21 -0600209 col (terminal.Colour): Colour object to use
Simon Glass3db916d2020-10-29 21:46:35 -0600210 rtags (dict): review tags to show
211 key: Response tag (e.g. 'Reviewed-by')
212 value: Set of people who gave that response, each a name/email string
213 indent (str): Indentation string to write before each line
214 is_new (bool): True if this output should be highlighted
215
216 Returns:
217 int: Number of review tags displayed
218 """
Simon Glass3db916d2020-10-29 21:46:35 -0600219 count = 0
Simon Glass2112d072020-10-29 21:46:38 -0600220 for tag in sorted(rtags.keys()):
221 people = rtags[tag]
222 for who in sorted(people):
Simon Glass02811582022-01-29 14:14:18 -0700223 terminal.tprint(indent + '%s %s: ' % ('+' if is_new else ' ', tag),
Simon Glassd4d3fb42025-04-29 07:22:21 -0600224 newline=False, colour=col.GREEN, bright=is_new,
225 col=col)
226 terminal.tprint(who, colour=col.WHITE, bright=is_new, col=col)
Simon Glass3db916d2020-10-29 21:46:35 -0600227 count += 1
228 return count
229
Simon Glassd0a0a582020-10-29 21:46:36 -0600230def create_branch(series, new_rtag_list, branch, dest_branch, overwrite,
231 repo=None):
232 """Create a new branch with review tags added
233
234 Args:
235 series (Series): Series object for the existing branch
236 new_rtag_list (list): List of review tags to add, one for each commit,
237 each a dict:
238 key: Response tag (e.g. 'Reviewed-by')
239 value: Set of people who gave that response, each a name/email
240 string
241 branch (str): Existing branch to update
242 dest_branch (str): Name of new branch to create
243 overwrite (bool): True to force overwriting dest_branch if it exists
244 repo (pygit2.Repository): Repo to use (use None unless testing)
245
246 Returns:
247 int: Total number of review tags added across all commits
248
249 Raises:
250 ValueError: if the destination branch name is the same as the original
251 branch, or it already exists and @overwrite is False
252 """
253 if branch == dest_branch:
254 raise ValueError(
255 'Destination branch must not be the same as the original branch')
256 if not repo:
257 repo = pygit2.Repository('.')
258 count = len(series.commits)
259 new_br = repo.branches.get(dest_branch)
260 if new_br:
261 if not overwrite:
262 raise ValueError("Branch '%s' already exists (-f to overwrite)" %
263 dest_branch)
264 new_br.delete()
265 if not branch:
266 branch = 'HEAD'
267 target = repo.revparse_single('%s~%d' % (branch, count))
268 repo.branches.local.create(dest_branch, target)
269
270 num_added = 0
271 for seq in range(count):
272 parent = repo.branches.get(dest_branch)
273 cherry = repo.revparse_single('%s~%d' % (branch, count - seq - 1))
274
275 repo.merge_base(cherry.oid, parent.target)
276 base_tree = cherry.parents[0].tree
277
278 index = repo.merge_trees(base_tree, parent, cherry)
279 tree_id = index.write_tree(repo)
280
281 lines = []
282 if new_rtag_list[seq]:
283 for tag, people in new_rtag_list[seq].items():
284 for who in people:
285 lines.append('%s: %s' % (tag, who))
286 num_added += 1
287 message = patchstream.insert_tags(cherry.message.rstrip(),
288 sorted(lines))
289
290 repo.create_commit(
291 parent.name, cherry.author, cherry.committer, message, tree_id,
292 [parent.target])
293 return num_added
294
Simon Glass3729b8b2025-04-29 07:22:24 -0600295def _check_status(series, series_id, pwork):
Simon Glass3db916d2020-10-29 21:46:35 -0600296 """Check the status of a series on Patchwork
297
298 This finds review tags and comments for a series in Patchwork, displaying
299 them to show what is new compared to the local series.
300
301 Args:
302 series (Series): Series object for the existing branch
303 series_id (str): Patch series ID number
Simon Glass25b91c12025-04-29 07:22:19 -0600304 pwork (Patchwork): Patchwork object to use for reading
Simon Glass27280f42025-04-29 07:22:17 -0600305
306 Return:
307 tuple:
308 list of Patch: List of patches sorted by sequence number
309 dict: Patches for commit
310 key: Commit number (0...n-1)
311 value: Patch object for that commit
312 list of dict: review tags:
313 key: Response tag (e.g. 'Reviewed-by')
314 value: Set of people who gave that response, each a name/email
315 string
316 list for each patch, each a:
317 list of Review objects for the patch
Simon Glass3db916d2020-10-29 21:46:35 -0600318 """
Simon Glass25b91c12025-04-29 07:22:19 -0600319 patches = collect_patches(series_id, pwork)
Simon Glass3db916d2020-10-29 21:46:35 -0600320 count = len(series.commits)
321 new_rtag_list = [None] * count
Simon Glass2112d072020-10-29 21:46:38 -0600322 review_list = [None] * count
Simon Glass3db916d2020-10-29 21:46:35 -0600323
324 patch_for_commit, _, warnings = compare_with_series(series, patches)
325 for warn in warnings:
Simon Glass011f1b32022-01-29 14:14:15 -0700326 tout.warning(warn)
Simon Glass3db916d2020-10-29 21:46:35 -0600327
328 patch_list = [patch_for_commit.get(c) for c in range(len(series.commits))]
329
330 with concurrent.futures.ThreadPoolExecutor(max_workers=16) as executor:
331 futures = executor.map(
Simon Glass2112d072020-10-29 21:46:38 -0600332 find_new_responses, repeat(new_rtag_list), repeat(review_list),
Simon Glass25b91c12025-04-29 07:22:19 -0600333 range(count), series.commits, patch_list, repeat(pwork))
Simon Glass3db916d2020-10-29 21:46:35 -0600334 for fresponse in futures:
335 if fresponse:
336 raise fresponse.exception()
Simon Glass27280f42025-04-29 07:22:17 -0600337 return patches, patch_for_commit, new_rtag_list, review_list
Simon Glass3db916d2020-10-29 21:46:35 -0600338
Simon Glass27280f42025-04-29 07:22:17 -0600339
340def check_patch_count(num_commits, num_patches):
341 """Check the number of commits and patches agree
342
343 Args:
344 num_commits (int): Number of commits
345 num_patches (int): Number of patches
346 """
347 if num_patches != num_commits:
348 tout.warning(f'Warning: Patchwork reports {num_patches} patches, '
349 f'series has {num_commits}')
350
351
Simon Glass3729b8b2025-04-29 07:22:24 -0600352def do_show_status(patches, series, branch, show_comments, col,
353 warnings_on_stderr=True):
354 """Check the status of a series on Patchwork
355
356 This finds review tags and comments for a series in Patchwork, displaying
357 them to show what is new compared to the local series.
358
359 Args:
360 patches (list of Patch): Patch objects in the series
361 series (Series): Series object for the existing branch
362 branch (str): Existing branch to update, or None
363 show_comments (bool): True to show the comments on each patch
364 col (terminal.Colour): Colour object
365
366 Return: tuple:
367 int: Number of new review tags to add
368 list: List of review tags to add, one item for each commit, each a
369 dict:
370 key: Response tag (e.g. 'Reviewed-by')
371 value: Set of people who gave that response, each a name/email
372 string
373 list of PATCH objects
374 """
375 compare = []
376 for pw_patch in patches:
377 patch = patchwork.Patch(pw_patch.id)
378 patch.parse_subject(pw_patch.series_data['name'])
379 compare.append(patch)
380
381 count = len(series.commits)
382 new_rtag_list = [None] * count
383 review_list = [None] * count
384
385 patch_for_commit, _, warnings = compare_with_series(series, compare)
386 for warn in warnings:
387 tout.do_output(tout.WARNING if warnings_on_stderr else tout.INFO, warn)
388
389 for seq, pw_patch in enumerate(patches):
390 compare[seq].patch = pw_patch
391
392 for i in range(count):
393 pat = patch_for_commit.get(i)
394 if pat:
395 patch_data = pat.patch.data
396 comment_data = pat.patch.comments
397 new_rtag_list[i], review_list[i] = process_reviews(
398 patch_data['content'], comment_data, series.commits[i].rtags)
399 num_to_add = _do_show_status(series, patch_for_commit, show_comments, new_rtag_list,
400 review_list, col)
401 return num_to_add, new_rtag_list, patches
402
403
404def _do_show_status(series, patch_for_commit, show_comments, new_rtag_list,
Simon Glass27280f42025-04-29 07:22:17 -0600405 review_list, col):
Simon Glass3db916d2020-10-29 21:46:35 -0600406 num_to_add = 0
407 for seq, cmt in enumerate(series.commits):
408 patch = patch_for_commit.get(seq)
409 if not patch:
410 continue
Simon Glass02811582022-01-29 14:14:18 -0700411 terminal.tprint('%3d %s' % (patch.seq, patch.subject[:50]),
Simon Glassd4d3fb42025-04-29 07:22:21 -0600412 colour=col.YELLOW, col=col)
Simon Glass3db916d2020-10-29 21:46:35 -0600413 cmt = series.commits[seq]
414 base_rtags = cmt.rtags
415 new_rtags = new_rtag_list[seq]
416
417 indent = ' ' * 2
Simon Glassd4d3fb42025-04-29 07:22:21 -0600418 show_responses(col, base_rtags, indent, False)
419 num_to_add += show_responses(col, new_rtags, indent, True)
Simon Glass2112d072020-10-29 21:46:38 -0600420 if show_comments:
421 for review in review_list[seq]:
Simon Glass3729b8b2025-04-29 07:22:24 -0600422 terminal.tprint('Review: %s' % review.meta, colour=col.RED,
423 col=col)
Simon Glass2112d072020-10-29 21:46:38 -0600424 for snippet in review.snippets:
425 for line in snippet:
426 quoted = line.startswith('>')
Simon Glass3729b8b2025-04-29 07:22:24 -0600427 terminal.tprint(
428 f' {line}',
429 colour=col.MAGENTA if quoted else None, col=col)
Simon Glass02811582022-01-29 14:14:18 -0700430 terminal.tprint()
Simon Glass27280f42025-04-29 07:22:17 -0600431 return num_to_add
432
433
Simon Glass3729b8b2025-04-29 07:22:24 -0600434def show_status(series, branch, dest_branch, force, patches, show_comments,
435 test_repo=None):
436 """Check the status of a series on Patchwork
437
438 This finds review tags and comments for a series in Patchwork, displaying
439 them to show what is new compared to the local series.
Simon Glass27280f42025-04-29 07:22:17 -0600440
441 Args:
442 series (Series): Series object for the existing branch
443 branch (str): Existing branch to update, or None
444 dest_branch (str): Name of new branch to create, or None
445 force (bool): True to force overwriting dest_branch if it exists
446 patches (list of Patch): Patches sorted by sequence number
Simon Glass3729b8b2025-04-29 07:22:24 -0600447 show_comments (bool): True to show the comments on each patch
Simon Glass27280f42025-04-29 07:22:17 -0600448 test_repo (pygit2.Repository): Repo to use (use None unless testing)
449 """
450 col = terminal.Color()
451 check_patch_count(len(series.commits), len(patches))
Simon Glass3729b8b2025-04-29 07:22:24 -0600452 num_to_add, new_rtag_list, _ = do_show_status(
453 patches, series, branch, show_comments, col)
Simon Glass3db916d2020-10-29 21:46:35 -0600454
Simon Glass3729b8b2025-04-29 07:22:24 -0600455 if not dest_branch and num_to_add:
456 msg = ' (use -d to write them to a new branch)'
457 else:
458 msg = ''
459 terminal.tprint(
460 f"{num_to_add} new response{'s' if num_to_add != 1 else ''} "
461 f'available in patchwork{msg}')
Simon Glassd0a0a582020-10-29 21:46:36 -0600462
463 if dest_branch:
Simon Glass3729b8b2025-04-29 07:22:24 -0600464 num_added = create_branch(series, new_rtag_list, branch,
465 dest_branch, force, test_repo)
Simon Glass02811582022-01-29 14:14:18 -0700466 terminal.tprint(
Simon Glass3729b8b2025-04-29 07:22:24 -0600467 f"{num_added} response{'s' if num_added != 1 else ''} added "
468 f"from patchwork into new branch '{dest_branch}'")
Simon Glass27280f42025-04-29 07:22:17 -0600469
470
Simon Glass3729b8b2025-04-29 07:22:24 -0600471async def check_status(link, pwork, read_comments=False):
472 """Set up an HTTP session and get the required state
473
474 Args:
475 link (str): Patch series ID number
476 pwork (Patchwork): Patchwork object to use for reading
477 read_comments (bool): True to read comments and state for each patch
478
479 Return: tuple:
480 list of Patch objects
481 """
482 async with aiohttp.ClientSession() as client:
483 patches = await pwork.series_get_state(client, link, read_comments)
484 return patches
485
486
Simon Glass27280f42025-04-29 07:22:17 -0600487def check_and_show_status(series, link, branch, dest_branch, force,
Simon Glass25b91c12025-04-29 07:22:19 -0600488 show_comments, pwork, test_repo=None):
Simon Glass27280f42025-04-29 07:22:17 -0600489 """Read the series status from patchwork and show it to the user
490
491 Args:
492 series (Series): Series object for the existing branch
493 link (str): Patch series ID number
494 branch (str): Existing branch to update, or None
495 dest_branch (str): Name of new branch to create, or None
496 force (bool): True to force overwriting dest_branch if it exists
Simon Glass3729b8b2025-04-29 07:22:24 -0600497 show_comments (bool): True to show the comments on each patch
Simon Glass25b91c12025-04-29 07:22:19 -0600498 pwork (Patchwork): Patchwork object to use for reading
Simon Glass27280f42025-04-29 07:22:17 -0600499 test_repo (pygit2.Repository): Repo to use (use None unless testing)
500 """
Simon Glass3729b8b2025-04-29 07:22:24 -0600501 patches = asyncio.run(check_status(link, pwork, True))
502
503 show_status(series, branch, dest_branch, force, patches, show_comments,
504 test_repo=test_repo)