| # SPDX-License-Identifier: GPL-2.0+ |
| # |
| # Copyright 2020 Google LLC |
| # |
| """Talks to the patchwork service to figure out what patches have been reviewed |
| and commented on. Provides a way to display review tags and comments. |
| Allows creation of a new branch based on the old but with the review tags |
| collected from patchwork. |
| """ |
| |
| import collections |
| import concurrent.futures |
| from itertools import repeat |
| |
| import pygit2 |
| import requests |
| |
| from u_boot_pylib import terminal |
| from u_boot_pylib import tout |
| from patman import patchstream |
| from patman import patchwork |
| from patman.patchstream import PatchStream |
| |
| |
| def to_int(vals): |
| """Convert a list of strings into integers, using 0 if not an integer |
| |
| Args: |
| vals (list): List of strings |
| |
| Returns: |
| list: List of integers, one for each input string |
| """ |
| out = [int(val) if val.isdigit() else 0 for val in vals] |
| return out |
| |
| |
| def process_reviews(content, comment_data, base_rtags): |
| """Process and return review data |
| |
| Args: |
| content (str): Content text of the patch itself - see pwork.get_patch() |
| comment_data (list of dict): Comments for the patch - see |
| pwork._get_patch_comments() |
| base_rtags (dict): base review tags (before any comments) |
| key: Response tag (e.g. 'Reviewed-by') |
| value: Set of people who gave that response, each a name/email |
| string |
| |
| Return: tuple: |
| dict: new review tags (noticed since the base_rtags) |
| key: Response tag (e.g. 'Reviewed-by') |
| value: Set of people who gave that response, each a name/email |
| string |
| list of patchwork.Review: reviews received on the patch |
| """ |
| pstrm = patchstream.PatchStream.process_text(content, True) |
| rtags = collections.defaultdict(set) |
| for response, people in pstrm.commit.rtags.items(): |
| rtags[response].update(people) |
| |
| reviews = [] |
| for comment in comment_data: |
| pstrm = patchstream.PatchStream.process_text(comment['content'], True) |
| if pstrm.snippets: |
| submitter = comment['submitter'] |
| person = f"{submitter['name']} <{submitter['email']}>" |
| reviews.append(patchwork.Review(person, pstrm.snippets)) |
| for response, people in pstrm.commit.rtags.items(): |
| rtags[response].update(people) |
| |
| # Find the tags that are not in the commit |
| new_rtags = collections.defaultdict(set) |
| for tag, people in rtags.items(): |
| for who in people: |
| is_new = (tag not in base_rtags or |
| who not in base_rtags[tag]) |
| if is_new: |
| new_rtags[tag].add(who) |
| return new_rtags, reviews |
| |
| |
| def compare_with_series(series, patches): |
| """Compare a list of patches with a series it came from |
| |
| This prints any problems as warnings |
| |
| Args: |
| series (Series): Series to compare against |
| patches (:type: list of Patch): list of Patch objects to compare with |
| |
| Returns: |
| tuple |
| dict: |
| key: Commit number (0...n-1) |
| value: Patch object for that commit |
| dict: |
| key: Patch number (0...n-1) |
| value: Commit object for that patch |
| """ |
| # Check the names match |
| warnings = [] |
| patch_for_commit = {} |
| all_patches = set(patches) |
| for seq, cmt in enumerate(series.commits): |
| pmatch = [p for p in all_patches if p.subject == cmt.subject] |
| if len(pmatch) == 1: |
| patch_for_commit[seq] = pmatch[0] |
| all_patches.remove(pmatch[0]) |
| elif len(pmatch) > 1: |
| warnings.append("Multiple patches match commit %d ('%s'):\n %s" % |
| (seq + 1, cmt.subject, |
| '\n '.join([p.subject for p in pmatch]))) |
| else: |
| warnings.append("Cannot find patch for commit %d ('%s')" % |
| (seq + 1, cmt.subject)) |
| |
| |
| # Check the names match |
| commit_for_patch = {} |
| all_commits = set(series.commits) |
| for seq, patch in enumerate(patches): |
| cmatch = [c for c in all_commits if c.subject == patch.subject] |
| if len(cmatch) == 1: |
| commit_for_patch[seq] = cmatch[0] |
| all_commits.remove(cmatch[0]) |
| elif len(cmatch) > 1: |
| warnings.append("Multiple commits match patch %d ('%s'):\n %s" % |
| (seq + 1, patch.subject, |
| '\n '.join([c.subject for c in cmatch]))) |
| else: |
| warnings.append("Cannot find commit for patch %d ('%s')" % |
| (seq + 1, patch.subject)) |
| |
| return patch_for_commit, commit_for_patch, warnings |
| |
| def collect_patches(series_id, pwork): |
| """Collect patch information about a series from patchwork |
| |
| Uses the Patchwork REST API to collect information provided by patchwork |
| about the status of each patch. |
| |
| Args: |
| series_id (str): Patch series ID number |
| pwork (Patchwork): Patchwork object to use for reading |
| |
| Returns: |
| list of Patch: List of patches sorted by sequence number |
| |
| Raises: |
| ValueError: if the URL could not be read or the web page does not follow |
| the expected structure |
| """ |
| data = pwork.request('series/%s/' % series_id) |
| |
| # Get all the rows, which are patches |
| patch_dict = data['patches'] |
| count = len(patch_dict) |
| |
| patches = [] |
| |
| # Work through each row (patch) one at a time, collecting the information |
| warn_count = 0 |
| for pw_patch in patch_dict: |
| patch = patchwork.Patch(pw_patch['id']) |
| patch.parse_subject(pw_patch['name']) |
| patches.append(patch) |
| if warn_count > 1: |
| tout.warning(' (total of %d warnings)' % warn_count) |
| |
| # Sort patches by patch number |
| patches = sorted(patches, key=lambda x: x.seq) |
| return patches |
| |
| def find_new_responses(new_rtag_list, review_list, seq, cmt, patch, pwork): |
| """Find new rtags collected by patchwork that we don't know about |
| |
| This is designed to be run in parallel, once for each commit/patch |
| |
| Args: |
| new_rtag_list (list): New rtags are written to new_rtag_list[seq] |
| list, each a dict: |
| key: Response tag (e.g. 'Reviewed-by') |
| value: Set of people who gave that response, each a name/email |
| string |
| review_list (list): New reviews are written to review_list[seq] |
| list, each a |
| List of reviews for the patch, each a Review |
| seq (int): Position in new_rtag_list to update |
| cmt (Commit): Commit object for this commit |
| patch (Patch): Corresponding Patch object for this patch |
| pwork (Patchwork): Patchwork object to use for reading |
| """ |
| if not patch: |
| return |
| |
| # Get the content for the patch email itself as well as all comments |
| data = pwork.request('patches/%s/' % patch.id) |
| comment_data = pwork.request('patches/%s/comments/' % patch.id) |
| |
| new_rtags, reviews = process_reviews(data['content'], comment_data, |
| cmt.rtags) |
| new_rtag_list[seq] = new_rtags |
| review_list[seq] = reviews |
| |
| def show_responses(col, rtags, indent, is_new): |
| """Show rtags collected |
| |
| Args: |
| col (terminal.Colour): Colour object to use |
| rtags (dict): review tags to show |
| key: Response tag (e.g. 'Reviewed-by') |
| value: Set of people who gave that response, each a name/email string |
| indent (str): Indentation string to write before each line |
| is_new (bool): True if this output should be highlighted |
| |
| Returns: |
| int: Number of review tags displayed |
| """ |
| count = 0 |
| for tag in sorted(rtags.keys()): |
| people = rtags[tag] |
| for who in sorted(people): |
| terminal.tprint(indent + '%s %s: ' % ('+' if is_new else ' ', tag), |
| newline=False, colour=col.GREEN, bright=is_new, |
| col=col) |
| terminal.tprint(who, colour=col.WHITE, bright=is_new, col=col) |
| count += 1 |
| return count |
| |
| def create_branch(series, new_rtag_list, branch, dest_branch, overwrite, |
| repo=None): |
| """Create a new branch with review tags added |
| |
| Args: |
| series (Series): Series object for the existing branch |
| new_rtag_list (list): List of review tags to add, one for each commit, |
| each a dict: |
| key: Response tag (e.g. 'Reviewed-by') |
| value: Set of people who gave that response, each a name/email |
| string |
| branch (str): Existing branch to update |
| dest_branch (str): Name of new branch to create |
| overwrite (bool): True to force overwriting dest_branch if it exists |
| repo (pygit2.Repository): Repo to use (use None unless testing) |
| |
| Returns: |
| int: Total number of review tags added across all commits |
| |
| Raises: |
| ValueError: if the destination branch name is the same as the original |
| branch, or it already exists and @overwrite is False |
| """ |
| if branch == dest_branch: |
| raise ValueError( |
| 'Destination branch must not be the same as the original branch') |
| if not repo: |
| repo = pygit2.Repository('.') |
| count = len(series.commits) |
| new_br = repo.branches.get(dest_branch) |
| if new_br: |
| if not overwrite: |
| raise ValueError("Branch '%s' already exists (-f to overwrite)" % |
| dest_branch) |
| new_br.delete() |
| if not branch: |
| branch = 'HEAD' |
| target = repo.revparse_single('%s~%d' % (branch, count)) |
| repo.branches.local.create(dest_branch, target) |
| |
| num_added = 0 |
| for seq in range(count): |
| parent = repo.branches.get(dest_branch) |
| cherry = repo.revparse_single('%s~%d' % (branch, count - seq - 1)) |
| |
| repo.merge_base(cherry.oid, parent.target) |
| base_tree = cherry.parents[0].tree |
| |
| index = repo.merge_trees(base_tree, parent, cherry) |
| tree_id = index.write_tree(repo) |
| |
| lines = [] |
| if new_rtag_list[seq]: |
| for tag, people in new_rtag_list[seq].items(): |
| for who in people: |
| lines.append('%s: %s' % (tag, who)) |
| num_added += 1 |
| message = patchstream.insert_tags(cherry.message.rstrip(), |
| sorted(lines)) |
| |
| repo.create_commit( |
| parent.name, cherry.author, cherry.committer, message, tree_id, |
| [parent.target]) |
| return num_added |
| |
| def check_status(series, series_id, pwork): |
| """Check the status of a series on Patchwork |
| |
| This finds review tags and comments for a series in Patchwork, displaying |
| them to show what is new compared to the local series. |
| |
| Args: |
| series (Series): Series object for the existing branch |
| series_id (str): Patch series ID number |
| pwork (Patchwork): Patchwork object to use for reading |
| |
| Return: |
| tuple: |
| list of Patch: List of patches sorted by sequence number |
| dict: Patches for commit |
| key: Commit number (0...n-1) |
| value: Patch object for that commit |
| list of dict: review tags: |
| key: Response tag (e.g. 'Reviewed-by') |
| value: Set of people who gave that response, each a name/email |
| string |
| list for each patch, each a: |
| list of Review objects for the patch |
| """ |
| patches = collect_patches(series_id, pwork) |
| count = len(series.commits) |
| new_rtag_list = [None] * count |
| review_list = [None] * count |
| |
| patch_for_commit, _, warnings = compare_with_series(series, patches) |
| for warn in warnings: |
| tout.warning(warn) |
| |
| patch_list = [patch_for_commit.get(c) for c in range(len(series.commits))] |
| |
| with concurrent.futures.ThreadPoolExecutor(max_workers=16) as executor: |
| futures = executor.map( |
| find_new_responses, repeat(new_rtag_list), repeat(review_list), |
| range(count), series.commits, patch_list, repeat(pwork)) |
| for fresponse in futures: |
| if fresponse: |
| raise fresponse.exception() |
| return patches, patch_for_commit, new_rtag_list, review_list |
| |
| |
| def check_patch_count(num_commits, num_patches): |
| """Check the number of commits and patches agree |
| |
| Args: |
| num_commits (int): Number of commits |
| num_patches (int): Number of patches |
| """ |
| if num_patches != num_commits: |
| tout.warning(f'Warning: Patchwork reports {num_patches} patches, ' |
| f'series has {num_commits}') |
| |
| |
| def do_show_status(series, patch_for_commit, show_comments, new_rtag_list, |
| review_list, col): |
| num_to_add = 0 |
| for seq, cmt in enumerate(series.commits): |
| patch = patch_for_commit.get(seq) |
| if not patch: |
| continue |
| terminal.tprint('%3d %s' % (patch.seq, patch.subject[:50]), |
| colour=col.YELLOW, col=col) |
| cmt = series.commits[seq] |
| base_rtags = cmt.rtags |
| new_rtags = new_rtag_list[seq] |
| |
| indent = ' ' * 2 |
| show_responses(col, base_rtags, indent, False) |
| num_to_add += show_responses(col, new_rtags, indent, True) |
| if show_comments: |
| for review in review_list[seq]: |
| terminal.tprint('Review: %s' % review.meta, colour=col.RED) |
| for snippet in review.snippets: |
| for line in snippet: |
| quoted = line.startswith('>') |
| terminal.tprint(' %s' % line, |
| colour=col.MAGENTA if quoted else None) |
| terminal.tprint() |
| return num_to_add |
| |
| |
| def show_status(series, branch, dest_branch, force, patches, patch_for_commit, |
| show_comments, new_rtag_list, review_list, test_repo=None): |
| """Show status to the user and allow a branch to be written |
| |
| Args: |
| series (Series): Series object for the existing branch |
| branch (str): Existing branch to update, or None |
| dest_branch (str): Name of new branch to create, or None |
| force (bool): True to force overwriting dest_branch if it exists |
| patches (list of Patch): Patches sorted by sequence number |
| patch_for_commit (dict): Patches for commit |
| key: Commit number (0...n-1) |
| value: Patch object for that commit |
| show_comments (bool): True to show patch comments |
| new_rtag_list (list of dict) review tags for each patch: |
| key: Response tag (e.g. 'Reviewed-by') |
| value: Set of people who gave that response, each a name/email |
| string |
| review_list (list of list): list for each patch, each a: |
| list of Review objects for the patch |
| test_repo (pygit2.Repository): Repo to use (use None unless testing) |
| """ |
| col = terminal.Color() |
| check_patch_count(len(series.commits), len(patches)) |
| num_to_add = do_show_status(series, patch_for_commit, show_comments, |
| new_rtag_list, review_list, col) |
| |
| terminal.tprint("%d new response%s available in patchwork%s" % |
| (num_to_add, 's' if num_to_add != 1 else '', |
| '' if dest_branch |
| else ' (use -d to write them to a new branch)')) |
| |
| if dest_branch: |
| num_added = create_branch(series, new_rtag_list, branch, dest_branch, |
| force, test_repo) |
| terminal.tprint( |
| "%d response%s added from patchwork into new branch '%s'" % |
| (num_added, 's' if num_added != 1 else '', dest_branch)) |
| |
| |
| def check_and_show_status(series, link, branch, dest_branch, force, |
| show_comments, pwork, test_repo=None): |
| """Read the series status from patchwork and show it to the user |
| |
| Args: |
| series (Series): Series object for the existing branch |
| link (str): Patch series ID number |
| branch (str): Existing branch to update, or None |
| dest_branch (str): Name of new branch to create, or None |
| force (bool): True to force overwriting dest_branch if it exists |
| show_comments (bool): True to show patch comments |
| pwork (Patchwork): Patchwork object to use for reading |
| test_repo (pygit2.Repository): Repo to use (use None unless testing) |
| """ |
| patches, patch_for_commit, new_rtag_list, review_list = check_status( |
| series, link, pwork) |
| show_status(series, branch, dest_branch, force, patches, patch_for_commit, |
| show_comments, new_rtag_list, review_list, test_repo) |