blob: e2e2a83e677323e63dac8741f1434a512094104d [file] [log] [blame]
Tom Rini10e47792018-05-06 17:58:06 -04001# SPDX-License-Identifier: GPL-2.0+
Simon Glass26132882012-01-14 15:12:45 +00002# Copyright (c) 2011 The Chromium OS Authors.
3#
Simon Glass26132882012-01-14 15:12:45 +00004
Simon Glassc42b1d02020-10-29 21:46:17 -06005"""Handles parsing a stream of commits/emails from 'git log' or other source"""
6
Simon Glassda8a2922020-10-29 21:46:37 -06007import collections
Douglas Anderson52b5ee82019-09-27 09:23:56 -07008import datetime
Simon Glass06202d62020-10-29 21:46:27 -06009import io
Wu, Josh393aaa22015-04-03 10:51:17 +080010import math
Simon Glass26132882012-01-14 15:12:45 +000011import os
12import re
Simon Glassda8a2922020-10-29 21:46:37 -060013import queue
Simon Glass26132882012-01-14 15:12:45 +000014import shutil
15import tempfile
16
Simon Glassa997ea52020-04-17 18:09:04 -060017from patman import commit
18from patman import gitutil
19from patman.series import Series
Simon Glass131444f2023-02-23 18:18:04 -070020from u_boot_pylib import command
Simon Glass26132882012-01-14 15:12:45 +000021
22# Tags that we detect and remove
Simon Glass06022262020-10-29 21:46:18 -060023RE_REMOVE = re.compile(r'^BUG=|^TEST=|^BRANCH=|^Review URL:'
Simon Glassc42b1d02020-10-29 21:46:17 -060024 r'|Reviewed-on:|Commit-\w*:')
Simon Glass26132882012-01-14 15:12:45 +000025
26# Lines which are allowed after a TEST= line
Simon Glass06022262020-10-29 21:46:18 -060027RE_ALLOWED_AFTER_TEST = re.compile('^Signed-off-by:')
Simon Glass26132882012-01-14 15:12:45 +000028
Ilya Yanok644c1842012-08-06 23:46:05 +000029# Signoffs
Simon Glass06022262020-10-29 21:46:18 -060030RE_SIGNOFF = re.compile('^Signed-off-by: *(.*)')
Ilya Yanok644c1842012-08-06 23:46:05 +000031
Sean Anderson48f46d62020-05-04 16:28:34 -040032# Cover letter tag
Simon Glass06022262020-10-29 21:46:18 -060033RE_COVER = re.compile('^Cover-([a-z-]*): *(.*)')
Simon Glassc72f3da2013-03-20 16:43:00 +000034
Simon Glass26132882012-01-14 15:12:45 +000035# Patch series tag
Simon Glass06022262020-10-29 21:46:18 -060036RE_SERIES_TAG = re.compile('^Series-([a-z-]*): *(.*)')
Albert ARIBAUDd880efd2013-11-12 11:14:41 +010037
Douglas Anderson52b5ee82019-09-27 09:23:56 -070038# Change-Id will be used to generate the Message-Id and then be stripped
Simon Glass06022262020-10-29 21:46:18 -060039RE_CHANGE_ID = re.compile('^Change-Id: *(.*)')
Douglas Anderson52b5ee82019-09-27 09:23:56 -070040
Albert ARIBAUDd880efd2013-11-12 11:14:41 +010041# Commit series tag
Simon Glass06022262020-10-29 21:46:18 -060042RE_COMMIT_TAG = re.compile('^Commit-([a-z-]*): *(.*)')
Simon Glass26132882012-01-14 15:12:45 +000043
44# Commit tags that we want to collect and keep
Simon Glass06022262020-10-29 21:46:18 -060045RE_TAG = re.compile('^(Tested-by|Acked-by|Reviewed-by|Patch-cc|Fixes): (.*)')
Simon Glass26132882012-01-14 15:12:45 +000046
47# The start of a new commit in the git log
Simon Glass06022262020-10-29 21:46:18 -060048RE_COMMIT = re.compile('^commit ([0-9a-f]*)$')
Simon Glass26132882012-01-14 15:12:45 +000049
50# We detect these since checkpatch doesn't always do it
Simon Glass06022262020-10-29 21:46:18 -060051RE_SPACE_BEFORE_TAB = re.compile('^[+].* \t')
Simon Glass26132882012-01-14 15:12:45 +000052
Sean Anderson1a32f922020-05-04 16:28:35 -040053# Match indented lines for changes
Simon Glass06022262020-10-29 21:46:18 -060054RE_LEADING_WHITESPACE = re.compile(r'^\s')
Sean Anderson1a32f922020-05-04 16:28:35 -040055
Simon Glassda8a2922020-10-29 21:46:37 -060056# Detect a 'diff' line
57RE_DIFF = re.compile(r'^>.*diff --git a/(.*) b/(.*)$')
58
59# Detect a context line, like '> @@ -153,8 +153,13 @@ CheckPatch
60RE_LINE = re.compile(r'>.*@@ \-(\d+),\d+ \+(\d+),\d+ @@ *(.*)')
61
Patrick Delaunay6bbdd0c2021-07-22 16:51:42 +020062# Detect line with invalid TAG
63RE_INV_TAG = re.compile('^Serie-([a-z-]*): *(.*)')
64
Simon Glass26132882012-01-14 15:12:45 +000065# States we can be in - can we use range() and still have comments?
66STATE_MSG_HEADER = 0 # Still in the message header
67STATE_PATCH_SUBJECT = 1 # In patch subject (first line of log for a commit)
68STATE_PATCH_HEADER = 2 # In patch header (after the subject)
69STATE_DIFFS = 3 # In the diff part (past --- line)
70
Maxim Cournoyer12f99fd2023-10-12 23:06:24 -040071
Simon Glass26132882012-01-14 15:12:45 +000072class PatchStream:
73 """Class for detecting/injecting tags in a patch or series of patches
74
75 We support processing the output of 'git log' to read out the tags we
76 are interested in. We can also process a patch file in order to remove
77 unwanted tags or inject additional ones. These correspond to the two
78 phases of processing.
79 """
Maxim Cournoyer12f99fd2023-10-12 23:06:24 -040080 def __init__(self, series, is_log=False, keep_change_id=False):
Simon Glass26132882012-01-14 15:12:45 +000081 self.skip_blank = False # True to skip a single blank line
82 self.found_test = False # Found a TEST= line
Sean Anderson48f46d62020-05-04 16:28:34 -040083 self.lines_after_test = 0 # Number of lines found after TEST=
Simon Glass26132882012-01-14 15:12:45 +000084 self.linenum = 1 # Output line number we are up to
85 self.in_section = None # Name of start...END section we are in
86 self.notes = [] # Series notes
87 self.section = [] # The current section...END section
88 self.series = series # Info about the patch series
89 self.is_log = is_log # True if indent like git log
Maxim Cournoyer12f99fd2023-10-12 23:06:24 -040090 self.keep_change_id = keep_change_id # True to keep Change-Id tags
Sean Anderson48f46d62020-05-04 16:28:34 -040091 self.in_change = None # Name of the change list we are in
92 self.change_version = 0 # Non-zero if we are in a change list
Sean Anderson1a32f922020-05-04 16:28:35 -040093 self.change_lines = [] # Lines of the current change
Simon Glass26132882012-01-14 15:12:45 +000094 self.blank_count = 0 # Number of blank lines stored up
95 self.state = STATE_MSG_HEADER # What state are we in?
Simon Glass26132882012-01-14 15:12:45 +000096 self.commit = None # Current commit
Simon Glass2112d072020-10-29 21:46:38 -060097 # List of unquoted test blocks, each a list of str lines
98 self.snippets = []
Simon Glassda8a2922020-10-29 21:46:37 -060099 self.cur_diff = None # Last 'diff' line seen (str)
100 self.cur_line = None # Last context (@@) line seen (str)
Simon Glass2112d072020-10-29 21:46:38 -0600101 self.recent_diff = None # 'diff' line for current snippet (str)
102 self.recent_line = None # '@@' line for current snippet (str)
Simon Glassda8a2922020-10-29 21:46:37 -0600103 self.recent_quoted = collections.deque([], 5)
104 self.recent_unquoted = queue.Queue()
105 self.was_quoted = None
Simon Glass26132882012-01-14 15:12:45 +0000106
Simon Glass06202d62020-10-29 21:46:27 -0600107 @staticmethod
108 def process_text(text, is_comment=False):
109 """Process some text through this class using a default Commit/Series
110
111 Args:
112 text (str): Text to parse
113 is_comment (bool): True if this is a comment rather than a patch.
114 If True, PatchStream doesn't expect a patch subject at the
115 start, but jumps straight into the body
116
117 Returns:
118 PatchStream: object with results
119 """
120 pstrm = PatchStream(Series())
121 pstrm.commit = commit.Commit(None)
122 infd = io.StringIO(text)
123 outfd = io.StringIO()
124 if is_comment:
125 pstrm.state = STATE_PATCH_HEADER
126 pstrm.process_stream(infd, outfd)
127 return pstrm
128
Simon Glassb5800392020-10-29 21:46:23 -0600129 def _add_warn(self, warn):
Simon Glass6d00f6c2020-10-29 21:46:24 -0600130 """Add a new warning to report to the user about the current commit
131
132 The new warning is added to the current commit if not already present.
Simon Glassb5800392020-10-29 21:46:23 -0600133
134 Args:
135 warn (str): Warning to report
Simon Glass6d00f6c2020-10-29 21:46:24 -0600136
137 Raises:
138 ValueError: Warning is generated with no commit associated
Simon Glassb5800392020-10-29 21:46:23 -0600139 """
Simon Glass6d00f6c2020-10-29 21:46:24 -0600140 if not self.commit:
Simon Glassb3106562021-03-22 18:01:42 +1300141 print('Warning outside commit: %s' % warn)
142 elif warn not in self.commit.warn:
Simon Glass6d00f6c2020-10-29 21:46:24 -0600143 self.commit.warn.append(warn)
Simon Glassb5800392020-10-29 21:46:23 -0600144
Simon Glass93f61c02020-10-29 21:46:19 -0600145 def _add_to_series(self, line, name, value):
Simon Glass26132882012-01-14 15:12:45 +0000146 """Add a new Series-xxx tag.
147
148 When a Series-xxx tag is detected, we come here to record it, if we
149 are scanning a 'git log'.
150
151 Args:
Simon Glass06127f92020-10-29 21:46:22 -0600152 line (str): Source line containing tag (useful for debug/error
153 messages)
154 name (str): Tag name (part after 'Series-')
155 value (str): Tag value (part after 'Series-xxx: ')
Simon Glass26132882012-01-14 15:12:45 +0000156 """
157 if name == 'notes':
158 self.in_section = name
159 self.skip_blank = False
160 if self.is_log:
Simon Glass8093a8f2020-10-29 21:46:25 -0600161 warn = self.series.AddTag(self.commit, line, name, value)
162 if warn:
163 self.commit.warn.append(warn)
Simon Glass26132882012-01-14 15:12:45 +0000164
Simon Glass5f6caa52020-10-29 21:46:21 -0600165 def _add_to_commit(self, name):
Albert ARIBAUDd880efd2013-11-12 11:14:41 +0100166 """Add a new Commit-xxx tag.
167
168 When a Commit-xxx tag is detected, we come here to record it.
169
170 Args:
Simon Glass06127f92020-10-29 21:46:22 -0600171 name (str): Tag name (part after 'Commit-')
Albert ARIBAUDd880efd2013-11-12 11:14:41 +0100172 """
173 if name == 'notes':
174 self.in_section = 'commit-' + name
175 self.skip_blank = False
176
Simon Glass93f61c02020-10-29 21:46:19 -0600177 def _add_commit_rtag(self, rtag_type, who):
Simon Glass08e91be2020-07-05 21:41:57 -0600178 """Add a response tag to the current commit
179
180 Args:
Simon Glass06127f92020-10-29 21:46:22 -0600181 rtag_type (str): rtag type (e.g. 'Reviewed-by')
182 who (str): Person who gave that rtag, e.g.
183 'Fred Bloggs <fred@bloggs.org>'
Simon Glass08e91be2020-07-05 21:41:57 -0600184 """
Simon Glass530ac272022-01-29 14:14:07 -0700185 self.commit.add_rtag(rtag_type, who)
Simon Glass08e91be2020-07-05 21:41:57 -0600186
Simon Glass93f61c02020-10-29 21:46:19 -0600187 def _close_commit(self):
Simon Glass26132882012-01-14 15:12:45 +0000188 """Save the current commit into our commit list, and reset our state"""
189 if self.commit and self.is_log:
190 self.series.AddCommit(self.commit)
191 self.commit = None
Bin Mengd23121c2016-06-26 23:24:30 -0700192 # If 'END' is missing in a 'Cover-letter' section, and that section
193 # happens to show up at the very end of the commit message, this is
194 # the chance for us to fix it up.
195 if self.in_section == 'cover' and self.is_log:
196 self.series.cover = self.section
197 self.in_section = None
198 self.skip_blank = True
199 self.section = []
Simon Glass26132882012-01-14 15:12:45 +0000200
Simon Glassda8a2922020-10-29 21:46:37 -0600201 self.cur_diff = None
202 self.recent_diff = None
203 self.recent_line = None
204
Simon Glass93f61c02020-10-29 21:46:19 -0600205 def _parse_version(self, value, line):
Sean Anderson48f46d62020-05-04 16:28:34 -0400206 """Parse a version from a *-changes tag
207
208 Args:
Simon Glass06127f92020-10-29 21:46:22 -0600209 value (str): Tag value (part after 'xxx-changes: '
210 line (str): Source line containing tag
Sean Anderson48f46d62020-05-04 16:28:34 -0400211
212 Returns:
Simon Glass06127f92020-10-29 21:46:22 -0600213 int: The version as an integer
214
215 Raises:
216 ValueError: the value cannot be converted
Sean Anderson48f46d62020-05-04 16:28:34 -0400217 """
218 try:
219 return int(value)
Simon Glassf9b6e0f2020-10-29 21:46:20 -0600220 except ValueError:
Sean Anderson48f46d62020-05-04 16:28:34 -0400221 raise ValueError("%s: Cannot decode version info '%s'" %
Simon Glassc42b1d02020-10-29 21:46:17 -0600222 (self.commit.hash, line))
Sean Anderson48f46d62020-05-04 16:28:34 -0400223
Simon Glass93f61c02020-10-29 21:46:19 -0600224 def _finalise_change(self):
225 """_finalise a (multi-line) change and add it to the series or commit"""
Sean Anderson1a32f922020-05-04 16:28:35 -0400226 if not self.change_lines:
227 return
228 change = '\n'.join(self.change_lines)
229
230 if self.in_change == 'Series':
231 self.series.AddChange(self.change_version, self.commit, change)
232 elif self.in_change == 'Cover':
233 self.series.AddChange(self.change_version, None, change)
234 elif self.in_change == 'Commit':
Simon Glass530ac272022-01-29 14:14:07 -0700235 self.commit.add_change(self.change_version, change)
Sean Anderson1a32f922020-05-04 16:28:35 -0400236 self.change_lines = []
237
Simon Glassda8a2922020-10-29 21:46:37 -0600238 def _finalise_snippet(self):
239 """Finish off a snippet and add it to the list
240
241 This is called when we get to the end of a snippet, i.e. the we enter
242 the next block of quoted text:
243
244 This is a comment from someone.
245
246 Something else
247
248 > Now we have some code <----- end of snippet
249 > more code
250
251 Now a comment about the above code
252
253 This adds the snippet to our list
254 """
255 quoted_lines = []
256 while self.recent_quoted:
257 quoted_lines.append(self.recent_quoted.popleft())
258 unquoted_lines = []
259 valid = False
260 while not self.recent_unquoted.empty():
261 text = self.recent_unquoted.get()
262 if not (text.startswith('On ') and text.endswith('wrote:')):
263 unquoted_lines.append(text)
264 if text:
265 valid = True
266 if valid:
267 lines = []
268 if self.recent_diff:
269 lines.append('> File: %s' % self.recent_diff)
270 if self.recent_line:
271 out = '> Line: %s / %s' % self.recent_line[:2]
272 if self.recent_line[2]:
273 out += ': %s' % self.recent_line[2]
274 lines.append(out)
275 lines += quoted_lines + unquoted_lines
276 if lines:
277 self.snippets.append(lines)
278
Simon Glass93f61c02020-10-29 21:46:19 -0600279 def process_line(self, line):
Simon Glass26132882012-01-14 15:12:45 +0000280 """Process a single line of a patch file or commit log
281
282 This process a line and returns a list of lines to output. The list
283 may be empty or may contain multiple output lines.
284
285 This is where all the complicated logic is located. The class's
286 state is used to move between different states and detect things
287 properly.
288
289 We can be in one of two modes:
290 self.is_log == True: This is 'git log' mode, where most output is
291 indented by 4 characters and we are scanning for tags
292
293 self.is_log == False: This is 'patch' mode, where we already have
294 all the tags, and are processing patches to remove junk we
295 don't want, and add things we think are required.
296
297 Args:
Simon Glass06127f92020-10-29 21:46:22 -0600298 line (str): text line to process
Simon Glass26132882012-01-14 15:12:45 +0000299
300 Returns:
Simon Glass06127f92020-10-29 21:46:22 -0600301 list: list of output lines, or [] if nothing should be output
302
303 Raises:
304 ValueError: a fatal error occurred while parsing, e.g. an END
305 without a starting tag, or two commits with two change IDs
Simon Glass26132882012-01-14 15:12:45 +0000306 """
307 # Initially we have no output. Prepare the input line string
308 out = []
309 line = line.rstrip('\n')
Scott Wood616a0ab2014-09-25 14:30:46 -0500310
Simon Glass06022262020-10-29 21:46:18 -0600311 commit_match = RE_COMMIT.match(line) if self.is_log else None
Scott Wood616a0ab2014-09-25 14:30:46 -0500312
Simon Glass26132882012-01-14 15:12:45 +0000313 if self.is_log:
314 if line[:4] == ' ':
315 line = line[4:]
316
317 # Handle state transition and skipping blank lines
Simon Glass06022262020-10-29 21:46:18 -0600318 series_tag_match = RE_SERIES_TAG.match(line)
319 change_id_match = RE_CHANGE_ID.match(line)
320 commit_tag_match = RE_COMMIT_TAG.match(line)
321 cover_match = RE_COVER.match(line)
322 signoff_match = RE_SIGNOFF.match(line)
323 leading_whitespace_match = RE_LEADING_WHITESPACE.match(line)
Simon Glassda8a2922020-10-29 21:46:37 -0600324 diff_match = RE_DIFF.match(line)
325 line_match = RE_LINE.match(line)
Patrick Delaunay6bbdd0c2021-07-22 16:51:42 +0200326 invalid_match = RE_INV_TAG.match(line)
Simon Glass26132882012-01-14 15:12:45 +0000327 tag_match = None
328 if self.state == STATE_PATCH_HEADER:
Simon Glass06022262020-10-29 21:46:18 -0600329 tag_match = RE_TAG.match(line)
Simon Glass26132882012-01-14 15:12:45 +0000330 is_blank = not line.strip()
331 if is_blank:
332 if (self.state == STATE_MSG_HEADER
333 or self.state == STATE_PATCH_SUBJECT):
334 self.state += 1
335
336 # We don't have a subject in the text stream of patch files
337 # It has its own line with a Subject: tag
338 if not self.is_log and self.state == STATE_PATCH_SUBJECT:
339 self.state += 1
340 elif commit_match:
341 self.state = STATE_MSG_HEADER
342
Bin Meng9228cc22016-06-26 23:24:32 -0700343 # If a tag is detected, or a new commit starts
Douglas Anderson52b5ee82019-09-27 09:23:56 -0700344 if series_tag_match or commit_tag_match or change_id_match or \
Sean Anderson48f46d62020-05-04 16:28:34 -0400345 cover_match or signoff_match or self.state == STATE_MSG_HEADER:
Bin Meng50ffc982016-06-26 23:24:31 -0700346 # but we are already in a section, this means 'END' is missing
347 # for that section, fix it up.
Bin Mengf83649f2016-06-26 23:24:29 -0700348 if self.in_section:
Simon Glassb5800392020-10-29 21:46:23 -0600349 self._add_warn("Missing 'END' in section '%s'" % self.in_section)
Bin Mengf83649f2016-06-26 23:24:29 -0700350 if self.in_section == 'cover':
351 self.series.cover = self.section
352 elif self.in_section == 'notes':
353 if self.is_log:
354 self.series.notes += self.section
355 elif self.in_section == 'commit-notes':
356 if self.is_log:
357 self.commit.notes += self.section
358 else:
Simon Glassb5800392020-10-29 21:46:23 -0600359 # This should not happen
360 raise ValueError("Unknown section '%s'" % self.in_section)
Bin Mengf83649f2016-06-26 23:24:29 -0700361 self.in_section = None
362 self.skip_blank = True
363 self.section = []
Bin Meng50ffc982016-06-26 23:24:31 -0700364 # but we are already in a change list, that means a blank line
365 # is missing, fix it up.
366 if self.in_change:
Simon Glassb5800392020-10-29 21:46:23 -0600367 self._add_warn("Missing 'blank line' in section '%s-changes'" %
368 self.in_change)
Simon Glass93f61c02020-10-29 21:46:19 -0600369 self._finalise_change()
Sean Anderson48f46d62020-05-04 16:28:34 -0400370 self.in_change = None
371 self.change_version = 0
Bin Mengf83649f2016-06-26 23:24:29 -0700372
Simon Glass26132882012-01-14 15:12:45 +0000373 # If we are in a section, keep collecting lines until we see END
374 if self.in_section:
375 if line == 'END':
376 if self.in_section == 'cover':
377 self.series.cover = self.section
378 elif self.in_section == 'notes':
379 if self.is_log:
380 self.series.notes += self.section
Albert ARIBAUDd880efd2013-11-12 11:14:41 +0100381 elif self.in_section == 'commit-notes':
382 if self.is_log:
383 self.commit.notes += self.section
Simon Glass26132882012-01-14 15:12:45 +0000384 else:
Simon Glassb5800392020-10-29 21:46:23 -0600385 # This should not happen
386 raise ValueError("Unknown section '%s'" % self.in_section)
Simon Glass26132882012-01-14 15:12:45 +0000387 self.in_section = None
388 self.skip_blank = True
389 self.section = []
390 else:
391 self.section.append(line)
392
Patrick Delaunay2870c1c2020-07-02 19:08:24 +0200393 # If we are not in a section, it is an unexpected END
394 elif line == 'END':
Simon Glassc42b1d02020-10-29 21:46:17 -0600395 raise ValueError("'END' wihout section")
Patrick Delaunay2870c1c2020-07-02 19:08:24 +0200396
Simon Glass26132882012-01-14 15:12:45 +0000397 # Detect the commit subject
398 elif not is_blank and self.state == STATE_PATCH_SUBJECT:
399 self.commit.subject = line
400
401 # Detect the tags we want to remove, and skip blank lines
Simon Glass06022262020-10-29 21:46:18 -0600402 elif RE_REMOVE.match(line) and not commit_tag_match:
Simon Glass26132882012-01-14 15:12:45 +0000403 self.skip_blank = True
404
405 # TEST= should be the last thing in the commit, so remove
406 # everything after it
407 if line.startswith('TEST='):
408 self.found_test = True
409 elif self.skip_blank and is_blank:
410 self.skip_blank = False
411
Sean Anderson48f46d62020-05-04 16:28:34 -0400412 # Detect Cover-xxx tags
Bin Mengd0bc03c2016-06-26 23:24:28 -0700413 elif cover_match:
Sean Anderson48f46d62020-05-04 16:28:34 -0400414 name = cover_match.group(1)
415 value = cover_match.group(2)
416 if name == 'letter':
417 self.in_section = 'cover'
418 self.skip_blank = False
419 elif name == 'letter-cc':
Simon Glass93f61c02020-10-29 21:46:19 -0600420 self._add_to_series(line, 'cover-cc', value)
Sean Anderson48f46d62020-05-04 16:28:34 -0400421 elif name == 'changes':
422 self.in_change = 'Cover'
Simon Glass93f61c02020-10-29 21:46:19 -0600423 self.change_version = self._parse_version(value, line)
Simon Glassc72f3da2013-03-20 16:43:00 +0000424
Simon Glass26132882012-01-14 15:12:45 +0000425 # If we are in a change list, key collected lines until a blank one
426 elif self.in_change:
427 if is_blank:
428 # Blank line ends this change list
Simon Glass93f61c02020-10-29 21:46:19 -0600429 self._finalise_change()
Sean Anderson48f46d62020-05-04 16:28:34 -0400430 self.in_change = None
431 self.change_version = 0
Simon Glass46b34212014-04-20 10:50:14 -0600432 elif line == '---':
Simon Glass93f61c02020-10-29 21:46:19 -0600433 self._finalise_change()
Sean Anderson48f46d62020-05-04 16:28:34 -0400434 self.in_change = None
435 self.change_version = 0
Simon Glass93f61c02020-10-29 21:46:19 -0600436 out = self.process_line(line)
Sean Anderson1a32f922020-05-04 16:28:35 -0400437 elif self.is_log:
438 if not leading_whitespace_match:
Simon Glass93f61c02020-10-29 21:46:19 -0600439 self._finalise_change()
Sean Anderson1a32f922020-05-04 16:28:35 -0400440 self.change_lines.append(line)
Simon Glass26132882012-01-14 15:12:45 +0000441 self.skip_blank = False
442
443 # Detect Series-xxx tags
Albert ARIBAUDd880efd2013-11-12 11:14:41 +0100444 elif series_tag_match:
445 name = series_tag_match.group(1)
446 value = series_tag_match.group(2)
Simon Glass26132882012-01-14 15:12:45 +0000447 if name == 'changes':
448 # value is the version number: e.g. 1, or 2
Sean Anderson48f46d62020-05-04 16:28:34 -0400449 self.in_change = 'Series'
Simon Glass93f61c02020-10-29 21:46:19 -0600450 self.change_version = self._parse_version(value, line)
Simon Glass26132882012-01-14 15:12:45 +0000451 else:
Simon Glass93f61c02020-10-29 21:46:19 -0600452 self._add_to_series(line, name, value)
Simon Glass26132882012-01-14 15:12:45 +0000453 self.skip_blank = True
454
Douglas Anderson52b5ee82019-09-27 09:23:56 -0700455 # Detect Change-Id tags
456 elif change_id_match:
Maxim Cournoyer12f99fd2023-10-12 23:06:24 -0400457 if self.keep_change_id:
458 out = [line]
Douglas Anderson52b5ee82019-09-27 09:23:56 -0700459 value = change_id_match.group(1)
460 if self.is_log:
461 if self.commit.change_id:
Simon Glassc42b1d02020-10-29 21:46:17 -0600462 raise ValueError(
Simon Glass718f1a82020-11-03 13:54:11 -0700463 "%s: Two Change-Ids: '%s' vs. '%s'" %
464 (self.commit.hash, self.commit.change_id, value))
Douglas Anderson52b5ee82019-09-27 09:23:56 -0700465 self.commit.change_id = value
466 self.skip_blank = True
467
Albert ARIBAUDd880efd2013-11-12 11:14:41 +0100468 # Detect Commit-xxx tags
469 elif commit_tag_match:
470 name = commit_tag_match.group(1)
471 value = commit_tag_match.group(2)
472 if name == 'notes':
Simon Glass5f6caa52020-10-29 21:46:21 -0600473 self._add_to_commit(name)
Albert ARIBAUDd880efd2013-11-12 11:14:41 +0100474 self.skip_blank = True
Sean Anderson48f46d62020-05-04 16:28:34 -0400475 elif name == 'changes':
476 self.in_change = 'Commit'
Simon Glass93f61c02020-10-29 21:46:19 -0600477 self.change_version = self._parse_version(value, line)
Patrick Delaunayed73b312020-07-02 19:52:54 +0200478 else:
Simon Glassb5800392020-10-29 21:46:23 -0600479 self._add_warn('Line %d: Ignoring Commit-%s' %
480 (self.linenum, name))
Albert ARIBAUDd880efd2013-11-12 11:14:41 +0100481
Patrick Delaunay6bbdd0c2021-07-22 16:51:42 +0200482 # Detect invalid tags
483 elif invalid_match:
484 raise ValueError("Line %d: Invalid tag = '%s'" %
485 (self.linenum, line))
486
Simon Glass26132882012-01-14 15:12:45 +0000487 # Detect the start of a new commit
488 elif commit_match:
Simon Glass93f61c02020-10-29 21:46:19 -0600489 self._close_commit()
Simon Glassbaec6842014-10-15 02:27:00 -0600490 self.commit = commit.Commit(commit_match.group(1))
Simon Glass26132882012-01-14 15:12:45 +0000491
492 # Detect tags in the commit message
493 elif tag_match:
Simon Glass08e91be2020-07-05 21:41:57 -0600494 rtag_type, who = tag_match.groups()
Simon Glass93f61c02020-10-29 21:46:19 -0600495 self._add_commit_rtag(rtag_type, who)
Simon Glass26132882012-01-14 15:12:45 +0000496 # Remove Tested-by self, since few will take much notice
Simon Glass08e91be2020-07-05 21:41:57 -0600497 if (rtag_type == 'Tested-by' and
498 who.find(os.getenv('USER') + '@') != -1):
Simon Glass3b762cc2020-10-29 21:46:28 -0600499 self._add_warn("Ignoring '%s'" % line)
Simon Glass08e91be2020-07-05 21:41:57 -0600500 elif rtag_type == 'Patch-cc':
Simon Glass530ac272022-01-29 14:14:07 -0700501 self.commit.add_cc(who.split(','))
Simon Glass26132882012-01-14 15:12:45 +0000502 else:
Simon Glass27374d32014-08-28 09:43:38 -0600503 out = [line]
Simon Glass26132882012-01-14 15:12:45 +0000504
Simon Glass46b34212014-04-20 10:50:14 -0600505 # Suppress duplicate signoffs
506 elif signoff_match:
Simon Glassb0cd3412014-08-28 09:43:35 -0600507 if (self.is_log or not self.commit or
Simon Glass530ac272022-01-29 14:14:07 -0700508 self.commit.check_duplicate_signoff(signoff_match.group(1))):
Simon Glass46b34212014-04-20 10:50:14 -0600509 out = [line]
510
Simon Glass26132882012-01-14 15:12:45 +0000511 # Well that means this is an ordinary line
512 else:
Simon Glass26132882012-01-14 15:12:45 +0000513 # Look for space before tab
Simon Glassf9b6e0f2020-10-29 21:46:20 -0600514 mat = RE_SPACE_BEFORE_TAB.match(line)
515 if mat:
Simon Glassb5800392020-10-29 21:46:23 -0600516 self._add_warn('Line %d/%d has space before tab' %
517 (self.linenum, mat.start()))
Simon Glass26132882012-01-14 15:12:45 +0000518
519 # OK, we have a valid non-blank line
520 out = [line]
521 self.linenum += 1
522 self.skip_blank = False
Simon Glassda8a2922020-10-29 21:46:37 -0600523
524 if diff_match:
525 self.cur_diff = diff_match.group(1)
526
527 # If this is quoted, keep recent lines
528 if not diff_match and self.linenum > 1 and line:
529 if line.startswith('>'):
530 if not self.was_quoted:
531 self._finalise_snippet()
532 self.recent_line = None
533 if not line_match:
534 self.recent_quoted.append(line)
535 self.was_quoted = True
536 self.recent_diff = self.cur_diff
537 else:
538 self.recent_unquoted.put(line)
539 self.was_quoted = False
540
541 if line_match:
542 self.recent_line = line_match.groups()
543
Simon Glass26132882012-01-14 15:12:45 +0000544 if self.state == STATE_DIFFS:
545 pass
546
547 # If this is the start of the diffs section, emit our tags and
548 # change log
549 elif line == '---':
550 self.state = STATE_DIFFS
551
Sean Anderson48f46d62020-05-04 16:28:34 -0400552 # Output the tags (signoff first), then change list
Simon Glass26132882012-01-14 15:12:45 +0000553 out = []
Simon Glass26132882012-01-14 15:12:45 +0000554 log = self.series.MakeChangeLog(self.commit)
Simon Glassb0cd3412014-08-28 09:43:35 -0600555 out += [line]
556 if self.commit:
557 out += self.commit.notes
558 out += [''] + log
Simon Glass26132882012-01-14 15:12:45 +0000559 elif self.found_test:
Simon Glass06022262020-10-29 21:46:18 -0600560 if not RE_ALLOWED_AFTER_TEST.match(line):
Simon Glass26132882012-01-14 15:12:45 +0000561 self.lines_after_test += 1
562
563 return out
564
Simon Glass93f61c02020-10-29 21:46:19 -0600565 def finalise(self):
Simon Glass26132882012-01-14 15:12:45 +0000566 """Close out processing of this patch stream"""
Simon Glassda8a2922020-10-29 21:46:37 -0600567 self._finalise_snippet()
Simon Glass93f61c02020-10-29 21:46:19 -0600568 self._finalise_change()
569 self._close_commit()
Simon Glass26132882012-01-14 15:12:45 +0000570 if self.lines_after_test:
Simon Glassb5800392020-10-29 21:46:23 -0600571 self._add_warn('Found %d lines after TEST=' % self.lines_after_test)
Simon Glass26132882012-01-14 15:12:45 +0000572
Simon Glass93f61c02020-10-29 21:46:19 -0600573 def _write_message_id(self, outfd):
Douglas Anderson52b5ee82019-09-27 09:23:56 -0700574 """Write the Message-Id into the output.
575
576 This is based on the Change-Id in the original patch, the version,
577 and the prefix.
578
579 Args:
Simon Glass06127f92020-10-29 21:46:22 -0600580 outfd (io.IOBase): Output stream file object
Douglas Anderson52b5ee82019-09-27 09:23:56 -0700581 """
582 if not self.commit.change_id:
583 return
584
585 # If the count is -1 we're testing, so use a fixed time
586 if self.commit.count == -1:
587 time_now = datetime.datetime(1999, 12, 31, 23, 59, 59)
588 else:
589 time_now = datetime.datetime.now()
590
591 # In theory there is email.utils.make_msgid() which would be nice
592 # to use, but it already produces something way too long and thus
593 # will produce ugly commit lines if someone throws this into
594 # a "Link:" tag in the final commit. So (sigh) roll our own.
595
596 # Start with the time; presumably we wouldn't send the same series
597 # with the same Change-Id at the exact same second.
598 parts = [time_now.strftime("%Y%m%d%H%M%S")]
599
600 # These seem like they would be nice to include.
601 if 'prefix' in self.series:
602 parts.append(self.series['prefix'])
Sean Andersondc1cd132021-10-22 19:07:04 -0400603 if 'postfix' in self.series:
Simon Glass547cba62022-02-11 13:23:18 -0700604 parts.append(self.series['postfix'])
Douglas Anderson52b5ee82019-09-27 09:23:56 -0700605 if 'version' in self.series:
606 parts.append("v%s" % self.series['version'])
607
608 parts.append(str(self.commit.count + 1))
609
610 # The Change-Id must be last, right before the @
611 parts.append(self.commit.change_id)
612
613 # Join parts together with "." and write it out.
614 outfd.write('Message-Id: <%s@changeid>\n' % '.'.join(parts))
615
Simon Glass93f61c02020-10-29 21:46:19 -0600616 def process_stream(self, infd, outfd):
Simon Glass26132882012-01-14 15:12:45 +0000617 """Copy a stream from infd to outfd, filtering out unwanting things.
618
619 This is used to process patch files one at a time.
620
621 Args:
Simon Glass06127f92020-10-29 21:46:22 -0600622 infd (io.IOBase): Input stream file object
623 outfd (io.IOBase): Output stream file object
Simon Glass26132882012-01-14 15:12:45 +0000624 """
625 # Extract the filename from each diff, for nice warnings
626 fname = None
627 last_fname = None
628 re_fname = re.compile('diff --git a/(.*) b/.*')
Douglas Anderson52b5ee82019-09-27 09:23:56 -0700629
Simon Glass93f61c02020-10-29 21:46:19 -0600630 self._write_message_id(outfd)
Douglas Anderson52b5ee82019-09-27 09:23:56 -0700631
Simon Glass26132882012-01-14 15:12:45 +0000632 while True:
633 line = infd.readline()
634 if not line:
635 break
Simon Glass93f61c02020-10-29 21:46:19 -0600636 out = self.process_line(line)
Simon Glass26132882012-01-14 15:12:45 +0000637
638 # Try to detect blank lines at EOF
639 for line in out:
640 match = re_fname.match(line)
641 if match:
642 last_fname = fname
643 fname = match.group(1)
644 if line == '+':
645 self.blank_count += 1
646 else:
647 if self.blank_count and (line == '-- ' or match):
Simon Glassb5800392020-10-29 21:46:23 -0600648 self._add_warn("Found possible blank line(s) at end of file '%s'" %
649 last_fname)
Simon Glass26132882012-01-14 15:12:45 +0000650 outfd.write('+\n' * self.blank_count)
651 outfd.write(line + '\n')
652 self.blank_count = 0
Simon Glass93f61c02020-10-29 21:46:19 -0600653 self.finalise()
Simon Glass26132882012-01-14 15:12:45 +0000654
Simon Glassd0a0a582020-10-29 21:46:36 -0600655def insert_tags(msg, tags_to_emit):
656 """Add extra tags to a commit message
657
658 The tags are added after an existing block of tags if found, otherwise at
659 the end.
660
661 Args:
662 msg (str): Commit message
663 tags_to_emit (list): List of tags to emit, each a str
664
665 Returns:
666 (str) new message
667 """
668 out = []
669 done = False
670 emit_tags = False
Simon Glass6a222e62021-08-01 16:02:39 -0600671 emit_blank = False
Simon Glassd0a0a582020-10-29 21:46:36 -0600672 for line in msg.splitlines():
673 if not done:
674 signoff_match = RE_SIGNOFF.match(line)
675 tag_match = RE_TAG.match(line)
676 if tag_match or signoff_match:
677 emit_tags = True
678 if emit_tags and not tag_match and not signoff_match:
679 out += tags_to_emit
680 emit_tags = False
681 done = True
Simon Glass6a222e62021-08-01 16:02:39 -0600682 emit_blank = not (signoff_match or tag_match)
683 else:
684 emit_blank = line
Simon Glassd0a0a582020-10-29 21:46:36 -0600685 out.append(line)
686 if not done:
Simon Glass6a222e62021-08-01 16:02:39 -0600687 if emit_blank:
688 out.append('')
Simon Glassd0a0a582020-10-29 21:46:36 -0600689 out += tags_to_emit
690 return '\n'.join(out)
691
692def get_list(commit_range, git_dir=None, count=None):
693 """Get a log of a list of comments
694
695 This returns the output of 'git log' for the selected commits
696
697 Args:
698 commit_range (str): Range of commits to count (e.g. 'HEAD..base')
699 git_dir (str): Path to git repositiory (None to use default)
700 count (int): Number of commits to list, or None for no limit
701
702 Returns
703 str: String containing the contents of the git log
704 """
Simon Glass761648b2022-01-29 14:14:11 -0700705 params = gitutil.log_cmd(commit_range, reverse=True, count=count,
Simon Glassd0a0a582020-10-29 21:46:36 -0600706 git_dir=git_dir)
Simon Glass840be732022-01-29 14:14:05 -0700707 return command.run_pipe([params], capture=True).stdout
Simon Glass26132882012-01-14 15:12:45 +0000708
Simon Glass93f61c02020-10-29 21:46:19 -0600709def get_metadata_for_list(commit_range, git_dir=None, count=None,
710 series=None, allow_overwrite=False):
Simon Glass26132882012-01-14 15:12:45 +0000711 """Reads out patch series metadata from the commits
712
713 This does a 'git log' on the relevant commits and pulls out the tags we
714 are interested in.
715
716 Args:
Simon Glass06127f92020-10-29 21:46:22 -0600717 commit_range (str): Range of commits to count (e.g. 'HEAD..base')
718 git_dir (str): Path to git repositiory (None to use default)
719 count (int): Number of commits to list, or None for no limit
720 series (Series): Object to add information into. By default a new series
Simon Glass680da3c2012-12-15 10:42:06 +0000721 is started.
Simon Glass06127f92020-10-29 21:46:22 -0600722 allow_overwrite (bool): Allow tags to overwrite an existing tag
723
Simon Glass680da3c2012-12-15 10:42:06 +0000724 Returns:
Simon Glass06127f92020-10-29 21:46:22 -0600725 Series: Object containing information about the commits.
Simon Glass26132882012-01-14 15:12:45 +0000726 """
Simon Glass4e5d6bc2014-09-05 19:00:19 -0600727 if not series:
728 series = Series()
Simon Glass359b55a62014-09-05 19:00:23 -0600729 series.allow_overwrite = allow_overwrite
Simon Glassd0a0a582020-10-29 21:46:36 -0600730 stdout = get_list(commit_range, git_dir, count)
Simon Glassf9b6e0f2020-10-29 21:46:20 -0600731 pst = PatchStream(series, is_log=True)
Simon Glass26132882012-01-14 15:12:45 +0000732 for line in stdout.splitlines():
Simon Glassf9b6e0f2020-10-29 21:46:20 -0600733 pst.process_line(line)
734 pst.finalise()
Simon Glass26132882012-01-14 15:12:45 +0000735 return series
736
Simon Glass93f61c02020-10-29 21:46:19 -0600737def get_metadata(branch, start, count):
Simon Glass680da3c2012-12-15 10:42:06 +0000738 """Reads out patch series metadata from the commits
739
740 This does a 'git log' on the relevant commits and pulls out the tags we
741 are interested in.
742
743 Args:
Simon Glass06127f92020-10-29 21:46:22 -0600744 branch (str): Branch to use (None for current branch)
745 start (int): Commit to start from: 0=branch HEAD, 1=next one, etc.
746 count (int): Number of commits to list
747
748 Returns:
749 Series: Object containing information about the commits.
Simon Glass680da3c2012-12-15 10:42:06 +0000750 """
Simon Glass93f61c02020-10-29 21:46:19 -0600751 return get_metadata_for_list(
752 '%s~%d' % (branch if branch else 'HEAD', start), None, count)
Simon Glass680da3c2012-12-15 10:42:06 +0000753
Simon Glass93f61c02020-10-29 21:46:19 -0600754def get_metadata_for_test(text):
Simon Glassdf1bc5c2017-05-29 15:31:31 -0600755 """Process metadata from a file containing a git log. Used for tests
756
757 Args:
758 text:
Simon Glass06127f92020-10-29 21:46:22 -0600759
760 Returns:
761 Series: Object containing information about the commits.
Simon Glassdf1bc5c2017-05-29 15:31:31 -0600762 """
763 series = Series()
Simon Glassf9b6e0f2020-10-29 21:46:20 -0600764 pst = PatchStream(series, is_log=True)
Simon Glassdf1bc5c2017-05-29 15:31:31 -0600765 for line in text.splitlines():
Simon Glassf9b6e0f2020-10-29 21:46:20 -0600766 pst.process_line(line)
767 pst.finalise()
Simon Glassdf1bc5c2017-05-29 15:31:31 -0600768 return series
769
Maxim Cournoyer12f99fd2023-10-12 23:06:24 -0400770def fix_patch(backup_dir, fname, series, cmt, keep_change_id=False):
Simon Glass26132882012-01-14 15:12:45 +0000771 """Fix up a patch file, by adding/removing as required.
772
773 We remove our tags from the patch file, insert changes lists, etc.
774 The patch file is processed in place, and overwritten.
775
776 A backup file is put into backup_dir (if not None).
777
778 Args:
Simon Glass06127f92020-10-29 21:46:22 -0600779 backup_dir (str): Path to directory to use to backup the file
780 fname (str): Filename to patch file to process
781 series (Series): Series information about this patch set
782 cmt (Commit): Commit object for this patch file
Maxim Cournoyer12f99fd2023-10-12 23:06:24 -0400783 keep_change_id (bool): Keep the Change-Id tag.
Simon Glass06127f92020-10-29 21:46:22 -0600784
Simon Glass26132882012-01-14 15:12:45 +0000785 Return:
Simon Glass06127f92020-10-29 21:46:22 -0600786 list: A list of errors, each str, or [] if all ok.
Simon Glass26132882012-01-14 15:12:45 +0000787 """
788 handle, tmpname = tempfile.mkstemp()
Simon Glassf544a2d2019-10-31 07:42:51 -0600789 outfd = os.fdopen(handle, 'w', encoding='utf-8')
790 infd = open(fname, 'r', encoding='utf-8')
Maxim Cournoyer12f99fd2023-10-12 23:06:24 -0400791 pst = PatchStream(series, keep_change_id=keep_change_id)
Simon Glassf9b6e0f2020-10-29 21:46:20 -0600792 pst.commit = cmt
793 pst.process_stream(infd, outfd)
Simon Glass26132882012-01-14 15:12:45 +0000794 infd.close()
795 outfd.close()
796
797 # Create a backup file if required
798 if backup_dir:
799 shutil.copy(fname, os.path.join(backup_dir, os.path.basename(fname)))
800 shutil.move(tmpname, fname)
Simon Glass6d00f6c2020-10-29 21:46:24 -0600801 return cmt.warn
Simon Glass26132882012-01-14 15:12:45 +0000802
Maxim Cournoyer12f99fd2023-10-12 23:06:24 -0400803def fix_patches(series, fnames, keep_change_id=False):
Simon Glass26132882012-01-14 15:12:45 +0000804 """Fix up a list of patches identified by filenames
805
806 The patch files are processed in place, and overwritten.
807
808 Args:
Simon Glass06127f92020-10-29 21:46:22 -0600809 series (Series): The Series object
810 fnames (:type: list of str): List of patch files to process
Maxim Cournoyer12f99fd2023-10-12 23:06:24 -0400811 keep_change_id (bool): Keep the Change-Id tag.
Simon Glass26132882012-01-14 15:12:45 +0000812 """
813 # Current workflow creates patches, so we shouldn't need a backup
814 backup_dir = None #tempfile.mkdtemp('clean-patch')
815 count = 0
816 for fname in fnames:
Simon Glassf9b6e0f2020-10-29 21:46:20 -0600817 cmt = series.commits[count]
818 cmt.patch = fname
819 cmt.count = count
Maxim Cournoyer12f99fd2023-10-12 23:06:24 -0400820 result = fix_patch(backup_dir, fname, series, cmt,
821 keep_change_id=keep_change_id)
Simon Glass26132882012-01-14 15:12:45 +0000822 if result:
Simon Glass891316b2020-10-29 21:46:30 -0600823 print('%d warning%s for %s:' %
824 (len(result), 's' if len(result) > 1 else '', fname))
Simon Glass26132882012-01-14 15:12:45 +0000825 for warn in result:
Simon Glass891316b2020-10-29 21:46:30 -0600826 print('\t%s' % warn)
827 print()
Simon Glass26132882012-01-14 15:12:45 +0000828 count += 1
Simon Glass891316b2020-10-29 21:46:30 -0600829 print('Cleaned %d patch%s' % (count, 'es' if count > 1 else ''))
Simon Glass26132882012-01-14 15:12:45 +0000830
Simon Glass93f61c02020-10-29 21:46:19 -0600831def insert_cover_letter(fname, series, count):
Simon Glass26132882012-01-14 15:12:45 +0000832 """Inserts a cover letter with the required info into patch 0
833
834 Args:
Simon Glass06127f92020-10-29 21:46:22 -0600835 fname (str): Input / output filename of the cover letter file
836 series (Series): Series object
837 count (int): Number of patches in the series
Simon Glass26132882012-01-14 15:12:45 +0000838 """
Simon Glassf9b6e0f2020-10-29 21:46:20 -0600839 fil = open(fname, 'r')
840 lines = fil.readlines()
841 fil.close()
Simon Glass26132882012-01-14 15:12:45 +0000842
Simon Glassf9b6e0f2020-10-29 21:46:20 -0600843 fil = open(fname, 'w')
Simon Glass26132882012-01-14 15:12:45 +0000844 text = series.cover
845 prefix = series.GetPatchPrefix()
846 for line in lines:
847 if line.startswith('Subject:'):
Wu, Josh393aaa22015-04-03 10:51:17 +0800848 # if more than 10 or 100 patches, it should say 00/xx, 000/xxx, etc
849 zero_repeat = int(math.log10(count)) + 1
850 zero = '0' * zero_repeat
851 line = 'Subject: [%s %s/%d] %s\n' % (prefix, zero, count, text[0])
Simon Glass26132882012-01-14 15:12:45 +0000852
853 # Insert our cover letter
854 elif line.startswith('*** BLURB HERE ***'):
855 # First the blurb test
856 line = '\n'.join(text[1:]) + '\n'
857 if series.get('notes'):
858 line += '\n'.join(series.notes) + '\n'
859
860 # Now the change list
861 out = series.MakeChangeLog(None)
862 line += '\n' + '\n'.join(out)
Simon Glassf9b6e0f2020-10-29 21:46:20 -0600863 fil.write(line)
864 fil.close()