blob: 47d7be28fdf739667ed489eed95aa15658cabee4 [file] [log] [blame]
Simon Glass2c266d82025-04-29 07:22:13 -06001# SPDX-License-Identifier: GPL-2.0+
2#
3# Copyright 2025 Simon Glass <sjg@chromium.org>
4#
5"""Provides a basic API for the patchwork server
6"""
7
8import asyncio
Simon Glass232eefd2025-04-29 07:22:14 -06009import re
Simon Glass2c266d82025-04-29 07:22:13 -060010
11import aiohttp
Simon Glass3729b8b2025-04-29 07:22:24 -060012from u_boot_pylib import terminal
Simon Glass2c266d82025-04-29 07:22:13 -060013
14# Number of retries
15RETRIES = 3
16
17# Max concurrent request
18MAX_CONCURRENT = 50
19
Simon Glass232eefd2025-04-29 07:22:14 -060020# Patches which are part of a multi-patch series are shown with a prefix like
21# [prefix, version, sequence], for example '[RFC, v2, 3/5]'. All but the last
22# part is optional. This decodes the string into groups. For single patches
23# the [] part is not present:
24# Groups: (ignore, ignore, ignore, prefix, version, sequence, subject)
25RE_PATCH = re.compile(r'(\[(((.*),)?(.*),)?(.*)\]\s)?(.*)$')
26
27# This decodes the sequence string into a patch number and patch count
28RE_SEQ = re.compile(r'(\d+)/(\d+)')
29
30
31class Patch(dict):
32 """Models a patch in patchwork
33
34 This class records information obtained from patchwork
35
36 Some of this information comes from the 'Patch' column:
37
38 [RFC,v2,1/3] dm: Driver and uclass changes for tiny-dm
39
40 This shows the prefix, version, seq, count and subject.
41
42 The other properties come from other columns in the display.
43
44 Properties:
45 pid (str): ID of the patch (typically an integer)
46 seq (int): Sequence number within series (1=first) parsed from sequence
47 string
48 count (int): Number of patches in series, parsed from sequence string
49 raw_subject (str): Entire subject line, e.g.
50 "[1/2,v2] efi_loader: Sort header file ordering"
51 prefix (str): Prefix string or None (e.g. 'RFC')
52 version (str): Version string or None (e.g. 'v2')
53 raw_subject (str): Raw patch subject
54 subject (str): Patch subject with [..] part removed (same as commit
55 subject)
Simon Glassa1046c82025-04-29 07:22:22 -060056 data (dict or None): Patch data:
Simon Glass232eefd2025-04-29 07:22:14 -060057 """
Simon Glassa1046c82025-04-29 07:22:22 -060058 def __init__(self, pid, state=None, data=None, comments=None,
59 series_data=None):
Simon Glass232eefd2025-04-29 07:22:14 -060060 super().__init__()
61 self.id = pid # Use 'id' to match what the Rest API provides
62 self.seq = None
63 self.count = None
64 self.prefix = None
65 self.version = None
66 self.raw_subject = None
67 self.subject = None
Simon Glassa1046c82025-04-29 07:22:22 -060068 self.state = state
69 self.data = data
70 self.comments = comments
71 self.series_data = series_data
72 self.name = None
Simon Glass232eefd2025-04-29 07:22:14 -060073
74 # These make us more like a dictionary
75 def __setattr__(self, name, value):
76 self[name] = value
77
78 def __getattr__(self, name):
79 return self[name]
80
81 def __hash__(self):
82 return hash(frozenset(self.items()))
83
84 def __str__(self):
85 return self.raw_subject
86
87 def parse_subject(self, raw_subject):
88 """Parse the subject of a patch into its component parts
89
90 See RE_PATCH for details. The parsed info is placed into seq, count,
91 prefix, version, subject
92
93 Args:
94 raw_subject (str): Subject string to parse
95
96 Raises:
97 ValueError: the subject cannot be parsed
98 """
99 self.raw_subject = raw_subject.strip()
100 mat = RE_PATCH.search(raw_subject.strip())
101 if not mat:
102 raise ValueError(f"Cannot parse subject '{raw_subject}'")
103 self.prefix, self.version, seq_info, self.subject = mat.groups()[3:]
104 mat_seq = RE_SEQ.match(seq_info) if seq_info else False
105 if mat_seq is None:
106 self.version = seq_info
107 seq_info = None
108 if self.version and not self.version.startswith('v'):
109 self.prefix = self.version
110 self.version = None
111 if seq_info:
112 if mat_seq:
113 self.seq = int(mat_seq.group(1))
114 self.count = int(mat_seq.group(2))
115 else:
116 self.seq = 1
117 self.count = 1
118
119
120class Review:
121 """Represents a single review email collected in Patchwork
122
123 Patches can attract multiple reviews. Each consists of an author/date and
124 a variable number of 'snippets', which are groups of quoted and unquoted
125 text.
126 """
127 def __init__(self, meta, snippets):
128 """Create new Review object
129
130 Args:
131 meta (str): Text containing review author and date
132 snippets (list): List of snippets in th review, each a list of text
133 lines
134 """
135 self.meta = ' : '.join([line for line in meta.splitlines() if line])
136 self.snippets = snippets
137
138
Simon Glass2c266d82025-04-29 07:22:13 -0600139class Patchwork:
140 """Class to handle communication with patchwork
141 """
Simon Glass802eeea2025-04-29 07:22:25 -0600142 def __init__(self, url, show_progress=True, single_thread=False):
Simon Glass2c266d82025-04-29 07:22:13 -0600143 """Set up a new patchwork handler
144
145 Args:
146 url (str): URL of patchwork server, e.g.
147 'https://patchwork.ozlabs.org'
148 """
149 self.url = url
Simon Glass25b91c12025-04-29 07:22:19 -0600150 self.fake_request = None
Simon Glass2c266d82025-04-29 07:22:13 -0600151 self.proj_id = None
152 self.link_name = None
153 self._show_progress = show_progress
Simon Glass802eeea2025-04-29 07:22:25 -0600154 self.semaphore = asyncio.Semaphore(
155 1 if single_thread else MAX_CONCURRENT)
Simon Glass2c266d82025-04-29 07:22:13 -0600156 self.request_count = 0
157
158 async def _request(self, client, subpath):
159 """Call the patchwork API and return the result as JSON
160
161 Args:
162 client (aiohttp.ClientSession): Session to use
163 subpath (str): URL subpath to use
164
165 Returns:
166 dict: Json result
167
168 Raises:
169 ValueError: the URL could not be read
170 """
171 # print('subpath', subpath)
172 self.request_count += 1
Simon Glass25b91c12025-04-29 07:22:19 -0600173 if self.fake_request:
174 return self.fake_request(subpath)
Simon Glass2c266d82025-04-29 07:22:13 -0600175
176 full_url = f'{self.url}/api/1.2/{subpath}'
177 async with self.semaphore:
178 # print('full_url', full_url)
179 for i in range(RETRIES + 1):
180 try:
181 async with client.get(full_url) as response:
182 if response.status != 200:
183 raise ValueError(
184 f"Could not read URL '{full_url}'")
185 result = await response.json()
186 # print('- done', full_url)
187 return result
188 break
189 except aiohttp.client_exceptions.ServerDisconnectedError:
190 if i == RETRIES:
191 raise
Simon Glass1568b692025-04-29 07:22:15 -0600192
Simon Glass25b91c12025-04-29 07:22:19 -0600193 @staticmethod
194 def for_testing(func):
195 """Get an instance to use for testing
196
197 Args:
198 func (function): Function to call to handle requests. The function
199 is passed a URL and is expected to return a dict with the
200 resulting data
201
202 Returns:
203 Patchwork: testing instance
204 """
205 pwork = Patchwork(None, show_progress=False)
206 pwork.fake_request = func
207 return pwork
208
Simon Glass1568b692025-04-29 07:22:15 -0600209 async def get_series(self, client, link):
210 """Read information about a series
211
212 Args:
213 client (aiohttp.ClientSession): Session to use
214 link (str): Patchwork series ID
215
216 Returns: dict containing patchwork's series information
217 id (int): series ID unique across patchwork instance, e.g. 3
218 url (str): Full URL, e.g.
219 'https://patchwork.ozlabs.org/api/1.2/series/3/'
220 web_url (str): Full URL, e.g.
221 'https://patchwork.ozlabs.org/project/uboot/list/?series=3
222 project (dict): project information (id, url, name, link_name,
223 list_id, list_email, etc.
224 name (str): Series name, e.g. '[U-Boot] moveconfig: fix error'
225 date (str): Date, e.g. '2017-08-27T08:00:51'
226 submitter (dict): id, url, name, email, e.g.:
227 "id": 6125,
228 "url": "https://patchwork.ozlabs.org/api/1.2/people/6125/",
229 "name": "Chris Packham",
230 "email": "judge.packham@gmail.com"
231 version (int): Version number
232 total (int): Total number of patches based on subject
233 received_total (int): Total patches received by patchwork
234 received_all (bool): True if all patches were received
235 mbox (str): URL of mailbox, e.g.
236 'https://patchwork.ozlabs.org/series/3/mbox/'
237 cover_letter (dict) or None, e.g.:
238 "id": 806215,
239 "url": "https://patchwork.ozlabs.org/api/1.2/covers/806215/",
240 "web_url": "https://patchwork.ozlabs.org/project/uboot/cover/
241 20170827094411.8583-1-judge.packham@gmail.com/",
242 "msgid": "<20170827094411.8583-1-judge.packham@gmail.com>",
243 "list_archive_url": null,
244 "date": "2017-08-27T09:44:07",
245 "name": "[U-Boot,v2,0/4] usb: net: Migrate USB Ethernet",
246 "mbox": "https://patchwork.ozlabs.org/project/uboot/cover/
247 20170827094411.8583-1-judge.packham@gmail.com/mbox/"
248 patches (list of dict), each e.g.:
249 "id": 806202,
250 "url": "https://patchwork.ozlabs.org/api/1.2/patches/806202/",
251 "web_url": "https://patchwork.ozlabs.org/project/uboot/patch/
252 20170827080051.816-1-judge.packham@gmail.com/",
253 "msgid": "<20170827080051.816-1-judge.packham@gmail.com>",
254 "list_archive_url": null,
255 "date": "2017-08-27T08:00:51",
256 "name": "[U-Boot] moveconfig: fix error message do_autoconf()",
257 "mbox": "https://patchwork.ozlabs.org/project/uboot/patch/
258 20170827080051.816-1-judge.packham@gmail.com/mbox/"
259 """
260 return await self._request(client, f'series/{link}/')
261
262 async def get_patch(self, client, patch_id):
263 """Read information about a patch
264
265 Args:
266 client (aiohttp.ClientSession): Session to use
267 patch_id (str): Patchwork patch ID
268
269 Returns: dict containing patchwork's patch information
270 "id": 185,
271 "url": "https://patchwork.ozlabs.org/api/1.2/patches/185/",
272 "web_url": "https://patchwork.ozlabs.org/project/cbe-oss-dev/patch/
273 200809050416.27831.adetsch@br.ibm.com/",
274 project (dict): project information (id, url, name, link_name,
275 list_id, list_email, etc.
276 "msgid": "<200809050416.27831.adetsch@br.ibm.com>",
277 "list_archive_url": null,
278 "date": "2008-09-05T07:16:27",
279 "name": "powerpc/spufs: Fix possible scheduling of a context",
280 "commit_ref": "b2e601d14deb2083e2a537b47869ab3895d23a28",
281 "pull_url": null,
282 "state": "accepted",
283 "archived": false,
284 "hash": "bc1c0b80d7cff66c0d1e5f3f8f4d10eb36176f0d",
285 "submitter": {
286 "id": 93,
287 "url": "https://patchwork.ozlabs.org/api/1.2/people/93/",
288 "name": "Andre Detsch",
289 "email": "adetsch@br.ibm.com"
290 },
291 "delegate": {
292 "id": 1,
293 "url": "https://patchwork.ozlabs.org/api/1.2/users/1/",
294 "username": "jk",
295 "first_name": "Jeremy",
296 "last_name": "Kerr",
297 "email": "jk@ozlabs.org"
298 },
299 "mbox": "https://patchwork.ozlabs.org/project/cbe-oss-dev/patch/
300 200809050416.27831.adetsch@br.ibm.com/mbox/",
301 "series": [],
302 "comments": "https://patchwork.ozlabs.org/api/patches/185/
303 comments/",
304 "check": "pending",
305 "checks": "https://patchwork.ozlabs.org/api/patches/185/checks/",
306 "tags": {},
307 "related": [],
308 "headers": {...}
309 "content": "We currently have a race when scheduling a context
310 after we have found a runnable context in spusched_tick, the
311 context may have been scheduled by spu_activate().
312
313 This may result in a panic if we try to unschedule a context
314 been freed in the meantime.
315
316 This change exits spu_schedule() if the context has already
317 scheduled, so we don't end up scheduling it twice.
318
319 Signed-off-by: Andre Detsch <adetsch@br.ibm.com>",
320 "diff": '''Index: spufs/arch/powerpc/platforms/cell/spufs/sched.c
321 =======================================================
322 --- spufs.orig/arch/powerpc/platforms/cell/spufs/sched.c
323 +++ spufs/arch/powerpc/platforms/cell/spufs/sched.c
324 @@ -727,7 +727,8 @@ static void spu_schedule(struct spu *spu
325 \t/* not a candidate for interruptible because it's called
326 \t from the scheduler thread or from spu_deactivate */
327 \tmutex_lock(&ctx->state_mutex);
328 -\t__spu_schedule(spu, ctx);
329 +\tif (ctx->state == SPU_STATE_SAVED)
330 +\t\t__spu_schedule(spu, ctx);
331 \tspu_release(ctx);
332 }
333 '''
334 "prefixes": ["3/3", ...]
335 """
336 return await self._request(client, f'patches/{patch_id}/')
337
338 async def _get_patch_comments(self, client, patch_id):
339 """Read comments about a patch
340
341 Args:
342 client (aiohttp.ClientSession): Session to use
343 patch_id (str): Patchwork patch ID
344
345 Returns: list of dict: list of comments:
346 id (int): series ID unique across patchwork instance, e.g. 3331924
347 web_url (str): Full URL, e.g.
348 'https://patchwork.ozlabs.org/comment/3331924/'
349 msgid (str): Message ID, e.g.
350 '<d2526c98-8198-4b8b-ab10-20bda0151da1@gmx.de>'
351 list_archive_url: (unknown?)
352 date (str): Date, e.g. '2024-06-20T13:38:03'
353 subject (str): email subject, e.g. 'Re: [PATCH 3/5] buildman:
354 Support building within a Python venv'
355 date (str): Date, e.g. '2017-08-27T08:00:51'
356 submitter (dict): id, url, name, email, e.g.:
357 "id": 61270,
358 "url": "https://patchwork.ozlabs.org/api/people/61270/",
359 "name": "Heinrich Schuchardt",
360 "email": "xypron.glpk@gmx.de"
361 content (str): Content of email, e.g. 'On 20.06.24 15:19,
362 Simon Glass wrote:
363 >...'
364 headers: dict: email headers, see get_cover() for an example
365 """
366 return await self._request(client, f'patches/{patch_id}/comments/')
367
368 async def get_patch_comments(self, patch_id):
369 async with aiohttp.ClientSession() as client:
370 return await self._get_patch_comments(client, patch_id)
371
372 async def _get_patch_status(self, client, patch_id):
373 """Get the patch status
374
375 Args:
376 client (aiohttp.ClientSession): Session to use
377 patch_id (int): Patch ID to look up in patchwork
378
379 Return:
380 PATCH: Patch information
381
382 Requests:
383 1 for patch, 1 for patch comments
384 """
385 data = await self.get_patch(client, patch_id)
386 state = data['state']
387 comment_data = await self._get_patch_comments(client, patch_id)
388
389 return Patch(patch_id, state, data, comment_data)
390
391 async def series_get_state(self, client, link, read_comments):
392 """Sync the series information against patchwork, to find patch status
393
394 Args:
395 client (aiohttp.ClientSession): Session to use
396 link (str): Patchwork series ID
397 read_comments (bool): True to read the comments on the patches
398
399 Return: tuple:
400 list of Patch objects
401 """
402 data = await self.get_series(client, link)
403 patch_list = list(data['patches'])
404
405 count = len(patch_list)
406 patches = []
407 if read_comments:
408 # Returns a list of Patch objects
409 tasks = [self._get_patch_status(client, patch_list[i]['id'])
410 for i in range(count)]
411
412 patch_status = await asyncio.gather(*tasks)
413 for patch_data, status in zip(patch_list, patch_status):
414 status.series_data = patch_data
415 patches.append(status)
416 else:
417 for i in range(count):
418 info = patch_list[i]
419 pat = Patch(info['id'], series_data=info)
420 pat.raw_subject = info['name']
421 patches.append(pat)
422 if self._show_progress:
423 terminal.print_clear()
424
425 return patches