blob: e1e2046343c074b230ab1f07b57ad60c779a68fb [file] [log] [blame]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001#
2# Copyright (C) 2008 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
Shawn O. Pearcec12c3602009-04-17 21:03:32 -070016import cPickle
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070017import os
18import re
Shawn O. Pearcefb231612009-04-10 18:53:46 -070019import subprocess
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070020import sys
Shawn O. Pearcefb231612009-04-10 18:53:46 -070021import time
22from signal import SIGTERM
Shawn O. Pearceb54a3922009-01-05 16:18:58 -080023from urllib2 import urlopen, HTTPError
24from error import GitError, UploadError
Shawn O. Pearcead3193a2009-04-18 09:54:51 -070025from trace import Trace
Shawn O. Pearcefb231612009-04-10 18:53:46 -070026from git_command import GitCommand, _ssh_sock
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070027
28R_HEADS = 'refs/heads/'
29R_TAGS = 'refs/tags/'
30ID_RE = re.compile('^[0-9a-f]{40}$')
31
Shawn O. Pearce146fe902009-03-25 14:06:43 -070032REVIEW_CACHE = dict()
33
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070034def IsId(rev):
35 return ID_RE.match(rev)
36
Shawn O. Pearcef8e32732009-04-17 11:00:31 -070037def _key(name):
38 parts = name.split('.')
39 if len(parts) < 2:
40 return name.lower()
41 parts[ 0] = parts[ 0].lower()
42 parts[-1] = parts[-1].lower()
43 return '.'.join(parts)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070044
45class GitConfig(object):
Shawn O. Pearce90be5c02008-10-29 15:21:24 -070046 _ForUser = None
47
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070048 @classmethod
49 def ForUser(cls):
Shawn O. Pearce90be5c02008-10-29 15:21:24 -070050 if cls._ForUser is None:
51 cls._ForUser = cls(file = os.path.expanduser('~/.gitconfig'))
52 return cls._ForUser
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070053
54 @classmethod
55 def ForRepository(cls, gitdir, defaults=None):
56 return cls(file = os.path.join(gitdir, 'config'),
57 defaults = defaults)
58
Shawn O. Pearce1b34c912009-05-21 18:52:49 -070059 def __init__(self, file, defaults=None, pickleFile=None):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070060 self.file = file
61 self.defaults = defaults
62 self._cache_dict = None
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -070063 self._section_dict = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070064 self._remotes = {}
65 self._branches = {}
Shawn O. Pearce1b34c912009-05-21 18:52:49 -070066
67 if pickleFile is None:
68 self._pickle = os.path.join(
69 os.path.dirname(self.file),
70 '.repopickle_' + os.path.basename(self.file))
71 else:
72 self._pickle = pickleFile
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070073
74 def Has(self, name, include_defaults = True):
75 """Return true if this configuration file has the key.
76 """
Shawn O. Pearcef8e32732009-04-17 11:00:31 -070077 if _key(name) in self._cache:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070078 return True
79 if include_defaults and self.defaults:
80 return self.defaults.Has(name, include_defaults = True)
81 return False
82
83 def GetBoolean(self, name):
84 """Returns a boolean from the configuration file.
85 None : The value was not defined, or is not a boolean.
86 True : The value was set to true or yes.
87 False: The value was set to false or no.
88 """
89 v = self.GetString(name)
90 if v is None:
91 return None
92 v = v.lower()
93 if v in ('true', 'yes'):
94 return True
95 if v in ('false', 'no'):
96 return False
97 return None
98
99 def GetString(self, name, all=False):
100 """Get the first value for a key, or None if it is not defined.
101
102 This configuration file is used first, if the key is not
103 defined or all = True then the defaults are also searched.
104 """
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700105 try:
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700106 v = self._cache[_key(name)]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700107 except KeyError:
108 if self.defaults:
109 return self.defaults.GetString(name, all = all)
110 v = []
111
112 if not all:
113 if v:
114 return v[0]
115 return None
116
117 r = []
118 r.extend(v)
119 if self.defaults:
120 r.extend(self.defaults.GetString(name, all = True))
121 return r
122
123 def SetString(self, name, value):
124 """Set the value(s) for a key.
125 Only this configuration file is modified.
126
127 The supplied value should be either a string,
128 or a list of strings (to store multiple values).
129 """
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700130 key = _key(name)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700131
132 try:
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700133 old = self._cache[key]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700134 except KeyError:
135 old = []
136
137 if value is None:
138 if old:
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700139 del self._cache[key]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700140 self._do('--unset-all', name)
141
142 elif isinstance(value, list):
143 if len(value) == 0:
144 self.SetString(name, None)
145
146 elif len(value) == 1:
147 self.SetString(name, value[0])
148
149 elif old != value:
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700150 self._cache[key] = list(value)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700151 self._do('--replace-all', name, value[0])
152 for i in xrange(1, len(value)):
153 self._do('--add', name, value[i])
154
155 elif len(old) != 1 or old[0] != value:
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700156 self._cache[key] = [value]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700157 self._do('--replace-all', name, value)
158
159 def GetRemote(self, name):
160 """Get the remote.$name.* configuration values as an object.
161 """
162 try:
163 r = self._remotes[name]
164 except KeyError:
165 r = Remote(self, name)
166 self._remotes[r.name] = r
167 return r
168
169 def GetBranch(self, name):
170 """Get the branch.$name.* configuration values as an object.
171 """
172 try:
173 b = self._branches[name]
174 except KeyError:
175 b = Branch(self, name)
176 self._branches[b.name] = b
177 return b
178
Shawn O. Pearce366ad212009-05-19 12:47:37 -0700179 def GetSubSections(self, section):
180 """List all subsection names matching $section.*.*
181 """
182 return self._sections.get(section, set())
183
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -0700184 def HasSection(self, section, subsection = ''):
185 """Does at least one key in section.subsection exist?
186 """
187 try:
188 return subsection in self._sections[section]
189 except KeyError:
190 return False
191
192 @property
193 def _sections(self):
194 d = self._section_dict
195 if d is None:
196 d = {}
197 for name in self._cache.keys():
198 p = name.split('.')
199 if 2 == len(p):
200 section = p[0]
201 subsect = ''
202 else:
203 section = p[0]
204 subsect = '.'.join(p[1:-1])
205 if section not in d:
206 d[section] = set()
207 d[section].add(subsect)
208 self._section_dict = d
209 return d
210
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700211 @property
212 def _cache(self):
213 if self._cache_dict is None:
214 self._cache_dict = self._Read()
215 return self._cache_dict
216
217 def _Read(self):
Shawn O. Pearcec12c3602009-04-17 21:03:32 -0700218 d = self._ReadPickle()
219 if d is None:
220 d = self._ReadGit()
221 self._SavePickle(d)
222 return d
223
224 def _ReadPickle(self):
225 try:
226 if os.path.getmtime(self._pickle) \
227 <= os.path.getmtime(self.file):
228 os.remove(self._pickle)
229 return None
230 except OSError:
231 return None
232 try:
Shawn O. Pearcead3193a2009-04-18 09:54:51 -0700233 Trace(': unpickle %s', self.file)
Shawn O. Pearce76ca9f82009-04-18 14:48:03 -0700234 fd = open(self._pickle, 'rb')
235 try:
236 return cPickle.load(fd)
237 finally:
238 fd.close()
Shawn O. Pearce2a3a81b2009-06-12 09:10:07 -0700239 except EOFError:
240 os.remove(self._pickle)
241 return None
Shawn O. Pearcec12c3602009-04-17 21:03:32 -0700242 except IOError:
243 os.remove(self._pickle)
244 return None
245 except cPickle.PickleError:
246 os.remove(self._pickle)
247 return None
248
249 def _SavePickle(self, cache):
250 try:
Shawn O. Pearce76ca9f82009-04-18 14:48:03 -0700251 fd = open(self._pickle, 'wb')
252 try:
253 cPickle.dump(cache, fd, cPickle.HIGHEST_PROTOCOL)
254 finally:
255 fd.close()
Shawn O. Pearcec12c3602009-04-17 21:03:32 -0700256 except IOError:
257 os.remove(self._pickle)
258 except cPickle.PickleError:
259 os.remove(self._pickle)
260
261 def _ReadGit(self):
David Aguilar438c5472009-06-28 15:09:16 -0700262 """
263 Read configuration data from git.
264
265 This internal method populates the GitConfig cache.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700266
David Aguilar438c5472009-06-28 15:09:16 -0700267 """
David Aguilar438c5472009-06-28 15:09:16 -0700268 c = {}
Shawn O. Pearcec24c7202009-07-02 16:12:57 -0700269 d = self._do('--null', '--list')
270 if d is None:
271 return c
272 for line in d.rstrip('\0').split('\0'):
David Aguilar438c5472009-06-28 15:09:16 -0700273 if '\n' in line:
274 key, val = line.split('\n', 1)
275 else:
276 key = line
277 val = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700278
279 if key in c:
280 c[key].append(val)
281 else:
282 c[key] = [val]
283
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700284 return c
285
286 def _do(self, *args):
287 command = ['config', '--file', self.file]
288 command.extend(args)
289
290 p = GitCommand(None,
291 command,
292 capture_stdout = True,
293 capture_stderr = True)
294 if p.Wait() == 0:
295 return p.stdout
296 else:
297 GitError('git config %s: %s' % (str(args), p.stderr))
298
299
300class RefSpec(object):
301 """A Git refspec line, split into its components:
302
303 forced: True if the line starts with '+'
304 src: Left side of the line
305 dst: Right side of the line
306 """
307
308 @classmethod
309 def FromString(cls, rs):
310 lhs, rhs = rs.split(':', 2)
311 if lhs.startswith('+'):
312 lhs = lhs[1:]
313 forced = True
314 else:
315 forced = False
316 return cls(forced, lhs, rhs)
317
318 def __init__(self, forced, lhs, rhs):
319 self.forced = forced
320 self.src = lhs
321 self.dst = rhs
322
323 def SourceMatches(self, rev):
324 if self.src:
325 if rev == self.src:
326 return True
327 if self.src.endswith('/*') and rev.startswith(self.src[:-1]):
328 return True
329 return False
330
331 def DestMatches(self, ref):
332 if self.dst:
333 if ref == self.dst:
334 return True
335 if self.dst.endswith('/*') and ref.startswith(self.dst[:-1]):
336 return True
337 return False
338
339 def MapSource(self, rev):
340 if self.src.endswith('/*'):
341 return self.dst[:-1] + rev[len(self.src) - 1:]
342 return self.dst
343
344 def __str__(self):
345 s = ''
346 if self.forced:
347 s += '+'
348 if self.src:
349 s += self.src
350 if self.dst:
351 s += ':'
352 s += self.dst
353 return s
354
355
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700356_ssh_cache = {}
357_ssh_master = True
358
Shawn O. Pearce896d5df2009-04-21 14:51:04 -0700359def _open_ssh(host, port):
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700360 global _ssh_master
361
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700362 key = '%s:%s' % (host, port)
363 if key in _ssh_cache:
364 return True
365
366 if not _ssh_master \
367 or 'GIT_SSH' in os.environ \
Shawn O. Pearce2b5b4ac2009-04-23 17:22:18 -0700368 or sys.platform in ('win32', 'cygwin'):
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700369 # failed earlier, or cygwin ssh can't do this
370 #
371 return False
372
373 command = ['ssh',
374 '-o','ControlPath %s' % _ssh_sock(),
375 '-p',str(port),
376 '-M',
377 '-N',
378 host]
379 try:
380 Trace(': %s', ' '.join(command))
381 p = subprocess.Popen(command)
382 except Exception, e:
383 _ssh_master = False
384 print >>sys.stderr, \
385 '\nwarn: cannot enable ssh control master for %s:%s\n%s' \
386 % (host,port, str(e))
387 return False
388
389 _ssh_cache[key] = p
390 time.sleep(1)
391 return True
392
393def close_ssh():
394 for key,p in _ssh_cache.iteritems():
Shawn O. Pearce26120ca2009-06-16 11:49:10 -0700395 try:
396 os.kill(p.pid, SIGTERM)
397 p.wait()
Shawn O. Pearcefb5c8fd2009-06-16 14:57:46 -0700398 except OSError:
Shawn O. Pearce26120ca2009-06-16 11:49:10 -0700399 pass
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700400 _ssh_cache.clear()
401
402 d = _ssh_sock(create=False)
403 if d:
404 try:
405 os.rmdir(os.path.dirname(d))
406 except OSError:
407 pass
408
409URI_SCP = re.compile(r'^([^@:]*@?[^:/]{1,}):')
Shawn O. Pearce2f968c92009-04-30 14:30:28 -0700410URI_ALL = re.compile(r'^([a-z][a-z+]*)://([^@/]*@?[^/]*)/')
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700411
412def _preconnect(url):
413 m = URI_ALL.match(url)
414 if m:
415 scheme = m.group(1)
416 host = m.group(2)
417 if ':' in host:
418 host, port = host.split(':')
Shawn O. Pearce896d5df2009-04-21 14:51:04 -0700419 else:
420 port = 22
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700421 if scheme in ('ssh', 'git+ssh', 'ssh+git'):
422 return _open_ssh(host, port)
423 return False
424
425 m = URI_SCP.match(url)
426 if m:
427 host = m.group(1)
Shawn O. Pearce896d5df2009-04-21 14:51:04 -0700428 return _open_ssh(host, 22)
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700429
Shawn O. Pearce7b4f4352009-06-12 09:06:35 -0700430 return False
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700431
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700432class Remote(object):
433 """Configuration options related to a remote.
434 """
435 def __init__(self, config, name):
436 self._config = config
437 self.name = name
438 self.url = self._Get('url')
439 self.review = self._Get('review')
Shawn O. Pearce339ba9f2008-11-06 09:52:51 -0800440 self.projectname = self._Get('projectname')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700441 self.fetch = map(lambda x: RefSpec.FromString(x),
442 self._Get('fetch', all=True))
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800443 self._review_protocol = None
444
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700445 def PreConnectFetch(self):
446 return _preconnect(self.url)
447
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800448 @property
449 def ReviewProtocol(self):
450 if self._review_protocol is None:
451 if self.review is None:
452 return None
453
454 u = self.review
455 if not u.startswith('http:') and not u.startswith('https:'):
456 u = 'http://%s' % u
Shawn O. Pearce13cc3842009-03-25 13:54:54 -0700457 if u.endswith('/Gerrit'):
458 u = u[:len(u) - len('/Gerrit')]
459 if not u.endswith('/ssh_info'):
460 if not u.endswith('/'):
461 u += '/'
462 u += 'ssh_info'
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800463
Shawn O. Pearce146fe902009-03-25 14:06:43 -0700464 if u in REVIEW_CACHE:
465 info = REVIEW_CACHE[u]
466 self._review_protocol = info[0]
467 self._review_host = info[1]
468 self._review_port = info[2]
469 else:
470 try:
471 info = urlopen(u).read()
472 if info == 'NOT_AVAILABLE':
473 raise UploadError('Upload over ssh unavailable')
474 if '<' in info:
475 # Assume the server gave us some sort of HTML
476 # response back, like maybe a login page.
477 #
478 raise UploadError('Cannot read %s:\n%s' % (u, info))
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800479
Shawn O. Pearce146fe902009-03-25 14:06:43 -0700480 self._review_protocol = 'ssh'
481 self._review_host = info.split(" ")[0]
482 self._review_port = info.split(" ")[1]
483 except HTTPError, e:
484 if e.code == 404:
485 self._review_protocol = 'http-post'
486 self._review_host = None
487 self._review_port = None
488 else:
489 raise UploadError('Cannot guess Gerrit version')
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800490
Shawn O. Pearce146fe902009-03-25 14:06:43 -0700491 REVIEW_CACHE[u] = (
492 self._review_protocol,
493 self._review_host,
494 self._review_port)
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800495 return self._review_protocol
496
497 def SshReviewUrl(self, userEmail):
498 if self.ReviewProtocol != 'ssh':
499 return None
500 return 'ssh://%s@%s:%s/%s' % (
501 userEmail.split("@")[0],
502 self._review_host,
503 self._review_port,
504 self.projectname)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700505
506 def ToLocal(self, rev):
507 """Convert a remote revision string to something we have locally.
508 """
509 if IsId(rev):
510 return rev
511 if rev.startswith(R_TAGS):
512 return rev
513
514 if not rev.startswith('refs/'):
515 rev = R_HEADS + rev
516
517 for spec in self.fetch:
518 if spec.SourceMatches(rev):
519 return spec.MapSource(rev)
520 raise GitError('remote %s does not have %s' % (self.name, rev))
521
522 def WritesTo(self, ref):
523 """True if the remote stores to the tracking ref.
524 """
525 for spec in self.fetch:
526 if spec.DestMatches(ref):
527 return True
528 return False
529
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800530 def ResetFetch(self, mirror=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700531 """Set the fetch refspec to its default value.
532 """
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800533 if mirror:
534 dst = 'refs/heads/*'
535 else:
536 dst = 'refs/remotes/%s/*' % self.name
537 self.fetch = [RefSpec(True, 'refs/heads/*', dst)]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700538
539 def Save(self):
540 """Save this remote to the configuration.
541 """
542 self._Set('url', self.url)
543 self._Set('review', self.review)
Shawn O. Pearce339ba9f2008-11-06 09:52:51 -0800544 self._Set('projectname', self.projectname)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700545 self._Set('fetch', map(lambda x: str(x), self.fetch))
546
547 def _Set(self, key, value):
548 key = 'remote.%s.%s' % (self.name, key)
549 return self._config.SetString(key, value)
550
551 def _Get(self, key, all=False):
552 key = 'remote.%s.%s' % (self.name, key)
553 return self._config.GetString(key, all = all)
554
555
556class Branch(object):
557 """Configuration options related to a single branch.
558 """
559 def __init__(self, config, name):
560 self._config = config
561 self.name = name
562 self.merge = self._Get('merge')
563
564 r = self._Get('remote')
565 if r:
566 self.remote = self._config.GetRemote(r)
567 else:
568 self.remote = None
569
570 @property
571 def LocalMerge(self):
572 """Convert the merge spec to a local name.
573 """
574 if self.remote and self.merge:
575 return self.remote.ToLocal(self.merge)
576 return None
577
578 def Save(self):
579 """Save this branch back into the configuration.
580 """
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -0700581 if self._config.HasSection('branch', self.name):
582 if self.remote:
583 self._Set('remote', self.remote.name)
584 else:
585 self._Set('remote', None)
586 self._Set('merge', self.merge)
587
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700588 else:
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -0700589 fd = open(self._config.file, 'ab')
590 try:
591 fd.write('[branch "%s"]\n' % self.name)
592 if self.remote:
593 fd.write('\tremote = %s\n' % self.remote.name)
594 if self.merge:
595 fd.write('\tmerge = %s\n' % self.merge)
596 finally:
597 fd.close()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700598
599 def _Set(self, key, value):
600 key = 'branch.%s.%s' % (self.name, key)
601 return self._config.SetString(key, value)
602
603 def _Get(self, key, all=False):
604 key = 'branch.%s.%s' % (self.name, key)
605 return self._config.GetString(key, all = all)