blob: fb4377cf0db479f72790746f3d6667ce5c6f22d8 [file] [log] [blame]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001#
2# Copyright (C) 2008 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
Sarah Owenscecd1d82012-11-01 22:59:27 -070016from __future__ import print_function
Chirayu Desai217ea7d2013-03-01 19:14:38 +053017
Dan Willemsen0745bb22015-08-17 13:41:45 -070018import contextlib
19import errno
Anthony King85b24ac2014-05-06 15:57:48 +010020import json
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070021import os
22import re
Łukasz Gardońbed59ce2017-08-08 10:18:11 +020023import ssl
Shawn O. Pearcefb231612009-04-10 18:53:46 -070024import subprocess
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070025import sys
Doug Anderson0048b692010-12-21 13:39:23 -080026try:
27 import threading as _threading
28except ImportError:
29 import dummy_threading as _threading
Shawn O. Pearcefb231612009-04-10 18:53:46 -070030import time
David Pursehouse59bbb582013-05-17 10:49:33 +090031
32from pyversion import is_python3
33if is_python3():
Sarah Owens1f7627f2012-10-31 09:21:55 -070034 import urllib.request
35 import urllib.error
36else:
David Pursehouse59bbb582013-05-17 10:49:33 +090037 import urllib2
Sarah Owens1f7627f2012-10-31 09:21:55 -070038 import imp
39 urllib = imp.new_module('urllib')
40 urllib.request = urllib2
41 urllib.error = urllib2
Shawn O. Pearcef00e0ce2009-08-22 18:39:49 -070042
Shawn O. Pearcefb231612009-04-10 18:53:46 -070043from signal import SIGTERM
Shawn O. Pearceb54a3922009-01-05 16:18:58 -080044from error import GitError, UploadError
Shawn O. Pearcead3193a2009-04-18 09:54:51 -070045from trace import Trace
David Pursehouseecf8f2b2013-05-24 12:12:23 +090046if is_python3():
47 from http.client import HTTPException
48else:
49 from httplib import HTTPException
Shawn O. Pearceca8c32c2010-05-11 18:21:33 -070050
51from git_command import GitCommand
52from git_command import ssh_sock
53from git_command import terminate_ssh_clients
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070054
55R_HEADS = 'refs/heads/'
56R_TAGS = 'refs/tags/'
David Pursehouse1d947b32012-10-25 12:23:11 +090057ID_RE = re.compile(r'^[0-9a-f]{40}$')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070058
Shawn O. Pearce146fe902009-03-25 14:06:43 -070059REVIEW_CACHE = dict()
60
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070061def IsId(rev):
62 return ID_RE.match(rev)
63
Shawn O. Pearcef8e32732009-04-17 11:00:31 -070064def _key(name):
65 parts = name.split('.')
66 if len(parts) < 2:
67 return name.lower()
68 parts[ 0] = parts[ 0].lower()
69 parts[-1] = parts[-1].lower()
70 return '.'.join(parts)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070071
72class GitConfig(object):
Shawn O. Pearce90be5c02008-10-29 15:21:24 -070073 _ForUser = None
74
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070075 @classmethod
76 def ForUser(cls):
Shawn O. Pearce90be5c02008-10-29 15:21:24 -070077 if cls._ForUser is None:
David Pursehouse8a68ff92012-09-24 12:15:13 +090078 cls._ForUser = cls(configfile = os.path.expanduser('~/.gitconfig'))
Shawn O. Pearce90be5c02008-10-29 15:21:24 -070079 return cls._ForUser
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070080
81 @classmethod
82 def ForRepository(cls, gitdir, defaults=None):
David Pursehouse8a68ff92012-09-24 12:15:13 +090083 return cls(configfile = os.path.join(gitdir, 'config'),
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070084 defaults = defaults)
85
Anthony King85b24ac2014-05-06 15:57:48 +010086 def __init__(self, configfile, defaults=None, jsonFile=None):
David Pursehouse8a68ff92012-09-24 12:15:13 +090087 self.file = configfile
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070088 self.defaults = defaults
89 self._cache_dict = None
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -070090 self._section_dict = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070091 self._remotes = {}
92 self._branches = {}
Shawn O. Pearce1b34c912009-05-21 18:52:49 -070093
Anthony King85b24ac2014-05-06 15:57:48 +010094 self._json = jsonFile
95 if self._json is None:
96 self._json = os.path.join(
Shawn O. Pearce1b34c912009-05-21 18:52:49 -070097 os.path.dirname(self.file),
Anthony King85b24ac2014-05-06 15:57:48 +010098 '.repo_' + os.path.basename(self.file) + '.json')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070099
100 def Has(self, name, include_defaults = True):
101 """Return true if this configuration file has the key.
102 """
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700103 if _key(name) in self._cache:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700104 return True
105 if include_defaults and self.defaults:
106 return self.defaults.Has(name, include_defaults = True)
107 return False
108
109 def GetBoolean(self, name):
110 """Returns a boolean from the configuration file.
111 None : The value was not defined, or is not a boolean.
112 True : The value was set to true or yes.
113 False: The value was set to false or no.
114 """
115 v = self.GetString(name)
116 if v is None:
117 return None
118 v = v.lower()
119 if v in ('true', 'yes'):
120 return True
121 if v in ('false', 'no'):
122 return False
123 return None
124
David Pursehouse8a68ff92012-09-24 12:15:13 +0900125 def GetString(self, name, all_keys=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700126 """Get the first value for a key, or None if it is not defined.
127
128 This configuration file is used first, if the key is not
David Pursehouse8a68ff92012-09-24 12:15:13 +0900129 defined or all_keys = True then the defaults are also searched.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700130 """
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700131 try:
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700132 v = self._cache[_key(name)]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700133 except KeyError:
134 if self.defaults:
David Pursehouse8a68ff92012-09-24 12:15:13 +0900135 return self.defaults.GetString(name, all_keys = all_keys)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700136 v = []
137
David Pursehouse8a68ff92012-09-24 12:15:13 +0900138 if not all_keys:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700139 if v:
140 return v[0]
141 return None
142
143 r = []
144 r.extend(v)
145 if self.defaults:
David Pursehouse8a68ff92012-09-24 12:15:13 +0900146 r.extend(self.defaults.GetString(name, all_keys = True))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700147 return r
148
149 def SetString(self, name, value):
150 """Set the value(s) for a key.
151 Only this configuration file is modified.
152
153 The supplied value should be either a string,
154 or a list of strings (to store multiple values).
155 """
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700156 key = _key(name)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700157
158 try:
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700159 old = self._cache[key]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700160 except KeyError:
161 old = []
162
163 if value is None:
164 if old:
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700165 del self._cache[key]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700166 self._do('--unset-all', name)
167
168 elif isinstance(value, list):
169 if len(value) == 0:
170 self.SetString(name, None)
171
172 elif len(value) == 1:
173 self.SetString(name, value[0])
174
175 elif old != value:
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700176 self._cache[key] = list(value)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700177 self._do('--replace-all', name, value[0])
Sarah Owensa6053d52012-11-01 13:36:50 -0700178 for i in range(1, len(value)):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700179 self._do('--add', name, value[i])
180
181 elif len(old) != 1 or old[0] != value:
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700182 self._cache[key] = [value]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700183 self._do('--replace-all', name, value)
184
185 def GetRemote(self, name):
186 """Get the remote.$name.* configuration values as an object.
187 """
188 try:
189 r = self._remotes[name]
190 except KeyError:
191 r = Remote(self, name)
192 self._remotes[r.name] = r
193 return r
194
195 def GetBranch(self, name):
196 """Get the branch.$name.* configuration values as an object.
197 """
198 try:
199 b = self._branches[name]
200 except KeyError:
201 b = Branch(self, name)
202 self._branches[b.name] = b
203 return b
204
Shawn O. Pearce366ad212009-05-19 12:47:37 -0700205 def GetSubSections(self, section):
206 """List all subsection names matching $section.*.*
207 """
208 return self._sections.get(section, set())
209
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -0700210 def HasSection(self, section, subsection = ''):
211 """Does at least one key in section.subsection exist?
212 """
213 try:
214 return subsection in self._sections[section]
215 except KeyError:
216 return False
217
Shawn O. Pearce13111b42011-09-19 11:00:31 -0700218 def UrlInsteadOf(self, url):
219 """Resolve any url.*.insteadof references.
220 """
221 for new_url in self.GetSubSections('url'):
Dan Willemsen4e4d40f2013-10-28 22:28:42 -0700222 for old_url in self.GetString('url.%s.insteadof' % new_url, True):
223 if old_url is not None and url.startswith(old_url):
224 return new_url + url[len(old_url):]
Shawn O. Pearce13111b42011-09-19 11:00:31 -0700225 return url
226
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -0700227 @property
228 def _sections(self):
229 d = self._section_dict
230 if d is None:
231 d = {}
232 for name in self._cache.keys():
233 p = name.split('.')
234 if 2 == len(p):
235 section = p[0]
236 subsect = ''
237 else:
238 section = p[0]
239 subsect = '.'.join(p[1:-1])
240 if section not in d:
241 d[section] = set()
242 d[section].add(subsect)
243 self._section_dict = d
244 return d
245
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700246 @property
247 def _cache(self):
248 if self._cache_dict is None:
249 self._cache_dict = self._Read()
250 return self._cache_dict
251
252 def _Read(self):
Anthony King85b24ac2014-05-06 15:57:48 +0100253 d = self._ReadJson()
Shawn O. Pearcec12c3602009-04-17 21:03:32 -0700254 if d is None:
255 d = self._ReadGit()
Anthony King85b24ac2014-05-06 15:57:48 +0100256 self._SaveJson(d)
Shawn O. Pearcec12c3602009-04-17 21:03:32 -0700257 return d
258
Anthony King85b24ac2014-05-06 15:57:48 +0100259 def _ReadJson(self):
Shawn O. Pearcec12c3602009-04-17 21:03:32 -0700260 try:
Anthony King85b24ac2014-05-06 15:57:48 +0100261 if os.path.getmtime(self._json) \
Shawn O. Pearcec12c3602009-04-17 21:03:32 -0700262 <= os.path.getmtime(self.file):
Anthony King85b24ac2014-05-06 15:57:48 +0100263 os.remove(self._json)
Shawn O. Pearcec12c3602009-04-17 21:03:32 -0700264 return None
265 except OSError:
266 return None
267 try:
Anthony King85b24ac2014-05-06 15:57:48 +0100268 Trace(': parsing %s', self.file)
269 fd = open(self._json)
Shawn O. Pearce76ca9f82009-04-18 14:48:03 -0700270 try:
Anthony King85b24ac2014-05-06 15:57:48 +0100271 return json.load(fd)
Shawn O. Pearce76ca9f82009-04-18 14:48:03 -0700272 finally:
273 fd.close()
Anthony King85b24ac2014-05-06 15:57:48 +0100274 except (IOError, ValueError):
275 os.remove(self._json)
Shawn O. Pearcec12c3602009-04-17 21:03:32 -0700276 return None
277
Anthony King85b24ac2014-05-06 15:57:48 +0100278 def _SaveJson(self, cache):
Shawn O. Pearcec12c3602009-04-17 21:03:32 -0700279 try:
Anthony King85b24ac2014-05-06 15:57:48 +0100280 fd = open(self._json, 'w')
Shawn O. Pearce76ca9f82009-04-18 14:48:03 -0700281 try:
Anthony King85b24ac2014-05-06 15:57:48 +0100282 json.dump(cache, fd, indent=2)
Shawn O. Pearce76ca9f82009-04-18 14:48:03 -0700283 finally:
284 fd.close()
Anthony King85b24ac2014-05-06 15:57:48 +0100285 except (IOError, TypeError):
Anthony Kingb1d1fd72015-06-03 17:02:26 +0100286 if os.path.exists(self._json):
Anthony King85b24ac2014-05-06 15:57:48 +0100287 os.remove(self._json)
Shawn O. Pearcec12c3602009-04-17 21:03:32 -0700288
289 def _ReadGit(self):
David Aguilar438c5472009-06-28 15:09:16 -0700290 """
291 Read configuration data from git.
292
293 This internal method populates the GitConfig cache.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700294
David Aguilar438c5472009-06-28 15:09:16 -0700295 """
David Aguilar438c5472009-06-28 15:09:16 -0700296 c = {}
Shawn O. Pearcec24c7202009-07-02 16:12:57 -0700297 d = self._do('--null', '--list')
298 if d is None:
299 return c
Chirayu Desai0eb35cb2013-11-19 18:46:29 +0530300 for line in d.decode('utf-8').rstrip('\0').split('\0'): # pylint: disable=W1401
301 # Backslash is not anomalous
David Aguilar438c5472009-06-28 15:09:16 -0700302 if '\n' in line:
David Pursehousec1b86a22012-11-14 11:36:51 +0900303 key, val = line.split('\n', 1)
David Aguilar438c5472009-06-28 15:09:16 -0700304 else:
David Pursehousec1b86a22012-11-14 11:36:51 +0900305 key = line
306 val = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700307
308 if key in c:
309 c[key].append(val)
310 else:
311 c[key] = [val]
312
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700313 return c
314
315 def _do(self, *args):
316 command = ['config', '--file', self.file]
317 command.extend(args)
318
319 p = GitCommand(None,
320 command,
321 capture_stdout = True,
322 capture_stderr = True)
323 if p.Wait() == 0:
324 return p.stdout
325 else:
326 GitError('git config %s: %s' % (str(args), p.stderr))
327
328
329class RefSpec(object):
330 """A Git refspec line, split into its components:
331
332 forced: True if the line starts with '+'
333 src: Left side of the line
334 dst: Right side of the line
335 """
336
337 @classmethod
338 def FromString(cls, rs):
339 lhs, rhs = rs.split(':', 2)
340 if lhs.startswith('+'):
341 lhs = lhs[1:]
342 forced = True
343 else:
344 forced = False
345 return cls(forced, lhs, rhs)
346
347 def __init__(self, forced, lhs, rhs):
348 self.forced = forced
349 self.src = lhs
350 self.dst = rhs
351
352 def SourceMatches(self, rev):
353 if self.src:
354 if rev == self.src:
355 return True
356 if self.src.endswith('/*') and rev.startswith(self.src[:-1]):
357 return True
358 return False
359
360 def DestMatches(self, ref):
361 if self.dst:
362 if ref == self.dst:
363 return True
364 if self.dst.endswith('/*') and ref.startswith(self.dst[:-1]):
365 return True
366 return False
367
368 def MapSource(self, rev):
369 if self.src.endswith('/*'):
370 return self.dst[:-1] + rev[len(self.src) - 1:]
371 return self.dst
372
373 def __str__(self):
374 s = ''
375 if self.forced:
376 s += '+'
377 if self.src:
378 s += self.src
379 if self.dst:
380 s += ':'
381 s += self.dst
382 return s
383
384
Doug Anderson06d029c2010-10-27 17:06:01 -0700385_master_processes = []
386_master_keys = set()
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700387_ssh_master = True
Doug Anderson0048b692010-12-21 13:39:23 -0800388_master_keys_lock = None
389
390def init_ssh():
391 """Should be called once at the start of repo to init ssh master handling.
392
393 At the moment, all we do is to create our lock.
394 """
395 global _master_keys_lock
396 assert _master_keys_lock is None, "Should only call init_ssh once"
397 _master_keys_lock = _threading.Lock()
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700398
Josh Guilfoyle71985722009-08-16 09:44:40 -0700399def _open_ssh(host, port=None):
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700400 global _ssh_master
401
Doug Anderson0048b692010-12-21 13:39:23 -0800402 # Acquire the lock. This is needed to prevent opening multiple masters for
403 # the same host when we're running "repo sync -jN" (for N > 1) _and_ the
404 # manifest <remote fetch="ssh://xyz"> specifies a different host from the
405 # one that was passed to repo init.
406 _master_keys_lock.acquire()
407 try:
Josh Guilfoyle71985722009-08-16 09:44:40 -0700408
Doug Anderson0048b692010-12-21 13:39:23 -0800409 # Check to see whether we already think that the master is running; if we
410 # think it's already running, return right away.
411 if port is not None:
412 key = '%s:%s' % (host, port)
413 else:
414 key = host
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700415
Doug Anderson0048b692010-12-21 13:39:23 -0800416 if key in _master_keys:
417 return True
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700418
Doug Anderson0048b692010-12-21 13:39:23 -0800419 if not _ssh_master \
420 or 'GIT_SSH' in os.environ \
421 or sys.platform in ('win32', 'cygwin'):
422 # failed earlier, or cygwin ssh can't do this
423 #
424 return False
Doug Anderson06d029c2010-10-27 17:06:01 -0700425
Doug Anderson0048b692010-12-21 13:39:23 -0800426 # We will make two calls to ssh; this is the common part of both calls.
427 command_base = ['ssh',
428 '-o','ControlPath %s' % ssh_sock(),
429 host]
430 if port is not None:
David Pursehouse8f62fb72012-11-14 12:09:38 +0900431 command_base[1:1] = ['-p', str(port)]
Doug Anderson06d029c2010-10-27 17:06:01 -0700432
Doug Anderson0048b692010-12-21 13:39:23 -0800433 # Since the key wasn't in _master_keys, we think that master isn't running.
434 # ...but before actually starting a master, we'll double-check. This can
435 # be important because we can't tell that that 'git@myhost.com' is the same
436 # as 'myhost.com' where "User git" is setup in the user's ~/.ssh/config file.
437 check_command = command_base + ['-O','check']
438 try:
439 Trace(': %s', ' '.join(check_command))
440 check_process = subprocess.Popen(check_command,
441 stdout=subprocess.PIPE,
442 stderr=subprocess.PIPE)
443 check_process.communicate() # read output, but ignore it...
444 isnt_running = check_process.wait()
Josh Guilfoyle71985722009-08-16 09:44:40 -0700445
Doug Anderson0048b692010-12-21 13:39:23 -0800446 if not isnt_running:
447 # Our double-check found that the master _was_ infact running. Add to
448 # the list of keys.
449 _master_keys.add(key)
450 return True
451 except Exception:
452 # Ignore excpetions. We we will fall back to the normal command and print
453 # to the log there.
454 pass
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700455
Doug Anderson0048b692010-12-21 13:39:23 -0800456 command = command_base[:1] + \
457 ['-M', '-N'] + \
458 command_base[1:]
459 try:
460 Trace(': %s', ' '.join(command))
461 p = subprocess.Popen(command)
Sarah Owensa5be53f2012-09-09 15:37:57 -0700462 except Exception as e:
Doug Anderson0048b692010-12-21 13:39:23 -0800463 _ssh_master = False
Sarah Owenscecd1d82012-11-01 22:59:27 -0700464 print('\nwarn: cannot enable ssh control master for %s:%s\n%s'
465 % (host,port, str(e)), file=sys.stderr)
Doug Anderson0048b692010-12-21 13:39:23 -0800466 return False
467
Timo Lotterbach05dc46b2016-07-15 16:48:42 +0200468 time.sleep(1)
469 ssh_died = (p.poll() is not None)
470 if ssh_died:
471 return False
472
Doug Anderson0048b692010-12-21 13:39:23 -0800473 _master_processes.append(p)
474 _master_keys.add(key)
Doug Anderson0048b692010-12-21 13:39:23 -0800475 return True
476 finally:
477 _master_keys_lock.release()
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700478
479def close_ssh():
Doug Anderson0048b692010-12-21 13:39:23 -0800480 global _master_keys_lock
481
Shawn O. Pearceca8c32c2010-05-11 18:21:33 -0700482 terminate_ssh_clients()
483
Doug Anderson06d029c2010-10-27 17:06:01 -0700484 for p in _master_processes:
Shawn O. Pearce26120ca2009-06-16 11:49:10 -0700485 try:
486 os.kill(p.pid, SIGTERM)
487 p.wait()
Shawn O. Pearcefb5c8fd2009-06-16 14:57:46 -0700488 except OSError:
Shawn O. Pearce26120ca2009-06-16 11:49:10 -0700489 pass
Doug Anderson06d029c2010-10-27 17:06:01 -0700490 del _master_processes[:]
491 _master_keys.clear()
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700492
Nico Sallembien1c85f4e2010-04-27 14:35:27 -0700493 d = ssh_sock(create=False)
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700494 if d:
495 try:
496 os.rmdir(os.path.dirname(d))
497 except OSError:
498 pass
499
Doug Anderson0048b692010-12-21 13:39:23 -0800500 # We're done with the lock, so we can delete it.
501 _master_keys_lock = None
502
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700503URI_SCP = re.compile(r'^([^@:]*@?[^:/]{1,}):')
Shawn O. Pearce898e12a2012-03-14 15:22:28 -0700504URI_ALL = re.compile(r'^([a-z][a-z+-]*)://([^@/]*@?[^/]*)/')
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700505
Shawn O. Pearcef322b9a2011-09-19 14:50:58 -0700506def GetSchemeFromUrl(url):
507 m = URI_ALL.match(url)
508 if m:
509 return m.group(1)
510 return None
511
Dan Willemsen0745bb22015-08-17 13:41:45 -0700512@contextlib.contextmanager
513def GetUrlCookieFile(url, quiet):
514 if url.startswith('persistent-'):
515 try:
516 p = subprocess.Popen(
517 ['git-remote-persistent-https', '-print_config', url],
518 stdin=subprocess.PIPE, stdout=subprocess.PIPE,
519 stderr=subprocess.PIPE)
520 try:
521 cookieprefix = 'http.cookiefile='
522 proxyprefix = 'http.proxy='
523 cookiefile = None
524 proxy = None
525 for line in p.stdout:
526 line = line.strip()
527 if line.startswith(cookieprefix):
528 cookiefile = line[len(cookieprefix):]
529 if line.startswith(proxyprefix):
530 proxy = line[len(proxyprefix):]
531 # Leave subprocess open, as cookie file may be transient.
532 if cookiefile or proxy:
533 yield cookiefile, proxy
534 return
535 finally:
536 p.stdin.close()
537 if p.wait():
538 err_msg = p.stderr.read()
539 if ' -print_config' in err_msg:
540 pass # Persistent proxy doesn't support -print_config.
541 elif not quiet:
542 print(err_msg, file=sys.stderr)
543 except OSError as e:
544 if e.errno == errno.ENOENT:
545 pass # No persistent proxy.
546 raise
547 yield GitConfig.ForUser().GetString('http.cookiefile'), None
548
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700549def _preconnect(url):
550 m = URI_ALL.match(url)
551 if m:
552 scheme = m.group(1)
553 host = m.group(2)
554 if ':' in host:
555 host, port = host.split(':')
Shawn O. Pearce896d5df2009-04-21 14:51:04 -0700556 else:
Josh Guilfoyle71985722009-08-16 09:44:40 -0700557 port = None
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700558 if scheme in ('ssh', 'git+ssh', 'ssh+git'):
559 return _open_ssh(host, port)
560 return False
561
562 m = URI_SCP.match(url)
563 if m:
564 host = m.group(1)
Josh Guilfoyle71985722009-08-16 09:44:40 -0700565 return _open_ssh(host)
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700566
Shawn O. Pearce7b4f4352009-06-12 09:06:35 -0700567 return False
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700568
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700569class Remote(object):
570 """Configuration options related to a remote.
571 """
572 def __init__(self, config, name):
573 self._config = config
574 self.name = name
575 self.url = self._Get('url')
Steve Raed6480452016-08-10 15:00:00 -0700576 self.pushUrl = self._Get('pushurl')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700577 self.review = self._Get('review')
Shawn O. Pearce339ba9f2008-11-06 09:52:51 -0800578 self.projectname = self._Get('projectname')
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530579 self.fetch = list(map(RefSpec.FromString,
580 self._Get('fetch', all_keys=True)))
Shawn O. Pearcec9571422012-01-11 14:58:54 -0800581 self._review_url = None
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800582
Ulrik Sjolinb6ea3bf2010-01-03 18:20:17 +0100583 def _InsteadOf(self):
584 globCfg = GitConfig.ForUser()
585 urlList = globCfg.GetSubSections('url')
586 longest = ""
587 longestUrl = ""
588
589 for url in urlList:
590 key = "url." + url + ".insteadOf"
David Pursehouse8a68ff92012-09-24 12:15:13 +0900591 insteadOfList = globCfg.GetString(key, all_keys=True)
Ulrik Sjolinb6ea3bf2010-01-03 18:20:17 +0100592
593 for insteadOf in insteadOfList:
594 if self.url.startswith(insteadOf) \
595 and len(insteadOf) > len(longest):
596 longest = insteadOf
597 longestUrl = url
598
599 if len(longest) == 0:
600 return self.url
601
602 return self.url.replace(longest, longestUrl, 1)
603
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700604 def PreConnectFetch(self):
Ulrik Sjolinb6ea3bf2010-01-03 18:20:17 +0100605 connectionUrl = self._InsteadOf()
606 return _preconnect(connectionUrl)
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700607
Łukasz Gardońbed59ce2017-08-08 10:18:11 +0200608 def ReviewUrl(self, userEmail, validate_certs):
Shawn O. Pearcec9571422012-01-11 14:58:54 -0800609 if self._review_url is None:
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800610 if self.review is None:
611 return None
612
613 u = self.review
Conley Owens7e12e0a2014-10-23 15:40:00 -0700614 if u.startswith('persistent-'):
615 u = u[len('persistent-'):]
Steve Pucci143d8a72014-01-30 09:45:53 -0800616 if u.split(':')[0] not in ('http', 'https', 'sso'):
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800617 u = 'http://%s' % u
Shawn O. Pearce13cc3842009-03-25 13:54:54 -0700618 if u.endswith('/Gerrit'):
619 u = u[:len(u) - len('/Gerrit')]
Shawn O. Pearcec9571422012-01-11 14:58:54 -0800620 if u.endswith('/ssh_info'):
621 u = u[:len(u) - len('/ssh_info')]
622 if not u.endswith('/'):
David Pursehouse8a68ff92012-09-24 12:15:13 +0900623 u += '/'
Shawn O. Pearcec9571422012-01-11 14:58:54 -0800624 http_url = u
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800625
Shawn O. Pearce146fe902009-03-25 14:06:43 -0700626 if u in REVIEW_CACHE:
Shawn O. Pearcec9571422012-01-11 14:58:54 -0800627 self._review_url = REVIEW_CACHE[u]
Shawn O. Pearce1a68dc52011-10-11 14:12:46 -0700628 elif 'REPO_HOST_PORT_INFO' in os.environ:
Shawn O. Pearcec9571422012-01-11 14:58:54 -0800629 host, port = os.environ['REPO_HOST_PORT_INFO'].split()
630 self._review_url = self._SshReviewUrl(userEmail, host, port)
631 REVIEW_CACHE[u] = self._review_url
Steve Pucci143d8a72014-01-30 09:45:53 -0800632 elif u.startswith('sso:'):
633 self._review_url = u # Assume it's right
634 REVIEW_CACHE[u] = self._review_url
Timo Lotterbacheec726c2016-10-07 10:52:08 +0200635 elif 'REPO_IGNORE_SSH_INFO' in os.environ:
636 self._review_url = http_url
637 REVIEW_CACHE[u] = self._review_url
Shawn O. Pearce146fe902009-03-25 14:06:43 -0700638 else:
639 try:
Shawn O. Pearcec9571422012-01-11 14:58:54 -0800640 info_url = u + 'ssh_info'
Łukasz Gardońbed59ce2017-08-08 10:18:11 +0200641 if not validate_certs:
642 context = ssl._create_unverified_context()
643 info = urllib.request.urlopen(info_url, context=context).read()
644 else:
645 info = urllib.request.urlopen(info_url).read()
Conley Owens745a39b2013-06-05 13:16:18 -0700646 if info == 'NOT_AVAILABLE' or '<' in info:
647 # If `info` contains '<', we assume the server gave us some sort
648 # of HTML response back, like maybe a login page.
Shawn O. Pearce146fe902009-03-25 14:06:43 -0700649 #
Conley Owens745a39b2013-06-05 13:16:18 -0700650 # Assume HTTP if SSH is not enabled or ssh_info doesn't look right.
Conley Owens2cd38a02014-02-04 15:32:29 -0800651 self._review_url = http_url
Shawn O. Pearce146fe902009-03-25 14:06:43 -0700652 else:
Shawn O. Pearcec9571422012-01-11 14:58:54 -0800653 host, port = info.split()
Dan Willemsen16889ba2016-09-22 16:39:06 +0000654 self._review_url = self._SshReviewUrl(userEmail, host, port)
Sarah Owens1f7627f2012-10-31 09:21:55 -0700655 except urllib.error.HTTPError as e:
Shawn O. Pearcec9571422012-01-11 14:58:54 -0800656 raise UploadError('%s: %s' % (self.review, str(e)))
Sarah Owens1f7627f2012-10-31 09:21:55 -0700657 except urllib.error.URLError as e:
Shawn O. Pearcebf1fbb22011-10-11 09:31:58 -0700658 raise UploadError('%s: %s' % (self.review, str(e)))
David Pursehouseecf8f2b2013-05-24 12:12:23 +0900659 except HTTPException as e:
660 raise UploadError('%s: %s' % (self.review, e.__class__.__name__))
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800661
Shawn O. Pearcec9571422012-01-11 14:58:54 -0800662 REVIEW_CACHE[u] = self._review_url
663 return self._review_url + self.projectname
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800664
Shawn O. Pearcec9571422012-01-11 14:58:54 -0800665 def _SshReviewUrl(self, userEmail, host, port):
Shawn O. Pearce3575b8f2010-07-15 17:00:14 -0700666 username = self._config.GetString('review.%s.username' % self.review)
667 if username is None:
Shawn O. Pearcec9571422012-01-11 14:58:54 -0800668 username = userEmail.split('@')[0]
669 return 'ssh://%s@%s:%s/' % (username, host, port)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700670
671 def ToLocal(self, rev):
672 """Convert a remote revision string to something we have locally.
673 """
Yann Droneaud936183a2013-09-12 10:51:18 +0200674 if self.name == '.' or IsId(rev):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700675 return rev
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700676
677 if not rev.startswith('refs/'):
678 rev = R_HEADS + rev
679
680 for spec in self.fetch:
681 if spec.SourceMatches(rev):
682 return spec.MapSource(rev)
Nasser Grainawi909d58b2014-09-19 12:13:04 -0600683
684 if not rev.startswith(R_HEADS):
685 return rev
686
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700687 raise GitError('remote %s does not have %s' % (self.name, rev))
688
689 def WritesTo(self, ref):
690 """True if the remote stores to the tracking ref.
691 """
692 for spec in self.fetch:
693 if spec.DestMatches(ref):
694 return True
695 return False
696
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800697 def ResetFetch(self, mirror=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700698 """Set the fetch refspec to its default value.
699 """
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800700 if mirror:
701 dst = 'refs/heads/*'
702 else:
703 dst = 'refs/remotes/%s/*' % self.name
704 self.fetch = [RefSpec(True, 'refs/heads/*', dst)]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700705
706 def Save(self):
707 """Save this remote to the configuration.
708 """
709 self._Set('url', self.url)
Steve Raed6480452016-08-10 15:00:00 -0700710 if self.pushUrl is not None:
711 self._Set('pushurl', self.pushUrl + '/' + self.projectname)
712 else:
713 self._Set('pushurl', self.pushUrl)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700714 self._Set('review', self.review)
Shawn O. Pearce339ba9f2008-11-06 09:52:51 -0800715 self._Set('projectname', self.projectname)
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530716 self._Set('fetch', list(map(str, self.fetch)))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700717
718 def _Set(self, key, value):
719 key = 'remote.%s.%s' % (self.name, key)
720 return self._config.SetString(key, value)
721
David Pursehouse8a68ff92012-09-24 12:15:13 +0900722 def _Get(self, key, all_keys=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700723 key = 'remote.%s.%s' % (self.name, key)
David Pursehouse8a68ff92012-09-24 12:15:13 +0900724 return self._config.GetString(key, all_keys = all_keys)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700725
726
727class Branch(object):
728 """Configuration options related to a single branch.
729 """
730 def __init__(self, config, name):
731 self._config = config
732 self.name = name
733 self.merge = self._Get('merge')
734
735 r = self._Get('remote')
736 if r:
737 self.remote = self._config.GetRemote(r)
738 else:
739 self.remote = None
740
741 @property
742 def LocalMerge(self):
743 """Convert the merge spec to a local name.
744 """
745 if self.remote and self.merge:
746 return self.remote.ToLocal(self.merge)
747 return None
748
749 def Save(self):
750 """Save this branch back into the configuration.
751 """
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -0700752 if self._config.HasSection('branch', self.name):
753 if self.remote:
754 self._Set('remote', self.remote.name)
755 else:
756 self._Set('remote', None)
757 self._Set('merge', self.merge)
758
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700759 else:
Chirayu Desai303a82f2014-08-19 22:57:17 +0530760 fd = open(self._config.file, 'a')
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -0700761 try:
762 fd.write('[branch "%s"]\n' % self.name)
763 if self.remote:
764 fd.write('\tremote = %s\n' % self.remote.name)
765 if self.merge:
766 fd.write('\tmerge = %s\n' % self.merge)
767 finally:
768 fd.close()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700769
770 def _Set(self, key, value):
771 key = 'branch.%s.%s' % (self.name, key)
772 return self._config.SetString(key, value)
773
David Pursehouse8a68ff92012-09-24 12:15:13 +0900774 def _Get(self, key, all_keys=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700775 key = 'branch.%s.%s' % (self.name, key)
David Pursehouse8a68ff92012-09-24 12:15:13 +0900776 return self._config.GetString(key, all_keys = all_keys)