Merge "Support smart-sync through persistent-http[s]"
diff --git a/git_config.py b/git_config.py
index 8ded7c2..0379181 100644
--- a/git_config.py
+++ b/git_config.py
@@ -15,6 +15,8 @@
from __future__ import print_function
+import contextlib
+import errno
import json
import os
import re
@@ -502,6 +504,43 @@
return m.group(1)
return None
+@contextlib.contextmanager
+def GetUrlCookieFile(url, quiet):
+ if url.startswith('persistent-'):
+ try:
+ p = subprocess.Popen(
+ ['git-remote-persistent-https', '-print_config', url],
+ stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ try:
+ cookieprefix = 'http.cookiefile='
+ proxyprefix = 'http.proxy='
+ cookiefile = None
+ proxy = None
+ for line in p.stdout:
+ line = line.strip()
+ if line.startswith(cookieprefix):
+ cookiefile = line[len(cookieprefix):]
+ if line.startswith(proxyprefix):
+ proxy = line[len(proxyprefix):]
+ # Leave subprocess open, as cookie file may be transient.
+ if cookiefile or proxy:
+ yield cookiefile, proxy
+ return
+ finally:
+ p.stdin.close()
+ if p.wait():
+ err_msg = p.stderr.read()
+ if ' -print_config' in err_msg:
+ pass # Persistent proxy doesn't support -print_config.
+ elif not quiet:
+ print(err_msg, file=sys.stderr)
+ except OSError as e:
+ if e.errno == errno.ENOENT:
+ pass # No persistent proxy.
+ raise
+ yield GitConfig.ForUser().GetString('http.cookiefile'), None
+
def _preconnect(url):
m = URI_ALL.match(url)
if m:
diff --git a/project.py b/project.py
index d82f3d7..24cac8b 100644
--- a/project.py
+++ b/project.py
@@ -13,7 +13,6 @@
# limitations under the License.
from __future__ import print_function
-import contextlib
import errno
import filecmp
import glob
@@ -31,7 +30,7 @@
from color import Coloring
from git_command import GitCommand, git_require
-from git_config import GitConfig, IsId, GetSchemeFromUrl, ID_RE
+from git_config import GitConfig, IsId, GetSchemeFromUrl, GetUrlCookieFile, ID_RE
from error import GitError, HookError, UploadError, DownloadError
from error import ManifestInvalidRevisionError
from error import NoManifestException
@@ -2030,7 +2029,7 @@
os.remove(tmpPath)
if 'http_proxy' in os.environ and 'darwin' == sys.platform:
cmd += ['--proxy', os.environ['http_proxy']]
- with self._GetBundleCookieFile(srcUrl, quiet) as cookiefile:
+ with GetUrlCookieFile(srcUrl, quiet) as (cookiefile, proxy):
if cookiefile:
cmd += ['--cookie', cookiefile, '--cookie-jar', cookiefile]
if srcUrl.startswith('persistent-'):
@@ -2078,40 +2077,6 @@
except OSError:
return False
- @contextlib.contextmanager
- def _GetBundleCookieFile(self, url, quiet):
- if url.startswith('persistent-'):
- try:
- p = subprocess.Popen(
- ['git-remote-persistent-https', '-print_config', url],
- stdin=subprocess.PIPE, stdout=subprocess.PIPE,
- stderr=subprocess.PIPE)
- try:
- prefix = 'http.cookiefile='
- cookiefile = None
- for line in p.stdout:
- line = line.strip()
- if line.startswith(prefix):
- cookiefile = line[len(prefix):]
- break
- # Leave subprocess open, as cookie file may be transient.
- if cookiefile:
- yield cookiefile
- return
- finally:
- p.stdin.close()
- if p.wait():
- err_msg = p.stderr.read()
- if ' -print_config' in err_msg:
- pass # Persistent proxy doesn't support -print_config.
- elif not quiet:
- print(err_msg, file=sys.stderr)
- except OSError as e:
- if e.errno == errno.ENOENT:
- pass # No persistent proxy.
- raise
- yield GitConfig.ForUser().GetString('http.cookiefile')
-
def _Checkout(self, rev, quiet=False):
cmd = ['checkout']
if quiet:
diff --git a/subcmds/sync.py b/subcmds/sync.py
index 652a0c0..0fc493c 100644
--- a/subcmds/sync.py
+++ b/subcmds/sync.py
@@ -23,18 +23,26 @@
import socket
import subprocess
import sys
+import tempfile
import time
from pyversion import is_python3
if is_python3():
+ import http.cookiejar as cookielib
+ import urllib.error
import urllib.parse
+ import urllib.request
import xmlrpc.client
else:
+ import cookielib
import imp
+ import urllib2
import urlparse
import xmlrpclib
urllib = imp.new_module('urllib')
+ urllib.error = urllib2
urllib.parse = urlparse
+ urllib.request = urllib2
xmlrpc = imp.new_module('xmlrpc')
xmlrpc.client = xmlrpclib
@@ -57,6 +65,7 @@
multiprocessing = None
from git_command import GIT, git_require
+from git_config import GetSchemeFromUrl, GetUrlCookieFile
from git_refs import R_HEADS, HEAD
import gitc_utils
from project import Project
@@ -598,8 +607,12 @@
(username, password),
1)
+ transport = PersistentTransport(manifest_server)
+ if manifest_server.startswith('persistent-'):
+ manifest_server = manifest_server[len('persistent-'):]
+
try:
- server = xmlrpc.client.Server(manifest_server)
+ server = xmlrpc.client.Server(manifest_server, transport=transport)
if opt.smart_sync:
p = self.manifest.manifestProject
b = p.GetBranch(p.CurrentBranch)
@@ -879,3 +892,93 @@
os.remove(self._path)
except OSError:
pass
+
+# This is a replacement for xmlrpc.client.Transport using urllib2
+# and supporting persistent-http[s]. It cannot change hosts from
+# request to request like the normal transport, the real url
+# is passed during initialization.
+class PersistentTransport(xmlrpc.client.Transport):
+ def __init__(self, orig_host):
+ self.orig_host = orig_host
+
+ def request(self, host, handler, request_body, verbose=False):
+ with GetUrlCookieFile(self.orig_host, not verbose) as (cookiefile, proxy):
+ # Python doesn't understand cookies with the #HttpOnly_ prefix
+ # Since we're only using them for HTTP, copy the file temporarily,
+ # stripping those prefixes away.
+ tmpcookiefile = tempfile.NamedTemporaryFile()
+ try:
+ with open(cookiefile) as f:
+ for line in f:
+ if line.startswith("#HttpOnly_"):
+ line = line[len("#HttpOnly_"):]
+ tmpcookiefile.write(line)
+ tmpcookiefile.flush()
+
+ cookiejar = cookielib.MozillaCookieJar(tmpcookiefile.name)
+ cookiejar.load()
+ finally:
+ tmpcookiefile.close()
+
+ proxyhandler = urllib.request.ProxyHandler
+ if proxy:
+ proxyhandler = urllib.request.ProxyHandler({
+ "http": proxy,
+ "https": proxy })
+
+ opener = urllib.request.build_opener(
+ urllib.request.HTTPCookieProcessor(cookiejar),
+ proxyhandler)
+
+ url = urllib.parse.urljoin(self.orig_host, handler)
+ parse_results = urllib.parse.urlparse(url)
+
+ scheme = parse_results.scheme
+ if scheme == 'persistent-http':
+ scheme = 'http'
+ if scheme == 'persistent-https':
+ # If we're proxying through persistent-https, use http. The
+ # proxy itself will do the https.
+ if proxy:
+ scheme = 'http'
+ else:
+ scheme = 'https'
+
+ # Parse out any authentication information using the base class
+ host, extra_headers, _ = self.get_host_info(parse_results.netloc)
+
+ url = urllib.parse.urlunparse((
+ scheme,
+ host,
+ parse_results.path,
+ parse_results.params,
+ parse_results.query,
+ parse_results.fragment))
+
+ request = urllib.request.Request(url, request_body)
+ if extra_headers is not None:
+ for (name, header) in extra_headers:
+ request.add_header(name, header)
+ request.add_header('Content-Type', 'text/xml')
+ try:
+ response = opener.open(request)
+ except urllib.error.HTTPError as e:
+ if e.code == 501:
+ # We may have been redirected through a login process
+ # but our POST turned into a GET. Retry.
+ response = opener.open(request)
+ else:
+ raise
+
+ p, u = xmlrpc.client.getparser()
+ while 1:
+ data = response.read(1024)
+ if not data:
+ break
+ p.feed(data)
+ p.close()
+ return u.close()
+
+ def close(self):
+ pass
+