blob: ecb89a1074af0db8f5bf709f473ad693da3d3171 [file] [log] [blame]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001#
2# Copyright (C) 2008 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
Shawn O. Pearce2a1ccb22009-04-10 16:51:53 -070016from optparse import SUPPRESS_HELP
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070017import os
18import re
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -070019import shutil
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070020import subprocess
21import sys
Shawn O. Pearcef6906872009-04-18 10:49:00 -070022import time
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070023
24from git_command import GIT
Shawn O. Pearcee756c412009-04-13 11:51:15 -070025from project import HEAD
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -070026from project import Project
27from project import RemoteSpec
Shawn O. Pearcec95583b2009-03-03 17:47:06 -080028from command import Command, MirrorSafeCommand
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070029from error import RepoChangedException, GitError
30from project import R_HEADS
Shawn O. Pearce350cde42009-04-16 11:21:18 -070031from project import SyncBuffer
Shawn O. Pearce68194f42009-04-10 16:48:52 -070032from progress import Progress
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070033
Shawn O. Pearcec95583b2009-03-03 17:47:06 -080034class Sync(Command, MirrorSafeCommand):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070035 common = True
36 helpSummary = "Update working tree to the latest revision"
37 helpUsage = """
38%prog [<project>...]
39"""
40 helpDescription = """
41The '%prog' command synchronizes local project directories
42with the remote repositories specified in the manifest. If a local
43project does not yet exist, it will clone a new local directory from
44the remote repository and set up tracking branches as specified in
45the manifest. If the local project already exists, '%prog'
46will update the remote branches and rebase any new local changes
47on top of the new remote changes.
48
49'%prog' will synchronize all projects listed at the command
50line. Projects can be specified either by name, or by a relative
51or absolute path to the project's local directory. If no projects
52are specified, '%prog' will synchronize all projects listed in
53the manifest.
Shawn O. Pearce3e768c92009-04-10 16:59:36 -070054
55The -d/--detach option can be used to switch specified projects
56back to the manifest revision. This option is especially helpful
57if the project is currently on a topic branch, but the manifest
58revision is temporarily needed.
Shawn O. Pearceeb7af872009-04-21 08:02:04 -070059
60SSH Connections
61---------------
62
63If at least one project remote URL uses an SSH connection (ssh://,
64git+ssh://, or user@host:path syntax) repo will automatically
65enable the SSH ControlMaster option when connecting to that host.
66This feature permits other projects in the same '%prog' session to
67reuse the same SSH tunnel, saving connection setup overheads.
68
69To disable this behavior on UNIX platforms, set the GIT_SSH
70environment variable to 'ssh'. For example:
71
72 export GIT_SSH=ssh
73 %prog
74
75Compatibility
76~~~~~~~~~~~~~
77
78This feature is automatically disabled on Windows, due to the lack
79of UNIX domain socket support.
80
81This feature is not compatible with url.insteadof rewrites in the
82user's ~/.gitconfig. '%prog' is currently not able to perform the
83rewrite early enough to establish the ControlMaster tunnel.
84
85If the remote SSH daemon is Gerrit Code Review, version 2.0.10 or
86later is required to fix a server side protocol bug.
87
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070088"""
89
90 def _Options(self, p):
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -070091 p.add_option('-l','--local-only',
92 dest='local_only', action='store_true',
93 help="only update working tree, don't fetch")
Shawn O. Pearce96fdcef2009-04-10 16:29:20 -070094 p.add_option('-n','--network-only',
95 dest='network_only', action='store_true',
96 help="fetch only, don't update working tree")
Shawn O. Pearce3e768c92009-04-10 16:59:36 -070097 p.add_option('-d','--detach',
98 dest='detach_head', action='store_true',
99 help='detach projects back to manifest revision')
100
Shawn O. Pearcefd89b672009-04-18 11:28:57 -0700101 g = p.add_option_group('repo Version options')
102 g.add_option('--no-repo-verify',
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700103 dest='no_repo_verify', action='store_true',
104 help='do not verify repo source code')
Shawn O. Pearcefd89b672009-04-18 11:28:57 -0700105 g.add_option('--repo-upgraded',
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800106 dest='repo_upgraded', action='store_true',
Shawn O. Pearce2a1ccb22009-04-10 16:51:53 -0700107 help=SUPPRESS_HELP)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700108
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700109 def _Fetch(self, projects):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700110 fetched = set()
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700111 pm = Progress('Fetching projects', len(projects))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700112 for project in projects:
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700113 pm.update()
114
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700115 if project.Sync_NetworkHalf():
116 fetched.add(project.gitdir)
117 else:
118 print >>sys.stderr, 'error: Cannot fetch %s' % project.name
119 sys.exit(1)
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700120 pm.end()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700121 return fetched
122
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700123 def UpdateProjectList(self):
124 new_project_paths = []
125 for project in self.manifest.projects.values():
126 new_project_paths.append(project.relpath)
127 file_name = 'project.list'
128 file_path = os.path.join(self.manifest.repodir, file_name)
129 old_project_paths = []
130
131 if os.path.exists(file_path):
132 fd = open(file_path, 'r')
133 try:
134 old_project_paths = fd.read().split('\n')
135 finally:
136 fd.close()
137 for path in old_project_paths:
138 if path not in new_project_paths:
139 project = Project(
140 manifest = self.manifest,
141 name = path,
142 remote = RemoteSpec('origin'),
143 gitdir = os.path.join(self.manifest.topdir,
144 path, '.git'),
145 worktree = os.path.join(self.manifest.topdir, path),
146 relpath = path,
147 revisionExpr = 'HEAD',
148 revisionId = None)
149 if project.IsDirty():
150 print >>sys.stderr, 'error: Cannot remove project "%s": \
151uncommitted changes are present' % project.relpath
152 print >>sys.stderr, ' commit changes, then run sync again'
153 return -1
154 else:
155 print >>sys.stderr, 'Deleting obsolete path %s' % project.worktree
156 shutil.rmtree(project.worktree)
Jaikumar Ganesh8135cdc2009-06-02 15:07:44 -0700157 # Try deleting parent subdirs if they are empty
158 dir = os.path.dirname(project.worktree)
159 while dir != self.manifest.topdir:
160 try:
161 os.rmdir(dir)
162 except OSError:
163 break
164 dir = os.path.dirname(dir)
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700165
166 fd = open(file_path, 'w')
167 try:
168 fd.write('\n'.join(new_project_paths))
169 finally:
170 fd.close()
171 return 0
172
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700173 def Execute(self, opt, args):
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700174 if opt.network_only and opt.detach_head:
175 print >>sys.stderr, 'error: cannot combine -n and -d'
176 sys.exit(1)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700177 if opt.network_only and opt.local_only:
178 print >>sys.stderr, 'error: cannot combine -n and -l'
179 sys.exit(1)
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700180
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700181 rp = self.manifest.repoProject
182 rp.PreSync()
183
184 mp = self.manifest.manifestProject
185 mp.PreSync()
186
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800187 if opt.repo_upgraded:
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700188 _PostRepoUpgrade(self.manifest)
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800189
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700190 all = self.GetProjects(args, missing_ok=True)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700191
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700192 if not opt.local_only:
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700193 to_fetch = []
194 now = time.time()
195 if (24 * 60 * 60) <= (now - rp.LastFetch):
196 to_fetch.append(rp)
197 to_fetch.append(mp)
198 to_fetch.extend(all)
199
200 fetched = self._Fetch(to_fetch)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700201 _PostRepoFetch(rp, opt.no_repo_verify)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700202 if opt.network_only:
203 # bail out now; the rest touches the working tree
204 return
205
206 if mp.HasChanges:
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700207 syncbuf = SyncBuffer(mp.config)
208 mp.Sync_LocalHalf(syncbuf)
209 if not syncbuf.Finish():
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700210 sys.exit(1)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700211
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700212 self.manifest._Unload()
213 all = self.GetProjects(args, missing_ok=True)
214 missing = []
215 for project in all:
216 if project.gitdir not in fetched:
217 missing.append(project)
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700218 self._Fetch(missing)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700219
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700220 if self.UpdateProjectList():
221 sys.exit(1)
222
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700223 syncbuf = SyncBuffer(mp.config,
224 detach_head = opt.detach_head)
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700225 pm = Progress('Syncing work tree', len(all))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700226 for project in all:
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700227 pm.update()
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800228 if project.worktree:
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700229 project.Sync_LocalHalf(syncbuf)
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700230 pm.end()
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700231 print >>sys.stderr
232 if not syncbuf.Finish():
233 sys.exit(1)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700234
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700235
236def _PostRepoUpgrade(manifest):
237 for project in manifest.projects.values():
238 if project.Exists:
239 project.PostRepoUpgrade()
240
241def _PostRepoFetch(rp, no_repo_verify=False, verbose=False):
242 if rp.HasChanges:
243 print >>sys.stderr, 'info: A new version of repo is available'
244 print >>sys.stderr, ''
245 if no_repo_verify or _VerifyTag(rp):
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700246 syncbuf = SyncBuffer(rp.config)
247 rp.Sync_LocalHalf(syncbuf)
248 if not syncbuf.Finish():
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700249 sys.exit(1)
250 print >>sys.stderr, 'info: Restarting repo with latest version'
251 raise RepoChangedException(['--repo-upgraded'])
252 else:
253 print >>sys.stderr, 'warning: Skipped upgrade to unverified version'
254 else:
255 if verbose:
256 print >>sys.stderr, 'repo version %s is current' % rp.work_git.describe(HEAD)
257
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700258def _VerifyTag(project):
259 gpg_dir = os.path.expanduser('~/.repoconfig/gnupg')
260 if not os.path.exists(gpg_dir):
261 print >>sys.stderr,\
262"""warning: GnuPG was not available during last "repo init"
263warning: Cannot automatically authenticate repo."""
264 return True
265
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700266 try:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700267 cur = project.bare_git.describe(project.GetRevisionId())
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700268 except GitError:
269 cur = None
270
271 if not cur \
272 or re.compile(r'^.*-[0-9]{1,}-g[0-9a-f]{1,}$').match(cur):
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700273 rev = project.revisionExpr
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700274 if rev.startswith(R_HEADS):
275 rev = rev[len(R_HEADS):]
276
277 print >>sys.stderr
278 print >>sys.stderr,\
279 "warning: project '%s' branch '%s' is not signed" \
280 % (project.name, rev)
281 return False
282
283 env = dict(os.environ)
284 env['GIT_DIR'] = project.gitdir
285 env['GNUPGHOME'] = gpg_dir
286
287 cmd = [GIT, 'tag', '-v', cur]
288 proc = subprocess.Popen(cmd,
289 stdout = subprocess.PIPE,
290 stderr = subprocess.PIPE,
291 env = env)
292 out = proc.stdout.read()
293 proc.stdout.close()
294
295 err = proc.stderr.read()
296 proc.stderr.close()
297
298 if proc.wait() != 0:
299 print >>sys.stderr
300 print >>sys.stderr, out
301 print >>sys.stderr, err
302 print >>sys.stderr
303 return False
304 return True