merge v1.12.37
1. 4350791 On project cleanup, don't remove nested projects
2. revert 76a4a9d project: Set config option to skip lfs smudge filter
diff --git a/manifest_xml.py b/manifest_xml.py
index 9c882af..8bcc616 100644
--- a/manifest_xml.py
+++ b/manifest_xml.py
@@ -314,6 +314,9 @@
if p.sync_s:
e.setAttribute('sync-s', 'true')
+ if p.lfs_fetch:
+ e.setAttribute('lfs-fetch', 'true')
+
if p.clone_depth:
e.setAttribute('clone-depth', str(p.clone_depth))
@@ -811,6 +814,11 @@
if node.getAttribute('force-path').lower() in ("yes", "true", "1"):
gitdir = os.path.join(self.topdir, '%s.git' % path)
+ lfs_fetch = node.getAttribute('lfs-fetch')
+ if not lfs_fetch:
+ lfe_fetch = False
+ else:
+ lfs_fetch = lfs_fetch.lower() in ("yes", "true", "1")
project = Project(manifest = self,
name = name,
remote = remote.ToRemoteSpec(name),
@@ -828,6 +836,7 @@
upstream = upstream,
parent = parent,
dest_branch = dest_branch,
+ lfs_fetch = lfs_fetch,
**extra_proj_attrs)
for n in node.childNodes:
diff --git a/project.py b/project.py
index 142258e..5479985 100644
--- a/project.py
+++ b/project.py
@@ -630,7 +630,7 @@
class Project(object):
# These objects can be shared between several working trees.
shareable_files = ['description', 'info']
- shareable_dirs = ['hooks', 'objects', 'rr-cache', 'svn']
+ shareable_dirs = ['hooks', 'objects', 'rr-cache', 'svn', 'lfs']
# These objects can only be used by a single working tree.
working_tree_files = ['config', 'packed-refs', 'shallow']
working_tree_dirs = ['logs', 'refs']
@@ -655,7 +655,8 @@
is_derived=False,
dest_branch=None,
optimized_fetch=False,
- old_revision=None):
+ old_revision=None,
+ lfs_fetch=False):
"""Init a Project object.
Args:
@@ -680,6 +681,7 @@
optimized_fetch: If True, when a project is set to a sha1 revision, only
fetch from the remote if the sha1 is not present locally.
old_revision: saved git commit id for open GITC projects.
+ lfs_fetch: git lfs fetch
"""
self.manifest = manifest
self.name = name
@@ -731,6 +733,7 @@
# This will be filled in if a project is later identified to be the
# project containing repo hooks.
self.enabled_repo_hooks = []
+ self.lfs_fetch = lfs_fetch
@property
def Derived(self):
@@ -1166,6 +1169,31 @@
message=msg)
+ def __FetchLfsObjects(self, name, refs):
+ if 'refs/heads/*' in refs:
+ refs = []
+ for ref in GitRefs(self.objdir).all:
+ if ref.startswith(R_HEADS):
+ refs.append(ref)
+ cmd = ['lfs', 'fetch']
+ cmd.append(name)
+ cmd.extend(refs)
+ gitcmd = GitCommand(self, cmd, bare=True)
+ ret = gitcmd.Wait()
+
+ def _CreateLocalMergeBranch(self, branch, revid):
+ if not branch and not ID_RE.match(revid):
+ return
+ ref = os.path.join(self.gitdir, branch)
+ if os.path.exists(ref): return
+ try:
+ self.work_git.rev_parse(branch)
+ except GitError:
+ try:
+ os.makedirs(os.path.dirname(ref))
+ except OSError:
+ pass
+ _lwrite(ref, '%s\n' % revid)
# Sync ##
def _ExtractArchive(self, tarpath, path=None):
@@ -1280,13 +1308,42 @@
def PostRepoUpgrade(self):
self._InitHooks()
+ def RemoveOldCopyAndLinkFiles(self, path = None):
+ old_copylink = []
+ file_path = os.path.join(self.gitdir if not path else path, '.repo_copylink')
+ if os.path.exists(file_path):
+ fd = open(file_path, 'r')
+ try:
+ old_copylink = fd.read().split('\n')
+ finally:
+ fd.close()
+ for dest in old_copylink:
+ if not dest: continue
+ target = os.path.join(self.manifest.topdir, dest)
+ if os.path.lexists(target):
+ if IsTrace():
+ Trace('rm %s', target)
+ os.remove(target)
+
def _CopyAndLinkFiles(self):
if self.manifest.isGitcClient:
return
+ self.RemoveOldCopyAndLinkFiles()
+ new_copylink = []
for copyfile in self.copyfiles:
copyfile._Copy()
+ new_copylink.append(copyfile.dest)
for linkfile in self.linkfiles:
linkfile._Link()
+ new_copylink.append(linkfile.dest)
+ file_path = os.path.join(self.gitdir, '.repo_copylink')
+ fd = open(file_path, 'w')
+ try:
+ fd.write('\n'.join(new_copylink))
+ fd.write('\n')
+ finally:
+ fd.close()
+
def GetCommitRevisionId(self):
"""Get revisionId of a commit.
@@ -1536,6 +1593,9 @@
branch.merge = branch_merge
if not branch.merge.startswith('refs/') and not ID_RE.match(branch_merge):
branch.merge = R_HEADS + branch_merge
+ '''make sure local merge branch exists'''
+ self._CreateLocalMergeBranch(branch.LocalMerge, self.revisionExpr)
+
revid = self.GetRevisionId(all_refs)
if head.startswith(R_HEADS):
@@ -1996,6 +2056,7 @@
else:
cmd.append('--tags')
+ refs = []
if prune:
cmd.append('--prune')
@@ -2003,16 +2064,18 @@
if not current_branch_only:
# Fetch whole repo
spec.append(str((u'+refs/heads/*:') + remote.ToLocal('refs/heads/*')))
+ if self.manifest.IsMirror: refs.append('refs/heads/*')
elif tag_name is not None:
spec.append('tag')
spec.append(tag_name)
+ refs.append(tag_name)
if not self.manifest.IsMirror:
branch = self.revisionExpr
if is_sha1 and depth and git_require((1, 8, 3)):
# Shallow checkout of a specific commit, fetch from that commit and not
# the heads only as the commit might be deeper in the history.
- spec.append(branch)
+ spec.append(branch)
else:
if is_sha1:
branch = self.upstream
@@ -2020,6 +2083,7 @@
if not branch.startswith('refs/'):
branch = R_HEADS + branch
spec.append(str((u'+%s:' % branch) + remote.ToLocal(branch)))
+ refs.append(remote.ToLocal(branch))
cmd.extend(spec)
ok = False
@@ -2072,7 +2136,8 @@
return self._RemoteFetch(name=name,
current_branch_only=current_branch_only,
initial=False, quiet=quiet, alt_dir=alt_dir)
-
+ if self.lfs_fetch:
+ self.__FetchLfsObjects(name, refs)
return ok
def _ApplyCloneBundle(self, initial=False, quiet=False):
@@ -2294,12 +2359,17 @@
m = self.manifest.manifestProject.config
for key in ['user.name', 'user.email']:
if m.Has(key, include_defaults=False):
- self.config.SetString(key, m.GetString(key))
- self.config.SetString('filter.lfs.smudge', 'git-lfs smudge --skip -- %f')
+ self.config.SetString(key, m.GetString(key))
if self.manifest.IsMirror:
self.config.SetString('core.bare', 'true')
else:
self.config.SetString('core.bare', None)
+
+ if self.lfs_fetch:
+ self.config.SetString('filter.lfs.clean', 'git-lfs clean %f')
+ self.config.SetString('filter.lfs.smudge', 'git-lfs smudge %f')
+ self.config.SetString('filter.lfs.required', 'true')
+
except Exception:
if init_obj_dir and os.path.exists(self.objdir):
shutil.rmtree(self.objdir)
diff --git a/subcmds/sync.py b/subcmds/sync.py
index 7ba9ebf..138eaf2 100644
--- a/subcmds/sync.py
+++ b/subcmds/sync.py
@@ -26,6 +26,8 @@
import tempfile
import time
+from urllib import quote as url_quote
+
from pyversion import is_python3
if is_python3():
import http.cookiejar as cookielib
@@ -255,7 +257,7 @@
dest='repo_upgraded', action='store_true',
help=SUPPRESS_HELP)
- def _FetchProjectList(self, opt, projects, *args, **kwargs):
+ def _FetchProjectList(self, opt, projects, lock, fetched, pm, sem, err_event):
"""Main function of the fetch threads when jobs are > 1.
Delegates most of the work to _FetchHelper.
@@ -266,10 +268,13 @@
*args, **kwargs: Remaining arguments to pass to _FetchHelper. See the
_FetchHelper docstring for details.
"""
- for project in projects:
- success = self._FetchHelper(opt, project, *args, **kwargs)
- if not success and not opt.force_broken:
- break
+ try:
+ for project in projects:
+ success = self._FetchHelper(opt, project, lock, fetched, pm, sem, err_event)
+ if not success and not opt.force_broken:
+ break
+ finally:
+ sem.release()
def _FetchHelper(self, opt, project, lock, fetched, pm, sem, err_event):
"""Fetch git objects for a single project.
@@ -340,7 +345,7 @@
finally:
if did_lock:
lock.release()
- sem.release()
+ #sem.release()
return success
@@ -556,8 +561,11 @@
print(' commit changes, then run sync again',
file=sys.stderr)
return -1
- elif self._DeleteProject(project.worktree):
- return -1
+ else:
+ if self._DeleteProject(project.worktree) == 0:
+ project.RemoveOldCopyAndLinkFiles(os.path.join(self.manifest.repodir, 'projects', '%s.git' % path))
+ else:
+ return -1
new_project_paths.sort()
fd = open(file_path, 'w')
@@ -641,7 +649,7 @@
if (username and password):
manifest_server = manifest_server.replace('://', '://%s:%s@' %
- (username, password),
+ (username, url_quote(password, safe='')),
1)
transport = PersistentTransport(manifest_server)