Use JSON instead of pickle
Use JSON as it is shown to be much faster than pickle.
Also clean up the loading and saving functions.
Change-Id: I45b3dee7b4d59a1c0e0d38d4a83b543ac5839390
diff --git a/git_config.py b/git_config.py
index 32879ec..380bdd2 100644
--- a/git_config.py
+++ b/git_config.py
@@ -15,8 +15,8 @@
from __future__ import print_function
+import json
import os
-import pickle
import re
import subprocess
import sys
@@ -80,7 +80,7 @@
return cls(configfile = os.path.join(gitdir, 'config'),
defaults = defaults)
- def __init__(self, configfile, defaults=None, pickleFile=None):
+ def __init__(self, configfile, defaults=None, jsonFile=None):
self.file = configfile
self.defaults = defaults
self._cache_dict = None
@@ -88,12 +88,11 @@
self._remotes = {}
self._branches = {}
- if pickleFile is None:
- self._pickle = os.path.join(
+ self._json = jsonFile
+ if self._json is None:
+ self._json = os.path.join(
os.path.dirname(self.file),
- '.repopickle_' + os.path.basename(self.file))
- else:
- self._pickle = pickleFile
+ '.repo_' + os.path.basename(self.file) + '.json')
def Has(self, name, include_defaults = True):
"""Return true if this configuration file has the key.
@@ -248,50 +247,41 @@
return self._cache_dict
def _Read(self):
- d = self._ReadPickle()
+ d = self._ReadJson()
if d is None:
d = self._ReadGit()
- self._SavePickle(d)
+ self._SaveJson(d)
return d
- def _ReadPickle(self):
+ def _ReadJson(self):
try:
- if os.path.getmtime(self._pickle) \
+ if os.path.getmtime(self._json) \
<= os.path.getmtime(self.file):
- os.remove(self._pickle)
+ os.remove(self._json)
return None
except OSError:
return None
try:
- Trace(': unpickle %s', self.file)
- fd = open(self._pickle, 'rb')
+ Trace(': parsing %s', self.file)
+ fd = open(self._json)
try:
- return pickle.load(fd)
+ return json.load(fd)
finally:
fd.close()
- except EOFError:
- os.remove(self._pickle)
- return None
- except IOError:
- os.remove(self._pickle)
- return None
- except pickle.PickleError:
- os.remove(self._pickle)
+ except (IOError, ValueError):
+ os.remove(self._json)
return None
- def _SavePickle(self, cache):
+ def _SaveJson(self, cache):
try:
- fd = open(self._pickle, 'wb')
+ fd = open(self._json, 'w')
try:
- pickle.dump(cache, fd, pickle.HIGHEST_PROTOCOL)
+ json.dump(cache, fd, indent=2)
finally:
fd.close()
- except IOError:
- if os.path.exists(self._pickle):
- os.remove(self._pickle)
- except pickle.PickleError:
- if os.path.exists(self._pickle):
- os.remove(self._pickle)
+ except (IOError, TypeError):
+ if os.path.exists(self.json):
+ os.remove(self._json)
def _ReadGit(self):
"""
diff --git a/subcmds/sync.py b/subcmds/sync.py
index a0a6896..6f77310 100644
--- a/subcmds/sync.py
+++ b/subcmds/sync.py
@@ -14,10 +14,10 @@
# limitations under the License.
from __future__ import print_function
+import json
import netrc
from optparse import SUPPRESS_HELP
import os
-import pickle
import re
import shutil
import socket
@@ -760,7 +760,7 @@
_ALPHA = 0.5
def __init__(self, manifest):
- self._path = os.path.join(manifest.repodir, '.repopickle_fetchtimes')
+ self._path = os.path.join(manifest.repodir, '.repo_fetchtimes.json')
self._times = None
self._seen = set()
@@ -779,22 +779,17 @@
def _Load(self):
if self._times is None:
try:
- f = open(self._path, 'rb')
- except IOError:
- self._times = {}
- return self._times
- try:
+ f = open(self._path)
try:
- self._times = pickle.load(f)
- except IOError:
- try:
- os.remove(self._path)
- except OSError:
- pass
- self._times = {}
- finally:
- f.close()
- return self._times
+ self._times = json.load(f)
+ finally:
+ f.close()
+ except (IOError, ValueError):
+ try:
+ os.remove(self._path)
+ except OSError:
+ pass
+ self._times = {}
def Save(self):
if self._times is None:
@@ -808,13 +803,13 @@
del self._times[name]
try:
- f = open(self._path, 'wb')
+ f = open(self._path, 'w')
try:
- pickle.dump(self._times, f)
- except (IOError, OSError, pickle.PickleError):
- try:
- os.remove(self._path)
- except OSError:
- pass
- finally:
- f.close()
+ json.dump(self._times, f, indent=2)
+ finally:
+ f.close()
+ except (IOError, TypeError):
+ try:
+ os.remove(self._path)
+ except OSError:
+ pass