diff --git a/setup.py b/setup.py index 32fc9d2..5b7763a 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ "long_description (%s)\n" % readme_file) sys.exit(1) -install_requires = ['Pygments', 'mock'] +install_requires = ['Pygments', 'mock', 'lockfile>=0.9.1'] if sys.version_info < (2, 7): install_requires.append('unittest2') tests_require = install_requires + ['dulwich', 'mercurial'] diff --git a/vcs/backends/base.py b/vcs/backends/base.py index 208f4df..ffa0e17 100644 --- a/vcs/backends/base.py +++ b/vcs/backends/base.py @@ -1,5 +1,9 @@ # -*- coding: utf-8 -*- +from __future__ import with_statement +import os +import gzip import datetime +import lockfile import itertools from vcs.utils import author_name, author_email @@ -45,10 +49,12 @@ class BaseRepository(object): tags as list of changesets """ scm = None + use_revisions_cache = False DEFAULT_BRANCH_NAME = None EMPTY_CHANGESET = '0' * 40 - def __init__(self, repo_path, create=False, **kwargs): + def __init__(self, repo_path, create=False, src_url=None, + use_revisions_cache=False): """ Initializes repository. Raises RepositoryError if repository could not be find at the given ``repo_path`` or directory at ``repo_path`` @@ -60,6 +66,8 @@ def __init__(self, repo_path, create=False, **kwargs): would be cloned; requires ``create`` parameter to be set to True - raises RepositoryError if src_url is set and create evaluates to False + :param use_revisions_cache: if set to True, would try to use cached + revisions list (and saves it if cache does not exist). """ raise NotImplementedError @@ -79,6 +87,60 @@ def __eq__(self, other): def __ne__(self, other): return not self.__eq__(other) + def _get_all_revisions(self): + raise NotImplementedError + + def invalidate_revisions(self): + """ + Marks ``revisions`` attribute to be re-fetched next time it's accessed. + """ + self._revisions = None + + def _revisions_get(self): + """ + Returns list of revisions' ids, in ascending order. Being lazy + attribute allows external tools to inject shas from cache. + """ + if getattr(self, '_revisions', None) is None: + if self.use_revisions_cache: + cache_path = self.get_revisions_cache_path() + if not os.path.isfile(cache_path): + self.cache_revisions() + self._revisions = self.get_cached_revisions() + else: + self._revisions = self._get_all_revisions() + return self._revisions + + def _revisions_set(self, revs): + self._revisions = revs + + revisions = property(_revisions_get, _revisions_set) + + def get_revisions_cache_path(self): + cache_filename = '.vcs.%s.revisions.cache' % self.scm + return os.path.join(self.path, cache_filename) + + def cache_revisions(self): + with self.get_revisions_lock(): + revisions = self._get_all_revisions() + try: + fout = gzip.open(self.get_revisions_cache_path(), 'w') + for revision in revisions: + fout.write('%s\n' % revision) + finally: + fout.close() + + def get_cached_revisions(self): + return gzip.open(self.get_revisions_cache_path()).read().splitlines() + + def get_revisions_lock(self): + """ + Returns ``lockfile.LockFile`` lock. + """ + lock_filename = '.vcs.%s.revisions.lock' % self.scm + lockpath = os.path.join(self.path, lock_filename) + return lockfile.LockFile(lockpath) + @LazyProperty def alias(self): for k, v in settings.BACKENDS.items(): diff --git a/vcs/backends/git/inmemory.py b/vcs/backends/git/inmemory.py index e940a2f..a001458 100644 --- a/vcs/backends/git/inmemory.py +++ b/vcs/backends/git/inmemory.py @@ -150,10 +150,10 @@ def commit(self, message, author, parents=None, branch=None, date=None, ref = 'refs/heads/%s' % branch repo.refs[ref] = commit.id - # Update vcs repository object & recreate dulwich repo - self.repository.revisions.append(commit.id) # invalidate parsed refs after commit self.repository._parsed_refs = self.repository._get_parsed_refs() + # Update vcs repository object & recreate dulwich repo + self.repository.invalidate_revisions() tip = self.repository.get_changeset() self.reset() return tip diff --git a/vcs/backends/git/repository.py b/vcs/backends/git/repository.py index 95183a8..300c0e0 100644 --- a/vcs/backends/git/repository.py +++ b/vcs/backends/git/repository.py @@ -43,8 +43,8 @@ class GitRepository(BaseRepository): scm = 'git' def __init__(self, repo_path, create=False, src_url=None, - update_after_clone=False, bare=False): - + update_after_clone=False, bare=False, use_revisions_cache=False): + self.use_revisions_cache = use_revisions_cache self.path = abspath(repo_path) repo = self._get_repo(create, src_url, update_after_clone, bare) self.bare = repo.bare @@ -68,14 +68,6 @@ def head(self): except KeyError: return None - @LazyProperty - def revisions(self): - """ - Returns list of revisions' ids, in ascending order. Being lazy - attribute allows external tools to inject shas from cache. - """ - return self._get_all_revisions() - @classmethod def _run_git_command(cls, cmd, **opts): """ diff --git a/vcs/backends/hg/inmemory.py b/vcs/backends/hg/inmemory.py index 540b18b..f6e3831 100644 --- a/vcs/backends/hg/inmemory.py +++ b/vcs/backends/hg/inmemory.py @@ -4,7 +4,7 @@ from vcs.backends.base import BaseInMemoryChangeset from vcs.exceptions import RepositoryError -from vcs.utils.hgcompat import memfilectx, memctx, hex, tolocal +from vcs.utils.hgcompat import memfilectx, memctx, tolocal class MercurialInMemoryChangeset(BaseInMemoryChangeset): @@ -98,16 +98,17 @@ def filectxfn(_repo, memctx, path): commit_ctx._date = date # TODO: Catch exceptions! - n = self.repository._repo.commitctx(commit_ctx) + self.repository._repo.commitctx(commit_ctx) # Returns mercurial node self._commit_ctx = commit_ctx # For reference # Update vcs repository object & recreate mercurial _repo # new_ctx = self.repository._repo[node] # new_tip = self.repository.get_changeset(new_ctx.hex()) - new_id = hex(n) - self.repository.revisions.append(new_id) self._repo = self.repository._get_repo(create=False) + self.repository.invalidate_revisions() self.repository.branches = self.repository._get_branches() tip = self.repository.get_changeset() self.reset() return tip + +# invalidate diff --git a/vcs/backends/hg/repository.py b/vcs/backends/hg/repository.py index 97fd643..ba5c3da 100644 --- a/vcs/backends/hg/repository.py +++ b/vcs/backends/hg/repository.py @@ -47,7 +47,7 @@ class MercurialRepository(BaseRepository): scm = 'hg' def __init__(self, repo_path, create=False, baseui=None, src_url=None, - update_after_clone=False): + update_after_clone=False, use_revisions_cache=False): """ Raises RepositoryError if repository could not be find at the given ``repo_path``. @@ -59,6 +59,8 @@ def __init__(self, repo_path, create=False, baseui=None, src_url=None, :param src_url=None: would try to clone repository from given location :param update_after_clone=False: sets update of working copy after making a clone + :param use_revisions_cache: if set to True, would try to use cached + revisions list (and saves it if cache does not exist). """ if not isinstance(repo_path, str): @@ -66,6 +68,7 @@ def __init__(self, repo_path, create=False, baseui=None, src_url=None, 'be instance of got %s instead' % type(repo_path)) + self.use_revisions_cache = use_revisions_cache self.path = abspath(repo_path) self.baseui = baseui or ui.ui() # We've set path and ui, now we can set _repo itself @@ -80,14 +83,6 @@ def _empty(self): # return len(self._repo.changelog) == 0 return len(self.revisions) == 0 - @LazyProperty - def revisions(self): - """ - Returns list of revisions' ids, in ascending order. Being lazy - attribute allows external tools to inject shas from cache. - """ - return self._get_all_revisions() - @LazyProperty def name(self): return os.path.basename(self.path) @@ -549,7 +544,7 @@ def get_user_name(self, config_file=None): :param config_file: A path to file which should be used to retrieve configuration from (might also be a list of file paths) """ - username = self.get_config_value('ui', 'username') + username = self.get_config_value('ui', 'username', config_file) if username: return author_name(username) return None @@ -561,7 +556,7 @@ def get_user_email(self, config_file=None): :param config_file: A path to file which should be used to retrieve configuration from (might also be a list of file paths) """ - username = self.get_config_value('ui', 'username') + username = self.get_config_value('ui', 'username', config_file) if username: return author_email(username) return None diff --git a/vcs/tests/conf.py b/vcs/tests/conf.py index 927149f..a5f1799 100644 --- a/vcs/tests/conf.py +++ b/vcs/tests/conf.py @@ -58,5 +58,6 @@ def get_new_dir(title): PACKAGE_DIR = os.path.abspath(os.path.join( os.path.dirname(__file__), '..')) _dest = jn(TEST_TMP_PATH, 'aconfig') -shutil.copy(jn(THIS, 'aconfig'), _dest) +TEST_USER_CONFIG_FILE_SRC = jn(THIS, 'aconfig') +shutil.copy(TEST_USER_CONFIG_FILE_SRC, _dest) TEST_USER_CONFIG_FILE = _dest diff --git a/vcs/tests/test_repository.py b/vcs/tests/test_repository.py index da81c47..6a9bcda 100644 --- a/vcs/tests/test_repository.py +++ b/vcs/tests/test_repository.py @@ -1,15 +1,18 @@ from __future__ import with_statement +import os +import gzip import datetime +from mock import Mock from vcs.tests.base import BackendTestMixin from vcs.tests.conf import SCM_TESTS -from vcs.tests.conf import TEST_USER_CONFIG_FILE +from vcs.tests.conf import TEST_USER_CONFIG_FILE_SRC from vcs.nodes import FileNode from vcs.utils.compat import unittest from vcs.exceptions import ChangesetDoesNotExistError class RepositoryBaseTest(BackendTestMixin): - recreate_repo_per_test = False + recreate_repo_per_test = True @classmethod def _get_commits(cls): @@ -17,18 +20,18 @@ def _get_commits(cls): def test_get_config_value(self): self.assertEqual(self.repo.get_config_value('universal', 'foo', - TEST_USER_CONFIG_FILE), 'bar') + TEST_USER_CONFIG_FILE_SRC), 'bar') def test_get_config_value_defaults_to_None(self): self.assertEqual(self.repo.get_config_value('universal', 'nonexist', - TEST_USER_CONFIG_FILE), None) + TEST_USER_CONFIG_FILE_SRC), None) def test_get_user_name(self): - self.assertEqual(self.repo.get_user_name(TEST_USER_CONFIG_FILE), + self.assertEqual(self.repo.get_user_name(TEST_USER_CONFIG_FILE_SRC), 'Foo Bar') def test_get_user_email(self): - self.assertEqual(self.repo.get_user_email(TEST_USER_CONFIG_FILE), + self.assertEqual(self.repo.get_user_email(TEST_USER_CONFIG_FILE_SRC), 'foo.bar@example.com') def test_repo_equality(self): @@ -45,6 +48,58 @@ class dummy(object): path = self.repo.path self.assertTrue(self.repo != dummy()) + def test_repo_invalidate_revisions(self): + revisions = self.repo.revisions[:] # copy + # at least in one test make sure revisions list is not empty + self.assertTrue(len(revisions) > 0) + self.repo.revisions = 'this should be recreated anyway' + self.repo.invalidate_revisions() + self.assertEqual(self.repo.revisions, revisions) + + def test_repo_invalidate_revisions_itself_does_not_access_revisions(self): + self.repo._get_all_revisions = Mock() + self.repo.invalidate_revisions() + self.assertFalse(self.repo._get_all_revisions.called) + self.repo.revisions # access attribute + self.assertTrue(self.repo._get_all_revisions.called) + + def test_repo_respects_use_revisions_cache(self): + revisions = self.repo.revisions[:] # copy + self.repo.use_revisions_cache = True + self.repo.invalidate_revisions() + self.repo.revisions # access attribute + cached = gzip.open(self.repo.get_revisions_cache_path()).read().splitlines() + self.assertEqual(revisions, cached) + + def test_get_cached_revisions(self): + self.repo.cache_revisions() + cache_path = self.repo.get_revisions_cache_path() + try: + fout = gzip.open(cache_path, 'w') + fout.write('foo\nbar') + finally: + fout.close() + + self.assertEqual(self.repo.get_cached_revisions(), ['foo', 'bar']) + + def test_cache_revisions(self): + revisions = self.repo.revisions[:] # copy + self.repo.cache_revisions() + cache_path = self.repo.get_revisions_cache_path() + cached_revisions = gzip.open(cache_path).read().splitlines() + self.assertEqual(revisions, cached_revisions) + + def test_repo_invalidate_recreates_cache(self): + self.repo.use_revisions_cache = True + self.repo.invalidate_revisions() + self.repo.revisions # access attribute + revisions = self.repo.revisions[:] # copy + os.remove(self.repo.get_revisions_cache_path()) + self.repo.invalidate_revisions() + self.repo.revisions # access attribute + cached = gzip.open(self.repo.get_revisions_cache_path()).read().splitlines() + self.assertEqual(revisions, cached) + class RepositoryGetDiffTest(BackendTestMixin):