X-Git-Url: http://git.scottworley.com/git-cache/blobdiff_plain/bef7ce53ce70499b4f13985cc9d4b64e2d8aace4..35000f728e66533de9b0cf6e8b040f9cdb82c1a7:/git_cache.py diff --git a/git_cache.py b/git_cache.py index d1923cd..1363f11 100644 --- a/git_cache.py +++ b/git_cache.py @@ -4,17 +4,35 @@ # implementation details for my comfort. So we re-implement here half of # nix's builtins.fetchGit. :( +import functools import hashlib import logging import os import subprocess +import sys +import time -from typing import Tuple +from typing import Iterator, NamedTuple, Optional, TypeVar, Tuple, Union -Path = str # eg: "/home/user/.cache/git-cache/v1" -Repo = str # eg: "https://github.com/NixOS/nixpkgs.git" -Ref = str # eg: "master" or "v1.0.0" -Rev = str # eg: "53a27350551844e1ed1a9257690294767389ef0d" +import backoff + +Path = str # eg: "/home/user/.cache/git-cache/v1" +Repo = str # eg: "https://github.com/NixOS/nixpkgs.git" +Ref = str # eg: "master" or "v1.0.0" +Rev = str # eg: "53a27350551844e1ed1a9257690294767389ef0d" +RefOrRev = Union[Ref, Rev] + + +class _LogEntry(NamedTuple): + ref: Ref + rev: Rev + + +T = TypeVar('T') + + +def _repo_hashname(repo: Repo) -> str: + return hashlib.sha256(repo.encode()).hexdigest() def git_cachedir(repo: Repo) -> Path: @@ -25,49 +43,105 @@ def git_cachedir(repo: Repo) -> Path: return Path(os.path.join( XDG_CACHE_HOME, 'git-cache/v1', - hashlib.sha256(repo.encode()).hexdigest())) + _repo_hashname(repo))) + + +def _log_filename(repo: Repo) -> Path: + # Use xdg module when it's less painful to have as a dependency + XDG_DATA_HOME = Path( + os.environ.get('XDG_DATA_HOME', os.path.expanduser('~/.local/share'))) + + return Path(os.path.join( + XDG_DATA_HOME, + 'git-cache/v1', + _repo_hashname(repo))) -def verify_ancestry(repo: Repo, ref: Ref, rev: Rev) -> None: +def is_ancestor(repo: Repo, descendant: RefOrRev, ancestor: RefOrRev) -> bool: cachedir = git_cachedir(repo) - logging.debug('Verifying rev %s is an ancestor of ref "%s"', rev, ref) - subprocess.run(['git', '-C', cachedir, 'merge-base', '--is-ancestor', - rev, ref], check=True) + logging.debug('Checking if %s is an ancestor of %s', ancestor, descendant) + process = subprocess.run(['git', + '-C', + cachedir, + 'merge-base', + '--is-ancestor', + ancestor, + descendant], + check=False) + return process.returncode == 0 + + +def verify_ancestry( + repo: Repo, + descendant: RefOrRev, + ancestor: RefOrRev) -> None: + if not is_ancestor(repo, descendant, ancestor): + raise Exception('%s is not an ancestor of %s' % (ancestor, descendant)) + + +def _read_fetch_log(repo: Repo) -> Iterator[_LogEntry]: + filename = _log_filename(repo) + if not os.path.exists(filename): + return + with open(filename, 'r') as f: + for line in f: + _, _, rev, ref = line.strip().split(maxsplit=3) + yield _LogEntry(ref, rev) + + +def _last(it: Iterator[T]) -> Optional[T]: + return functools.reduce(lambda a, b: b, it, None) + + +def _previous_fetched_rev(repo: Repo, ref: Ref) -> Optional[Rev]: + return _last(entry.rev for entry in _read_fetch_log( + repo) if entry.ref == ref) + + +def _log_fetch(repo: Repo, ref: Ref, rev: Rev) -> None: + prev_rev = _previous_fetched_rev(repo, ref) + if prev_rev is not None: + verify_ancestry(repo, rev, prev_rev) + filename = _log_filename(repo) + os.makedirs(os.path.dirname(filename), exist_ok=True) + with open(filename, 'a') as f: + f.write('%s fetch %s %s\n' % + (time.strftime('%Y-%m%d-%H:%M:%S%z'), rev, ref)) + + +@backoff.on_exception( + backoff.expo, + subprocess.CalledProcessError, + max_time=lambda: int(os.environ.get('BACKOFF_MAX_TIME', '30'))) +def _git_fetch(cachedir: Path, repo: Repo, ref: Ref) -> None: + # We don't use --force here because we want to abort and freak out if forced + # updates are happening. + subprocess.run(['git', '-C', cachedir, 'fetch', repo, + '%s:%s' % (ref, ref)], check=True) def fetch(repo: Repo, ref: Ref) -> Tuple[Path, Rev]: cachedir = git_cachedir(repo) if not os.path.exists(cachedir): logging.debug("Initializing git repo") - subprocess.run(['git', 'init', '--bare', cachedir], check=True) + subprocess.run(['git', 'init', '--bare', cachedir], + check=True, stdout=sys.stderr) logging.debug('Fetching ref "%s" from %s', ref, repo) - # We don't use --force here because we want to abort and freak out if forced - # updates are happening. - subprocess.run(['git', '-C', cachedir, 'fetch', repo, - '%s:%s' % (ref, ref)], check=True) + _git_fetch(cachedir, repo, ref) with open(os.path.join(cachedir, 'refs', 'heads', ref)) as rev_file: rev = Rev(rev_file.read(999).strip()) verify_ancestry(repo, ref, rev) + _log_fetch(repo, ref, rev) return cachedir, rev def ensure_rev_available(repo: Repo, ref: Ref, rev: Rev) -> Path: cachedir = git_cachedir(repo) - if os.path.exists(cachedir): - logging.debug('Checking if we already have rev %s', rev) - process = subprocess.run( - ['git', '-C', cachedir, 'cat-file', '-e', rev], check=False) - if process.returncode == 0: - logging.debug('We already have rev %s', rev) - verify_ancestry(repo, ref, rev) - return cachedir - if process.returncode != 1: - raise Exception( - 'Could not test for presence of rev %s. Is cache dir "%s" messed up?' % - (rev, cachedir)) + if os.path.exists(cachedir) and is_ancestor(repo, ref, rev): + return cachedir logging.debug( 'We do not have rev %s. We will fetch ref "%s" and hope it appears.', @@ -75,5 +149,19 @@ def ensure_rev_available(repo: Repo, ref: Ref, rev: Rev) -> Path: fetch(repo, ref) logging.debug('Verifying that fetch retrieved rev %s', rev) subprocess.run(['git', '-C', cachedir, 'cat-file', '-e', rev], check=True) + verify_ancestry(repo, ref, rev) return cachedir + + +def _main() -> None: + if len(sys.argv) == 3: + print('{1} {0}'.format(*fetch(Repo(sys.argv[1]), Ref(sys.argv[2])))) + elif len(sys.argv) == 4: + print(ensure_rev_available( + Repo(sys.argv[1]), Ref(sys.argv[2]), Rev(sys.argv[3]))) + else: + usage = '''usage: git-cache repo ref [rev] +example: git-cache https://github.com/NixOS/nixpkgs.git master''' + print(usage, file=sys.stderr) + sys.exit(1)