git.scottworley.com Git - git-cache/blob - git_cache.py

   1 # It would be nice if we could share the nix git cache, but as of the
   2 # time of writing it is transitioning from gitv2 (deprecated) to gitv3
   3 # (not ready yet), and trying to straddle them both is too far into nix
   4 # implementation details for my comfort.  So we re-implement here half of
   5 # nix's builtins.fetchGit.  :(
   6
   7 import argparse
   8 import functools
   9 import hashlib
  10 import logging
  11 import os
  12 import subprocess
  13 import sys
  14 import time
  15
  16 from typing import Iterator, NamedTuple, Optional, TypeVar, Tuple, Union
  17
  18 import backoff
  19
  20 Path = str   # eg: "/home/user/.cache/git-cache/v1"
  21 Repo = str   # eg: "https://github.com/NixOS/nixpkgs.git"
  22 Ref = str    # eg: "master" or "v1.0.0"
  23 Rev = str    # eg: "53a27350551844e1ed1a9257690294767389ef0d"
  24 RefOrRev = Union[Ref, Rev]
  25
  26
  27 class _LogEntry(NamedTuple):
  28     ref: Ref
  29     rev: Rev
  30
  31
  32 T = TypeVar('T')
  33
  34
  35 def _repo_hashname(repo: Repo) -> str:
  36     return hashlib.sha256(repo.encode()).hexdigest()
  37
  38
  39 def git_cachedir(repo: Repo) -> Path:
  40     # Use xdg module when it's less painful to have as a dependency
  41     XDG_CACHE_HOME = Path(
  42         os.environ.get('XDG_CACHE_HOME', os.path.expanduser('~/.cache')))
  43
  44     return Path(os.path.join(
  45         XDG_CACHE_HOME,
  46         'git-cache/v1',
  47         _repo_hashname(repo)))
  48
  49
  50 def _log_filename(repo: Repo) -> Path:
  51     # Use xdg module when it's less painful to have as a dependency
  52     XDG_DATA_HOME = Path(
  53         os.environ.get('XDG_DATA_HOME', os.path.expanduser('~/.local/share')))
  54
  55     return Path(os.path.join(
  56         XDG_DATA_HOME,
  57         'git-cache/v1',
  58         _repo_hashname(repo)))
  59
  60
  61 def is_ancestor(repo: Repo, descendant: RefOrRev, ancestor: RefOrRev) -> bool:
  62     cachedir = git_cachedir(repo)
  63     logging.debug('Checking if %s is an ancestor of %s', ancestor, descendant)
  64     process = subprocess.run(['git',
  65                               '-C',
  66                               cachedir,
  67                               'merge-base',
  68                               '--is-ancestor',
  69                               ancestor,
  70                               descendant],
  71                              check=False)
  72     return process.returncode == 0
  73
  74
  75 def verify_ancestry(
  76         repo: Repo,
  77         descendant: RefOrRev,
  78         ancestor: RefOrRev,
  79         force: bool = False) -> None:
  80     if not force and not is_ancestor(repo, descendant, ancestor):
  81         raise Exception('%s is not an ancestor of %s' % (ancestor, descendant))
  82
  83
  84 def _read_fetch_log(repo: Repo) -> Iterator[_LogEntry]:
  85     filename = _log_filename(repo)
  86     if not os.path.exists(filename):
  87         return
  88     with open(filename, 'r') as f:
  89         for line in f:
  90             _, _, rev, ref = line.strip().split(maxsplit=3)
  91             yield _LogEntry(ref, rev)
  92
  93
  94 def _last(it: Iterator[T]) -> Optional[T]:
  95     return functools.reduce(lambda a, b: b, it, None)
  96
  97
  98 def _previous_fetched_rev(repo: Repo, ref: Ref) -> Optional[Rev]:
  99     return _last(entry.rev for entry in _read_fetch_log(
 100         repo) if entry.ref == ref)
 101
 102
 103 def _log_fetch(repo: Repo, ref: Ref, rev: Rev, force: bool = False) -> None:
 104     if not force:
 105         prev_rev = _previous_fetched_rev(repo, ref)
 106         if prev_rev is not None:
 107             verify_ancestry(repo, rev, prev_rev)
 108     filename = _log_filename(repo)
 109     os.makedirs(os.path.dirname(filename), exist_ok=True)
 110     with open(filename, 'a') as f:
 111         f.write('%s %s %s %s\n' %
 112                 (time.strftime('%Y-%m%d-%H:%M:%S%z'),
 113                  ('FORCEDFETCH' if force else 'fetch'), rev, ref))
 114
 115
 116 def _show_force_warning() -> None:
 117     print('''
 118 **************************************************************************
 119 *               WARNING: git-cache INVOKED WITH --force!                 *
 120 *                                                                        *
 121 * This mode allows previously-fetched refs to be overwritten to point to *
 122 * non-descendants -- commits that don't have the previous version of the *
 123 * the ref in their history!                                              *
 124 *                                                                        *
 125 * This should only be invoked by a human operator who knows what they're *
 126 * doing to correct a specific, known, problem.  Care should be taken to  *
 127 * prevent recurrence.                                                    *
 128 *                                                                        *
 129 * Press ^C to abort.                                                     *
 130 *                                                                        *
 131 ''', end='', file=sys.stderr)
 132     warn_time_override = os.environ.get('FORCE_WARNING_TIME', None)
 133     warn_time: int
 134     if warn_time_override is None:
 135         warn_time = 15
 136     else:
 137         warn_time = int(warn_time_override)
 138         print(
 139             '''*   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!   *
 140 *   !!              WARNING DISPLAY TIME OVERRIDDEN                 !!   *
 141 *   !!                                                              !!   *
 142 *   !! This message is intended to be displayed long enough for a   !!   *
 143 *   !! human operator to read it and have a chance to abort.  An    !!   *
 144 *   !! override for the delay time is provided FOR THE UNIT TESTS   !!   *
 145 *   !! to avoid delaying software builds unnecessarily.  This is    !!   *
 146 *   !! INTENDED FOR USE IN UNIT TESTS ONLY; THIS MESSAGE SHOULD     !!   *
 147 *   !! NEVER BE SEEN OUTSIDE BUILD LOGS!                            !!   *
 148 *   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!   *
 149 *                                                                        *
 150 ''', end='', file=sys.stderr)
 151
 152     for i in range(warn_time, 0, -1):
 153         msg = '* %-70s *' % ("Continuing in %d seconds..." % i)
 154         print(msg, file=sys.stderr)
 155         time.sleep(1)
 156     print('*' * 74, file=sys.stderr)
 157
 158
 159 @backoff.on_exception(
 160     backoff.expo,
 161     subprocess.CalledProcessError,
 162     max_time=lambda: int(os.environ.get('BACKOFF_MAX_TIME', '30')))
 163 def _git_fetch(
 164         cachedir: Path,
 165         repo: Repo,
 166         ref: Ref,
 167         force: bool = False) -> None:
 168     subprocess.run(['git', '-C', cachedir, 'fetch'] +
 169                    (['--force'] if force else []) +
 170                    [repo, '%s:%s' % (ref, ref)], check=True)
 171
 172
 173 def fetch(repo: Repo, ref: Ref, force: bool = False) -> Tuple[Path, Rev]:
 174     if force:
 175         _show_force_warning()
 176     cachedir = git_cachedir(repo)
 177     if not os.path.exists(cachedir):
 178         logging.debug("Initializing git repo")
 179         subprocess.run(['git', 'init', '--bare', cachedir],
 180                        check=True, stdout=sys.stderr)
 181
 182     logging.debug('Fetching ref "%s" from %s', ref, repo)
 183     _git_fetch(cachedir, repo, ref, force=force)
 184
 185     with open(os.path.join(cachedir, 'refs', 'heads', ref)) as rev_file:
 186         rev = Rev(rev_file.read(999).strip())
 187     verify_ancestry(repo, ref, rev, force=force)
 188     _log_fetch(repo, ref, rev, force=force)
 189
 190     return cachedir, rev
 191
 192
 193 def ensure_rev_available(
 194         repo: Repo,
 195         ref: Ref,
 196         rev: Rev,
 197         force: bool = False) -> Path:
 198     cachedir = git_cachedir(repo)
 199     if os.path.exists(cachedir) and is_ancestor(repo, ref, rev):
 200         return cachedir
 201
 202     logging.debug(
 203         'We do not have rev %s.  We will fetch ref "%s" and hope it appears.',
 204         rev, ref)
 205     fetch(repo, ref, force=force)
 206     logging.debug('Verifying that fetch retrieved rev %s', rev)
 207     subprocess.run(['git', '-C', cachedir, 'cat-file', '-e', rev], check=True)
 208     verify_ancestry(repo, ref, rev, force=force)
 209
 210     return cachedir
 211
 212
 213 def _main() -> None:
 214     parser = argparse.ArgumentParser(
 215         description='Cache remote git repositories locally.',
 216         epilog='example usage: git-cache https://github.com/NixOS/nixpkgs.git master')
 217     parser.add_argument(
 218         '--force',
 219         action='store_true',
 220         help='Recover from a force-push in the remote repo')
 221     parser.add_argument(
 222         'repo',
 223         metavar='Repo',
 224         type=Repo,
 225         help='Git repository URL')
 226     parser.add_argument(
 227         'ref',
 228         metavar='Ref',
 229         type=Ref,
 230         help='Ref (branch or tag) in the git repo')
 231     parser.add_argument(
 232         'rev',
 233         metavar='Rev',
 234         type=Rev,
 235         nargs='?',
 236         help='Ensure that this revision is present.  ' +
 237         'If this revision is already present locally, no network operations are performed.')
 238     args = parser.parse_args()
 239
 240     if args.rev is None:
 241         print('{1} {0}'.format(*fetch(args.repo, args.ref, force=args.force)))
 242     else:
 243         print(
 244             ensure_rev_available(
 245                 args.repo,
 246                 args.ref,
 247                 args.rev,
 248                 force=args.force))