Rely on nixpkgs' python3Packages.backoff

[git-cache] / git_cache.py

diff --git a/git_cache.py b/git_cache.py
index 077406c63275d97d701e1e9498e1efca989045c1..c7bcdd215e2cf52bda04483ecb267d0b72ade596 100644 (file)
--- a/git_cache.py
+++ b/git_cache.py
@@ -4,23 +4,38 @@
 # implementation details for my comfort.  So we re-implement here half of
 # nix's builtins.fetchGit.  :(
 
 # implementation details for my comfort.  So we re-implement here half of
 # nix's builtins.fetchGit.  :(
 

+import argparse
+import functools

 import hashlib
 import logging
 import os
 import subprocess
 import sys
 import hashlib
 import logging
 import os
 import subprocess
 import sys

+import time

 
 

-from typing import Tuple, Union
+from typing import Iterator, NamedTuple, Optional, TypeVar, Tuple, Union

 
 import backoff
 
 
 import backoff
 

-Path = str  # eg: "/home/user/.cache/git-cache/v1"
-Repo = str  # eg: "https://github.com/NixOS/nixpkgs.git"
-Ref = str   # eg: "master" or "v1.0.0"
-Rev = str   # eg: "53a27350551844e1ed1a9257690294767389ef0d"
+Path = str   # eg: "/home/user/.cache/git-cache/v1"
+Repo = str   # eg: "https://github.com/NixOS/nixpkgs.git"
+Ref = str    # eg: "master" or "v1.0.0"
+Rev = str    # eg: "53a27350551844e1ed1a9257690294767389ef0d"

 RefOrRev = Union[Ref, Rev]
 
 
 RefOrRev = Union[Ref, Rev]
 
 

+class _LogEntry(NamedTuple):
+    ref: Ref
+    rev: Rev
+
+
+T = TypeVar('T')
+
+
+def _repo_hashname(repo: Repo) -> str:
+    return hashlib.sha256(repo.encode()).hexdigest()
+
+

 def git_cachedir(repo: Repo) -> Path:
     # Use xdg module when it's less painful to have as a dependency
     XDG_CACHE_HOME = Path(
 def git_cachedir(repo: Repo) -> Path:
     # Use xdg module when it's less painful to have as a dependency
     XDG_CACHE_HOME = Path(


@@ -29,7 +44,18 @@ def git_cachedir(repo: Repo) -> Path:
     return Path(os.path.join(
         XDG_CACHE_HOME,
         'git-cache/v1',
     return Path(os.path.join(
         XDG_CACHE_HOME,
         'git-cache/v1',

-        hashlib.sha256(repo.encode()).hexdigest()))
+        _repo_hashname(repo)))
+
+
+def _log_filename(repo: Repo) -> Path:
+    # Use xdg module when it's less painful to have as a dependency
+    XDG_DATA_HOME = Path(
+        os.environ.get('XDG_DATA_HOME', os.path.expanduser('~/.local/share')))
+
+    return Path(os.path.join(
+        XDG_DATA_HOME,
+        'git-cache/v1',
+        _repo_hashname(repo)))

 
 
 def is_ancestor(repo: Repo, descendant: RefOrRev, ancestor: RefOrRev) -> bool:
 
 
 def is_ancestor(repo: Repo, descendant: RefOrRev, ancestor: RefOrRev) -> bool:


@@ -49,40 +75,133 @@ def is_ancestor(repo: Repo, descendant: RefOrRev, ancestor: RefOrRev) -> bool:
 def verify_ancestry(
         repo: Repo,
         descendant: RefOrRev,
 def verify_ancestry(
         repo: Repo,
         descendant: RefOrRev,

-        ancestor: RefOrRev) -> None:
-    if not is_ancestor(repo, descendant, ancestor):
-        raise Exception('%s is not an ancestor of %s' % (ancestor, descendant))
+        ancestor: RefOrRev,
+        force: bool = False) -> None:
+    if not force and not is_ancestor(repo, descendant, ancestor):
+        raise Exception(f'{ancestor} is not an ancestor of {descendant}')
+
+
+def _read_fetch_log(repo: Repo) -> Iterator[_LogEntry]:
+    filename = _log_filename(repo)
+    if not os.path.exists(filename):
+        return
+    with open(filename, 'r', encoding='utf-8') as f:
+        for line in f:
+            _, _, rev, ref = line.strip().split(maxsplit=3)
+            yield _LogEntry(ref, rev)
+
+
+def _last(it: Iterator[T]) -> Optional[T]:
+    return functools.reduce(lambda a, b: b, it, None)
+
+
+def _previous_fetched_rev(repo: Repo, ref: Ref) -> Optional[Rev]:
+    return _last(entry.rev for entry in _read_fetch_log(
+        repo) if entry.ref == ref)
+
+
+def _log_fetch(repo: Repo, ref: Ref, rev: Rev, force: bool = False) -> None:
+    if not force:
+        prev_rev = _previous_fetched_rev(repo, ref)
+        if prev_rev is not None:
+            verify_ancestry(repo, rev, prev_rev)
+    filename = _log_filename(repo)
+    os.makedirs(os.path.dirname(filename), exist_ok=True)
+    with open(filename, 'a', encoding='utf-8') as f:
+        f.write(
+            f'{time.strftime("%Y-%m%d-%H:%M:%S%z")} '
+            f'{"FORCEDFETCH" if force else "fetch"} {rev} {ref}\n'
+        )
+
+
+def _show_force_warning() -> None:
+    print('''
+**************************************************************************
+*               WARNING: git-cache INVOKED WITH --force!                 *
+*                                                                        *
+* This mode allows previously-fetched refs to be overwritten to point to *
+* non-descendants -- commits that don't have the previous version of the *
+* the ref in their history!                                              *
+*                                                                        *
+* This should only be invoked by a human operator who knows what they're *
+* doing to correct a specific, known, problem.  Care should be taken to  *
+* prevent recurrence.                                                    *
+*                                                                        *
+* Press ^C to abort.                                                     *
+*                                                                        *
+''', end='', file=sys.stderr)
+    warn_time_override = os.environ.get('FORCE_WARNING_TIME', None)
+    warn_time: int
+    if warn_time_override is None:
+        warn_time = 15
+    else:
+        warn_time = int(warn_time_override)
+        print(
+            '''*   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!   *
+*   !!              WARNING DISPLAY TIME OVERRIDDEN                 !!   *
+*   !!                                                              !!   *
+*   !! This message is intended to be displayed long enough for a   !!   *
+*   !! human operator to read it and have a chance to abort.  An    !!   *
+*   !! override for the delay time is provided FOR THE UNIT TESTS   !!   *
+*   !! to avoid delaying software builds unnecessarily.  This is    !!   *
+*   !! INTENDED FOR USE IN UNIT TESTS ONLY; THIS MESSAGE SHOULD     !!   *
+*   !! NEVER BE SEEN OUTSIDE BUILD LOGS!                            !!   *
+*   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!   *
+*                                                                        *
+''', end='', file=sys.stderr)
+
+    for i in range(warn_time, 0, -1):
+        msg = f'* {f"Continuing in {i} seconds...":-70s} *'
+        print(msg, file=sys.stderr)
+        time.sleep(1)
+    print('*' * 74, file=sys.stderr)

 
 
 @backoff.on_exception(
     backoff.expo,
     subprocess.CalledProcessError,
     max_time=lambda: int(os.environ.get('BACKOFF_MAX_TIME', '30')))
 
 
 @backoff.on_exception(
     backoff.expo,
     subprocess.CalledProcessError,
     max_time=lambda: int(os.environ.get('BACKOFF_MAX_TIME', '30')))

-def _git_fetch(cachedir: Path, repo: Repo, ref: Ref) -> None:
-    # We don't use --force here because we want to abort and freak out if forced
-    # updates are happening.
-    subprocess.run(['git', '-C', cachedir, 'fetch', repo,
-                    '%s:%s' % (ref, ref)], check=True)
+def _git_fetch(
+        cachedir: Path,
+        repo: Repo,
+        ref: Ref,
+        force: bool = False) -> None:
+    subprocess.run(['git', '-C', cachedir, 'fetch'] +
+                   (['--force'] if force else []) +
+                   [repo, f'{ref}:{ref}'], check=True)

 
 
 
 

-def fetch(repo: Repo, ref: Ref) -> Tuple[Path, Rev]:
+def fetch(repo: Repo, ref: Ref, force: bool = False) -> Tuple[Path, Rev]:
+    if force:
+        _show_force_warning()

     cachedir = git_cachedir(repo)
     if not os.path.exists(cachedir):
         logging.debug("Initializing git repo")
     cachedir = git_cachedir(repo)
     if not os.path.exists(cachedir):
         logging.debug("Initializing git repo")

-        subprocess.run(['git', 'init', '--bare', cachedir],
-                       check=True, stdout=sys.stderr)
+        subprocess.run(['git',
+                        '-c',
+                        'init.defaultBranch=git-cache--no-default-branch',
+                        'init',
+                        '--bare',
+                        cachedir],
+                       check=True,
+                       stdout=sys.stderr)

 
     logging.debug('Fetching ref "%s" from %s', ref, repo)
 
     logging.debug('Fetching ref "%s" from %s', ref, repo)

-    _git_fetch(cachedir, repo, ref)
+    _git_fetch(cachedir, repo, ref, force=force)

 
 

-    with open(os.path.join(cachedir, 'refs', 'heads', ref)) as rev_file:
+    with open(os.path.join(cachedir, 'refs', 'heads', ref), encoding='utf-8') as rev_file:

         rev = Rev(rev_file.read(999).strip())
         rev = Rev(rev_file.read(999).strip())

-    verify_ancestry(repo, ref, rev)
+    verify_ancestry(repo, ref, rev, force=force)
+    _log_fetch(repo, ref, rev, force=force)

 
     return cachedir, rev
 
 
 
     return cachedir, rev
 
 

-def ensure_rev_available(repo: Repo, ref: Ref, rev: Rev) -> Path:
+def ensure_rev_available(
+        repo: Repo,
+        ref: Ref,
+        rev: Rev,
+        force: bool = False) -> Path:

     cachedir = git_cachedir(repo)
     if os.path.exists(cachedir) and is_ancestor(repo, ref, rev):
         return cachedir
     cachedir = git_cachedir(repo)
     if os.path.exists(cachedir) and is_ancestor(repo, ref, rev):
         return cachedir


@@ -90,22 +209,48 @@ def ensure_rev_available(repo: Repo, ref: Ref, rev: Rev) -> Path:
     logging.debug(
         'We do not have rev %s.  We will fetch ref "%s" and hope it appears.',
         rev, ref)
     logging.debug(
         'We do not have rev %s.  We will fetch ref "%s" and hope it appears.',
         rev, ref)

-    fetch(repo, ref)
+    fetch(repo, ref, force=force)

     logging.debug('Verifying that fetch retrieved rev %s', rev)
     subprocess.run(['git', '-C', cachedir, 'cat-file', '-e', rev], check=True)
     logging.debug('Verifying that fetch retrieved rev %s', rev)
     subprocess.run(['git', '-C', cachedir, 'cat-file', '-e', rev], check=True)

-    verify_ancestry(repo, ref, rev)
+    verify_ancestry(repo, ref, rev, force=force)

 
     return cachedir
 
 
 def _main() -> None:
 
     return cachedir
 
 
 def _main() -> None:

-    if len(sys.argv) == 3:
-        print('{1} {0}'.format(*fetch(Repo(sys.argv[1]), Ref(sys.argv[2]))))
-    elif len(sys.argv) == 4:
-        print(ensure_rev_available(
-            Repo(sys.argv[1]), Ref(sys.argv[2]), Rev(sys.argv[3])))
+    parser = argparse.ArgumentParser(
+        description='Cache remote git repositories locally.',
+        epilog='example usage: git-cache https://github.com/NixOS/nixpkgs.git master')
+    parser.add_argument(
+        '--force',
+        action='store_true',
+        help='Recover from a force-push in the remote repo')
+    parser.add_argument(
+        'repo',
+        metavar='Repo',
+        type=Repo,
+        help='Git repository URL')
+    parser.add_argument(
+        'ref',
+        metavar='Ref',
+        type=Ref,
+        help='Ref (branch or tag) in the git repo')
+    parser.add_argument(
+        'rev',
+        metavar='Rev',
+        type=Rev,
+        nargs='?',
+        help='Ensure that this revision is present.  ' +
+        'If this revision is already present locally, no network operations are performed.')
+    args = parser.parse_args()
+
+    if args.rev is None:
+        cachedir, rev = fetch(args.repo, args.ref, force=args.force)
+        print(f'{rev} {cachedir}')

     else:
     else:

-        usage = '''usage: git-cache repo ref [rev]
-example: git-cache https://github.com/NixOS/nixpkgs.git master'''
-        print(usage, file=sys.stderr)
-        sys.exit(1)
+        print(
+            ensure_rev_available(
+                args.repo,
+                args.ref,
+                args.rev,
+                force=args.force))