def verify_ancestry(
repo: Repo,
descendant: RefOrRev,
- ancestor: RefOrRev) -> None:
- if not is_ancestor(repo, descendant, ancestor):
+ ancestor: RefOrRev,
+ force: bool = False) -> None:
+ if not force and not is_ancestor(repo, descendant, ancestor):
raise Exception('%s is not an ancestor of %s' % (ancestor, descendant))
repo) if entry.ref == ref)
-def _log_fetch(repo: Repo, ref: Ref, rev: Rev) -> None:
- prev_rev = _previous_fetched_rev(repo, ref)
- if prev_rev is not None:
- verify_ancestry(repo, rev, prev_rev)
+def _log_fetch(repo: Repo, ref: Ref, rev: Rev, force: bool = False) -> None:
+ if not force:
+ prev_rev = _previous_fetched_rev(repo, ref)
+ if prev_rev is not None:
+ verify_ancestry(repo, rev, prev_rev)
filename = _log_filename(repo)
os.makedirs(os.path.dirname(filename), exist_ok=True)
with open(filename, 'a') as f:
- f.write('%s fetch %s %s\n' %
- (time.strftime('%Y-%m%d-%H:%M:%S%z'), rev, ref))
+ f.write('%s %s %s %s\n' %
+ (time.strftime('%Y-%m%d-%H:%M:%S%z'),
+ ('FORCEDFETCH' if force else 'fetch'), rev, ref))
+
+
+def _show_force_warning() -> None:
+ print('''
+**************************************************************************
+* WARNING: git-cache INVOKED WITH --force! *
+* *
+* This mode allows previously-fetched refs to be overwritten to point to *
+* non-descendants -- commits that don't have the previous version of the *
+* the ref in their history! *
+* *
+* This should only be invoked by a human operator who knows what they're *
+* doing to correct a specific, known, problem. Care should be taken to *
+* prevent recurrence. *
+* *
+* Press ^C to abort. *
+* *
+''', end='', file=sys.stderr)
+ warn_time_override = os.environ.get('FORCE_WARNING_TIME', None)
+ warn_time: int
+ if warn_time_override is None:
+ warn_time = 15
+ else:
+ warn_time = int(warn_time_override)
+ print(
+ '''* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! *
+* !! WARNING DISPLAY TIME OVERRIDDEN !! *
+* !! !! *
+* !! This message is intended to be displayed long enough for a !! *
+* !! human operator to read it and have a chance to abort. An !! *
+* !! override for the delay time is provided FOR THE UNIT TESTS !! *
+* !! to avoid delaying software builds unnecessarily. This is !! *
+* !! INTENDED FOR USE IN UNIT TESTS ONLY; THIS MESSAGE SHOULD !! *
+* !! NEVER BE SEEN OUTSIDE BUILD LOGS! !! *
+* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! *
+* *
+''', end='', file=sys.stderr)
+
+ for i in range(warn_time, 0, -1):
+ msg = '* %-70s *' % ("Continuing in %d seconds..." % i)
+ print(msg, file=sys.stderr)
+ time.sleep(1)
+ print('*' * 74, file=sys.stderr)
+
@backoff.on_exception(
backoff.expo,
subprocess.CalledProcessError,
max_time=lambda: int(os.environ.get('BACKOFF_MAX_TIME', '30')))
-def _git_fetch(cachedir: Path, repo: Repo, ref: Ref) -> None:
- # We don't use --force here because we want to abort and freak out if forced
- # updates are happening.
- subprocess.run(['git', '-C', cachedir, 'fetch', repo,
- '%s:%s' % (ref, ref)], check=True)
+def _git_fetch(
+ cachedir: Path,
+ repo: Repo,
+ ref: Ref,
+ force: bool = False) -> None:
+ subprocess.run(['git', '-C', cachedir, 'fetch'] +
+ (['--force'] if force else []) +
+ [repo, '%s:%s' % (ref, ref)], check=True)
-def fetch(repo: Repo, ref: Ref) -> Tuple[Path, Rev]:
+def fetch(repo: Repo, ref: Ref, force: bool = False) -> Tuple[Path, Rev]:
+ if force:
+ _show_force_warning()
cachedir = git_cachedir(repo)
if not os.path.exists(cachedir):
logging.debug("Initializing git repo")
check=True, stdout=sys.stderr)
logging.debug('Fetching ref "%s" from %s', ref, repo)
- _git_fetch(cachedir, repo, ref)
+ _git_fetch(cachedir, repo, ref, force=force)
with open(os.path.join(cachedir, 'refs', 'heads', ref)) as rev_file:
rev = Rev(rev_file.read(999).strip())
- verify_ancestry(repo, ref, rev)
- _log_fetch(repo, ref, rev)
+ verify_ancestry(repo, ref, rev, force=force)
+ _log_fetch(repo, ref, rev, force=force)
return cachedir, rev
-def ensure_rev_available(repo: Repo, ref: Ref, rev: Rev) -> Path:
+def ensure_rev_available(
+ repo: Repo,
+ ref: Ref,
+ rev: Rev,
+ force: bool = False) -> Path:
cachedir = git_cachedir(repo)
if os.path.exists(cachedir) and is_ancestor(repo, ref, rev):
return cachedir
logging.debug(
'We do not have rev %s. We will fetch ref "%s" and hope it appears.',
rev, ref)
- fetch(repo, ref)
+ fetch(repo, ref, force=force)
logging.debug('Verifying that fetch retrieved rev %s', rev)
subprocess.run(['git', '-C', cachedir, 'cat-file', '-e', rev], check=True)
- verify_ancestry(repo, ref, rev)
+ verify_ancestry(repo, ref, rev, force=force)
return cachedir
parser = argparse.ArgumentParser(
description='Cache remote git repositories locally.',
epilog='example usage: git-cache https://github.com/NixOS/nixpkgs.git master')
+ parser.add_argument(
+ '--force',
+ action='store_true',
+ help='Recover from a force-push in the remote repo')
parser.add_argument(
'repo',
metavar='Repo',
args = parser.parse_args()
if args.rev is None:
- print('{1} {0}'.format(*fetch(args.repo, args.ref)))
+ print('{1} {0}'.format(*fetch(args.repo, args.ref, force=args.force)))
else:
- print(ensure_rev_available(args.repo, args.ref, args.rev))
+ print(
+ ensure_rev_available(
+ args.repo,
+ args.ref,
+ args.rev,
+ force=args.force))
os.environ['GIT_COMMITTER_EMAIL'] = 'test_git_cache@example.com'
os.environ['BACKOFF_MAX_TIME'] = '0'
+ os.environ['FORCE_WARNING_TIME'] = '0' # ONLY FOR TEST USE!
self.tempdir = tempfile.TemporaryDirectory(prefix='git_cache_test-')
self.upstream = os.path.join(self.tempdir.name, 'upstream')
with self.assertRaises(Exception):
git_cache.ensure_rev_available(self.upstream, 'master', rev)
+ def test_force_fetch_after_amend(self) -> None:
+ git_cache.fetch(self.upstream, 'master')
+ _git(self.upstream, 'commit', '--amend', '-m', 'Amended')
+ git_cache.fetch(self.upstream, 'master', force=True)
+
+ def test_force_ensure_after_amend(self) -> None:
+ git_cache.fetch(self.upstream, 'master')
+ _git(self.upstream, 'commit', '--amend', '-m', 'Amended')
+ rev = _git(self.upstream, 'log', '--format=%H', '-n1').strip().decode()
+ git_cache.ensure_rev_available(
+ self.upstream, 'master', rev, force=True)
+
+ def test_force_fetch_after_amend_and_cache_delete(self) -> None:
+ d, _ = git_cache.fetch(self.upstream, 'master')
+ shutil.rmtree(d)
+ _git(self.upstream, 'commit', '--amend', '-m', 'Amended')
+ git_cache.fetch(self.upstream, 'master', force=True)
+
+ def test_force_ensure_after_amend_and_cache_delete(self) -> None:
+ d, _ = git_cache.fetch(self.upstream, 'master')
+ shutil.rmtree(d)
+ _git(self.upstream, 'commit', '--amend', '-m', 'Amended')
+ rev = _git(self.upstream, 'log', '--format=%H', '-n1').strip().decode()
+ git_cache.ensure_rev_available(
+ self.upstream, 'master', rev, force=True)
+
if __name__ == '__main__':
unittest.main()
projects / git-cache / commitdiff