From 083b90e7f36d292909d0ce573e3681ddfb011a77 Mon Sep 17 00:00:00 2001 From: Scott Worley Date: Mon, 21 Jun 2021 17:28:28 -0700 Subject: [PATCH] --force flag to recover from remote force-push --- Changelog | 3 ++ git_cache.py | 109 ++++++++++++++++++++++++++++++++++++---------- test_git_cache.py | 27 ++++++++++++ 3 files changed, 117 insertions(+), 22 deletions(-) diff --git a/Changelog b/Changelog index 3e6b108..6ee741a 100644 --- a/Changelog +++ b/Changelog @@ -2,6 +2,9 @@ ### Changed - Refuse force-push updates even if XDG_CACHE_HOME is cleared +### Added +- --force flag to recover from remote force-push + ## [1.3.1] - 2021-06-07 ### Changed diff --git a/git_cache.py b/git_cache.py index d43df26..b0f992e 100644 --- a/git_cache.py +++ b/git_cache.py @@ -75,8 +75,9 @@ def is_ancestor(repo: Repo, descendant: RefOrRev, ancestor: RefOrRev) -> bool: def verify_ancestry( repo: Repo, descendant: RefOrRev, - ancestor: RefOrRev) -> None: - if not is_ancestor(repo, descendant, ancestor): + ancestor: RefOrRev, + force: bool = False) -> None: + if not force and not is_ancestor(repo, descendant, ancestor): raise Exception('%s is not an ancestor of %s' % (ancestor, descendant)) @@ -99,29 +100,80 @@ def _previous_fetched_rev(repo: Repo, ref: Ref) -> Optional[Rev]: repo) if entry.ref == ref) -def _log_fetch(repo: Repo, ref: Ref, rev: Rev) -> None: - prev_rev = _previous_fetched_rev(repo, ref) - if prev_rev is not None: - verify_ancestry(repo, rev, prev_rev) +def _log_fetch(repo: Repo, ref: Ref, rev: Rev, force: bool = False) -> None: + if not force: + prev_rev = _previous_fetched_rev(repo, ref) + if prev_rev is not None: + verify_ancestry(repo, rev, prev_rev) filename = _log_filename(repo) os.makedirs(os.path.dirname(filename), exist_ok=True) with open(filename, 'a') as f: - f.write('%s fetch %s %s\n' % - (time.strftime('%Y-%m%d-%H:%M:%S%z'), rev, ref)) + f.write('%s %s %s %s\n' % + (time.strftime('%Y-%m%d-%H:%M:%S%z'), + ('FORCEDFETCH' if force else 'fetch'), rev, ref)) + + +def _show_force_warning() -> None: + print(''' +************************************************************************** +* WARNING: git-cache INVOKED WITH --force! * +* * +* This mode allows previously-fetched refs to be overwritten to point to * +* non-descendants -- commits that don't have the previous version of the * +* the ref in their history! * +* * +* This should only be invoked by a human operator who knows what they're * +* doing to correct a specific, known, problem. Care should be taken to * +* prevent recurrence. * +* * +* Press ^C to abort. * +* * +''', end='', file=sys.stderr) + warn_time_override = os.environ.get('FORCE_WARNING_TIME', None) + warn_time: int + if warn_time_override is None: + warn_time = 15 + else: + warn_time = int(warn_time_override) + print( + '''* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! * +* !! WARNING DISPLAY TIME OVERRIDDEN !! * +* !! !! * +* !! This message is intended to be displayed long enough for a !! * +* !! human operator to read it and have a chance to abort. An !! * +* !! override for the delay time is provided FOR THE UNIT TESTS !! * +* !! to avoid delaying software builds unnecessarily. This is !! * +* !! INTENDED FOR USE IN UNIT TESTS ONLY; THIS MESSAGE SHOULD !! * +* !! NEVER BE SEEN OUTSIDE BUILD LOGS! !! * +* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! * +* * +''', end='', file=sys.stderr) + + for i in range(warn_time, 0, -1): + msg = '* %-70s *' % ("Continuing in %d seconds..." % i) + print(msg, file=sys.stderr) + time.sleep(1) + print('*' * 74, file=sys.stderr) + @backoff.on_exception( backoff.expo, subprocess.CalledProcessError, max_time=lambda: int(os.environ.get('BACKOFF_MAX_TIME', '30'))) -def _git_fetch(cachedir: Path, repo: Repo, ref: Ref) -> None: - # We don't use --force here because we want to abort and freak out if forced - # updates are happening. - subprocess.run(['git', '-C', cachedir, 'fetch', repo, - '%s:%s' % (ref, ref)], check=True) +def _git_fetch( + cachedir: Path, + repo: Repo, + ref: Ref, + force: bool = False) -> None: + subprocess.run(['git', '-C', cachedir, 'fetch'] + + (['--force'] if force else []) + + [repo, '%s:%s' % (ref, ref)], check=True) -def fetch(repo: Repo, ref: Ref) -> Tuple[Path, Rev]: +def fetch(repo: Repo, ref: Ref, force: bool = False) -> Tuple[Path, Rev]: + if force: + _show_force_warning() cachedir = git_cachedir(repo) if not os.path.exists(cachedir): logging.debug("Initializing git repo") @@ -129,17 +181,21 @@ def fetch(repo: Repo, ref: Ref) -> Tuple[Path, Rev]: check=True, stdout=sys.stderr) logging.debug('Fetching ref "%s" from %s', ref, repo) - _git_fetch(cachedir, repo, ref) + _git_fetch(cachedir, repo, ref, force=force) with open(os.path.join(cachedir, 'refs', 'heads', ref)) as rev_file: rev = Rev(rev_file.read(999).strip()) - verify_ancestry(repo, ref, rev) - _log_fetch(repo, ref, rev) + verify_ancestry(repo, ref, rev, force=force) + _log_fetch(repo, ref, rev, force=force) return cachedir, rev -def ensure_rev_available(repo: Repo, ref: Ref, rev: Rev) -> Path: +def ensure_rev_available( + repo: Repo, + ref: Ref, + rev: Rev, + force: bool = False) -> Path: cachedir = git_cachedir(repo) if os.path.exists(cachedir) and is_ancestor(repo, ref, rev): return cachedir @@ -147,10 +203,10 @@ def ensure_rev_available(repo: Repo, ref: Ref, rev: Rev) -> Path: logging.debug( 'We do not have rev %s. We will fetch ref "%s" and hope it appears.', rev, ref) - fetch(repo, ref) + fetch(repo, ref, force=force) logging.debug('Verifying that fetch retrieved rev %s', rev) subprocess.run(['git', '-C', cachedir, 'cat-file', '-e', rev], check=True) - verify_ancestry(repo, ref, rev) + verify_ancestry(repo, ref, rev, force=force) return cachedir @@ -159,6 +215,10 @@ def _main() -> None: parser = argparse.ArgumentParser( description='Cache remote git repositories locally.', epilog='example usage: git-cache https://github.com/NixOS/nixpkgs.git master') + parser.add_argument( + '--force', + action='store_true', + help='Recover from a force-push in the remote repo') parser.add_argument( 'repo', metavar='Repo', @@ -179,6 +239,11 @@ def _main() -> None: args = parser.parse_args() if args.rev is None: - print('{1} {0}'.format(*fetch(args.repo, args.ref))) + print('{1} {0}'.format(*fetch(args.repo, args.ref, force=args.force))) else: - print(ensure_rev_available(args.repo, args.ref, args.rev)) + print( + ensure_rev_available( + args.repo, + args.ref, + args.rev, + force=args.force)) diff --git a/test_git_cache.py b/test_git_cache.py index b7e0ede..2990f56 100644 --- a/test_git_cache.py +++ b/test_git_cache.py @@ -52,6 +52,7 @@ class TestGitCache(unittest.TestCase): os.environ['GIT_COMMITTER_EMAIL'] = 'test_git_cache@example.com' os.environ['BACKOFF_MAX_TIME'] = '0' + os.environ['FORCE_WARNING_TIME'] = '0' # ONLY FOR TEST USE! self.tempdir = tempfile.TemporaryDirectory(prefix='git_cache_test-') self.upstream = os.path.join(self.tempdir.name, 'upstream') @@ -225,6 +226,32 @@ class TestGitCache(unittest.TestCase): with self.assertRaises(Exception): git_cache.ensure_rev_available(self.upstream, 'master', rev) + def test_force_fetch_after_amend(self) -> None: + git_cache.fetch(self.upstream, 'master') + _git(self.upstream, 'commit', '--amend', '-m', 'Amended') + git_cache.fetch(self.upstream, 'master', force=True) + + def test_force_ensure_after_amend(self) -> None: + git_cache.fetch(self.upstream, 'master') + _git(self.upstream, 'commit', '--amend', '-m', 'Amended') + rev = _git(self.upstream, 'log', '--format=%H', '-n1').strip().decode() + git_cache.ensure_rev_available( + self.upstream, 'master', rev, force=True) + + def test_force_fetch_after_amend_and_cache_delete(self) -> None: + d, _ = git_cache.fetch(self.upstream, 'master') + shutil.rmtree(d) + _git(self.upstream, 'commit', '--amend', '-m', 'Amended') + git_cache.fetch(self.upstream, 'master', force=True) + + def test_force_ensure_after_amend_and_cache_delete(self) -> None: + d, _ = git_cache.fetch(self.upstream, 'master') + shutil.rmtree(d) + _git(self.upstream, 'commit', '--amend', '-m', 'Amended') + rev = _git(self.upstream, 'log', '--format=%H', '-n1').strip().decode() + git_cache.ensure_rev_available( + self.upstream, 'master', rev, force=True) + if __name__ == '__main__': unittest.main() -- 2.44.1