--force flag to recover from remote force-push

diff --git a/Changelog b/Changelog
index 3e6b10866a9cfff5d2aa01e22b6bf29074173e6a..6ee741a57c4e5d69925366683de11611e9d1318f 100644 (file)
--- a/Changelog
+++ b/Changelog
@@ -2,6 +2,9 @@
 ### Changed
 - Refuse force-push updates even if XDG_CACHE_HOME is cleared
 
 ### Changed
 - Refuse force-push updates even if XDG_CACHE_HOME is cleared
 

+### Added
+- --force flag to recover from remote force-push
+

 
 ## [1.3.1] - 2021-06-07
 ### Changed
 
 ## [1.3.1] - 2021-06-07
 ### Changed

diff --git a/git_cache.py b/git_cache.py
index d43df2624fd8e9d9bc92536a4617c21eeb69d249..b0f992e76589666ee86f5f8d8e27f51142af71c5 100644 (file)
--- a/git_cache.py
+++ b/git_cache.py
@@ -75,8 +75,9 @@ def is_ancestor(repo: Repo, descendant: RefOrRev, ancestor: RefOrRev) -> bool:
 def verify_ancestry(
         repo: Repo,
         descendant: RefOrRev,
 def verify_ancestry(
         repo: Repo,
         descendant: RefOrRev,

-        ancestor: RefOrRev) -> None:
-    if not is_ancestor(repo, descendant, ancestor):
+        ancestor: RefOrRev,
+        force: bool = False) -> None:
+    if not force and not is_ancestor(repo, descendant, ancestor):

         raise Exception('%s is not an ancestor of %s' % (ancestor, descendant))
 
 
         raise Exception('%s is not an ancestor of %s' % (ancestor, descendant))
 
 


@@ -99,29 +100,80 @@ def _previous_fetched_rev(repo: Repo, ref: Ref) -> Optional[Rev]:
         repo) if entry.ref == ref)
 
 
         repo) if entry.ref == ref)
 
 

-def _log_fetch(repo: Repo, ref: Ref, rev: Rev) -> None:
-    prev_rev = _previous_fetched_rev(repo, ref)
-    if prev_rev is not None:
-        verify_ancestry(repo, rev, prev_rev)
+def _log_fetch(repo: Repo, ref: Ref, rev: Rev, force: bool = False) -> None:
+    if not force:
+        prev_rev = _previous_fetched_rev(repo, ref)
+        if prev_rev is not None:
+            verify_ancestry(repo, rev, prev_rev)

     filename = _log_filename(repo)
     os.makedirs(os.path.dirname(filename), exist_ok=True)
     with open(filename, 'a') as f:
     filename = _log_filename(repo)
     os.makedirs(os.path.dirname(filename), exist_ok=True)
     with open(filename, 'a') as f:

-        f.write('%s fetch %s %s\n' %
-                (time.strftime('%Y-%m%d-%H:%M:%S%z'), rev, ref))
+        f.write('%s %s %s %s\n' %
+                (time.strftime('%Y-%m%d-%H:%M:%S%z'),
+                 ('FORCEDFETCH' if force else 'fetch'), rev, ref))
+
+
+def _show_force_warning() -> None:
+    print('''
+**************************************************************************
+*               WARNING: git-cache INVOKED WITH --force!                 *
+*                                                                        *
+* This mode allows previously-fetched refs to be overwritten to point to *
+* non-descendants -- commits that don't have the previous version of the *
+* the ref in their history!                                              *
+*                                                                        *
+* This should only be invoked by a human operator who knows what they're *
+* doing to correct a specific, known, problem.  Care should be taken to  *
+* prevent recurrence.                                                    *
+*                                                                        *
+* Press ^C to abort.                                                     *
+*                                                                        *
+''', end='', file=sys.stderr)
+    warn_time_override = os.environ.get('FORCE_WARNING_TIME', None)
+    warn_time: int
+    if warn_time_override is None:
+        warn_time = 15
+    else:
+        warn_time = int(warn_time_override)
+        print(
+            '''*   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!   *
+*   !!              WARNING DISPLAY TIME OVERRIDDEN                 !!   *
+*   !!                                                              !!   *
+*   !! This message is intended to be displayed long enough for a   !!   *
+*   !! human operator to read it and have a chance to abort.  An    !!   *
+*   !! override for the delay time is provided FOR THE UNIT TESTS   !!   *
+*   !! to avoid delaying software builds unnecessarily.  This is    !!   *
+*   !! INTENDED FOR USE IN UNIT TESTS ONLY; THIS MESSAGE SHOULD     !!   *
+*   !! NEVER BE SEEN OUTSIDE BUILD LOGS!                            !!   *
+*   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!   *
+*                                                                        *
+''', end='', file=sys.stderr)
+
+    for i in range(warn_time, 0, -1):
+        msg = '* %-70s *' % ("Continuing in %d seconds..." % i)
+        print(msg, file=sys.stderr)
+        time.sleep(1)
+    print('*' * 74, file=sys.stderr)
+

 
 
 @backoff.on_exception(
     backoff.expo,
     subprocess.CalledProcessError,
     max_time=lambda: int(os.environ.get('BACKOFF_MAX_TIME', '30')))
 
 
 @backoff.on_exception(
     backoff.expo,
     subprocess.CalledProcessError,
     max_time=lambda: int(os.environ.get('BACKOFF_MAX_TIME', '30')))

-def _git_fetch(cachedir: Path, repo: Repo, ref: Ref) -> None:
-    # We don't use --force here because we want to abort and freak out if forced
-    # updates are happening.
-    subprocess.run(['git', '-C', cachedir, 'fetch', repo,
-                    '%s:%s' % (ref, ref)], check=True)
+def _git_fetch(
+        cachedir: Path,
+        repo: Repo,
+        ref: Ref,
+        force: bool = False) -> None:
+    subprocess.run(['git', '-C', cachedir, 'fetch'] +
+                   (['--force'] if force else []) +
+                   [repo, '%s:%s' % (ref, ref)], check=True)

 
 
 
 

-def fetch(repo: Repo, ref: Ref) -> Tuple[Path, Rev]:
+def fetch(repo: Repo, ref: Ref, force: bool = False) -> Tuple[Path, Rev]:
+    if force:
+        _show_force_warning()

     cachedir = git_cachedir(repo)
     if not os.path.exists(cachedir):
         logging.debug("Initializing git repo")
     cachedir = git_cachedir(repo)
     if not os.path.exists(cachedir):
         logging.debug("Initializing git repo")


@@ -129,17 +181,21 @@ def fetch(repo: Repo, ref: Ref) -> Tuple[Path, Rev]:
                        check=True, stdout=sys.stderr)
 
     logging.debug('Fetching ref "%s" from %s', ref, repo)
                        check=True, stdout=sys.stderr)
 
     logging.debug('Fetching ref "%s" from %s', ref, repo)

-    _git_fetch(cachedir, repo, ref)
+    _git_fetch(cachedir, repo, ref, force=force)

 
     with open(os.path.join(cachedir, 'refs', 'heads', ref)) as rev_file:
         rev = Rev(rev_file.read(999).strip())
 
     with open(os.path.join(cachedir, 'refs', 'heads', ref)) as rev_file:
         rev = Rev(rev_file.read(999).strip())

-    verify_ancestry(repo, ref, rev)
-    _log_fetch(repo, ref, rev)
+    verify_ancestry(repo, ref, rev, force=force)
+    _log_fetch(repo, ref, rev, force=force)

 
     return cachedir, rev
 
 
 
     return cachedir, rev
 
 

-def ensure_rev_available(repo: Repo, ref: Ref, rev: Rev) -> Path:
+def ensure_rev_available(
+        repo: Repo,
+        ref: Ref,
+        rev: Rev,
+        force: bool = False) -> Path:

     cachedir = git_cachedir(repo)
     if os.path.exists(cachedir) and is_ancestor(repo, ref, rev):
         return cachedir
     cachedir = git_cachedir(repo)
     if os.path.exists(cachedir) and is_ancestor(repo, ref, rev):
         return cachedir


@@ -147,10 +203,10 @@ def ensure_rev_available(repo: Repo, ref: Ref, rev: Rev) -> Path:
     logging.debug(
         'We do not have rev %s.  We will fetch ref "%s" and hope it appears.',
         rev, ref)
     logging.debug(
         'We do not have rev %s.  We will fetch ref "%s" and hope it appears.',
         rev, ref)

-    fetch(repo, ref)
+    fetch(repo, ref, force=force)

     logging.debug('Verifying that fetch retrieved rev %s', rev)
     subprocess.run(['git', '-C', cachedir, 'cat-file', '-e', rev], check=True)
     logging.debug('Verifying that fetch retrieved rev %s', rev)
     subprocess.run(['git', '-C', cachedir, 'cat-file', '-e', rev], check=True)

-    verify_ancestry(repo, ref, rev)
+    verify_ancestry(repo, ref, rev, force=force)

 
     return cachedir
 
 
     return cachedir
 


@@ -159,6 +215,10 @@ def _main() -> None:
     parser = argparse.ArgumentParser(
         description='Cache remote git repositories locally.',
         epilog='example usage: git-cache https://github.com/NixOS/nixpkgs.git master')
     parser = argparse.ArgumentParser(
         description='Cache remote git repositories locally.',
         epilog='example usage: git-cache https://github.com/NixOS/nixpkgs.git master')

+    parser.add_argument(
+        '--force',
+        action='store_true',
+        help='Recover from a force-push in the remote repo')

     parser.add_argument(
         'repo',
         metavar='Repo',
     parser.add_argument(
         'repo',
         metavar='Repo',


@@ -179,6 +239,11 @@ def _main() -> None:
     args = parser.parse_args()
 
     if args.rev is None:
     args = parser.parse_args()
 
     if args.rev is None:

-        print('{1} {0}'.format(*fetch(args.repo, args.ref)))
+        print('{1} {0}'.format(*fetch(args.repo, args.ref, force=args.force)))

     else:
     else:

-        print(ensure_rev_available(args.repo, args.ref, args.rev))
+        print(
+            ensure_rev_available(
+                args.repo,
+                args.ref,
+                args.rev,
+                force=args.force))

diff --git a/test_git_cache.py b/test_git_cache.py
index b7e0edeb149ff4d478d85f235f0ff7ed1ae17e3f..2990f56d50d35cbea956da36237c9967003e3199 100644 (file)
--- a/test_git_cache.py
+++ b/test_git_cache.py
@@ -52,6 +52,7 @@ class TestGitCache(unittest.TestCase):
         os.environ['GIT_COMMITTER_EMAIL'] = 'test_git_cache@example.com'
 
         os.environ['BACKOFF_MAX_TIME'] = '0'
         os.environ['GIT_COMMITTER_EMAIL'] = 'test_git_cache@example.com'
 
         os.environ['BACKOFF_MAX_TIME'] = '0'

+        os.environ['FORCE_WARNING_TIME'] = '0'  # ONLY FOR TEST USE!

 
         self.tempdir = tempfile.TemporaryDirectory(prefix='git_cache_test-')
         self.upstream = os.path.join(self.tempdir.name, 'upstream')
 
         self.tempdir = tempfile.TemporaryDirectory(prefix='git_cache_test-')
         self.upstream = os.path.join(self.tempdir.name, 'upstream')


@@ -225,6 +226,32 @@ class TestGitCache(unittest.TestCase):
         with self.assertRaises(Exception):
             git_cache.ensure_rev_available(self.upstream, 'master', rev)
 
         with self.assertRaises(Exception):
             git_cache.ensure_rev_available(self.upstream, 'master', rev)
 

+    def test_force_fetch_after_amend(self) -> None:
+        git_cache.fetch(self.upstream, 'master')
+        _git(self.upstream, 'commit', '--amend', '-m', 'Amended')
+        git_cache.fetch(self.upstream, 'master', force=True)
+
+    def test_force_ensure_after_amend(self) -> None:
+        git_cache.fetch(self.upstream, 'master')
+        _git(self.upstream, 'commit', '--amend', '-m', 'Amended')
+        rev = _git(self.upstream, 'log', '--format=%H', '-n1').strip().decode()
+        git_cache.ensure_rev_available(
+            self.upstream, 'master', rev, force=True)
+
+    def test_force_fetch_after_amend_and_cache_delete(self) -> None:
+        d, _ = git_cache.fetch(self.upstream, 'master')
+        shutil.rmtree(d)
+        _git(self.upstream, 'commit', '--amend', '-m', 'Amended')
+        git_cache.fetch(self.upstream, 'master', force=True)
+
+    def test_force_ensure_after_amend_and_cache_delete(self) -> None:
+        d, _ = git_cache.fetch(self.upstream, 'master')
+        shutil.rmtree(d)
+        _git(self.upstream, 'commit', '--amend', '-m', 'Amended')
+        rev = _git(self.upstream, 'log', '--format=%H', '-n1').strip().decode()
+        git_cache.ensure_rev_available(
+            self.upstream, 'master', rev, force=True)
+

 
 if __name__ == '__main__':
     unittest.main()
 
 if __name__ == '__main__':
     unittest.main()