]> git.scottworley.com Git - git-cache/blob - git_cache.py
a34c3291b3e6f359e4430a3e9d5720f6923487ce
[git-cache] / git_cache.py
1 # git-cache: Cache git content locally
2 #
3 # This program is free software: you can redistribute it and/or modify it
4 # under the terms of the GNU General Public License as published by the
5 # Free Software Foundation, version 3.
6
7
8 # It would be nice if we could share the nix git cache, but as of the
9 # time of writing it is transitioning from gitv2 (deprecated) to gitv3
10 # (not ready yet), and trying to straddle them both is too far into nix
11 # implementation details for my comfort. So we re-implement here half of
12 # nix's builtins.fetchGit. :(
13
14 import argparse
15 import functools
16 import hashlib
17 import logging
18 import os
19 import subprocess
20 import sys
21 import time
22
23 from typing import Iterator, NamedTuple, Optional, TypeVar, Tuple, Union
24
25 import backoff
26
27 Path = str # eg: "/home/user/.cache/git-cache/v1"
28 Repo = str # eg: "https://github.com/NixOS/nixpkgs.git"
29 Ref = str # eg: "master" or "v1.0.0"
30 Rev = str # eg: "53a27350551844e1ed1a9257690294767389ef0d"
31 RefOrRev = Union[Ref, Rev]
32
33
34 class _LogEntry(NamedTuple):
35 ref: Ref
36 rev: Rev
37
38
39 T = TypeVar('T')
40
41
42 def _repo_hashname(repo: Repo) -> str:
43 return hashlib.sha256(repo.encode()).hexdigest()
44
45
46 def git_cachedir(repo: Repo) -> Path:
47 # Use xdg module when it's less painful to have as a dependency
48 XDG_CACHE_HOME = Path(
49 os.environ.get('XDG_CACHE_HOME', os.path.expanduser('~/.cache')))
50
51 return Path(os.path.join(
52 XDG_CACHE_HOME,
53 'git-cache/v1',
54 _repo_hashname(repo)))
55
56
57 def _log_filename(repo: Repo) -> Path:
58 # Use xdg module when it's less painful to have as a dependency
59 XDG_DATA_HOME = Path(
60 os.environ.get('XDG_DATA_HOME', os.path.expanduser('~/.local/share')))
61
62 return Path(os.path.join(
63 XDG_DATA_HOME,
64 'git-cache/v1',
65 _repo_hashname(repo)))
66
67
68 def is_ancestor(repo: Repo, descendant: RefOrRev, ancestor: RefOrRev) -> bool:
69 cachedir = git_cachedir(repo)
70 logging.debug('Checking if %s is an ancestor of %s', ancestor, descendant)
71 process = subprocess.run(['git',
72 '-C',
73 cachedir,
74 'merge-base',
75 '--is-ancestor',
76 ancestor,
77 descendant],
78 check=False)
79 return process.returncode == 0
80
81
82 def verify_ancestry(
83 repo: Repo,
84 descendant: RefOrRev,
85 ancestor: RefOrRev,
86 force: bool = False) -> None:
87 if not force and not is_ancestor(repo, descendant, ancestor):
88 raise Exception(f'{ancestor} is not an ancestor of {descendant}')
89
90
91 def _read_fetch_log(repo: Repo) -> Iterator[_LogEntry]:
92 filename = _log_filename(repo)
93 if not os.path.exists(filename):
94 return
95 with open(filename, 'r', encoding='utf-8') as f:
96 for line in f:
97 _, _, rev, ref = line.strip().split(maxsplit=3)
98 yield _LogEntry(ref, rev)
99
100
101 def _last(it: Iterator[T]) -> Optional[T]:
102 return functools.reduce(lambda a, b: b, it, None)
103
104
105 def _previous_fetched_rev(repo: Repo, ref: Ref) -> Optional[Rev]:
106 return _last(entry.rev for entry in _read_fetch_log(
107 repo) if entry.ref == ref)
108
109
110 def _log_fetch(repo: Repo, ref: Ref, rev: Rev, force: bool = False) -> None:
111 if not force:
112 prev_rev = _previous_fetched_rev(repo, ref)
113 if prev_rev is not None:
114 verify_ancestry(repo, rev, prev_rev)
115 filename = _log_filename(repo)
116 os.makedirs(os.path.dirname(filename), exist_ok=True)
117 with open(filename, 'a', encoding='utf-8') as f:
118 f.write(
119 f'{time.strftime("%Y-%m%d-%H:%M:%S%z")} '
120 f'{"FORCEDFETCH" if force else "fetch"} {rev} {ref}\n'
121 )
122
123
124 def _show_force_warning() -> None:
125 print('''
126 **************************************************************************
127 * WARNING: git-cache INVOKED WITH --force! *
128 * *
129 * This mode allows previously-fetched refs to be overwritten to point to *
130 * non-descendants -- commits that don't have the previous version of the *
131 * the ref in their history! *
132 * *
133 * This should only be invoked by a human operator who knows what they're *
134 * doing to correct a specific, known, problem. Care should be taken to *
135 * prevent recurrence. *
136 * *
137 * Press ^C to abort. *
138 * *
139 ''', end='', file=sys.stderr)
140 warn_time_override = os.environ.get('FORCE_WARNING_TIME', None)
141 warn_time: int
142 if warn_time_override is None:
143 warn_time = 15
144 else:
145 warn_time = int(warn_time_override)
146 print(
147 '''* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! *
148 * !! WARNING DISPLAY TIME OVERRIDDEN !! *
149 * !! !! *
150 * !! This message is intended to be displayed long enough for a !! *
151 * !! human operator to read it and have a chance to abort. An !! *
152 * !! override for the delay time is provided FOR THE UNIT TESTS !! *
153 * !! to avoid delaying software builds unnecessarily. This is !! *
154 * !! INTENDED FOR USE IN UNIT TESTS ONLY; THIS MESSAGE SHOULD !! *
155 * !! NEVER BE SEEN OUTSIDE BUILD LOGS! !! *
156 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! *
157 * *
158 ''', end='', file=sys.stderr)
159
160 for i in range(warn_time, 0, -1):
161 msg = f'* {f"Continuing in {i} seconds...":-70s} *'
162 print(msg, file=sys.stderr)
163 time.sleep(1)
164 print('*' * 74, file=sys.stderr)
165
166
167 @backoff.on_exception(
168 backoff.expo,
169 subprocess.CalledProcessError,
170 max_time=lambda: int(os.environ.get('BACKOFF_MAX_TIME', '30')))
171 def _git_fetch(
172 cachedir: Path,
173 repo: Repo,
174 ref: Ref,
175 force: bool = False) -> None:
176 subprocess.run(['git', '-C', cachedir, 'fetch'] +
177 (['--force'] if force else []) +
178 [repo, f'{ref}:{ref}'], check=True)
179
180
181 def fetch(repo: Repo, ref: Ref, force: bool = False) -> Tuple[Path, Rev]:
182 if force:
183 _show_force_warning()
184 cachedir = git_cachedir(repo)
185 if not os.path.exists(cachedir):
186 logging.debug("Initializing git repo")
187 subprocess.run(['git',
188 '-c',
189 'init.defaultBranch=git-cache--no-default-branch',
190 'init',
191 '--bare',
192 cachedir],
193 check=True,
194 stdout=sys.stderr)
195
196 logging.debug('Fetching ref "%s" from %s', ref, repo)
197 _git_fetch(cachedir, repo, ref, force=force)
198
199 with open(os.path.join(cachedir, 'refs', 'heads', ref), encoding='utf-8') as rev_file:
200 rev = Rev(rev_file.read(999).strip())
201 verify_ancestry(repo, ref, rev, force=force)
202 _log_fetch(repo, ref, rev, force=force)
203
204 return cachedir, rev
205
206
207 def ensure_rev_available(
208 repo: Repo,
209 ref: Ref,
210 rev: Rev,
211 force: bool = False) -> Path:
212 cachedir = git_cachedir(repo)
213 if os.path.exists(cachedir) and is_ancestor(repo, ref, rev):
214 return cachedir
215
216 logging.debug(
217 'We do not have rev %s. We will fetch ref "%s" and hope it appears.',
218 rev, ref)
219 fetch(repo, ref, force=force)
220 logging.debug('Verifying that fetch retrieved rev %s', rev)
221 subprocess.run(['git', '-C', cachedir, 'cat-file', '-e', rev], check=True)
222 verify_ancestry(repo, ref, rev, force=force)
223
224 return cachedir
225
226
227 def _main() -> None:
228 parser = argparse.ArgumentParser(
229 description='Cache remote git repositories locally.',
230 epilog='example usage: git-cache https://github.com/NixOS/nixpkgs.git master')
231 parser.add_argument(
232 '--force',
233 action='store_true',
234 help='Recover from a force-push in the remote repo')
235 parser.add_argument(
236 'repo',
237 metavar='Repo',
238 type=Repo,
239 help='Git repository URL')
240 parser.add_argument(
241 'ref',
242 metavar='Ref',
243 type=Ref,
244 help='Ref (branch or tag) in the git repo')
245 parser.add_argument(
246 'rev',
247 metavar='Rev',
248 type=Rev,
249 nargs='?',
250 help='Ensure that this revision is present. ' +
251 'If this revision is already present locally, no network operations are performed.')
252 args = parser.parse_args()
253
254 if args.rev is None:
255 cachedir, rev = fetch(args.repo, args.ref, force=args.force)
256 print(f'{rev} {cachedir}')
257 else:
258 print(
259 ensure_rev_available(
260 args.repo,
261 args.ref,
262 args.rev,
263 force=args.force))