]> git.scottworley.com Git - git-cache/blame - git_cache.py
Release 1.4.2
[git-cache] / git_cache.py
CommitLineData
bef7ce53
SW
1# It would be nice if we could share the nix git cache, but as of the
2# time of writing it is transitioning from gitv2 (deprecated) to gitv3
3# (not ready yet), and trying to straddle them both is too far into nix
4# implementation details for my comfort. So we re-implement here half of
5# nix's builtins.fetchGit. :(
6
d1ab0853 7import argparse
35000f72 8import functools
bef7ce53
SW
9import hashlib
10import logging
11import os
12import subprocess
347be7cf 13import sys
35000f72 14import time
bef7ce53 15
35000f72 16from typing import Iterator, NamedTuple, Optional, TypeVar, Tuple, Union
bef7ce53 17
f36d5c6f
SW
18import backoff
19
35000f72
SW
20Path = str # eg: "/home/user/.cache/git-cache/v1"
21Repo = str # eg: "https://github.com/NixOS/nixpkgs.git"
22Ref = str # eg: "master" or "v1.0.0"
23Rev = str # eg: "53a27350551844e1ed1a9257690294767389ef0d"
a5d42d8d 24RefOrRev = Union[Ref, Rev]
bef7ce53
SW
25
26
35000f72
SW
27class _LogEntry(NamedTuple):
28 ref: Ref
29 rev: Rev
30
31
32T = TypeVar('T')
33
34
35def _repo_hashname(repo: Repo) -> str:
36 return hashlib.sha256(repo.encode()).hexdigest()
37
38
bef7ce53
SW
39def git_cachedir(repo: Repo) -> Path:
40 # Use xdg module when it's less painful to have as a dependency
41 XDG_CACHE_HOME = Path(
42 os.environ.get('XDG_CACHE_HOME', os.path.expanduser('~/.cache')))
43
44 return Path(os.path.join(
45 XDG_CACHE_HOME,
46 'git-cache/v1',
35000f72
SW
47 _repo_hashname(repo)))
48
49
50def _log_filename(repo: Repo) -> Path:
51 # Use xdg module when it's less painful to have as a dependency
52 XDG_DATA_HOME = Path(
53 os.environ.get('XDG_DATA_HOME', os.path.expanduser('~/.local/share')))
54
55 return Path(os.path.join(
56 XDG_DATA_HOME,
57 'git-cache/v1',
58 _repo_hashname(repo)))
bef7ce53
SW
59
60
a5d42d8d 61def is_ancestor(repo: Repo, descendant: RefOrRev, ancestor: RefOrRev) -> bool:
bef7ce53 62 cachedir = git_cachedir(repo)
a5d42d8d
SW
63 logging.debug('Checking if %s is an ancestor of %s', ancestor, descendant)
64 process = subprocess.run(['git',
65 '-C',
66 cachedir,
67 'merge-base',
68 '--is-ancestor',
69 ancestor,
70 descendant],
71 check=False)
eb638847
SW
72 return process.returncode == 0
73
74
a5d42d8d
SW
75def verify_ancestry(
76 repo: Repo,
77 descendant: RefOrRev,
083b90e7
SW
78 ancestor: RefOrRev,
79 force: bool = False) -> None:
80 if not force and not is_ancestor(repo, descendant, ancestor):
f580771a 81 raise Exception(f'{ancestor} is not an ancestor of {descendant}')
bef7ce53
SW
82
83
35000f72
SW
84def _read_fetch_log(repo: Repo) -> Iterator[_LogEntry]:
85 filename = _log_filename(repo)
86 if not os.path.exists(filename):
87 return
a28f4bb7 88 with open(filename, 'r', encoding='utf-8') as f:
35000f72
SW
89 for line in f:
90 _, _, rev, ref = line.strip().split(maxsplit=3)
91 yield _LogEntry(ref, rev)
92
93
94def _last(it: Iterator[T]) -> Optional[T]:
95 return functools.reduce(lambda a, b: b, it, None)
96
97
98def _previous_fetched_rev(repo: Repo, ref: Ref) -> Optional[Rev]:
99 return _last(entry.rev for entry in _read_fetch_log(
100 repo) if entry.ref == ref)
101
102
083b90e7
SW
103def _log_fetch(repo: Repo, ref: Ref, rev: Rev, force: bool = False) -> None:
104 if not force:
105 prev_rev = _previous_fetched_rev(repo, ref)
106 if prev_rev is not None:
107 verify_ancestry(repo, rev, prev_rev)
35000f72
SW
108 filename = _log_filename(repo)
109 os.makedirs(os.path.dirname(filename), exist_ok=True)
a28f4bb7 110 with open(filename, 'a', encoding='utf-8') as f:
f580771a
SW
111 f.write(
112 f'{time.strftime("%Y-%m%d-%H:%M:%S%z")} '
113 f'{"FORCEDFETCH" if force else "fetch"} {rev} {ref}\n'
114 )
083b90e7
SW
115
116
117def _show_force_warning() -> None:
118 print('''
119**************************************************************************
120* WARNING: git-cache INVOKED WITH --force! *
121* *
122* This mode allows previously-fetched refs to be overwritten to point to *
123* non-descendants -- commits that don't have the previous version of the *
124* the ref in their history! *
125* *
126* This should only be invoked by a human operator who knows what they're *
127* doing to correct a specific, known, problem. Care should be taken to *
128* prevent recurrence. *
129* *
130* Press ^C to abort. *
131* *
132''', end='', file=sys.stderr)
133 warn_time_override = os.environ.get('FORCE_WARNING_TIME', None)
134 warn_time: int
135 if warn_time_override is None:
136 warn_time = 15
137 else:
138 warn_time = int(warn_time_override)
139 print(
140 '''* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! *
141* !! WARNING DISPLAY TIME OVERRIDDEN !! *
142* !! !! *
143* !! This message is intended to be displayed long enough for a !! *
144* !! human operator to read it and have a chance to abort. An !! *
145* !! override for the delay time is provided FOR THE UNIT TESTS !! *
146* !! to avoid delaying software builds unnecessarily. This is !! *
147* !! INTENDED FOR USE IN UNIT TESTS ONLY; THIS MESSAGE SHOULD !! *
148* !! NEVER BE SEEN OUTSIDE BUILD LOGS! !! *
149* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! *
150* *
151''', end='', file=sys.stderr)
152
153 for i in range(warn_time, 0, -1):
f580771a 154 msg = f'* {f"Continuing in {i} seconds...":-70s} *'
083b90e7
SW
155 print(msg, file=sys.stderr)
156 time.sleep(1)
157 print('*' * 74, file=sys.stderr)
158
35000f72 159
f36d5c6f
SW
160@backoff.on_exception(
161 backoff.expo,
162 subprocess.CalledProcessError,
163 max_time=lambda: int(os.environ.get('BACKOFF_MAX_TIME', '30')))
083b90e7
SW
164def _git_fetch(
165 cachedir: Path,
166 repo: Repo,
167 ref: Ref,
168 force: bool = False) -> None:
169 subprocess.run(['git', '-C', cachedir, 'fetch'] +
170 (['--force'] if force else []) +
f580771a 171 [repo, f'{ref}:{ref}'], check=True)
f36d5c6f
SW
172
173
083b90e7
SW
174def fetch(repo: Repo, ref: Ref, force: bool = False) -> Tuple[Path, Rev]:
175 if force:
176 _show_force_warning()
bef7ce53
SW
177 cachedir = git_cachedir(repo)
178 if not os.path.exists(cachedir):
179 logging.debug("Initializing git repo")
21971f7f
SW
180 subprocess.run(['git',
181 '-c',
182 'init.defaultBranch=git-cache--no-default-branch',
183 'init',
184 '--bare',
185 cachedir],
186 check=True,
187 stdout=sys.stderr)
bef7ce53
SW
188
189 logging.debug('Fetching ref "%s" from %s', ref, repo)
083b90e7 190 _git_fetch(cachedir, repo, ref, force=force)
bef7ce53 191
a28f4bb7 192 with open(os.path.join(cachedir, 'refs', 'heads', ref), encoding='utf-8') as rev_file:
bef7ce53 193 rev = Rev(rev_file.read(999).strip())
083b90e7
SW
194 verify_ancestry(repo, ref, rev, force=force)
195 _log_fetch(repo, ref, rev, force=force)
bef7ce53
SW
196
197 return cachedir, rev
198
199
083b90e7
SW
200def ensure_rev_available(
201 repo: Repo,
202 ref: Ref,
203 rev: Rev,
204 force: bool = False) -> Path:
bef7ce53 205 cachedir = git_cachedir(repo)
eb638847
SW
206 if os.path.exists(cachedir) and is_ancestor(repo, ref, rev):
207 return cachedir
bef7ce53
SW
208
209 logging.debug(
210 'We do not have rev %s. We will fetch ref "%s" and hope it appears.',
211 rev, ref)
083b90e7 212 fetch(repo, ref, force=force)
bef7ce53
SW
213 logging.debug('Verifying that fetch retrieved rev %s', rev)
214 subprocess.run(['git', '-C', cachedir, 'cat-file', '-e', rev], check=True)
083b90e7 215 verify_ancestry(repo, ref, rev, force=force)
bef7ce53
SW
216
217 return cachedir
347be7cf
SW
218
219
220def _main() -> None:
d1ab0853
SW
221 parser = argparse.ArgumentParser(
222 description='Cache remote git repositories locally.',
223 epilog='example usage: git-cache https://github.com/NixOS/nixpkgs.git master')
083b90e7
SW
224 parser.add_argument(
225 '--force',
226 action='store_true',
227 help='Recover from a force-push in the remote repo')
d1ab0853
SW
228 parser.add_argument(
229 'repo',
230 metavar='Repo',
231 type=Repo,
232 help='Git repository URL')
233 parser.add_argument(
234 'ref',
235 metavar='Ref',
236 type=Ref,
237 help='Ref (branch or tag) in the git repo')
238 parser.add_argument(
239 'rev',
240 metavar='Rev',
241 type=Rev,
242 nargs='?',
243 help='Ensure that this revision is present. ' +
244 'If this revision is already present locally, no network operations are performed.')
245 args = parser.parse_args()
246
247 if args.rev is None:
f580771a
SW
248 cachedir, rev = fetch(args.repo, args.ref, force=args.force)
249 print(f'{rev} {cachedir}')
347be7cf 250 else:
083b90e7
SW
251 print(
252 ensure_rev_available(
253 args.repo,
254 args.ref,
255 args.rev,
256 force=args.force))