]> git.scottworley.com Git - git-cache/blob - git_cache.py
d54041be67fe1d3336cce7f1e03cf272288695ee
[git-cache] / git_cache.py
1 # It would be nice if we could share the nix git cache, but as of the
2 # time of writing it is transitioning from gitv2 (deprecated) to gitv3
3 # (not ready yet), and trying to straddle them both is too far into nix
4 # implementation details for my comfort. So we re-implement here half of
5 # nix's builtins.fetchGit. :(
6
7 import argparse
8 import functools
9 import hashlib
10 import logging
11 import os
12 import subprocess
13 import sys
14 import time
15
16 from typing import Iterator, NamedTuple, Optional, TypeVar, Tuple, Union
17
18 import backoff
19
20 Path = str # eg: "/home/user/.cache/git-cache/v1"
21 Repo = str # eg: "https://github.com/NixOS/nixpkgs.git"
22 Ref = str # eg: "master" or "v1.0.0"
23 Rev = str # eg: "53a27350551844e1ed1a9257690294767389ef0d"
24 RefOrRev = Union[Ref, Rev]
25
26
27 class _LogEntry(NamedTuple):
28 ref: Ref
29 rev: Rev
30
31
32 T = TypeVar('T')
33
34
35 def _repo_hashname(repo: Repo) -> str:
36 return hashlib.sha256(repo.encode()).hexdigest()
37
38
39 def git_cachedir(repo: Repo) -> Path:
40 # Use xdg module when it's less painful to have as a dependency
41 XDG_CACHE_HOME = Path(
42 os.environ.get('XDG_CACHE_HOME', os.path.expanduser('~/.cache')))
43
44 return Path(os.path.join(
45 XDG_CACHE_HOME,
46 'git-cache/v1',
47 _repo_hashname(repo)))
48
49
50 def _log_filename(repo: Repo) -> Path:
51 # Use xdg module when it's less painful to have as a dependency
52 XDG_DATA_HOME = Path(
53 os.environ.get('XDG_DATA_HOME', os.path.expanduser('~/.local/share')))
54
55 return Path(os.path.join(
56 XDG_DATA_HOME,
57 'git-cache/v1',
58 _repo_hashname(repo)))
59
60
61 def is_ancestor(repo: Repo, descendant: RefOrRev, ancestor: RefOrRev) -> bool:
62 cachedir = git_cachedir(repo)
63 logging.debug('Checking if %s is an ancestor of %s', ancestor, descendant)
64 process = subprocess.run(['git',
65 '-C',
66 cachedir,
67 'merge-base',
68 '--is-ancestor',
69 ancestor,
70 descendant],
71 check=False)
72 return process.returncode == 0
73
74
75 def verify_ancestry(
76 repo: Repo,
77 descendant: RefOrRev,
78 ancestor: RefOrRev,
79 force: bool = False) -> None:
80 if not force and not is_ancestor(repo, descendant, ancestor):
81 raise Exception('%s is not an ancestor of %s' % (ancestor, descendant))
82
83
84 def _read_fetch_log(repo: Repo) -> Iterator[_LogEntry]:
85 filename = _log_filename(repo)
86 if not os.path.exists(filename):
87 return
88 with open(filename, 'r') as f:
89 for line in f:
90 _, _, rev, ref = line.strip().split(maxsplit=3)
91 yield _LogEntry(ref, rev)
92
93
94 def _last(it: Iterator[T]) -> Optional[T]:
95 return functools.reduce(lambda a, b: b, it, None)
96
97
98 def _previous_fetched_rev(repo: Repo, ref: Ref) -> Optional[Rev]:
99 return _last(entry.rev for entry in _read_fetch_log(
100 repo) if entry.ref == ref)
101
102
103 def _log_fetch(repo: Repo, ref: Ref, rev: Rev, force: bool = False) -> None:
104 if not force:
105 prev_rev = _previous_fetched_rev(repo, ref)
106 if prev_rev is not None:
107 verify_ancestry(repo, rev, prev_rev)
108 filename = _log_filename(repo)
109 os.makedirs(os.path.dirname(filename), exist_ok=True)
110 with open(filename, 'a') as f:
111 f.write('%s %s %s %s\n' %
112 (time.strftime('%Y-%m%d-%H:%M:%S%z'),
113 ('FORCEDFETCH' if force else 'fetch'), rev, ref))
114
115
116 def _show_force_warning() -> None:
117 print('''
118 **************************************************************************
119 * WARNING: git-cache INVOKED WITH --force! *
120 * *
121 * This mode allows previously-fetched refs to be overwritten to point to *
122 * non-descendants -- commits that don't have the previous version of the *
123 * the ref in their history! *
124 * *
125 * This should only be invoked by a human operator who knows what they're *
126 * doing to correct a specific, known, problem. Care should be taken to *
127 * prevent recurrence. *
128 * *
129 * Press ^C to abort. *
130 * *
131 ''', end='', file=sys.stderr)
132 warn_time_override = os.environ.get('FORCE_WARNING_TIME', None)
133 warn_time: int
134 if warn_time_override is None:
135 warn_time = 15
136 else:
137 warn_time = int(warn_time_override)
138 print(
139 '''* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! *
140 * !! WARNING DISPLAY TIME OVERRIDDEN !! *
141 * !! !! *
142 * !! This message is intended to be displayed long enough for a !! *
143 * !! human operator to read it and have a chance to abort. An !! *
144 * !! override for the delay time is provided FOR THE UNIT TESTS !! *
145 * !! to avoid delaying software builds unnecessarily. This is !! *
146 * !! INTENDED FOR USE IN UNIT TESTS ONLY; THIS MESSAGE SHOULD !! *
147 * !! NEVER BE SEEN OUTSIDE BUILD LOGS! !! *
148 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! *
149 * *
150 ''', end='', file=sys.stderr)
151
152 for i in range(warn_time, 0, -1):
153 msg = '* %-70s *' % ("Continuing in %d seconds..." % i)
154 print(msg, file=sys.stderr)
155 time.sleep(1)
156 print('*' * 74, file=sys.stderr)
157
158
159 @backoff.on_exception(
160 backoff.expo,
161 subprocess.CalledProcessError,
162 max_time=lambda: int(os.environ.get('BACKOFF_MAX_TIME', '30')))
163 def _git_fetch(
164 cachedir: Path,
165 repo: Repo,
166 ref: Ref,
167 force: bool = False) -> None:
168 subprocess.run(['git', '-C', cachedir, 'fetch'] +
169 (['--force'] if force else []) +
170 [repo, '%s:%s' % (ref, ref)], check=True)
171
172
173 def fetch(repo: Repo, ref: Ref, force: bool = False) -> Tuple[Path, Rev]:
174 if force:
175 _show_force_warning()
176 cachedir = git_cachedir(repo)
177 if not os.path.exists(cachedir):
178 logging.debug("Initializing git repo")
179 subprocess.run(['git', 'init', '--bare', cachedir],
180 check=True, stdout=sys.stderr)
181
182 logging.debug('Fetching ref "%s" from %s', ref, repo)
183 _git_fetch(cachedir, repo, ref, force=force)
184
185 with open(os.path.join(cachedir, 'refs', 'heads', ref)) as rev_file:
186 rev = Rev(rev_file.read(999).strip())
187 verify_ancestry(repo, ref, rev, force=force)
188 _log_fetch(repo, ref, rev, force=force)
189
190 return cachedir, rev
191
192
193 def ensure_rev_available(
194 repo: Repo,
195 ref: Ref,
196 rev: Rev,
197 force: bool = False) -> Path:
198 cachedir = git_cachedir(repo)
199 if os.path.exists(cachedir) and is_ancestor(repo, ref, rev):
200 return cachedir
201
202 logging.debug(
203 'We do not have rev %s. We will fetch ref "%s" and hope it appears.',
204 rev, ref)
205 fetch(repo, ref, force=force)
206 logging.debug('Verifying that fetch retrieved rev %s', rev)
207 subprocess.run(['git', '-C', cachedir, 'cat-file', '-e', rev], check=True)
208 verify_ancestry(repo, ref, rev, force=force)
209
210 return cachedir
211
212
213 def _main() -> None:
214 parser = argparse.ArgumentParser(
215 description='Cache remote git repositories locally.',
216 epilog='example usage: git-cache https://github.com/NixOS/nixpkgs.git master')
217 parser.add_argument(
218 '--force',
219 action='store_true',
220 help='Recover from a force-push in the remote repo')
221 parser.add_argument(
222 'repo',
223 metavar='Repo',
224 type=Repo,
225 help='Git repository URL')
226 parser.add_argument(
227 'ref',
228 metavar='Ref',
229 type=Ref,
230 help='Ref (branch or tag) in the git repo')
231 parser.add_argument(
232 'rev',
233 metavar='Rev',
234 type=Rev,
235 nargs='?',
236 help='Ensure that this revision is present. ' +
237 'If this revision is already present locally, no network operations are performed.')
238 args = parser.parse_args()
239
240 if args.rev is None:
241 print('{1} {0}'.format(*fetch(args.repo, args.ref, force=args.force)))
242 else:
243 print(
244 ensure_rev_available(
245 args.repo,
246 args.ref,
247 args.rev,
248 force=args.force))