]> git.scottworley.com Git - pinch/blob - pinch.py
4a6158adb7a7907537d10a9f5273ccc6bd2fafc3
[pinch] / pinch.py
1 import argparse
2 import configparser
3 import filecmp
4 import functools
5 import getpass
6 import hashlib
7 import operator
8 import os
9 import os.path
10 import shlex
11 import shutil
12 import subprocess
13 import tempfile
14 import types
15 import urllib.parse
16 import urllib.request
17 import xml.dom.minidom
18
19 from typing import (
20 Dict,
21 Iterable,
22 List,
23 NewType,
24 Tuple,
25 )
26
27 Digest16 = NewType('Digest16', str)
28 Digest32 = NewType('Digest32', str)
29
30
31 class ChannelTableEntry(types.SimpleNamespace):
32 absolute_url: str
33 digest: Digest16
34 file: str
35 size: int
36 url: str
37
38
39 class Channel(types.SimpleNamespace):
40 channel_html: bytes
41 channel_url: str
42 forwarded_url: str
43 git_ref: str
44 git_repo: str
45 git_revision: str
46 old_git_revision: str
47 release_name: str
48 table: Dict[str, ChannelTableEntry]
49
50
51 class VerificationError(Exception):
52 pass
53
54
55 class Verification:
56
57 def __init__(self) -> None:
58 self.line_length = 0
59
60 def status(self, s: str) -> None:
61 print(s, end=' ', flush=True)
62 self.line_length += 1 + len(s) # Unicode??
63
64 @staticmethod
65 def _color(s: str, c: int) -> str:
66 return '\033[%2dm%s\033[00m' % (c, s)
67
68 def result(self, r: bool) -> None:
69 message, color = {True: ('OK ', 92), False: ('FAIL', 91)}[r]
70 length = len(message)
71 cols = shutil.get_terminal_size().columns
72 pad = (cols - (self.line_length + length)) % cols
73 print(' ' * pad + self._color(message, color))
74 self.line_length = 0
75 if not r:
76 raise VerificationError()
77
78 def check(self, s: str, r: bool) -> None:
79 self.status(s)
80 self.result(r)
81
82 def ok(self) -> None:
83 self.result(True)
84
85
86 def compare(a: str, b: str) -> Tuple[List[str], List[str], List[str]]:
87
88 def throw(error: OSError) -> None:
89 raise error
90
91 def join(x: str, y: str) -> str:
92 return y if x == '.' else os.path.join(x, y)
93
94 def recursive_files(d: str) -> Iterable[str]:
95 all_files: List[str] = []
96 for path, dirs, files in os.walk(d, onerror=throw):
97 rel = os.path.relpath(path, start=d)
98 all_files.extend(join(rel, f) for f in files)
99 for dir_or_link in dirs:
100 if os.path.islink(join(path, dir_or_link)):
101 all_files.append(join(rel, dir_or_link))
102 return all_files
103
104 def exclude_dot_git(files: Iterable[str]) -> Iterable[str]:
105 return (f for f in files if not f.startswith('.git/'))
106
107 files = functools.reduce(
108 operator.or_, (set(
109 exclude_dot_git(
110 recursive_files(x))) for x in [a, b]))
111 return filecmp.cmpfiles(a, b, files, shallow=False)
112
113
114 def fetch(v: Verification, channel: Channel) -> None:
115 v.status('Fetching channel')
116 request = urllib.request.urlopen(channel.channel_url, timeout=10)
117 channel.channel_html = request.read()
118 channel.forwarded_url = request.geturl()
119 v.result(request.status == 200)
120 v.check('Got forwarded', channel.channel_url != channel.forwarded_url)
121
122
123 def parse_channel(v: Verification, channel: Channel) -> None:
124 v.status('Parsing channel description as XML')
125 d = xml.dom.minidom.parseString(channel.channel_html)
126 v.ok()
127
128 v.status('Extracting release name:')
129 title_name = d.getElementsByTagName(
130 'title')[0].firstChild.nodeValue.split()[2]
131 h1_name = d.getElementsByTagName('h1')[0].firstChild.nodeValue.split()[2]
132 v.status(title_name)
133 v.result(title_name == h1_name)
134 channel.release_name = title_name
135
136 v.status('Extracting git commit:')
137 git_commit_node = d.getElementsByTagName('tt')[0]
138 channel.git_revision = git_commit_node.firstChild.nodeValue
139 v.status(channel.git_revision)
140 v.ok()
141 v.status('Verifying git commit label')
142 v.result(git_commit_node.previousSibling.nodeValue == 'Git commit ')
143
144 v.status('Parsing table')
145 channel.table = {}
146 for row in d.getElementsByTagName('tr')[1:]:
147 name = row.childNodes[0].firstChild.firstChild.nodeValue
148 url = row.childNodes[0].firstChild.getAttribute('href')
149 size = int(row.childNodes[1].firstChild.nodeValue)
150 digest = Digest16(row.childNodes[2].firstChild.firstChild.nodeValue)
151 channel.table[name] = ChannelTableEntry(
152 url=url, digest=digest, size=size)
153 v.ok()
154
155
156 def digest_string(s: bytes) -> Digest16:
157 return Digest16(hashlib.sha256(s).hexdigest())
158
159
160 def digest_file(filename: str) -> Digest16:
161 hasher = hashlib.sha256()
162 with open(filename, 'rb') as f:
163 # pylint: disable=cell-var-from-loop
164 for block in iter(lambda: f.read(4096), b''):
165 hasher.update(block)
166 return Digest16(hasher.hexdigest())
167
168
169 def to_Digest16(v: Verification, digest32: Digest32) -> Digest16:
170 v.status('Converting digest to base16')
171 process = subprocess.run(
172 ['nix', 'to-base16', '--type', 'sha256', digest32], capture_output=True)
173 v.result(process.returncode == 0)
174 return Digest16(process.stdout.decode().strip())
175
176
177 def to_Digest32(v: Verification, digest16: Digest16) -> Digest32:
178 v.status('Converting digest to base32')
179 process = subprocess.run(
180 ['nix', 'to-base32', '--type', 'sha256', digest16], capture_output=True)
181 v.result(process.returncode == 0)
182 return Digest32(process.stdout.decode().strip())
183
184
185 def fetch_with_nix_prefetch_url(
186 v: Verification,
187 url: str,
188 digest: Digest16) -> str:
189 v.status('Fetching %s' % url)
190 process = subprocess.run(
191 ['nix-prefetch-url', '--print-path', url, digest], capture_output=True)
192 v.result(process.returncode == 0)
193 prefetch_digest, path, empty = process.stdout.decode().split('\n')
194 assert empty == ''
195 v.check("Verifying nix-prefetch-url's digest",
196 to_Digest16(v, Digest32(prefetch_digest)) == digest)
197 v.status("Verifying file digest")
198 file_digest = digest_file(path)
199 v.result(file_digest == digest)
200 return path
201
202
203 def fetch_resources(v: Verification, channel: Channel) -> None:
204 for resource in ['git-revision', 'nixexprs.tar.xz']:
205 fields = channel.table[resource]
206 fields.absolute_url = urllib.parse.urljoin(
207 channel.forwarded_url, fields.url)
208 fields.file = fetch_with_nix_prefetch_url(
209 v, fields.absolute_url, fields.digest)
210 v.status('Verifying git commit on main page matches git commit in table')
211 v.result(
212 open(
213 channel.table['git-revision'].file).read(999) == channel.git_revision)
214
215
216 def git_cachedir(git_repo: str) -> str:
217 # TODO: Consider using pyxdg to find this path.
218 return os.path.expanduser(
219 '~/.cache/pinch/git/%s' %
220 digest_string(
221 git_repo.encode()))
222
223
224 def verify_git_ancestry(v: Verification, channel: Channel) -> None:
225 cachedir = git_cachedir(channel.git_repo)
226 v.status('Verifying rev is an ancestor of ref')
227 process = subprocess.run(['git',
228 '-C',
229 cachedir,
230 'merge-base',
231 '--is-ancestor',
232 channel.git_revision,
233 channel.git_ref])
234 v.result(process.returncode == 0)
235
236 if hasattr(channel, 'old_git_revision'):
237 v.status(
238 'Verifying rev is an ancestor of previous rev %s' %
239 channel.old_git_revision)
240 process = subprocess.run(['git',
241 '-C',
242 cachedir,
243 'merge-base',
244 '--is-ancestor',
245 channel.old_git_revision,
246 channel.git_revision])
247 v.result(process.returncode == 0)
248
249
250 def git_fetch(v: Verification, channel: Channel) -> None:
251 # It would be nice if we could share the nix git cache, but as of the time
252 # of writing it is transitioning from gitv2 (deprecated) to gitv3 (not ready
253 # yet), and trying to straddle them both is too far into nix implementation
254 # details for my comfort. So we re-implement here half of nix.fetchGit.
255 # :(
256
257 cachedir = git_cachedir(channel.git_repo)
258 if not os.path.exists(cachedir):
259 v.status("Initializing git repo")
260 process = subprocess.run(
261 ['git', 'init', '--bare', cachedir])
262 v.result(process.returncode == 0)
263
264 v.status('Fetching ref "%s" from %s' % (channel.git_ref, channel.git_repo))
265 # We don't use --force here because we want to abort and freak out if forced
266 # updates are happening.
267 process = subprocess.run(['git',
268 '-C',
269 cachedir,
270 'fetch',
271 channel.git_repo,
272 '%s:%s' % (channel.git_ref,
273 channel.git_ref)])
274 v.result(process.returncode == 0)
275
276 if hasattr(channel, 'git_revision'):
277 v.status('Verifying that fetch retrieved this rev')
278 process = subprocess.run(
279 ['git', '-C', cachedir, 'cat-file', '-e', channel.git_revision])
280 v.result(process.returncode == 0)
281 else:
282 channel.git_revision = open(
283 os.path.join(
284 cachedir,
285 'refs',
286 'heads',
287 channel.git_ref)).read(999).strip()
288
289 verify_git_ancestry(v, channel)
290
291
292 def ensure_git_rev_available(v: Verification, channel: Channel) -> None:
293 cachedir = git_cachedir(channel.git_repo)
294 if os.path.exists(cachedir):
295 v.status('Checking if we already have this rev:')
296 process = subprocess.run(
297 ['git', '-C', cachedir, 'cat-file', '-e', channel.git_revision])
298 if process.returncode == 0:
299 v.status('yes')
300 if process.returncode == 1:
301 v.status('no')
302 v.result(process.returncode == 0 or process.returncode == 1)
303 if process.returncode == 0:
304 verify_git_ancestry(v, channel)
305 return
306 git_fetch(v, channel)
307
308
309 def compare_tarball_and_git(
310 v: Verification,
311 channel: Channel,
312 channel_contents: str,
313 git_contents: str) -> None:
314 v.status('Comparing channel tarball with git checkout')
315 match, mismatch, errors = compare(os.path.join(
316 channel_contents, channel.release_name), git_contents)
317 v.ok()
318 v.check('%d files match' % len(match), len(match) > 0)
319 v.check('%d files differ' % len(mismatch), len(mismatch) == 0)
320 expected_errors = [
321 '.git-revision',
322 '.version-suffix',
323 'nixpkgs',
324 'programs.sqlite',
325 'svn-revision']
326 benign_errors = []
327 for ee in expected_errors:
328 if ee in errors:
329 errors.remove(ee)
330 benign_errors.append(ee)
331 v.check(
332 '%d unexpected incomparable files' %
333 len(errors),
334 len(errors) == 0)
335 v.check(
336 '(%d of %d expected incomparable files)' %
337 (len(benign_errors),
338 len(expected_errors)),
339 len(benign_errors) == len(expected_errors))
340
341
342 def extract_tarball(v: Verification, channel: Channel, dest: str) -> None:
343 v.status('Extracting tarball %s' %
344 channel.table['nixexprs.tar.xz'].file)
345 shutil.unpack_archive(
346 channel.table['nixexprs.tar.xz'].file,
347 dest)
348 v.ok()
349
350
351 def git_checkout(v: Verification, channel: Channel, dest: str) -> None:
352 v.status('Checking out corresponding git revision')
353 git = subprocess.Popen(['git',
354 '-C',
355 git_cachedir(channel.git_repo),
356 'archive',
357 channel.git_revision],
358 stdout=subprocess.PIPE)
359 tar = subprocess.Popen(
360 ['tar', 'x', '-C', dest, '-f', '-'], stdin=git.stdout)
361 git.stdout.close()
362 tar.wait()
363 git.wait()
364 v.result(git.returncode == 0 and tar.returncode == 0)
365
366
367 def git_get_tarball(v: Verification, channel: Channel) -> str:
368 with tempfile.TemporaryDirectory() as output_dir:
369 output_filename = os.path.join(
370 output_dir, channel.release_name + '.tar.xz')
371 with open(output_filename, 'w') as output_file:
372 v.status(
373 'Generating tarball for git revision %s' %
374 channel.git_revision)
375 git = subprocess.Popen(['git',
376 '-C',
377 git_cachedir(channel.git_repo),
378 'archive',
379 '--prefix=%s/' % channel.release_name,
380 channel.git_revision],
381 stdout=subprocess.PIPE)
382 xz = subprocess.Popen(['xz'], stdin=git.stdout, stdout=output_file)
383 xz.wait()
384 git.wait()
385 v.result(git.returncode == 0 and xz.returncode == 0)
386
387 v.status('Putting tarball in Nix store')
388 process = subprocess.run(
389 ['nix-store', '--add', output_filename], capture_output=True)
390 v.result(process.returncode == 0)
391 return process.stdout.decode().strip()
392
393
394 def check_channel_metadata(
395 v: Verification,
396 channel: Channel,
397 channel_contents: str) -> None:
398 v.status('Verifying git commit in channel tarball')
399 v.result(
400 open(
401 os.path.join(
402 channel_contents,
403 channel.release_name,
404 '.git-revision')).read(999) == channel.git_revision)
405
406 v.status(
407 'Verifying version-suffix is a suffix of release name %s:' %
408 channel.release_name)
409 version_suffix = open(
410 os.path.join(
411 channel_contents,
412 channel.release_name,
413 '.version-suffix')).read(999)
414 v.status(version_suffix)
415 v.result(channel.release_name.endswith(version_suffix))
416
417
418 def check_channel_contents(v: Verification, channel: Channel) -> None:
419 with tempfile.TemporaryDirectory() as channel_contents, \
420 tempfile.TemporaryDirectory() as git_contents:
421
422 extract_tarball(v, channel, channel_contents)
423 check_channel_metadata(v, channel, channel_contents)
424
425 git_checkout(v, channel, git_contents)
426
427 compare_tarball_and_git(v, channel, channel_contents, git_contents)
428
429 v.status('Removing temporary directories')
430 v.ok()
431
432
433 def pin_channel(v: Verification, channel: Channel) -> None:
434 fetch(v, channel)
435 parse_channel(v, channel)
436 fetch_resources(v, channel)
437 ensure_git_rev_available(v, channel)
438 check_channel_contents(v, channel)
439
440
441 def git_revision_name(v: Verification, channel: Channel) -> str:
442 v.status('Getting commit date')
443 process = subprocess.run(['git',
444 '-C',
445 git_cachedir(channel.git_repo),
446 'lo',
447 '-n1',
448 '--format=%ct-%h',
449 '--abbrev=11',
450 channel.git_revision],
451 capture_output=True)
452 v.result(process.returncode == 0 and process.stdout != '')
453 return '%s-%s' % (os.path.basename(channel.git_repo),
454 process.stdout.decode().strip())
455
456
457 def pin(args: argparse.Namespace) -> None:
458 v = Verification()
459 config = configparser.ConfigParser()
460 config.read_file(open(args.channels_file), args.channels_file)
461 for section in config.sections():
462 if args.channels and section not in args.channels:
463 continue
464
465 channel = Channel(**dict(config[section].items()))
466 if hasattr(channel, 'git_revision'):
467 channel.old_git_revision = channel.git_revision
468 del channel.git_revision
469
470 if 'channel_url' in config[section]:
471 pin_channel(v, channel)
472 config[section]['release_name'] = channel.release_name
473 config[section]['tarball_url'] = channel.table['nixexprs.tar.xz'].absolute_url
474 config[section]['tarball_sha256'] = channel.table['nixexprs.tar.xz'].digest
475 else:
476 git_fetch(v, channel)
477 config[section]['release_name'] = git_revision_name(v, channel)
478 config[section]['git_revision'] = channel.git_revision
479
480 with open(args.channels_file, 'w') as configfile:
481 config.write(configfile)
482
483
484 def update(args: argparse.Namespace) -> None:
485 v = Verification()
486 config = configparser.ConfigParser()
487 config.read_file(open(args.channels_file), args.channels_file)
488 exprs = []
489 for section in config.sections():
490 if 'channel_url' in config[section]:
491 tarball = fetch_with_nix_prefetch_url(
492 v, config[section]['tarball_url'], Digest16(
493 config[section]['tarball_sha256']))
494 else:
495 channel = Channel(**dict(config[section].items()))
496 ensure_git_rev_available(v, channel)
497 tarball = git_get_tarball(v, channel)
498 exprs.append(
499 'f: f { name = "%s"; channelName = "%s"; src = builtins.storePath "%s"; }' %
500 (config[section]['release_name'], section, tarball))
501 command = [
502 'nix-env',
503 '--profile',
504 '/nix/var/nix/profiles/per-user/%s/channels' %
505 getpass.getuser(),
506 '--show-trace',
507 '--file',
508 '<nix/unpack-channel.nix>',
509 '--install',
510 '--from-expression'] + exprs
511 if args.dry_run:
512 print(' '.join(map(shlex.quote, command)))
513 else:
514 v.status('Installing channels with nix-env')
515 process = subprocess.run(command)
516 v.result(process.returncode == 0)
517
518
519 def main() -> None:
520 parser = argparse.ArgumentParser(prog='pinch')
521 subparsers = parser.add_subparsers(dest='mode', required=True)
522 parser_pin = subparsers.add_parser('pin')
523 parser_pin.add_argument('channels_file', type=str)
524 parser_pin.add_argument('channels', type=str, nargs='*')
525 parser_pin.set_defaults(func=pin)
526 parser_update = subparsers.add_parser('update')
527 parser_update.add_argument('--dry-run', action='store_true')
528 parser_update.add_argument('channels_file', type=str)
529 parser_update.set_defaults(func=update)
530 args = parser.parse_args()
531 args.func(args)
532
533
534 main()