]> git.scottworley.com Git - pinch/blob - pinch.py
Release: 1.5.1
[pinch] / pinch.py
1 import argparse
2 import configparser
3 import filecmp
4 import functools
5 import getpass
6 import hashlib
7 import operator
8 import os
9 import os.path
10 import shlex
11 import shutil
12 import subprocess
13 import sys
14 import tempfile
15 import types
16 import urllib.parse
17 import urllib.request
18 import xml.dom.minidom
19
20 from typing import (
21 Dict,
22 Iterable,
23 List,
24 NewType,
25 Tuple,
26 )
27
28 # Use xdg module when it's less painful to have as a dependency
29
30
31 class XDG(types.SimpleNamespace):
32 XDG_CACHE_HOME: str
33
34
35 xdg = XDG(
36 XDG_CACHE_HOME=os.getenv(
37 'XDG_CACHE_HOME',
38 os.path.expanduser('~/.cache')))
39
40
41 Digest16 = NewType('Digest16', str)
42 Digest32 = NewType('Digest32', str)
43
44
45 class ChannelTableEntry(types.SimpleNamespace):
46 absolute_url: str
47 digest: Digest16
48 file: str
49 size: int
50 url: str
51
52
53 class Channel(types.SimpleNamespace):
54 alias_of: str
55 channel_html: bytes
56 channel_url: str
57 forwarded_url: str
58 git_ref: str
59 git_repo: str
60 git_revision: str
61 old_git_revision: str
62 release_name: str
63 table: Dict[str, ChannelTableEntry]
64
65
66 class VerificationError(Exception):
67 pass
68
69
70 class Verification:
71
72 def __init__(self) -> None:
73 self.line_length = 0
74
75 def status(self, s: str) -> None:
76 print(s, end=' ', file=sys.stderr, flush=True)
77 self.line_length += 1 + len(s) # Unicode??
78
79 @staticmethod
80 def _color(s: str, c: int) -> str:
81 return '\033[%2dm%s\033[00m' % (c, s)
82
83 def result(self, r: bool) -> None:
84 message, color = {True: ('OK ', 92), False: ('FAIL', 91)}[r]
85 length = len(message)
86 cols = shutil.get_terminal_size().columns or 80
87 pad = (cols - (self.line_length + length)) % cols
88 print(' ' * pad + self._color(message, color), file=sys.stderr)
89 self.line_length = 0
90 if not r:
91 raise VerificationError()
92
93 def check(self, s: str, r: bool) -> None:
94 self.status(s)
95 self.result(r)
96
97 def ok(self) -> None:
98 self.result(True)
99
100
101 def compare(a: str, b: str) -> Tuple[List[str], List[str], List[str]]:
102
103 def throw(error: OSError) -> None:
104 raise error
105
106 def join(x: str, y: str) -> str:
107 return y if x == '.' else os.path.join(x, y)
108
109 def recursive_files(d: str) -> Iterable[str]:
110 all_files: List[str] = []
111 for path, dirs, files in os.walk(d, onerror=throw):
112 rel = os.path.relpath(path, start=d)
113 all_files.extend(join(rel, f) for f in files)
114 for dir_or_link in dirs:
115 if os.path.islink(join(path, dir_or_link)):
116 all_files.append(join(rel, dir_or_link))
117 return all_files
118
119 def exclude_dot_git(files: Iterable[str]) -> Iterable[str]:
120 return (f for f in files if not f.startswith('.git/'))
121
122 files = functools.reduce(
123 operator.or_, (set(
124 exclude_dot_git(
125 recursive_files(x))) for x in [a, b]))
126 return filecmp.cmpfiles(a, b, files, shallow=False)
127
128
129 def fetch(v: Verification, channel: Channel) -> None:
130 v.status('Fetching channel')
131 request = urllib.request.urlopen(channel.channel_url, timeout=10)
132 channel.channel_html = request.read()
133 channel.forwarded_url = request.geturl()
134 v.result(request.status == 200) # type: ignore # (for old mypy)
135 v.check('Got forwarded', channel.channel_url != channel.forwarded_url)
136
137
138 def parse_channel(v: Verification, channel: Channel) -> None:
139 v.status('Parsing channel description as XML')
140 d = xml.dom.minidom.parseString(channel.channel_html)
141 v.ok()
142
143 v.status('Extracting release name:')
144 title_name = d.getElementsByTagName(
145 'title')[0].firstChild.nodeValue.split()[2]
146 h1_name = d.getElementsByTagName('h1')[0].firstChild.nodeValue.split()[2]
147 v.status(title_name)
148 v.result(title_name == h1_name)
149 channel.release_name = title_name
150
151 v.status('Extracting git commit:')
152 git_commit_node = d.getElementsByTagName('tt')[0]
153 channel.git_revision = git_commit_node.firstChild.nodeValue
154 v.status(channel.git_revision)
155 v.ok()
156 v.status('Verifying git commit label')
157 v.result(git_commit_node.previousSibling.nodeValue == 'Git commit ')
158
159 v.status('Parsing table')
160 channel.table = {}
161 for row in d.getElementsByTagName('tr')[1:]:
162 name = row.childNodes[0].firstChild.firstChild.nodeValue
163 url = row.childNodes[0].firstChild.getAttribute('href')
164 size = int(row.childNodes[1].firstChild.nodeValue)
165 digest = Digest16(row.childNodes[2].firstChild.firstChild.nodeValue)
166 channel.table[name] = ChannelTableEntry(
167 url=url, digest=digest, size=size)
168 v.ok()
169
170
171 def digest_string(s: bytes) -> Digest16:
172 return Digest16(hashlib.sha256(s).hexdigest())
173
174
175 def digest_file(filename: str) -> Digest16:
176 hasher = hashlib.sha256()
177 with open(filename, 'rb') as f:
178 # pylint: disable=cell-var-from-loop
179 for block in iter(lambda: f.read(4096), b''):
180 hasher.update(block)
181 return Digest16(hasher.hexdigest())
182
183
184 def to_Digest16(v: Verification, digest32: Digest32) -> Digest16:
185 v.status('Converting digest to base16')
186 process = subprocess.run(
187 ['nix', 'to-base16', '--type', 'sha256', digest32], stdout=subprocess.PIPE)
188 v.result(process.returncode == 0)
189 return Digest16(process.stdout.decode().strip())
190
191
192 def to_Digest32(v: Verification, digest16: Digest16) -> Digest32:
193 v.status('Converting digest to base32')
194 process = subprocess.run(
195 ['nix', 'to-base32', '--type', 'sha256', digest16], stdout=subprocess.PIPE)
196 v.result(process.returncode == 0)
197 return Digest32(process.stdout.decode().strip())
198
199
200 def fetch_with_nix_prefetch_url(
201 v: Verification,
202 url: str,
203 digest: Digest16) -> str:
204 v.status('Fetching %s' % url)
205 process = subprocess.run(
206 ['nix-prefetch-url', '--print-path', url, digest], stdout=subprocess.PIPE)
207 v.result(process.returncode == 0)
208 prefetch_digest, path, empty = process.stdout.decode().split('\n')
209 assert empty == ''
210 v.check("Verifying nix-prefetch-url's digest",
211 to_Digest16(v, Digest32(prefetch_digest)) == digest)
212 v.status("Verifying file digest")
213 file_digest = digest_file(path)
214 v.result(file_digest == digest)
215 return path # type: ignore # (for old mypy)
216
217
218 def fetch_resources(v: Verification, channel: Channel) -> None:
219 for resource in ['git-revision', 'nixexprs.tar.xz']:
220 fields = channel.table[resource]
221 fields.absolute_url = urllib.parse.urljoin(
222 channel.forwarded_url, fields.url)
223 fields.file = fetch_with_nix_prefetch_url(
224 v, fields.absolute_url, fields.digest)
225 v.status('Verifying git commit on main page matches git commit in table')
226 v.result(
227 open(
228 channel.table['git-revision'].file).read(999) == channel.git_revision)
229
230
231 def git_cachedir(git_repo: str) -> str:
232 return os.path.join(
233 xdg.XDG_CACHE_HOME,
234 'pinch/git',
235 digest_string(git_repo.encode()))
236
237
238 def tarball_cache_file(channel: Channel) -> str:
239 return os.path.join(
240 xdg.XDG_CACHE_HOME,
241 'pinch/git-tarball',
242 '%s-%s-%s' %
243 (digest_string(channel.git_repo.encode()),
244 channel.git_revision,
245 channel.release_name))
246
247
248 def verify_git_ancestry(v: Verification, channel: Channel) -> None:
249 cachedir = git_cachedir(channel.git_repo)
250 v.status('Verifying rev is an ancestor of ref')
251 process = subprocess.run(['git',
252 '-C',
253 cachedir,
254 'merge-base',
255 '--is-ancestor',
256 channel.git_revision,
257 channel.git_ref])
258 v.result(process.returncode == 0)
259
260 if hasattr(channel, 'old_git_revision'):
261 v.status(
262 'Verifying rev is an ancestor of previous rev %s' %
263 channel.old_git_revision)
264 process = subprocess.run(['git',
265 '-C',
266 cachedir,
267 'merge-base',
268 '--is-ancestor',
269 channel.old_git_revision,
270 channel.git_revision])
271 v.result(process.returncode == 0)
272
273
274 def git_fetch(v: Verification, channel: Channel) -> None:
275 # It would be nice if we could share the nix git cache, but as of the time
276 # of writing it is transitioning from gitv2 (deprecated) to gitv3 (not ready
277 # yet), and trying to straddle them both is too far into nix implementation
278 # details for my comfort. So we re-implement here half of nix.fetchGit.
279 # :(
280
281 cachedir = git_cachedir(channel.git_repo)
282 if not os.path.exists(cachedir):
283 v.status("Initializing git repo")
284 process = subprocess.run(
285 ['git', 'init', '--bare', cachedir])
286 v.result(process.returncode == 0)
287
288 v.status('Fetching ref "%s" from %s' % (channel.git_ref, channel.git_repo))
289 # We don't use --force here because we want to abort and freak out if forced
290 # updates are happening.
291 process = subprocess.run(['git',
292 '-C',
293 cachedir,
294 'fetch',
295 channel.git_repo,
296 '%s:%s' % (channel.git_ref,
297 channel.git_ref)])
298 v.result(process.returncode == 0)
299
300 if hasattr(channel, 'git_revision'):
301 v.status('Verifying that fetch retrieved this rev')
302 process = subprocess.run(
303 ['git', '-C', cachedir, 'cat-file', '-e', channel.git_revision])
304 v.result(process.returncode == 0)
305 else:
306 channel.git_revision = open(
307 os.path.join(
308 cachedir,
309 'refs',
310 'heads',
311 channel.git_ref)).read(999).strip()
312
313 verify_git_ancestry(v, channel)
314
315
316 def ensure_git_rev_available(v: Verification, channel: Channel) -> None:
317 cachedir = git_cachedir(channel.git_repo)
318 if os.path.exists(cachedir):
319 v.status('Checking if we already have this rev:')
320 process = subprocess.run(
321 ['git', '-C', cachedir, 'cat-file', '-e', channel.git_revision])
322 if process.returncode == 0:
323 v.status('yes')
324 if process.returncode == 1:
325 v.status('no')
326 v.result(process.returncode == 0 or process.returncode == 1)
327 if process.returncode == 0:
328 verify_git_ancestry(v, channel)
329 return
330 git_fetch(v, channel)
331
332
333 def compare_tarball_and_git(
334 v: Verification,
335 channel: Channel,
336 channel_contents: str,
337 git_contents: str) -> None:
338 v.status('Comparing channel tarball with git checkout')
339 match, mismatch, errors = compare(os.path.join(
340 channel_contents, channel.release_name), git_contents)
341 v.ok()
342 v.check('%d files match' % len(match), len(match) > 0)
343 v.check('%d files differ' % len(mismatch), len(mismatch) == 0)
344 expected_errors = [
345 '.git-revision',
346 '.version-suffix',
347 'nixpkgs',
348 'programs.sqlite',
349 'svn-revision']
350 benign_errors = []
351 for ee in expected_errors:
352 if ee in errors:
353 errors.remove(ee)
354 benign_errors.append(ee)
355 v.check(
356 '%d unexpected incomparable files' %
357 len(errors),
358 len(errors) == 0)
359 v.check(
360 '(%d of %d expected incomparable files)' %
361 (len(benign_errors),
362 len(expected_errors)),
363 len(benign_errors) == len(expected_errors))
364
365
366 def extract_tarball(v: Verification, channel: Channel, dest: str) -> None:
367 v.status('Extracting tarball %s' %
368 channel.table['nixexprs.tar.xz'].file)
369 shutil.unpack_archive(
370 channel.table['nixexprs.tar.xz'].file,
371 dest)
372 v.ok()
373
374
375 def git_checkout(v: Verification, channel: Channel, dest: str) -> None:
376 v.status('Checking out corresponding git revision')
377 git = subprocess.Popen(['git',
378 '-C',
379 git_cachedir(channel.git_repo),
380 'archive',
381 channel.git_revision],
382 stdout=subprocess.PIPE)
383 tar = subprocess.Popen(
384 ['tar', 'x', '-C', dest, '-f', '-'], stdin=git.stdout)
385 if git.stdout:
386 git.stdout.close()
387 tar.wait()
388 git.wait()
389 v.result(git.returncode == 0 and tar.returncode == 0)
390
391
392 def git_get_tarball(v: Verification, channel: Channel) -> str:
393 cache_file = tarball_cache_file(channel)
394 if os.path.exists(cache_file):
395 cached_tarball = open(cache_file).read(9999)
396 if os.path.exists(cached_tarball):
397 return cached_tarball
398
399 with tempfile.TemporaryDirectory() as output_dir:
400 output_filename = os.path.join(
401 output_dir, channel.release_name + '.tar.xz')
402 with open(output_filename, 'w') as output_file:
403 v.status(
404 'Generating tarball for git revision %s' %
405 channel.git_revision)
406 git = subprocess.Popen(['git',
407 '-C',
408 git_cachedir(channel.git_repo),
409 'archive',
410 '--prefix=%s/' % channel.release_name,
411 channel.git_revision],
412 stdout=subprocess.PIPE)
413 xz = subprocess.Popen(['xz'], stdin=git.stdout, stdout=output_file)
414 xz.wait()
415 git.wait()
416 v.result(git.returncode == 0 and xz.returncode == 0)
417
418 v.status('Putting tarball in Nix store')
419 process = subprocess.run(
420 ['nix-store', '--add', output_filename], stdout=subprocess.PIPE)
421 v.result(process.returncode == 0)
422 store_tarball = process.stdout.decode().strip()
423
424 os.makedirs(os.path.dirname(cache_file), exist_ok=True)
425 open(cache_file, 'w').write(store_tarball)
426 return store_tarball # type: ignore # (for old mypy)
427
428
429 def check_channel_metadata(
430 v: Verification,
431 channel: Channel,
432 channel_contents: str) -> None:
433 v.status('Verifying git commit in channel tarball')
434 v.result(
435 open(
436 os.path.join(
437 channel_contents,
438 channel.release_name,
439 '.git-revision')).read(999) == channel.git_revision)
440
441 v.status(
442 'Verifying version-suffix is a suffix of release name %s:' %
443 channel.release_name)
444 version_suffix = open(
445 os.path.join(
446 channel_contents,
447 channel.release_name,
448 '.version-suffix')).read(999)
449 v.status(version_suffix)
450 v.result(channel.release_name.endswith(version_suffix))
451
452
453 def check_channel_contents(v: Verification, channel: Channel) -> None:
454 with tempfile.TemporaryDirectory() as channel_contents, \
455 tempfile.TemporaryDirectory() as git_contents:
456
457 extract_tarball(v, channel, channel_contents)
458 check_channel_metadata(v, channel, channel_contents)
459
460 git_checkout(v, channel, git_contents)
461
462 compare_tarball_and_git(v, channel, channel_contents, git_contents)
463
464 v.status('Removing temporary directories')
465 v.ok()
466
467
468 def pin_channel(v: Verification, channel: Channel) -> None:
469 fetch(v, channel)
470 parse_channel(v, channel)
471 fetch_resources(v, channel)
472 ensure_git_rev_available(v, channel)
473 check_channel_contents(v, channel)
474
475
476 def git_revision_name(v: Verification, channel: Channel) -> str:
477 v.status('Getting commit date')
478 process = subprocess.run(['git',
479 '-C',
480 git_cachedir(channel.git_repo),
481 'log',
482 '-n1',
483 '--format=%ct-%h',
484 '--abbrev=11',
485 '--no-show-signature',
486 channel.git_revision],
487 stdout=subprocess.PIPE)
488 v.result(process.returncode == 0 and process.stdout != b'')
489 return '%s-%s' % (os.path.basename(channel.git_repo),
490 process.stdout.decode().strip())
491
492
493 def read_config(filename: str) -> configparser.ConfigParser:
494 config = configparser.ConfigParser()
495 config.read_file(open(filename), filename)
496 return config
497
498
499 def pin(args: argparse.Namespace) -> None:
500 v = Verification()
501 config = read_config(args.channels_file)
502 for section in config.sections():
503 if args.channels and section not in args.channels:
504 continue
505
506 channel = Channel(**dict(config[section].items()))
507
508 if hasattr(channel, 'alias_of'):
509 assert not hasattr(channel, 'git_repo')
510 continue
511
512 if hasattr(channel, 'git_revision'):
513 channel.old_git_revision = channel.git_revision
514 del channel.git_revision
515
516 if 'channel_url' in config[section]:
517 pin_channel(v, channel)
518 config[section]['release_name'] = channel.release_name
519 config[section]['tarball_url'] = channel.table['nixexprs.tar.xz'].absolute_url
520 config[section]['tarball_sha256'] = channel.table['nixexprs.tar.xz'].digest
521 else:
522 git_fetch(v, channel)
523 config[section]['release_name'] = git_revision_name(v, channel)
524 config[section]['git_revision'] = channel.git_revision
525
526 with open(args.channels_file, 'w') as configfile:
527 config.write(configfile)
528
529
530 def fetch_channel(
531 v: Verification,
532 section: str,
533 conf: configparser.SectionProxy) -> str:
534 if 'git_repo' not in conf or 'release_name' not in conf:
535 raise Exception(
536 'Cannot update unpinned channel "%s" (Run "pin" before "update")' %
537 section)
538
539 if 'channel_url' in conf:
540 return fetch_with_nix_prefetch_url(
541 v, conf['tarball_url'], Digest16(
542 conf['tarball_sha256']))
543
544 channel = Channel(**dict(conf.items()))
545 ensure_git_rev_available(v, channel)
546 return git_get_tarball(v, channel)
547
548
549 def update(args: argparse.Namespace) -> None:
550 v = Verification()
551 config = configparser.ConfigParser()
552 exprs: Dict[str, str] = {}
553 configs = [read_config(filename) for filename in args.channels_file]
554 for config in configs:
555 for section in config.sections():
556 if 'alias_of' in config[section]:
557 assert 'git_repo' not in config[section]
558 continue
559 tarball = fetch_channel(v, section, config[section])
560 if section in exprs:
561 raise Exception('Duplicate channel "%s"' % section)
562 exprs[section] = (
563 'f: f { name = "%s"; channelName = "%%s"; src = builtins.storePath "%s"; }' %
564 (config[section]['release_name'], tarball))
565
566 for config in configs:
567 for section in config.sections():
568 if 'alias_of' in config[section]:
569 if section in exprs:
570 raise Exception('Duplicate channel "%s"' % section)
571 exprs[section] = exprs[str(config[section]['alias_of'])]
572
573 command = [
574 'nix-env',
575 '--profile',
576 '/nix/var/nix/profiles/per-user/%s/channels' %
577 getpass.getuser(),
578 '--show-trace',
579 '--file',
580 '<nix/unpack-channel.nix>',
581 '--install',
582 '--from-expression'] + [exprs[name] % name for name in sorted(exprs.keys())]
583 if args.dry_run:
584 print(' '.join(map(shlex.quote, command)))
585 else:
586 v.status('Installing channels with nix-env')
587 process = subprocess.run(command)
588 v.result(process.returncode == 0)
589
590
591 def main() -> None:
592 parser = argparse.ArgumentParser(prog='pinch')
593 subparsers = parser.add_subparsers(dest='mode', required=True)
594 parser_pin = subparsers.add_parser('pin')
595 parser_pin.add_argument('channels_file', type=str)
596 parser_pin.add_argument('channels', type=str, nargs='*')
597 parser_pin.set_defaults(func=pin)
598 parser_update = subparsers.add_parser('update')
599 parser_update.add_argument('--dry-run', action='store_true')
600 parser_update.add_argument('channels_file', type=str, nargs='+')
601 parser_update.set_defaults(func=update)
602 args = parser.parse_args()
603 args.func(args)
604
605
606 if __name__ == '__main__':
607 main()