]> git.scottworley.com Git - pinch/blame - pinch.py
Reject duplicate channel names
[pinch] / pinch.py
CommitLineData
0e5e611d 1import argparse
f15e458d 2import configparser
2f96f32a
SW
3import filecmp
4import functools
736c25eb 5import getpass
2f96f32a
SW
6import hashlib
7import operator
8import os
9import os.path
9a78329e 10import shlex
2f96f32a 11import shutil
73bec7e8 12import subprocess
9d7844bb 13import sys
2f96f32a 14import tempfile
89e79125 15import types
2f96f32a
SW
16import urllib.parse
17import urllib.request
18import xml.dom.minidom
19
20from typing import (
2f96f32a
SW
21 Dict,
22 Iterable,
23 List,
73bec7e8 24 NewType,
2f96f32a
SW
25 Tuple,
26)
27
73bec7e8
SW
28Digest16 = NewType('Digest16', str)
29Digest32 = NewType('Digest32', str)
30
2f96f32a 31
72d3478a 32class ChannelTableEntry(types.SimpleNamespace):
e434d96d 33 absolute_url: str
73bec7e8 34 digest: Digest16
89e79125
SW
35 file: str
36 size: int
37 url: str
38
39
72d3478a 40class Channel(types.SimpleNamespace):
17906b27 41 alias_of: str
89e79125 42 channel_html: bytes
b17def3f 43 channel_url: str
89e79125 44 forwarded_url: str
dc038df0
SW
45 git_ref: str
46 git_repo: str
89e79125 47 git_revision: str
8fca6c28 48 old_git_revision: str
3e6421c4 49 release_name: str
72d3478a 50 table: Dict[str, ChannelTableEntry]
89e79125
SW
51
52
2f96f32a
SW
53class VerificationError(Exception):
54 pass
55
56
57class Verification:
58
59 def __init__(self) -> None:
60 self.line_length = 0
61
62 def status(self, s: str) -> None:
9d7844bb 63 print(s, end=' ', file=sys.stderr, flush=True)
2f96f32a
SW
64 self.line_length += 1 + len(s) # Unicode??
65
66 @staticmethod
67 def _color(s: str, c: int) -> str:
68 return '\033[%2dm%s\033[00m' % (c, s)
69
70 def result(self, r: bool) -> None:
71 message, color = {True: ('OK ', 92), False: ('FAIL', 91)}[r]
72 length = len(message)
73 cols = shutil.get_terminal_size().columns
74 pad = (cols - (self.line_length + length)) % cols
9d7844bb 75 print(' ' * pad + self._color(message, color), file=sys.stderr)
2f96f32a
SW
76 self.line_length = 0
77 if not r:
78 raise VerificationError()
79
80 def check(self, s: str, r: bool) -> None:
81 self.status(s)
82 self.result(r)
83
84 def ok(self) -> None:
85 self.result(True)
86
87
dc038df0 88def compare(a: str, b: str) -> Tuple[List[str], List[str], List[str]]:
2f96f32a
SW
89
90 def throw(error: OSError) -> None:
91 raise error
92
93 def join(x: str, y: str) -> str:
94 return y if x == '.' else os.path.join(x, y)
95
96 def recursive_files(d: str) -> Iterable[str]:
97 all_files: List[str] = []
98 for path, dirs, files in os.walk(d, onerror=throw):
99 rel = os.path.relpath(path, start=d)
100 all_files.extend(join(rel, f) for f in files)
101 for dir_or_link in dirs:
102 if os.path.islink(join(path, dir_or_link)):
103 all_files.append(join(rel, dir_or_link))
104 return all_files
105
106 def exclude_dot_git(files: Iterable[str]) -> Iterable[str]:
107 return (f for f in files if not f.startswith('.git/'))
108
109 files = functools.reduce(
110 operator.or_, (set(
111 exclude_dot_git(
112 recursive_files(x))) for x in [a, b]))
113 return filecmp.cmpfiles(a, b, files, shallow=False)
114
115
ca2c3edd 116def fetch(v: Verification, channel: Channel) -> None:
2f96f32a 117 v.status('Fetching channel')
b17def3f 118 request = urllib.request.urlopen(channel.channel_url, timeout=10)
72d3478a
SW
119 channel.channel_html = request.read()
120 channel.forwarded_url = request.geturl()
2f96f32a 121 v.result(request.status == 200)
b17def3f 122 v.check('Got forwarded', channel.channel_url != channel.forwarded_url)
2f96f32a
SW
123
124
72d3478a 125def parse_channel(v: Verification, channel: Channel) -> None:
2f96f32a 126 v.status('Parsing channel description as XML')
72d3478a 127 d = xml.dom.minidom.parseString(channel.channel_html)
2f96f32a
SW
128 v.ok()
129
3e6421c4
SW
130 v.status('Extracting release name:')
131 title_name = d.getElementsByTagName(
132 'title')[0].firstChild.nodeValue.split()[2]
133 h1_name = d.getElementsByTagName('h1')[0].firstChild.nodeValue.split()[2]
134 v.status(title_name)
135 v.result(title_name == h1_name)
72d3478a 136 channel.release_name = title_name
3e6421c4
SW
137
138 v.status('Extracting git commit:')
2f96f32a 139 git_commit_node = d.getElementsByTagName('tt')[0]
61aaf799
SW
140 channel.git_revision = git_commit_node.firstChild.nodeValue
141 v.status(channel.git_revision)
2f96f32a
SW
142 v.ok()
143 v.status('Verifying git commit label')
144 v.result(git_commit_node.previousSibling.nodeValue == 'Git commit ')
145
146 v.status('Parsing table')
72d3478a 147 channel.table = {}
2f96f32a
SW
148 for row in d.getElementsByTagName('tr')[1:]:
149 name = row.childNodes[0].firstChild.firstChild.nodeValue
150 url = row.childNodes[0].firstChild.getAttribute('href')
151 size = int(row.childNodes[1].firstChild.nodeValue)
73bec7e8 152 digest = Digest16(row.childNodes[2].firstChild.firstChild.nodeValue)
dc038df0
SW
153 channel.table[name] = ChannelTableEntry(
154 url=url, digest=digest, size=size)
2f96f32a
SW
155 v.ok()
156
157
dc038df0
SW
158def digest_string(s: bytes) -> Digest16:
159 return Digest16(hashlib.sha256(s).hexdigest())
160
161
73bec7e8
SW
162def digest_file(filename: str) -> Digest16:
163 hasher = hashlib.sha256()
164 with open(filename, 'rb') as f:
165 # pylint: disable=cell-var-from-loop
166 for block in iter(lambda: f.read(4096), b''):
167 hasher.update(block)
168 return Digest16(hasher.hexdigest())
169
170
171def to_Digest16(v: Verification, digest32: Digest32) -> Digest16:
172 v.status('Converting digest to base16')
173 process = subprocess.run(
174 ['nix', 'to-base16', '--type', 'sha256', digest32], capture_output=True)
175 v.result(process.returncode == 0)
176 return Digest16(process.stdout.decode().strip())
177
178
179def to_Digest32(v: Verification, digest16: Digest16) -> Digest32:
180 v.status('Converting digest to base32')
181 process = subprocess.run(
182 ['nix', 'to-base32', '--type', 'sha256', digest16], capture_output=True)
183 v.result(process.returncode == 0)
184 return Digest32(process.stdout.decode().strip())
185
186
187def fetch_with_nix_prefetch_url(
188 v: Verification,
189 url: str,
190 digest: Digest16) -> str:
191 v.status('Fetching %s' % url)
192 process = subprocess.run(
193 ['nix-prefetch-url', '--print-path', url, digest], capture_output=True)
194 v.result(process.returncode == 0)
195 prefetch_digest, path, empty = process.stdout.decode().split('\n')
196 assert empty == ''
197 v.check("Verifying nix-prefetch-url's digest",
198 to_Digest16(v, Digest32(prefetch_digest)) == digest)
199 v.status("Verifying file digest")
200 file_digest = digest_file(path)
201 v.result(file_digest == digest)
202 return path
2f96f32a 203
73bec7e8 204
72d3478a 205def fetch_resources(v: Verification, channel: Channel) -> None:
2f96f32a 206 for resource in ['git-revision', 'nixexprs.tar.xz']:
72d3478a 207 fields = channel.table[resource]
e434d96d
SW
208 fields.absolute_url = urllib.parse.urljoin(
209 channel.forwarded_url, fields.url)
210 fields.file = fetch_with_nix_prefetch_url(
211 v, fields.absolute_url, fields.digest)
73bec7e8
SW
212 v.status('Verifying git commit on main page matches git commit in table')
213 v.result(
214 open(
61aaf799 215 channel.table['git-revision'].file).read(999) == channel.git_revision)
2f96f32a 216
971d3659 217
9836141c
SW
218def git_cachedir(git_repo: str) -> str:
219 # TODO: Consider using pyxdg to find this path.
971d3659 220 return os.path.expanduser(
4fdb3625 221 '~/.cache/pinch/git/%s' %
971d3659
SW
222 digest_string(
223 git_repo.encode()))
224
225
226def verify_git_ancestry(v: Verification, channel: Channel) -> None:
227 cachedir = git_cachedir(channel.git_repo)
228 v.status('Verifying rev is an ancestor of ref')
229 process = subprocess.run(['git',
230 '-C',
231 cachedir,
232 'merge-base',
233 '--is-ancestor',
234 channel.git_revision,
235 channel.git_ref])
236 v.result(process.returncode == 0)
237
238 if hasattr(channel, 'old_git_revision'):
239 v.status(
240 'Verifying rev is an ancestor of previous rev %s' %
241 channel.old_git_revision)
242 process = subprocess.run(['git',
243 '-C',
244 cachedir,
245 'merge-base',
246 '--is-ancestor',
247 channel.old_git_revision,
248 channel.git_revision])
249 v.result(process.returncode == 0)
9836141c 250
2f96f32a 251
dc038df0
SW
252def git_fetch(v: Verification, channel: Channel) -> None:
253 # It would be nice if we could share the nix git cache, but as of the time
254 # of writing it is transitioning from gitv2 (deprecated) to gitv3 (not ready
255 # yet), and trying to straddle them both is too far into nix implementation
256 # details for my comfort. So we re-implement here half of nix.fetchGit.
257 # :(
258
9836141c
SW
259 cachedir = git_cachedir(channel.git_repo)
260 if not os.path.exists(cachedir):
dc038df0
SW
261 v.status("Initializing git repo")
262 process = subprocess.run(
9836141c 263 ['git', 'init', '--bare', cachedir])
dc038df0
SW
264 v.result(process.returncode == 0)
265
971d3659
SW
266 v.status('Fetching ref "%s" from %s' % (channel.git_ref, channel.git_repo))
267 # We don't use --force here because we want to abort and freak out if forced
268 # updates are happening.
269 process = subprocess.run(['git',
270 '-C',
271 cachedir,
272 'fetch',
273 channel.git_repo,
274 '%s:%s' % (channel.git_ref,
275 channel.git_ref)])
276 v.result(process.returncode == 0)
277
8fca6c28 278 if hasattr(channel, 'git_revision'):
971d3659 279 v.status('Verifying that fetch retrieved this rev')
8fca6c28 280 process = subprocess.run(
9836141c 281 ['git', '-C', cachedir, 'cat-file', '-e', channel.git_revision])
dc038df0 282 v.result(process.returncode == 0)
971d3659 283 else:
8fca6c28
SW
284 channel.git_revision = open(
285 os.path.join(
9836141c 286 cachedir,
8fca6c28
SW
287 'refs',
288 'heads',
289 channel.git_ref)).read(999).strip()
dc038df0 290
971d3659 291 verify_git_ancestry(v, channel)
dc038df0 292
971d3659
SW
293
294def ensure_git_rev_available(v: Verification, channel: Channel) -> None:
295 cachedir = git_cachedir(channel.git_repo)
296 if os.path.exists(cachedir):
297 v.status('Checking if we already have this rev:')
298 process = subprocess.run(
299 ['git', '-C', cachedir, 'cat-file', '-e', channel.git_revision])
300 if process.returncode == 0:
301 v.status('yes')
302 if process.returncode == 1:
303 v.status('no')
304 v.result(process.returncode == 0 or process.returncode == 1)
305 if process.returncode == 0:
306 verify_git_ancestry(v, channel)
307 return
308 git_fetch(v, channel)
7d889b12 309
dc038df0 310
925c801b
SW
311def compare_tarball_and_git(
312 v: Verification,
313 channel: Channel,
314 channel_contents: str,
315 git_contents: str) -> None:
316 v.status('Comparing channel tarball with git checkout')
317 match, mismatch, errors = compare(os.path.join(
318 channel_contents, channel.release_name), git_contents)
319 v.ok()
320 v.check('%d files match' % len(match), len(match) > 0)
321 v.check('%d files differ' % len(mismatch), len(mismatch) == 0)
322 expected_errors = [
323 '.git-revision',
324 '.version-suffix',
325 'nixpkgs',
326 'programs.sqlite',
327 'svn-revision']
328 benign_errors = []
329 for ee in expected_errors:
330 if ee in errors:
331 errors.remove(ee)
332 benign_errors.append(ee)
333 v.check(
334 '%d unexpected incomparable files' %
335 len(errors),
336 len(errors) == 0)
337 v.check(
338 '(%d of %d expected incomparable files)' %
339 (len(benign_errors),
340 len(expected_errors)),
341 len(benign_errors) == len(expected_errors))
342
343
344def extract_tarball(v: Verification, channel: Channel, dest: str) -> None:
345 v.status('Extracting tarball %s' %
346 channel.table['nixexprs.tar.xz'].file)
347 shutil.unpack_archive(
348 channel.table['nixexprs.tar.xz'].file,
349 dest)
350 v.ok()
351
352
353def git_checkout(v: Verification, channel: Channel, dest: str) -> None:
354 v.status('Checking out corresponding git revision')
355 git = subprocess.Popen(['git',
356 '-C',
9836141c 357 git_cachedir(channel.git_repo),
925c801b 358 'archive',
61aaf799 359 channel.git_revision],
925c801b
SW
360 stdout=subprocess.PIPE)
361 tar = subprocess.Popen(
362 ['tar', 'x', '-C', dest, '-f', '-'], stdin=git.stdout)
363 git.stdout.close()
364 tar.wait()
365 git.wait()
366 v.result(git.returncode == 0 and tar.returncode == 0)
367
368
736c25eb
SW
369def git_get_tarball(v: Verification, channel: Channel) -> str:
370 with tempfile.TemporaryDirectory() as output_dir:
371 output_filename = os.path.join(
372 output_dir, channel.release_name + '.tar.xz')
373 with open(output_filename, 'w') as output_file:
374 v.status(
375 'Generating tarball for git revision %s' %
376 channel.git_revision)
377 git = subprocess.Popen(['git',
378 '-C',
379 git_cachedir(channel.git_repo),
380 'archive',
381 '--prefix=%s/' % channel.release_name,
382 channel.git_revision],
383 stdout=subprocess.PIPE)
384 xz = subprocess.Popen(['xz'], stdin=git.stdout, stdout=output_file)
385 xz.wait()
386 git.wait()
387 v.result(git.returncode == 0 and xz.returncode == 0)
388
389 v.status('Putting tarball in Nix store')
390 process = subprocess.run(
391 ['nix-store', '--add', output_filename], capture_output=True)
392 v.result(process.returncode == 0)
393 return process.stdout.decode().strip()
394
395
f9cd7bdc
SW
396def check_channel_metadata(
397 v: Verification,
398 channel: Channel,
399 channel_contents: str) -> None:
400 v.status('Verifying git commit in channel tarball')
401 v.result(
402 open(
403 os.path.join(
404 channel_contents,
405 channel.release_name,
61aaf799 406 '.git-revision')).read(999) == channel.git_revision)
f9cd7bdc
SW
407
408 v.status(
409 'Verifying version-suffix is a suffix of release name %s:' %
410 channel.release_name)
411 version_suffix = open(
412 os.path.join(
413 channel_contents,
414 channel.release_name,
415 '.version-suffix')).read(999)
416 v.status(version_suffix)
417 v.result(channel.release_name.endswith(version_suffix))
418
419
72d3478a 420def check_channel_contents(v: Verification, channel: Channel) -> None:
dc038df0
SW
421 with tempfile.TemporaryDirectory() as channel_contents, \
422 tempfile.TemporaryDirectory() as git_contents:
925c801b
SW
423
424 extract_tarball(v, channel, channel_contents)
f9cd7bdc
SW
425 check_channel_metadata(v, channel, channel_contents)
426
925c801b
SW
427 git_checkout(v, channel, git_contents)
428
429 compare_tarball_and_git(v, channel, channel_contents, git_contents)
430
dc038df0 431 v.status('Removing temporary directories')
2f96f32a
SW
432 v.ok()
433
434
8fca6c28
SW
435def pin_channel(v: Verification, channel: Channel) -> None:
436 fetch(v, channel)
437 parse_channel(v, channel)
438 fetch_resources(v, channel)
971d3659 439 ensure_git_rev_available(v, channel)
8fca6c28
SW
440 check_channel_contents(v, channel)
441
442
e3cae769
SW
443def git_revision_name(v: Verification, channel: Channel) -> str:
444 v.status('Getting commit date')
445 process = subprocess.run(['git',
446 '-C',
9836141c 447 git_cachedir(channel.git_repo),
e3cae769
SW
448 'lo',
449 '-n1',
450 '--format=%ct-%h',
451 '--abbrev=11',
452 channel.git_revision],
453 capture_output=True)
454 v.result(process.returncode == 0 and process.stdout != '')
455 return '%s-%s' % (os.path.basename(channel.git_repo),
456 process.stdout.decode().strip())
457
458
01ba0eb2
SW
459def read_config(filename: str) -> configparser.ConfigParser:
460 config = configparser.ConfigParser()
461 config.read_file(open(filename), filename)
462 return config
463
464
0e5e611d 465def pin(args: argparse.Namespace) -> None:
2f96f32a 466 v = Verification()
01ba0eb2 467 config = read_config(args.channels_file)
5cfa8e11 468 for section in config.sections():
98853153
SW
469 if args.channels and section not in args.channels:
470 continue
736c25eb
SW
471
472 channel = Channel(**dict(config[section].items()))
17906b27
SW
473
474 if hasattr(channel, 'alias_of'):
475 assert not hasattr(channel, 'git_repo')
476 continue
477
736c25eb
SW
478 if hasattr(channel, 'git_revision'):
479 channel.old_git_revision = channel.git_revision
480 del channel.git_revision
481
8fca6c28
SW
482 if 'channel_url' in config[section]:
483 pin_channel(v, channel)
736c25eb 484 config[section]['release_name'] = channel.release_name
8fca6c28
SW
485 config[section]['tarball_url'] = channel.table['nixexprs.tar.xz'].absolute_url
486 config[section]['tarball_sha256'] = channel.table['nixexprs.tar.xz'].digest
487 else:
488 git_fetch(v, channel)
736c25eb 489 config[section]['release_name'] = git_revision_name(v, channel)
8fca6c28
SW
490 config[section]['git_revision'] = channel.git_revision
491
0e5e611d 492 with open(args.channels_file, 'w') as configfile:
e434d96d 493 config.write(configfile)
2f96f32a
SW
494
495
736c25eb
SW
496def update(args: argparse.Namespace) -> None:
497 v = Verification()
498 config = configparser.ConfigParser()
17906b27 499 exprs = {}
01ba0eb2
SW
500 configs = [read_config(filename) for filename in args.channels_file]
501 for config in configs:
502 for section in config.sections():
503
504 if 'alias_of' in config[section]:
505 assert 'git_repo' not in config[section]
506 continue
507
508 if 'channel_url' in config[section]:
509 tarball = fetch_with_nix_prefetch_url(
510 v, config[section]['tarball_url'], Digest16(
511 config[section]['tarball_sha256']))
512 else:
513 channel = Channel(**dict(config[section].items()))
514 ensure_git_rev_available(v, channel)
515 tarball = git_get_tarball(v, channel)
516
ab7ebb2f
SW
517 if section in exprs:
518 raise Exception('Duplicate channel "%s"' % section)
01ba0eb2
SW
519 exprs[section] = (
520 'f: f { name = "%s"; channelName = "%%s"; src = builtins.storePath "%s"; }' %
521 (config[section]['release_name'], tarball))
522
523 for config in configs:
524 for section in config.sections():
525 if 'alias_of' in config[section]:
ab7ebb2f
SW
526 if section in exprs:
527 raise Exception('Duplicate channel "%s"' % section)
01ba0eb2 528 exprs[section] = exprs[str(config[section]['alias_of'])]
17906b27 529
9a78329e
SW
530 command = [
531 'nix-env',
532 '--profile',
533 '/nix/var/nix/profiles/per-user/%s/channels' %
534 getpass.getuser(),
535 '--show-trace',
536 '--file',
537 '<nix/unpack-channel.nix>',
538 '--install',
17906b27 539 '--from-expression'] + [exprs[name] % name for name in sorted(exprs.keys())]
9a78329e
SW
540 if args.dry_run:
541 print(' '.join(map(shlex.quote, command)))
542 else:
543 v.status('Installing channels with nix-env')
544 process = subprocess.run(command)
545 v.result(process.returncode == 0)
736c25eb
SW
546
547
0e5e611d
SW
548def main() -> None:
549 parser = argparse.ArgumentParser(prog='pinch')
550 subparsers = parser.add_subparsers(dest='mode', required=True)
551 parser_pin = subparsers.add_parser('pin')
552 parser_pin.add_argument('channels_file', type=str)
98853153 553 parser_pin.add_argument('channels', type=str, nargs='*')
0e5e611d 554 parser_pin.set_defaults(func=pin)
736c25eb 555 parser_update = subparsers.add_parser('update')
9a78329e 556 parser_update.add_argument('--dry-run', action='store_true')
01ba0eb2 557 parser_update.add_argument('channels_file', type=str, nargs='+')
736c25eb 558 parser_update.set_defaults(func=update)
0e5e611d
SW
559 args = parser.parse_args()
560 args.func(args)
561
562
563main()