]> git.scottworley.com Git - pinch/blob - pinch.py
46e5af30f9111043827bab3089040c1acaf1699f
[pinch] / pinch.py
1 import argparse
2 import configparser
3 import filecmp
4 import functools
5 import getpass
6 import hashlib
7 import operator
8 import os
9 import os.path
10 import shlex
11 import shutil
12 import subprocess
13 import sys
14 import tempfile
15 import types
16 import urllib.parse
17 import urllib.request
18 import xml.dom.minidom
19
20 from typing import (
21 Dict,
22 Iterable,
23 List,
24 NewType,
25 Tuple,
26 )
27
28 Digest16 = NewType('Digest16', str)
29 Digest32 = NewType('Digest32', str)
30
31
32 class ChannelTableEntry(types.SimpleNamespace):
33 absolute_url: str
34 digest: Digest16
35 file: str
36 size: int
37 url: str
38
39
40 class Channel(types.SimpleNamespace):
41 channel_html: bytes
42 channel_url: str
43 forwarded_url: str
44 git_ref: str
45 git_repo: str
46 git_revision: str
47 old_git_revision: str
48 release_name: str
49 table: Dict[str, ChannelTableEntry]
50
51
52 class VerificationError(Exception):
53 pass
54
55
56 class Verification:
57
58 def __init__(self) -> None:
59 self.line_length = 0
60
61 def status(self, s: str) -> None:
62 print(s, end=' ', file=sys.stderr, flush=True)
63 self.line_length += 1 + len(s) # Unicode??
64
65 @staticmethod
66 def _color(s: str, c: int) -> str:
67 return '\033[%2dm%s\033[00m' % (c, s)
68
69 def result(self, r: bool) -> None:
70 message, color = {True: ('OK ', 92), False: ('FAIL', 91)}[r]
71 length = len(message)
72 cols = shutil.get_terminal_size().columns
73 pad = (cols - (self.line_length + length)) % cols
74 print(' ' * pad + self._color(message, color), file=sys.stderr)
75 self.line_length = 0
76 if not r:
77 raise VerificationError()
78
79 def check(self, s: str, r: bool) -> None:
80 self.status(s)
81 self.result(r)
82
83 def ok(self) -> None:
84 self.result(True)
85
86
87 def compare(a: str, b: str) -> Tuple[List[str], List[str], List[str]]:
88
89 def throw(error: OSError) -> None:
90 raise error
91
92 def join(x: str, y: str) -> str:
93 return y if x == '.' else os.path.join(x, y)
94
95 def recursive_files(d: str) -> Iterable[str]:
96 all_files: List[str] = []
97 for path, dirs, files in os.walk(d, onerror=throw):
98 rel = os.path.relpath(path, start=d)
99 all_files.extend(join(rel, f) for f in files)
100 for dir_or_link in dirs:
101 if os.path.islink(join(path, dir_or_link)):
102 all_files.append(join(rel, dir_or_link))
103 return all_files
104
105 def exclude_dot_git(files: Iterable[str]) -> Iterable[str]:
106 return (f for f in files if not f.startswith('.git/'))
107
108 files = functools.reduce(
109 operator.or_, (set(
110 exclude_dot_git(
111 recursive_files(x))) for x in [a, b]))
112 return filecmp.cmpfiles(a, b, files, shallow=False)
113
114
115 def fetch(v: Verification, channel: Channel) -> None:
116 v.status('Fetching channel')
117 request = urllib.request.urlopen(channel.channel_url, timeout=10)
118 channel.channel_html = request.read()
119 channel.forwarded_url = request.geturl()
120 v.result(request.status == 200)
121 v.check('Got forwarded', channel.channel_url != channel.forwarded_url)
122
123
124 def parse_channel(v: Verification, channel: Channel) -> None:
125 v.status('Parsing channel description as XML')
126 d = xml.dom.minidom.parseString(channel.channel_html)
127 v.ok()
128
129 v.status('Extracting release name:')
130 title_name = d.getElementsByTagName(
131 'title')[0].firstChild.nodeValue.split()[2]
132 h1_name = d.getElementsByTagName('h1')[0].firstChild.nodeValue.split()[2]
133 v.status(title_name)
134 v.result(title_name == h1_name)
135 channel.release_name = title_name
136
137 v.status('Extracting git commit:')
138 git_commit_node = d.getElementsByTagName('tt')[0]
139 channel.git_revision = git_commit_node.firstChild.nodeValue
140 v.status(channel.git_revision)
141 v.ok()
142 v.status('Verifying git commit label')
143 v.result(git_commit_node.previousSibling.nodeValue == 'Git commit ')
144
145 v.status('Parsing table')
146 channel.table = {}
147 for row in d.getElementsByTagName('tr')[1:]:
148 name = row.childNodes[0].firstChild.firstChild.nodeValue
149 url = row.childNodes[0].firstChild.getAttribute('href')
150 size = int(row.childNodes[1].firstChild.nodeValue)
151 digest = Digest16(row.childNodes[2].firstChild.firstChild.nodeValue)
152 channel.table[name] = ChannelTableEntry(
153 url=url, digest=digest, size=size)
154 v.ok()
155
156
157 def digest_string(s: bytes) -> Digest16:
158 return Digest16(hashlib.sha256(s).hexdigest())
159
160
161 def digest_file(filename: str) -> Digest16:
162 hasher = hashlib.sha256()
163 with open(filename, 'rb') as f:
164 # pylint: disable=cell-var-from-loop
165 for block in iter(lambda: f.read(4096), b''):
166 hasher.update(block)
167 return Digest16(hasher.hexdigest())
168
169
170 def to_Digest16(v: Verification, digest32: Digest32) -> Digest16:
171 v.status('Converting digest to base16')
172 process = subprocess.run(
173 ['nix', 'to-base16', '--type', 'sha256', digest32], capture_output=True)
174 v.result(process.returncode == 0)
175 return Digest16(process.stdout.decode().strip())
176
177
178 def to_Digest32(v: Verification, digest16: Digest16) -> Digest32:
179 v.status('Converting digest to base32')
180 process = subprocess.run(
181 ['nix', 'to-base32', '--type', 'sha256', digest16], capture_output=True)
182 v.result(process.returncode == 0)
183 return Digest32(process.stdout.decode().strip())
184
185
186 def fetch_with_nix_prefetch_url(
187 v: Verification,
188 url: str,
189 digest: Digest16) -> str:
190 v.status('Fetching %s' % url)
191 process = subprocess.run(
192 ['nix-prefetch-url', '--print-path', url, digest], capture_output=True)
193 v.result(process.returncode == 0)
194 prefetch_digest, path, empty = process.stdout.decode().split('\n')
195 assert empty == ''
196 v.check("Verifying nix-prefetch-url's digest",
197 to_Digest16(v, Digest32(prefetch_digest)) == digest)
198 v.status("Verifying file digest")
199 file_digest = digest_file(path)
200 v.result(file_digest == digest)
201 return path
202
203
204 def fetch_resources(v: Verification, channel: Channel) -> None:
205 for resource in ['git-revision', 'nixexprs.tar.xz']:
206 fields = channel.table[resource]
207 fields.absolute_url = urllib.parse.urljoin(
208 channel.forwarded_url, fields.url)
209 fields.file = fetch_with_nix_prefetch_url(
210 v, fields.absolute_url, fields.digest)
211 v.status('Verifying git commit on main page matches git commit in table')
212 v.result(
213 open(
214 channel.table['git-revision'].file).read(999) == channel.git_revision)
215
216
217 def git_cachedir(git_repo: str) -> str:
218 # TODO: Consider using pyxdg to find this path.
219 return os.path.expanduser(
220 '~/.cache/pinch/git/%s' %
221 digest_string(
222 git_repo.encode()))
223
224
225 def verify_git_ancestry(v: Verification, channel: Channel) -> None:
226 cachedir = git_cachedir(channel.git_repo)
227 v.status('Verifying rev is an ancestor of ref')
228 process = subprocess.run(['git',
229 '-C',
230 cachedir,
231 'merge-base',
232 '--is-ancestor',
233 channel.git_revision,
234 channel.git_ref])
235 v.result(process.returncode == 0)
236
237 if hasattr(channel, 'old_git_revision'):
238 v.status(
239 'Verifying rev is an ancestor of previous rev %s' %
240 channel.old_git_revision)
241 process = subprocess.run(['git',
242 '-C',
243 cachedir,
244 'merge-base',
245 '--is-ancestor',
246 channel.old_git_revision,
247 channel.git_revision])
248 v.result(process.returncode == 0)
249
250
251 def git_fetch(v: Verification, channel: Channel) -> None:
252 # It would be nice if we could share the nix git cache, but as of the time
253 # of writing it is transitioning from gitv2 (deprecated) to gitv3 (not ready
254 # yet), and trying to straddle them both is too far into nix implementation
255 # details for my comfort. So we re-implement here half of nix.fetchGit.
256 # :(
257
258 cachedir = git_cachedir(channel.git_repo)
259 if not os.path.exists(cachedir):
260 v.status("Initializing git repo")
261 process = subprocess.run(
262 ['git', 'init', '--bare', cachedir])
263 v.result(process.returncode == 0)
264
265 v.status('Fetching ref "%s" from %s' % (channel.git_ref, channel.git_repo))
266 # We don't use --force here because we want to abort and freak out if forced
267 # updates are happening.
268 process = subprocess.run(['git',
269 '-C',
270 cachedir,
271 'fetch',
272 channel.git_repo,
273 '%s:%s' % (channel.git_ref,
274 channel.git_ref)])
275 v.result(process.returncode == 0)
276
277 if hasattr(channel, 'git_revision'):
278 v.status('Verifying that fetch retrieved this rev')
279 process = subprocess.run(
280 ['git', '-C', cachedir, 'cat-file', '-e', channel.git_revision])
281 v.result(process.returncode == 0)
282 else:
283 channel.git_revision = open(
284 os.path.join(
285 cachedir,
286 'refs',
287 'heads',
288 channel.git_ref)).read(999).strip()
289
290 verify_git_ancestry(v, channel)
291
292
293 def ensure_git_rev_available(v: Verification, channel: Channel) -> None:
294 cachedir = git_cachedir(channel.git_repo)
295 if os.path.exists(cachedir):
296 v.status('Checking if we already have this rev:')
297 process = subprocess.run(
298 ['git', '-C', cachedir, 'cat-file', '-e', channel.git_revision])
299 if process.returncode == 0:
300 v.status('yes')
301 if process.returncode == 1:
302 v.status('no')
303 v.result(process.returncode == 0 or process.returncode == 1)
304 if process.returncode == 0:
305 verify_git_ancestry(v, channel)
306 return
307 git_fetch(v, channel)
308
309
310 def compare_tarball_and_git(
311 v: Verification,
312 channel: Channel,
313 channel_contents: str,
314 git_contents: str) -> None:
315 v.status('Comparing channel tarball with git checkout')
316 match, mismatch, errors = compare(os.path.join(
317 channel_contents, channel.release_name), git_contents)
318 v.ok()
319 v.check('%d files match' % len(match), len(match) > 0)
320 v.check('%d files differ' % len(mismatch), len(mismatch) == 0)
321 expected_errors = [
322 '.git-revision',
323 '.version-suffix',
324 'nixpkgs',
325 'programs.sqlite',
326 'svn-revision']
327 benign_errors = []
328 for ee in expected_errors:
329 if ee in errors:
330 errors.remove(ee)
331 benign_errors.append(ee)
332 v.check(
333 '%d unexpected incomparable files' %
334 len(errors),
335 len(errors) == 0)
336 v.check(
337 '(%d of %d expected incomparable files)' %
338 (len(benign_errors),
339 len(expected_errors)),
340 len(benign_errors) == len(expected_errors))
341
342
343 def extract_tarball(v: Verification, channel: Channel, dest: str) -> None:
344 v.status('Extracting tarball %s' %
345 channel.table['nixexprs.tar.xz'].file)
346 shutil.unpack_archive(
347 channel.table['nixexprs.tar.xz'].file,
348 dest)
349 v.ok()
350
351
352 def git_checkout(v: Verification, channel: Channel, dest: str) -> None:
353 v.status('Checking out corresponding git revision')
354 git = subprocess.Popen(['git',
355 '-C',
356 git_cachedir(channel.git_repo),
357 'archive',
358 channel.git_revision],
359 stdout=subprocess.PIPE)
360 tar = subprocess.Popen(
361 ['tar', 'x', '-C', dest, '-f', '-'], stdin=git.stdout)
362 git.stdout.close()
363 tar.wait()
364 git.wait()
365 v.result(git.returncode == 0 and tar.returncode == 0)
366
367
368 def git_get_tarball(v: Verification, channel: Channel) -> str:
369 with tempfile.TemporaryDirectory() as output_dir:
370 output_filename = os.path.join(
371 output_dir, channel.release_name + '.tar.xz')
372 with open(output_filename, 'w') as output_file:
373 v.status(
374 'Generating tarball for git revision %s' %
375 channel.git_revision)
376 git = subprocess.Popen(['git',
377 '-C',
378 git_cachedir(channel.git_repo),
379 'archive',
380 '--prefix=%s/' % channel.release_name,
381 channel.git_revision],
382 stdout=subprocess.PIPE)
383 xz = subprocess.Popen(['xz'], stdin=git.stdout, stdout=output_file)
384 xz.wait()
385 git.wait()
386 v.result(git.returncode == 0 and xz.returncode == 0)
387
388 v.status('Putting tarball in Nix store')
389 process = subprocess.run(
390 ['nix-store', '--add', output_filename], capture_output=True)
391 v.result(process.returncode == 0)
392 return process.stdout.decode().strip()
393
394
395 def check_channel_metadata(
396 v: Verification,
397 channel: Channel,
398 channel_contents: str) -> None:
399 v.status('Verifying git commit in channel tarball')
400 v.result(
401 open(
402 os.path.join(
403 channel_contents,
404 channel.release_name,
405 '.git-revision')).read(999) == channel.git_revision)
406
407 v.status(
408 'Verifying version-suffix is a suffix of release name %s:' %
409 channel.release_name)
410 version_suffix = open(
411 os.path.join(
412 channel_contents,
413 channel.release_name,
414 '.version-suffix')).read(999)
415 v.status(version_suffix)
416 v.result(channel.release_name.endswith(version_suffix))
417
418
419 def check_channel_contents(v: Verification, channel: Channel) -> None:
420 with tempfile.TemporaryDirectory() as channel_contents, \
421 tempfile.TemporaryDirectory() as git_contents:
422
423 extract_tarball(v, channel, channel_contents)
424 check_channel_metadata(v, channel, channel_contents)
425
426 git_checkout(v, channel, git_contents)
427
428 compare_tarball_and_git(v, channel, channel_contents, git_contents)
429
430 v.status('Removing temporary directories')
431 v.ok()
432
433
434 def pin_channel(v: Verification, channel: Channel) -> None:
435 fetch(v, channel)
436 parse_channel(v, channel)
437 fetch_resources(v, channel)
438 ensure_git_rev_available(v, channel)
439 check_channel_contents(v, channel)
440
441
442 def git_revision_name(v: Verification, channel: Channel) -> str:
443 v.status('Getting commit date')
444 process = subprocess.run(['git',
445 '-C',
446 git_cachedir(channel.git_repo),
447 'lo',
448 '-n1',
449 '--format=%ct-%h',
450 '--abbrev=11',
451 channel.git_revision],
452 capture_output=True)
453 v.result(process.returncode == 0 and process.stdout != '')
454 return '%s-%s' % (os.path.basename(channel.git_repo),
455 process.stdout.decode().strip())
456
457
458 def pin(args: argparse.Namespace) -> None:
459 v = Verification()
460 config = configparser.ConfigParser()
461 config.read_file(open(args.channels_file), args.channels_file)
462 for section in config.sections():
463 if args.channels and section not in args.channels:
464 continue
465
466 channel = Channel(**dict(config[section].items()))
467 if hasattr(channel, 'git_revision'):
468 channel.old_git_revision = channel.git_revision
469 del channel.git_revision
470
471 if 'channel_url' in config[section]:
472 pin_channel(v, channel)
473 config[section]['release_name'] = channel.release_name
474 config[section]['tarball_url'] = channel.table['nixexprs.tar.xz'].absolute_url
475 config[section]['tarball_sha256'] = channel.table['nixexprs.tar.xz'].digest
476 else:
477 git_fetch(v, channel)
478 config[section]['release_name'] = git_revision_name(v, channel)
479 config[section]['git_revision'] = channel.git_revision
480
481 with open(args.channels_file, 'w') as configfile:
482 config.write(configfile)
483
484
485 def update(args: argparse.Namespace) -> None:
486 v = Verification()
487 config = configparser.ConfigParser()
488 config.read_file(open(args.channels_file), args.channels_file)
489 exprs = []
490 for section in config.sections():
491 if 'channel_url' in config[section]:
492 tarball = fetch_with_nix_prefetch_url(
493 v, config[section]['tarball_url'], Digest16(
494 config[section]['tarball_sha256']))
495 else:
496 channel = Channel(**dict(config[section].items()))
497 ensure_git_rev_available(v, channel)
498 tarball = git_get_tarball(v, channel)
499 exprs.append(
500 'f: f { name = "%s"; channelName = "%s"; src = builtins.storePath "%s"; }' %
501 (config[section]['release_name'], section, tarball))
502 command = [
503 'nix-env',
504 '--profile',
505 '/nix/var/nix/profiles/per-user/%s/channels' %
506 getpass.getuser(),
507 '--show-trace',
508 '--file',
509 '<nix/unpack-channel.nix>',
510 '--install',
511 '--from-expression'] + exprs
512 if args.dry_run:
513 print(' '.join(map(shlex.quote, command)))
514 else:
515 v.status('Installing channels with nix-env')
516 process = subprocess.run(command)
517 v.result(process.returncode == 0)
518
519
520 def main() -> None:
521 parser = argparse.ArgumentParser(prog='pinch')
522 subparsers = parser.add_subparsers(dest='mode', required=True)
523 parser_pin = subparsers.add_parser('pin')
524 parser_pin.add_argument('channels_file', type=str)
525 parser_pin.add_argument('channels', type=str, nargs='*')
526 parser_pin.set_defaults(func=pin)
527 parser_update = subparsers.add_parser('update')
528 parser_update.add_argument('--dry-run', action='store_true')
529 parser_update.add_argument('channels_file', type=str)
530 parser_update.set_defaults(func=update)
531 args = parser.parse_args()
532 args.func(args)
533
534
535 main()