git.scottworley.com Git - pinch/blob - pinch.py

   1 import argparse
   2 import configparser
   3 import filecmp
   4 import functools
   5 import getpass
   6 import hashlib
   7 import operator
   8 import os
   9 import os.path
  10 import shlex
  11 import shutil
  12 import subprocess
  13 import sys
  14 import tempfile
  15 import types
  16 import urllib.parse
  17 import urllib.request
  18 import xml.dom.minidom
  19
  20 from typing import (
  21     Dict,
  22     Iterable,
  23     List,
  24     NewType,
  25     Tuple,
  26 )
  27
  28 # Use xdg module when it's less painful to have as a dependency
  29
  30
  31 class XDG(types.SimpleNamespace):
  32     XDG_CACHE_HOME: str
  33
  34
  35 xdg = XDG(
  36     XDG_CACHE_HOME=os.getenv(
  37         'XDG_CACHE_HOME',
  38         os.path.expanduser('~/.cache')))
  39
  40
  41 Digest16 = NewType('Digest16', str)
  42 Digest32 = NewType('Digest32', str)
  43
  44
  45 class ChannelTableEntry(types.SimpleNamespace):
  46     absolute_url: str
  47     digest: Digest16
  48     file: str
  49     size: int
  50     url: str
  51
  52
  53 class Channel(types.SimpleNamespace):
  54     alias_of: str
  55     channel_html: bytes
  56     channel_url: str
  57     forwarded_url: str
  58     git_ref: str
  59     git_repo: str
  60     git_revision: str
  61     old_git_revision: str
  62     release_name: str
  63     table: Dict[str, ChannelTableEntry]
  64
  65
  66 class VerificationError(Exception):
  67     pass
  68
  69
  70 class Verification:
  71
  72     def __init__(self) -> None:
  73         self.line_length = 0
  74
  75     def status(self, s: str) -> None:
  76         print(s, end=' ', file=sys.stderr, flush=True)
  77         self.line_length += 1 + len(s)  # Unicode??
  78
  79     @staticmethod
  80     def _color(s: str, c: int) -> str:
  81         return '\033[%2dm%s\033[00m' % (c, s)
  82
  83     def result(self, r: bool) -> None:
  84         message, color = {True: ('OK ', 92), False: ('FAIL', 91)}[r]
  85         length = len(message)
  86         cols = shutil.get_terminal_size().columns or 80
  87         pad = (cols - (self.line_length + length)) % cols
  88         print(' ' * pad + self._color(message, color), file=sys.stderr)
  89         self.line_length = 0
  90         if not r:
  91             raise VerificationError()
  92
  93     def check(self, s: str, r: bool) -> None:
  94         self.status(s)
  95         self.result(r)
  96
  97     def ok(self) -> None:
  98         self.result(True)
  99
 100
 101 def compare(a: str, b: str) -> Tuple[List[str], List[str], List[str]]:
 102
 103     def throw(error: OSError) -> None:
 104         raise error
 105
 106     def join(x: str, y: str) -> str:
 107         return y if x == '.' else os.path.join(x, y)
 108
 109     def recursive_files(d: str) -> Iterable[str]:
 110         all_files: List[str] = []
 111         for path, dirs, files in os.walk(d, onerror=throw):
 112             rel = os.path.relpath(path, start=d)
 113             all_files.extend(join(rel, f) for f in files)
 114             for dir_or_link in dirs:
 115                 if os.path.islink(join(path, dir_or_link)):
 116                     all_files.append(join(rel, dir_or_link))
 117         return all_files
 118
 119     def exclude_dot_git(files: Iterable[str]) -> Iterable[str]:
 120         return (f for f in files if not f.startswith('.git/'))
 121
 122     files = functools.reduce(
 123         operator.or_, (set(
 124             exclude_dot_git(
 125                 recursive_files(x))) for x in [a, b]))
 126     return filecmp.cmpfiles(a, b, files, shallow=False)
 127
 128
 129 def fetch(v: Verification, channel: Channel) -> None:
 130     v.status('Fetching channel')
 131     request = urllib.request.urlopen(channel.channel_url, timeout=10)
 132     channel.channel_html = request.read()
 133     channel.forwarded_url = request.geturl()
 134     v.result(request.status == 200)
 135     v.check('Got forwarded', channel.channel_url != channel.forwarded_url)
 136
 137
 138 def parse_channel(v: Verification, channel: Channel) -> None:
 139     v.status('Parsing channel description as XML')
 140     d = xml.dom.minidom.parseString(channel.channel_html)
 141     v.ok()
 142
 143     v.status('Extracting release name:')
 144     title_name = d.getElementsByTagName(
 145         'title')[0].firstChild.nodeValue.split()[2]
 146     h1_name = d.getElementsByTagName('h1')[0].firstChild.nodeValue.split()[2]
 147     v.status(title_name)
 148     v.result(title_name == h1_name)
 149     channel.release_name = title_name
 150
 151     v.status('Extracting git commit:')
 152     git_commit_node = d.getElementsByTagName('tt')[0]
 153     channel.git_revision = git_commit_node.firstChild.nodeValue
 154     v.status(channel.git_revision)
 155     v.ok()
 156     v.status('Verifying git commit label')
 157     v.result(git_commit_node.previousSibling.nodeValue == 'Git commit ')
 158
 159     v.status('Parsing table')
 160     channel.table = {}
 161     for row in d.getElementsByTagName('tr')[1:]:
 162         name = row.childNodes[0].firstChild.firstChild.nodeValue
 163         url = row.childNodes[0].firstChild.getAttribute('href')
 164         size = int(row.childNodes[1].firstChild.nodeValue)
 165         digest = Digest16(row.childNodes[2].firstChild.firstChild.nodeValue)
 166         channel.table[name] = ChannelTableEntry(
 167             url=url, digest=digest, size=size)
 168     v.ok()
 169
 170
 171 def digest_string(s: bytes) -> Digest16:
 172     return Digest16(hashlib.sha256(s).hexdigest())
 173
 174
 175 def digest_file(filename: str) -> Digest16:
 176     hasher = hashlib.sha256()
 177     with open(filename, 'rb') as f:
 178         # pylint: disable=cell-var-from-loop
 179         for block in iter(lambda: f.read(4096), b''):
 180             hasher.update(block)
 181     return Digest16(hasher.hexdigest())
 182
 183
 184 def to_Digest16(v: Verification, digest32: Digest32) -> Digest16:
 185     v.status('Converting digest to base16')
 186     process = subprocess.run(
 187         ['nix', 'to-base16', '--type', 'sha256', digest32], capture_output=True)
 188     v.result(process.returncode == 0)
 189     return Digest16(process.stdout.decode().strip())
 190
 191
 192 def to_Digest32(v: Verification, digest16: Digest16) -> Digest32:
 193     v.status('Converting digest to base32')
 194     process = subprocess.run(
 195         ['nix', 'to-base32', '--type', 'sha256', digest16], capture_output=True)
 196     v.result(process.returncode == 0)
 197     return Digest32(process.stdout.decode().strip())
 198
 199
 200 def fetch_with_nix_prefetch_url(
 201         v: Verification,
 202         url: str,
 203         digest: Digest16) -> str:
 204     v.status('Fetching %s' % url)
 205     process = subprocess.run(
 206         ['nix-prefetch-url', '--print-path', url, digest], capture_output=True)
 207     v.result(process.returncode == 0)
 208     prefetch_digest, path, empty = process.stdout.decode().split('\n')
 209     assert empty == ''
 210     v.check("Verifying nix-prefetch-url's digest",
 211             to_Digest16(v, Digest32(prefetch_digest)) == digest)
 212     v.status("Verifying file digest")
 213     file_digest = digest_file(path)
 214     v.result(file_digest == digest)
 215     return path
 216
 217
 218 def fetch_resources(v: Verification, channel: Channel) -> None:
 219     for resource in ['git-revision', 'nixexprs.tar.xz']:
 220         fields = channel.table[resource]
 221         fields.absolute_url = urllib.parse.urljoin(
 222             channel.forwarded_url, fields.url)
 223         fields.file = fetch_with_nix_prefetch_url(
 224             v, fields.absolute_url, fields.digest)
 225     v.status('Verifying git commit on main page matches git commit in table')
 226     v.result(
 227         open(
 228             channel.table['git-revision'].file).read(999) == channel.git_revision)
 229
 230
 231 def git_cachedir(git_repo: str) -> str:
 232     return os.path.join(
 233         xdg.XDG_CACHE_HOME,
 234         'pinch/git',
 235         digest_string(git_repo.encode()))
 236
 237
 238 def tarball_cache_file(channel: Channel) -> str:
 239     return os.path.join(
 240         xdg.XDG_CACHE_HOME,
 241         'pinch/git-tarball',
 242         '%s-%s-%s' %
 243         (digest_string(channel.git_repo.encode()),
 244          channel.git_revision,
 245          channel.release_name))
 246
 247
 248 def verify_git_ancestry(v: Verification, channel: Channel) -> None:
 249     cachedir = git_cachedir(channel.git_repo)
 250     v.status('Verifying rev is an ancestor of ref')
 251     process = subprocess.run(['git',
 252                               '-C',
 253                               cachedir,
 254                               'merge-base',
 255                               '--is-ancestor',
 256                               channel.git_revision,
 257                               channel.git_ref])
 258     v.result(process.returncode == 0)
 259
 260     if hasattr(channel, 'old_git_revision'):
 261         v.status(
 262             'Verifying rev is an ancestor of previous rev %s' %
 263             channel.old_git_revision)
 264         process = subprocess.run(['git',
 265                                   '-C',
 266                                   cachedir,
 267                                   'merge-base',
 268                                   '--is-ancestor',
 269                                   channel.old_git_revision,
 270                                   channel.git_revision])
 271         v.result(process.returncode == 0)
 272
 273
 274 def git_fetch(v: Verification, channel: Channel) -> None:
 275     # It would be nice if we could share the nix git cache, but as of the time
 276     # of writing it is transitioning from gitv2 (deprecated) to gitv3 (not ready
 277     # yet), and trying to straddle them both is too far into nix implementation
 278     # details for my comfort.  So we re-implement here half of nix.fetchGit.
 279     # :(
 280
 281     cachedir = git_cachedir(channel.git_repo)
 282     if not os.path.exists(cachedir):
 283         v.status("Initializing git repo")
 284         process = subprocess.run(
 285             ['git', 'init', '--bare', cachedir])
 286         v.result(process.returncode == 0)
 287
 288     v.status('Fetching ref "%s" from %s' % (channel.git_ref, channel.git_repo))
 289     # We don't use --force here because we want to abort and freak out if forced
 290     # updates are happening.
 291     process = subprocess.run(['git',
 292                               '-C',
 293                               cachedir,
 294                               'fetch',
 295                               channel.git_repo,
 296                               '%s:%s' % (channel.git_ref,
 297                                          channel.git_ref)])
 298     v.result(process.returncode == 0)
 299
 300     if hasattr(channel, 'git_revision'):
 301         v.status('Verifying that fetch retrieved this rev')
 302         process = subprocess.run(
 303             ['git', '-C', cachedir, 'cat-file', '-e', channel.git_revision])
 304         v.result(process.returncode == 0)
 305     else:
 306         channel.git_revision = open(
 307             os.path.join(
 308                 cachedir,
 309                 'refs',
 310                 'heads',
 311                 channel.git_ref)).read(999).strip()
 312
 313     verify_git_ancestry(v, channel)
 314
 315
 316 def ensure_git_rev_available(v: Verification, channel: Channel) -> None:
 317     cachedir = git_cachedir(channel.git_repo)
 318     if os.path.exists(cachedir):
 319         v.status('Checking if we already have this rev:')
 320         process = subprocess.run(
 321             ['git', '-C', cachedir, 'cat-file', '-e', channel.git_revision])
 322         if process.returncode == 0:
 323             v.status('yes')
 324         if process.returncode == 1:
 325             v.status('no')
 326         v.result(process.returncode == 0 or process.returncode == 1)
 327         if process.returncode == 0:
 328             verify_git_ancestry(v, channel)
 329             return
 330     git_fetch(v, channel)
 331
 332
 333 def compare_tarball_and_git(
 334         v: Verification,
 335         channel: Channel,
 336         channel_contents: str,
 337         git_contents: str) -> None:
 338     v.status('Comparing channel tarball with git checkout')
 339     match, mismatch, errors = compare(os.path.join(
 340         channel_contents, channel.release_name), git_contents)
 341     v.ok()
 342     v.check('%d files match' % len(match), len(match) > 0)
 343     v.check('%d files differ' % len(mismatch), len(mismatch) == 0)
 344     expected_errors = [
 345         '.git-revision',
 346         '.version-suffix',
 347         'nixpkgs',
 348         'programs.sqlite',
 349         'svn-revision']
 350     benign_errors = []
 351     for ee in expected_errors:
 352         if ee in errors:
 353             errors.remove(ee)
 354             benign_errors.append(ee)
 355     v.check(
 356         '%d unexpected incomparable files' %
 357         len(errors),
 358         len(errors) == 0)
 359     v.check(
 360         '(%d of %d expected incomparable files)' %
 361         (len(benign_errors),
 362          len(expected_errors)),
 363         len(benign_errors) == len(expected_errors))
 364
 365
 366 def extract_tarball(v: Verification, channel: Channel, dest: str) -> None:
 367     v.status('Extracting tarball %s' %
 368              channel.table['nixexprs.tar.xz'].file)
 369     shutil.unpack_archive(
 370         channel.table['nixexprs.tar.xz'].file,
 371         dest)
 372     v.ok()
 373
 374
 375 def git_checkout(v: Verification, channel: Channel, dest: str) -> None:
 376     v.status('Checking out corresponding git revision')
 377     git = subprocess.Popen(['git',
 378                             '-C',
 379                             git_cachedir(channel.git_repo),
 380                             'archive',
 381                             channel.git_revision],
 382                            stdout=subprocess.PIPE)
 383     tar = subprocess.Popen(
 384         ['tar', 'x', '-C', dest, '-f', '-'], stdin=git.stdout)
 385     if git.stdout:
 386         git.stdout.close()
 387     tar.wait()
 388     git.wait()
 389     v.result(git.returncode == 0 and tar.returncode == 0)
 390
 391
 392 def git_get_tarball(v: Verification, channel: Channel) -> str:
 393     cache_file = tarball_cache_file(channel)
 394     if os.path.exists(cache_file):
 395         cached_tarball = open(cache_file).read(9999)
 396         if os.path.exists(cached_tarball):
 397             return cached_tarball
 398
 399     with tempfile.TemporaryDirectory() as output_dir:
 400         output_filename = os.path.join(
 401             output_dir, channel.release_name + '.tar.xz')
 402         with open(output_filename, 'w') as output_file:
 403             v.status(
 404                 'Generating tarball for git revision %s' %
 405                 channel.git_revision)
 406             git = subprocess.Popen(['git',
 407                                     '-C',
 408                                     git_cachedir(channel.git_repo),
 409                                     'archive',
 410                                     '--prefix=%s/' % channel.release_name,
 411                                     channel.git_revision],
 412                                    stdout=subprocess.PIPE)
 413             xz = subprocess.Popen(['xz'], stdin=git.stdout, stdout=output_file)
 414             xz.wait()
 415             git.wait()
 416             v.result(git.returncode == 0 and xz.returncode == 0)
 417
 418         v.status('Putting tarball in Nix store')
 419         process = subprocess.run(
 420             ['nix-store', '--add', output_filename], capture_output=True)
 421         v.result(process.returncode == 0)
 422         store_tarball = process.stdout.decode().strip()
 423
 424         os.makedirs(os.path.dirname(cache_file), exist_ok=True)
 425         open(cache_file, 'w').write(store_tarball)
 426         return store_tarball
 427
 428
 429 def check_channel_metadata(
 430         v: Verification,
 431         channel: Channel,
 432         channel_contents: str) -> None:
 433     v.status('Verifying git commit in channel tarball')
 434     v.result(
 435         open(
 436             os.path.join(
 437                 channel_contents,
 438                 channel.release_name,
 439                 '.git-revision')).read(999) == channel.git_revision)
 440
 441     v.status(
 442         'Verifying version-suffix is a suffix of release name %s:' %
 443         channel.release_name)
 444     version_suffix = open(
 445         os.path.join(
 446             channel_contents,
 447             channel.release_name,
 448             '.version-suffix')).read(999)
 449     v.status(version_suffix)
 450     v.result(channel.release_name.endswith(version_suffix))
 451
 452
 453 def check_channel_contents(v: Verification, channel: Channel) -> None:
 454     with tempfile.TemporaryDirectory() as channel_contents, \
 455             tempfile.TemporaryDirectory() as git_contents:
 456
 457         extract_tarball(v, channel, channel_contents)
 458         check_channel_metadata(v, channel, channel_contents)
 459
 460         git_checkout(v, channel, git_contents)
 461
 462         compare_tarball_and_git(v, channel, channel_contents, git_contents)
 463
 464         v.status('Removing temporary directories')
 465     v.ok()
 466
 467
 468 def pin_channel(v: Verification, channel: Channel) -> None:
 469     fetch(v, channel)
 470     parse_channel(v, channel)
 471     fetch_resources(v, channel)
 472     ensure_git_rev_available(v, channel)
 473     check_channel_contents(v, channel)
 474
 475
 476 def git_revision_name(v: Verification, channel: Channel) -> str:
 477     v.status('Getting commit date')
 478     process = subprocess.run(['git',
 479                               '-C',
 480                               git_cachedir(channel.git_repo),
 481                               'log',
 482                               '-n1',
 483                               '--format=%ct-%h',
 484                               '--abbrev=11',
 485                               channel.git_revision],
 486                              capture_output=True)
 487     v.result(process.returncode == 0 and process.stdout != b'')
 488     return '%s-%s' % (os.path.basename(channel.git_repo),
 489                       process.stdout.decode().strip())
 490
 491
 492 def read_config(filename: str) -> configparser.ConfigParser:
 493     config = configparser.ConfigParser()
 494     config.read_file(open(filename), filename)
 495     return config
 496
 497
 498 def pin(args: argparse.Namespace) -> None:
 499     v = Verification()
 500     config = read_config(args.channels_file)
 501     for section in config.sections():
 502         if args.channels and section not in args.channels:
 503             continue
 504
 505         channel = Channel(**dict(config[section].items()))
 506
 507         if hasattr(channel, 'alias_of'):
 508             assert not hasattr(channel, 'git_repo')
 509             continue
 510
 511         if hasattr(channel, 'git_revision'):
 512             channel.old_git_revision = channel.git_revision
 513             del channel.git_revision
 514
 515         if 'channel_url' in config[section]:
 516             pin_channel(v, channel)
 517             config[section]['release_name'] = channel.release_name
 518             config[section]['tarball_url'] = channel.table['nixexprs.tar.xz'].absolute_url
 519             config[section]['tarball_sha256'] = channel.table['nixexprs.tar.xz'].digest
 520         else:
 521             git_fetch(v, channel)
 522             config[section]['release_name'] = git_revision_name(v, channel)
 523         config[section]['git_revision'] = channel.git_revision
 524
 525     with open(args.channels_file, 'w') as configfile:
 526         config.write(configfile)
 527
 528
 529 def fetch_channel(
 530         v: Verification,
 531         section: str,
 532         conf: configparser.SectionProxy) -> str:
 533     if 'git_repo' not in conf or 'release_name' not in conf:
 534         raise Exception(
 535             'Cannot update unpinned channel "%s" (Run "pin" before "update")' %
 536             section)
 537
 538     if 'channel_url' in conf:
 539         return fetch_with_nix_prefetch_url(
 540             v, conf['tarball_url'], Digest16(
 541                 conf['tarball_sha256']))
 542
 543     channel = Channel(**dict(conf.items()))
 544     ensure_git_rev_available(v, channel)
 545     return git_get_tarball(v, channel)
 546
 547
 548 def update(args: argparse.Namespace) -> None:
 549     v = Verification()
 550     config = configparser.ConfigParser()
 551     exprs: Dict[str, str] = {}
 552     configs = [read_config(filename) for filename in args.channels_file]
 553     for config in configs:
 554         for section in config.sections():
 555             if 'alias_of' in config[section]:
 556                 assert 'git_repo' not in config[section]
 557                 continue
 558             tarball = fetch_channel(v, section, config[section])
 559             if section in exprs:
 560                 raise Exception('Duplicate channel "%s"' % section)
 561             exprs[section] = (
 562                 'f: f { name = "%s"; channelName = "%%s"; src = builtins.storePath "%s"; }' %
 563                 (config[section]['release_name'], tarball))
 564
 565     for config in configs:
 566         for section in config.sections():
 567             if 'alias_of' in config[section]:
 568                 if section in exprs:
 569                     raise Exception('Duplicate channel "%s"' % section)
 570                 exprs[section] = exprs[str(config[section]['alias_of'])]
 571
 572     command = [
 573         'nix-env',
 574         '--profile',
 575         '/nix/var/nix/profiles/per-user/%s/channels' %
 576         getpass.getuser(),
 577         '--show-trace',
 578         '--file',
 579         '<nix/unpack-channel.nix>',
 580         '--install',
 581         '--from-expression'] + [exprs[name] % name for name in sorted(exprs.keys())]
 582     if args.dry_run:
 583         print(' '.join(map(shlex.quote, command)))
 584     else:
 585         v.status('Installing channels with nix-env')
 586         process = subprocess.run(command)
 587         v.result(process.returncode == 0)
 588
 589
 590 def main() -> None:
 591     parser = argparse.ArgumentParser(prog='pinch')
 592     subparsers = parser.add_subparsers(dest='mode', required=True)
 593     parser_pin = subparsers.add_parser('pin')
 594     parser_pin.add_argument('channels_file', type=str)
 595     parser_pin.add_argument('channels', type=str, nargs='*')
 596     parser_pin.set_defaults(func=pin)
 597     parser_update = subparsers.add_parser('update')
 598     parser_update.add_argument('--dry-run', action='store_true')
 599     parser_update.add_argument('channels_file', type=str, nargs='+')
 600     parser_update.set_defaults(func=update)
 601     args = parser.parse_args()
 602     args.func(args)
 603
 604
 605 main()