git.scottworley.com Git - pinch/blob - pinch.py

   1 import argparse
   2 import configparser
   3 import filecmp
   4 import functools
   5 import getpass
   6 import hashlib
   7 import operator
   8 import os
   9 import os.path
  10 import shlex
  11 import shutil
  12 import subprocess
  13 import sys
  14 import tempfile
  15 import types
  16 import urllib.parse
  17 import urllib.request
  18 import xml.dom.minidom
  19
  20 from typing import (
  21     Dict,
  22     Iterable,
  23     List,
  24     NewType,
  25     Tuple,
  26 )
  27
  28 # Use xdg module when it's less painful to have as a dependency
  29
  30
  31 class XDG(types.SimpleNamespace):
  32     XDG_CACHE_HOME: str
  33
  34
  35 xdg = XDG(
  36     XDG_CACHE_HOME=os.getenv(
  37         'XDG_CACHE_HOME',
  38         os.path.expanduser('~/.cache')))
  39
  40
  41 Digest16 = NewType('Digest16', str)
  42 Digest32 = NewType('Digest32', str)
  43
  44
  45 class ChannelTableEntry(types.SimpleNamespace):
  46     absolute_url: str
  47     digest: Digest16
  48     file: str
  49     size: int
  50     url: str
  51
  52
  53 class Channel(types.SimpleNamespace):
  54     alias_of: str
  55     channel_html: bytes
  56     channel_url: str
  57     forwarded_url: str
  58     git_ref: str
  59     git_repo: str
  60     git_revision: str
  61     old_git_revision: str
  62     release_name: str
  63     table: Dict[str, ChannelTableEntry]
  64
  65
  66 class VerificationError(Exception):
  67     pass
  68
  69
  70 class Verification:
  71
  72     def __init__(self) -> None:
  73         self.line_length = 0
  74
  75     def status(self, s: str) -> None:
  76         print(s, end=' ', file=sys.stderr, flush=True)
  77         self.line_length += 1 + len(s)  # Unicode??
  78
  79     @staticmethod
  80     def _color(s: str, c: int) -> str:
  81         return '\033[%2dm%s\033[00m' % (c, s)
  82
  83     def result(self, r: bool) -> None:
  84         message, color = {True: ('OK ', 92), False: ('FAIL', 91)}[r]
  85         length = len(message)
  86         cols = shutil.get_terminal_size().columns or 80
  87         pad = (cols - (self.line_length + length)) % cols
  88         print(' ' * pad + self._color(message, color), file=sys.stderr)
  89         self.line_length = 0
  90         if not r:
  91             raise VerificationError()
  92
  93     def check(self, s: str, r: bool) -> None:
  94         self.status(s)
  95         self.result(r)
  96
  97     def ok(self) -> None:
  98         self.result(True)
  99
 100
 101 def compare(a: str, b: str) -> Tuple[List[str], List[str], List[str]]:
 102
 103     def throw(error: OSError) -> None:
 104         raise error
 105
 106     def join(x: str, y: str) -> str:
 107         return y if x == '.' else os.path.join(x, y)
 108
 109     def recursive_files(d: str) -> Iterable[str]:
 110         all_files: List[str] = []
 111         for path, dirs, files in os.walk(d, onerror=throw):
 112             rel = os.path.relpath(path, start=d)
 113             all_files.extend(join(rel, f) for f in files)
 114             for dir_or_link in dirs:
 115                 if os.path.islink(join(path, dir_or_link)):
 116                     all_files.append(join(rel, dir_or_link))
 117         return all_files
 118
 119     def exclude_dot_git(files: Iterable[str]) -> Iterable[str]:
 120         return (f for f in files if not f.startswith('.git/'))
 121
 122     files = functools.reduce(
 123         operator.or_, (set(
 124             exclude_dot_git(
 125                 recursive_files(x))) for x in [a, b]))
 126     return filecmp.cmpfiles(a, b, files, shallow=False)
 127
 128
 129 def fetch(v: Verification, channel: Channel) -> None:
 130     v.status('Fetching channel')
 131     request = urllib.request.urlopen(channel.channel_url, timeout=10)
 132     channel.channel_html = request.read()
 133     channel.forwarded_url = request.geturl()
 134     v.result(request.status == 200)  # type: ignore  # (for old mypy)
 135     v.check('Got forwarded', channel.channel_url != channel.forwarded_url)
 136
 137
 138 def parse_channel(v: Verification, channel: Channel) -> None:
 139     v.status('Parsing channel description as XML')
 140     d = xml.dom.minidom.parseString(channel.channel_html)
 141     v.ok()
 142
 143     v.status('Extracting release name:')
 144     title_name = d.getElementsByTagName(
 145         'title')[0].firstChild.nodeValue.split()[2]
 146     h1_name = d.getElementsByTagName('h1')[0].firstChild.nodeValue.split()[2]
 147     v.status(title_name)
 148     v.result(title_name == h1_name)
 149     channel.release_name = title_name
 150
 151     v.status('Extracting git commit:')
 152     git_commit_node = d.getElementsByTagName('tt')[0]
 153     channel.git_revision = git_commit_node.firstChild.nodeValue
 154     v.status(channel.git_revision)
 155     v.ok()
 156     v.status('Verifying git commit label')
 157     v.result(git_commit_node.previousSibling.nodeValue == 'Git commit ')
 158
 159     v.status('Parsing table')
 160     channel.table = {}
 161     for row in d.getElementsByTagName('tr')[1:]:
 162         name = row.childNodes[0].firstChild.firstChild.nodeValue
 163         url = row.childNodes[0].firstChild.getAttribute('href')
 164         size = int(row.childNodes[1].firstChild.nodeValue)
 165         digest = Digest16(row.childNodes[2].firstChild.firstChild.nodeValue)
 166         channel.table[name] = ChannelTableEntry(
 167             url=url, digest=digest, size=size)
 168     v.ok()
 169
 170
 171 def digest_string(s: bytes) -> Digest16:
 172     return Digest16(hashlib.sha256(s).hexdigest())
 173
 174
 175 def digest_file(filename: str) -> Digest16:
 176     hasher = hashlib.sha256()
 177     with open(filename, 'rb') as f:
 178         # pylint: disable=cell-var-from-loop
 179         for block in iter(lambda: f.read(4096), b''):
 180             hasher.update(block)
 181     return Digest16(hasher.hexdigest())
 182
 183
 184 def to_Digest16(v: Verification, digest32: Digest32) -> Digest16:
 185     v.status('Converting digest to base16')
 186     process = subprocess.run(
 187         ['nix', 'to-base16', '--type', 'sha256', digest32], stdout=subprocess.PIPE)
 188     v.result(process.returncode == 0)
 189     return Digest16(process.stdout.decode().strip())
 190
 191
 192 def to_Digest32(v: Verification, digest16: Digest16) -> Digest32:
 193     v.status('Converting digest to base32')
 194     process = subprocess.run(
 195         ['nix', 'to-base32', '--type', 'sha256', digest16], stdout=subprocess.PIPE)
 196     v.result(process.returncode == 0)
 197     return Digest32(process.stdout.decode().strip())
 198
 199
 200 def fetch_with_nix_prefetch_url(
 201         v: Verification,
 202         url: str,
 203         digest: Digest16) -> str:
 204     v.status('Fetching %s' % url)
 205     process = subprocess.run(
 206         ['nix-prefetch-url', '--print-path', url, digest], stdout=subprocess.PIPE)
 207     v.result(process.returncode == 0)
 208     prefetch_digest, path, empty = process.stdout.decode().split('\n')
 209     assert empty == ''
 210     v.check("Verifying nix-prefetch-url's digest",
 211             to_Digest16(v, Digest32(prefetch_digest)) == digest)
 212     v.status("Verifying file digest")
 213     file_digest = digest_file(path)
 214     v.result(file_digest == digest)
 215     return path  # type: ignore  # (for old mypy)
 216
 217
 218 def fetch_resources(v: Verification, channel: Channel) -> None:
 219     for resource in ['git-revision', 'nixexprs.tar.xz']:
 220         fields = channel.table[resource]
 221         fields.absolute_url = urllib.parse.urljoin(
 222             channel.forwarded_url, fields.url)
 223         fields.file = fetch_with_nix_prefetch_url(
 224             v, fields.absolute_url, fields.digest)
 225     v.status('Verifying git commit on main page matches git commit in table')
 226     v.result(
 227         open(
 228             channel.table['git-revision'].file).read(999) == channel.git_revision)
 229
 230
 231 def git_cachedir(git_repo: str) -> str:
 232     return os.path.join(
 233         xdg.XDG_CACHE_HOME,
 234         'pinch/git',
 235         digest_string(git_repo.encode()))
 236
 237
 238 def tarball_cache_file(channel: Channel) -> str:
 239     return os.path.join(
 240         xdg.XDG_CACHE_HOME,
 241         'pinch/git-tarball',
 242         '%s-%s-%s' %
 243         (digest_string(channel.git_repo.encode()),
 244          channel.git_revision,
 245          channel.release_name))
 246
 247
 248 def verify_git_ancestry(v: Verification, channel: Channel) -> None:
 249     cachedir = git_cachedir(channel.git_repo)
 250     v.status('Verifying rev is an ancestor of ref')
 251     process = subprocess.run(['git',
 252                               '-C',
 253                               cachedir,
 254                               'merge-base',
 255                               '--is-ancestor',
 256                               channel.git_revision,
 257                               channel.git_ref])
 258     v.result(process.returncode == 0)
 259
 260     if hasattr(channel, 'old_git_revision'):
 261         v.status(
 262             'Verifying rev is an ancestor of previous rev %s' %
 263             channel.old_git_revision)
 264         process = subprocess.run(['git',
 265                                   '-C',
 266                                   cachedir,
 267                                   'merge-base',
 268                                   '--is-ancestor',
 269                                   channel.old_git_revision,
 270                                   channel.git_revision])
 271         v.result(process.returncode == 0)
 272
 273
 274 def git_fetch(v: Verification, channel: Channel) -> None:
 275     # It would be nice if we could share the nix git cache, but as of the time
 276     # of writing it is transitioning from gitv2 (deprecated) to gitv3 (not ready
 277     # yet), and trying to straddle them both is too far into nix implementation
 278     # details for my comfort.  So we re-implement here half of nix.fetchGit.
 279     # :(
 280
 281     cachedir = git_cachedir(channel.git_repo)
 282     if not os.path.exists(cachedir):
 283         v.status("Initializing git repo")
 284         process = subprocess.run(
 285             ['git', 'init', '--bare', cachedir])
 286         v.result(process.returncode == 0)
 287
 288     v.status('Fetching ref "%s" from %s' % (channel.git_ref, channel.git_repo))
 289     # We don't use --force here because we want to abort and freak out if forced
 290     # updates are happening.
 291     process = subprocess.run(['git',
 292                               '-C',
 293                               cachedir,
 294                               'fetch',
 295                               channel.git_repo,
 296                               '%s:%s' % (channel.git_ref,
 297                                          channel.git_ref)])
 298     v.result(process.returncode == 0)
 299
 300     if hasattr(channel, 'git_revision'):
 301         v.status('Verifying that fetch retrieved this rev')
 302         process = subprocess.run(
 303             ['git', '-C', cachedir, 'cat-file', '-e', channel.git_revision])
 304         v.result(process.returncode == 0)
 305     else:
 306         channel.git_revision = open(
 307             os.path.join(
 308                 cachedir,
 309                 'refs',
 310                 'heads',
 311                 channel.git_ref)).read(999).strip()
 312
 313     verify_git_ancestry(v, channel)
 314
 315
 316 def ensure_git_rev_available(v: Verification, channel: Channel) -> None:
 317     cachedir = git_cachedir(channel.git_repo)
 318     if os.path.exists(cachedir):
 319         v.status('Checking if we already have this rev:')
 320         process = subprocess.run(
 321             ['git', '-C', cachedir, 'cat-file', '-e', channel.git_revision])
 322         if process.returncode == 0:
 323             v.status('yes')
 324         if process.returncode == 1:
 325             v.status('no')
 326         v.result(process.returncode == 0 or process.returncode == 1)
 327         if process.returncode == 0:
 328             verify_git_ancestry(v, channel)
 329             return
 330     git_fetch(v, channel)
 331
 332
 333 def compare_tarball_and_git(
 334         v: Verification,
 335         channel: Channel,
 336         channel_contents: str,
 337         git_contents: str) -> None:
 338     v.status('Comparing channel tarball with git checkout')
 339     match, mismatch, errors = compare(os.path.join(
 340         channel_contents, channel.release_name), git_contents)
 341     v.ok()
 342     v.check('%d files match' % len(match), len(match) > 0)
 343     v.check('%d files differ' % len(mismatch), len(mismatch) == 0)
 344     expected_errors = [
 345         '.git-revision',
 346         '.version-suffix',
 347         'nixpkgs',
 348         'programs.sqlite',
 349         'svn-revision']
 350     benign_errors = []
 351     for ee in expected_errors:
 352         if ee in errors:
 353             errors.remove(ee)
 354             benign_errors.append(ee)
 355     v.check(
 356         '%d unexpected incomparable files' %
 357         len(errors),
 358         len(errors) == 0)
 359     v.check(
 360         '(%d of %d expected incomparable files)' %
 361         (len(benign_errors),
 362          len(expected_errors)),
 363         len(benign_errors) == len(expected_errors))
 364
 365
 366 def extract_tarball(v: Verification, channel: Channel, dest: str) -> None:
 367     v.status('Extracting tarball %s' %
 368              channel.table['nixexprs.tar.xz'].file)
 369     shutil.unpack_archive(
 370         channel.table['nixexprs.tar.xz'].file,
 371         dest)
 372     v.ok()
 373
 374
 375 def git_checkout(v: Verification, channel: Channel, dest: str) -> None:
 376     v.status('Checking out corresponding git revision')
 377     git = subprocess.Popen(['git',
 378                             '-C',
 379                             git_cachedir(channel.git_repo),
 380                             'archive',
 381                             channel.git_revision],
 382                            stdout=subprocess.PIPE)
 383     tar = subprocess.Popen(
 384         ['tar', 'x', '-C', dest, '-f', '-'], stdin=git.stdout)
 385     if git.stdout:
 386         git.stdout.close()
 387     tar.wait()
 388     git.wait()
 389     v.result(git.returncode == 0 and tar.returncode == 0)
 390
 391
 392 def git_get_tarball(v: Verification, channel: Channel) -> str:
 393     cache_file = tarball_cache_file(channel)
 394     if os.path.exists(cache_file):
 395         cached_tarball = open(cache_file).read(9999)
 396         if os.path.exists(cached_tarball):
 397             return cached_tarball
 398
 399     with tempfile.TemporaryDirectory() as output_dir:
 400         output_filename = os.path.join(
 401             output_dir, channel.release_name + '.tar.xz')
 402         with open(output_filename, 'w') as output_file:
 403             v.status(
 404                 'Generating tarball for git revision %s' %
 405                 channel.git_revision)
 406             git = subprocess.Popen(['git',
 407                                     '-C',
 408                                     git_cachedir(channel.git_repo),
 409                                     'archive',
 410                                     '--prefix=%s/' % channel.release_name,
 411                                     channel.git_revision],
 412                                    stdout=subprocess.PIPE)
 413             xz = subprocess.Popen(['xz'], stdin=git.stdout, stdout=output_file)
 414             xz.wait()
 415             git.wait()
 416             v.result(git.returncode == 0 and xz.returncode == 0)
 417
 418         v.status('Putting tarball in Nix store')
 419         process = subprocess.run(
 420             ['nix-store', '--add', output_filename], stdout=subprocess.PIPE)
 421         v.result(process.returncode == 0)
 422         store_tarball = process.stdout.decode().strip()
 423
 424         os.makedirs(os.path.dirname(cache_file), exist_ok=True)
 425         open(cache_file, 'w').write(store_tarball)
 426         return store_tarball  # type: ignore  # (for old mypy)
 427
 428
 429 def check_channel_metadata(
 430         v: Verification,
 431         channel: Channel,
 432         channel_contents: str) -> None:
 433     v.status('Verifying git commit in channel tarball')
 434     v.result(
 435         open(
 436             os.path.join(
 437                 channel_contents,
 438                 channel.release_name,
 439                 '.git-revision')).read(999) == channel.git_revision)
 440
 441     v.status(
 442         'Verifying version-suffix is a suffix of release name %s:' %
 443         channel.release_name)
 444     version_suffix = open(
 445         os.path.join(
 446             channel_contents,
 447             channel.release_name,
 448             '.version-suffix')).read(999)
 449     v.status(version_suffix)
 450     v.result(channel.release_name.endswith(version_suffix))
 451
 452
 453 def check_channel_contents(v: Verification, channel: Channel) -> None:
 454     with tempfile.TemporaryDirectory() as channel_contents, \
 455             tempfile.TemporaryDirectory() as git_contents:
 456
 457         extract_tarball(v, channel, channel_contents)
 458         check_channel_metadata(v, channel, channel_contents)
 459
 460         git_checkout(v, channel, git_contents)
 461
 462         compare_tarball_and_git(v, channel, channel_contents, git_contents)
 463
 464         v.status('Removing temporary directories')
 465     v.ok()
 466
 467
 468 def pin_channel(v: Verification, channel: Channel) -> None:
 469     fetch(v, channel)
 470     parse_channel(v, channel)
 471     fetch_resources(v, channel)
 472     ensure_git_rev_available(v, channel)
 473     check_channel_contents(v, channel)
 474
 475
 476 def git_revision_name(v: Verification, channel: Channel) -> str:
 477     v.status('Getting commit date')
 478     process = subprocess.run(['git',
 479                               '-C',
 480                               git_cachedir(channel.git_repo),
 481                               'log',
 482                               '-n1',
 483                               '--format=%ct-%h',
 484                               '--abbrev=11',
 485                               '--no-show-signature',
 486                               channel.git_revision],
 487                              stdout=subprocess.PIPE)
 488     v.result(process.returncode == 0 and process.stdout != b'')
 489     return '%s-%s' % (os.path.basename(channel.git_repo),
 490                       process.stdout.decode().strip())
 491
 492
 493 def read_config(filename: str) -> configparser.ConfigParser:
 494     config = configparser.ConfigParser()
 495     config.read_file(open(filename), filename)
 496     return config
 497
 498
 499 def pin(args: argparse.Namespace) -> None:
 500     v = Verification()
 501     config = read_config(args.channels_file)
 502     for section in config.sections():
 503         if args.channels and section not in args.channels:
 504             continue
 505
 506         channel = Channel(**dict(config[section].items()))
 507
 508         if hasattr(channel, 'alias_of'):
 509             assert not hasattr(channel, 'git_repo')
 510             continue
 511
 512         if hasattr(channel, 'git_revision'):
 513             channel.old_git_revision = channel.git_revision
 514             del channel.git_revision
 515
 516         if 'channel_url' in config[section]:
 517             pin_channel(v, channel)
 518             config[section]['release_name'] = channel.release_name
 519             config[section]['tarball_url'] = channel.table['nixexprs.tar.xz'].absolute_url
 520             config[section]['tarball_sha256'] = channel.table['nixexprs.tar.xz'].digest
 521         else:
 522             git_fetch(v, channel)
 523             config[section]['release_name'] = git_revision_name(v, channel)
 524         config[section]['git_revision'] = channel.git_revision
 525
 526     with open(args.channels_file, 'w') as configfile:
 527         config.write(configfile)
 528
 529
 530 def fetch_channel(
 531         v: Verification,
 532         section: str,
 533         conf: configparser.SectionProxy) -> str:
 534     if 'git_repo' not in conf or 'release_name' not in conf:
 535         raise Exception(
 536             'Cannot update unpinned channel "%s" (Run "pin" before "update")' %
 537             section)
 538
 539     if 'channel_url' in conf:
 540         return fetch_with_nix_prefetch_url(
 541             v, conf['tarball_url'], Digest16(
 542                 conf['tarball_sha256']))
 543
 544     channel = Channel(**dict(conf.items()))
 545     ensure_git_rev_available(v, channel)
 546     return git_get_tarball(v, channel)
 547
 548
 549 def update(args: argparse.Namespace) -> None:
 550     v = Verification()
 551     config = configparser.ConfigParser()
 552     exprs: Dict[str, str] = {}
 553     configs = [read_config(filename) for filename in args.channels_file]
 554     for config in configs:
 555         for section in config.sections():
 556             if 'alias_of' in config[section]:
 557                 assert 'git_repo' not in config[section]
 558                 continue
 559             tarball = fetch_channel(v, section, config[section])
 560             if section in exprs:
 561                 raise Exception('Duplicate channel "%s"' % section)
 562             exprs[section] = (
 563                 'f: f { name = "%s"; channelName = "%%s"; src = builtins.storePath "%s"; }' %
 564                 (config[section]['release_name'], tarball))
 565
 566     for config in configs:
 567         for section in config.sections():
 568             if 'alias_of' in config[section]:
 569                 if section in exprs:
 570                     raise Exception('Duplicate channel "%s"' % section)
 571                 exprs[section] = exprs[str(config[section]['alias_of'])]
 572
 573     command = [
 574         'nix-env',
 575         '--profile',
 576         '/nix/var/nix/profiles/per-user/%s/channels' %
 577         getpass.getuser(),
 578         '--show-trace',
 579         '--file',
 580         '<nix/unpack-channel.nix>',
 581         '--install',
 582         '--from-expression'] + [exprs[name] % name for name in sorted(exprs.keys())]
 583     if args.dry_run:
 584         print(' '.join(map(shlex.quote, command)))
 585     else:
 586         v.status('Installing channels with nix-env')
 587         process = subprocess.run(command)
 588         v.result(process.returncode == 0)
 589
 590
 591 def main() -> None:
 592     parser = argparse.ArgumentParser(prog='pinch')
 593     subparsers = parser.add_subparsers(dest='mode', required=True)
 594     parser_pin = subparsers.add_parser('pin')
 595     parser_pin.add_argument('channels_file', type=str)
 596     parser_pin.add_argument('channels', type=str, nargs='*')
 597     parser_pin.set_defaults(func=pin)
 598     parser_update = subparsers.add_parser('update')
 599     parser_update.add_argument('--dry-run', action='store_true')
 600     parser_update.add_argument('channels_file', type=str, nargs='+')
 601     parser_update.set_defaults(func=update)
 602     args = parser.parse_args()
 603     args.func(args)
 604
 605
 606 if __name__ == '__main__':
 607     main()