From dc038df02de3df877d535f1c978ad7537eaf70a8 Mon Sep 17 00:00:00 2001 From: Scott Worley Date: Thu, 9 Apr 2020 15:56:45 -0700 Subject: [PATCH 1/1] Compare channel tarball with git --- pinch.py | 126 ++++++++++++++++++++++++++++++++++++++++++++++++++----- test.sh | 2 +- 2 files changed, 116 insertions(+), 12 deletions(-) diff --git a/pinch.py b/pinch.py index a8b6103..e00bfd8 100644 --- a/pinch.py +++ b/pinch.py @@ -17,7 +17,6 @@ from typing import ( Iterable, List, NewType, - Sequence, Tuple, ) @@ -35,6 +34,9 @@ class ChannelTableEntry(types.SimpleNamespace): class Channel(types.SimpleNamespace): channel_html: bytes forwarded_url: str + git_cachedir: str + git_ref: str + git_repo: str git_revision: str release_name: str table: Dict[str, ChannelTableEntry] @@ -76,10 +78,7 @@ class Verification: self.result(True) -def compare(a: str, - b: str) -> Tuple[Sequence[str], - Sequence[str], - Sequence[str]]: +def compare(a: str, b: str) -> Tuple[List[str], List[str], List[str]]: def throw(error: OSError) -> None: raise error @@ -144,10 +143,15 @@ def parse_channel(v: Verification, channel: Channel) -> None: url = row.childNodes[0].firstChild.getAttribute('href') size = int(row.childNodes[1].firstChild.nodeValue) digest = Digest16(row.childNodes[2].firstChild.firstChild.nodeValue) - channel.table[name] = ChannelTableEntry(url=url, digest=digest, size=size) + channel.table[name] = ChannelTableEntry( + url=url, digest=digest, size=size) v.ok() +def digest_string(s: bytes) -> Digest16: + return Digest16(hashlib.sha256(s).hexdigest()) + + def digest_file(filename: str) -> Digest16: hasher = hashlib.sha256() with open(filename, 'rb') as f: @@ -202,21 +206,121 @@ def fetch_resources(v: Verification, channel: Channel) -> None: channel.table['git-revision'].file).read(999) == channel.git_commit) +def git_fetch(v: Verification, channel: Channel) -> None: + # It would be nice if we could share the nix git cache, but as of the time + # of writing it is transitioning from gitv2 (deprecated) to gitv3 (not ready + # yet), and trying to straddle them both is too far into nix implementation + # details for my comfort. So we re-implement here half of nix.fetchGit. + # :( + + # TODO: Consider using pyxdg to find this path. + channel.git_cachedir = os.path.expanduser( + '~/.cache/nix-pin-channel/git/%s' % + digest_string( + channel.url.encode())) + if not os.path.exists(channel.git_cachedir): + v.status("Initializing git repo") + process = subprocess.run( + ['git', 'init', '--bare', channel.git_cachedir]) + v.result(process.returncode == 0) + + v.status('Checking if we already have this rev:') + process = subprocess.run( + ['git', '-C', channel.git_cachedir, 'cat-file', '-e', channel.git_commit]) + if process.returncode == 0: + v.status('yes') + if process.returncode == 1: + v.status('no') + v.result(process.returncode == 0 or process.returncode == 1) + if process.returncode == 1: + v.status('Fetching ref "%s"' % channel.git_ref) + # We don't use --force here because we want to abort and freak out if forced + # updates are happening. + process = subprocess.run(['git', + '-C', + channel.git_cachedir, + 'fetch', + channel.git_repo, + '%s:%s' % (channel.git_ref, + channel.git_ref)]) + v.result(process.returncode == 0) + v.status('Verifying that fetch retrieved this rev') + process = subprocess.run( + ['git', '-C', channel.git_cachedir, 'cat-file', '-e', channel.git_commit]) + v.result(process.returncode == 0) + + v.status('Verifying rev is an ancestor of ref') + process = subprocess.run(['git', + '-C', + channel.git_cachedir, + 'merge-base', + '--is-ancestor', + channel.git_commit, + channel.git_ref]) + v.result(process.returncode == 0) + + def check_channel_contents(v: Verification, channel: Channel) -> None: - with tempfile.TemporaryDirectory() as d: - v.status('Extracting %s' % channel.table['nixexprs.tar.xz'].file) - shutil.unpack_archive(channel.table['nixexprs.tar.xz'].file, d) + with tempfile.TemporaryDirectory() as channel_contents, \ + tempfile.TemporaryDirectory() as git_contents: + v.status('Extracting tarball %s' % + channel.table['nixexprs.tar.xz'].file) + shutil.unpack_archive( + channel.table['nixexprs.tar.xz'].file, + channel_contents) + v.ok() + v.status('Checking out corresponding git revision') + git = subprocess.Popen(['git', + '-C', + channel.git_cachedir, + 'archive', + channel.git_commit], + stdout=subprocess.PIPE) + tar = subprocess.Popen( + ['tar', 'x', '-C', git_contents, '-f', '-'], stdin=git.stdout) + git.stdout.close() + tar.wait() + git.wait() + v.result(git.returncode == 0 and tar.returncode == 0) + v.status('Comparing channel tarball with git checkout') + match, mismatch, errors = compare(os.path.join( + channel_contents, channel.release_name), git_contents) v.ok() - v.status('Removing temporary directory') + v.check('%d files match' % len(match), len(match) > 0) + v.check('%d files differ' % len(mismatch), len(mismatch) == 0) + expected_errors = [ + '.git-revision', + '.version-suffix', + 'nixpkgs', + 'programs.sqlite', + 'svn-revision'] + benign_errors = [] + for ee in expected_errors: + if ee in errors: + errors.remove(ee) + benign_errors.append(ee) + v.check( + '%d unexpected incomparable files' % + len(errors), + len(errors) == 0) + v.check( + '(%d of %d expected incomparable files)' % + (len(benign_errors), + len(expected_errors)), + len(benign_errors) == len(expected_errors)) + v.status('Removing temporary directories') v.ok() def main() -> None: v = Verification() - channel = Channel(url='https://channels.nixos.org/nixos-20.03') + channel = Channel(url='https://channels.nixos.org/nixos-20.03', + git_repo='https://github.com/NixOS/nixpkgs.git', + git_ref='nixos-20.03') fetch(v, channel) parse_channel(v, channel) fetch_resources(v, channel) + git_fetch(v, channel) check_channel_contents(v, channel) print(channel) diff --git a/test.sh b/test.sh index 779a65b..edab8d8 100755 --- a/test.sh +++ b/test.sh @@ -8,7 +8,7 @@ find . -name '*.py' -print0 | xargs -0 mypy --strict --ignore-missing-imports find . -name '*_test.py' -print0 | xargs -0 -r -n1 python3 -find . -name '*.py' -print0 | xargs -0 pylint --reports=n --persistent=n --ignore-imports=y -d invalid-name,missing-docstring,too-few-public-methods +find . -name '*.py' -print0 | xargs -0 pylint --reports=n --persistent=n --ignore-imports=y -d fixme,invalid-name,missing-docstring,too-few-public-methods formatting_needs_fixing=$( find . -name '*.py' -print0 | -- 2.44.1