Compare channel tarball with git

diff --git a/pinch.py b/pinch.py
index a8b6103c3bb78e157673afadeca0864ff85ce231..e00bfd8cf0c43b1b2dda67d770668d5ebaf44041 100644 (file)
--- a/pinch.py
+++ b/pinch.py
@@ -17,7 +17,6 @@ from typing import (
     Iterable,
     List,
     NewType,
-    Sequence,
     Tuple,
 )
 
@@ -35,6 +34,9 @@ class ChannelTableEntry(types.SimpleNamespace):
 class Channel(types.SimpleNamespace):
     channel_html: bytes
     forwarded_url: str
+    git_cachedir: str
+    git_ref: str
+    git_repo: str
     git_revision: str
     release_name: str
     table: Dict[str, ChannelTableEntry]
@@ -76,10 +78,7 @@ class Verification:
         self.result(True)
 
 
-def compare(a: str,
-            b: str) -> Tuple[Sequence[str],
-                             Sequence[str],
-                             Sequence[str]]:
+def compare(a: str, b: str) -> Tuple[List[str], List[str], List[str]]:
 
     def throw(error: OSError) -> None:
         raise error
@@ -144,10 +143,15 @@ def parse_channel(v: Verification, channel: Channel) -> None:
         url = row.childNodes[0].firstChild.getAttribute('href')
         size = int(row.childNodes[1].firstChild.nodeValue)
         digest = Digest16(row.childNodes[2].firstChild.firstChild.nodeValue)
-        channel.table[name] = ChannelTableEntry(url=url, digest=digest, size=size)
+        channel.table[name] = ChannelTableEntry(
+            url=url, digest=digest, size=size)
     v.ok()
 
 
+def digest_string(s: bytes) -> Digest16:
+    return Digest16(hashlib.sha256(s).hexdigest())
+
+
 def digest_file(filename: str) -> Digest16:
     hasher = hashlib.sha256()
     with open(filename, 'rb') as f:
@@ -202,21 +206,121 @@ def fetch_resources(v: Verification, channel: Channel) -> None:
             channel.table['git-revision'].file).read(999) == channel.git_commit)
 
 
+def git_fetch(v: Verification, channel: Channel) -> None:
+    # It would be nice if we could share the nix git cache, but as of the time
+    # of writing it is transitioning from gitv2 (deprecated) to gitv3 (not ready
+    # yet), and trying to straddle them both is too far into nix implementation
+    # details for my comfort.  So we re-implement here half of nix.fetchGit.
+    # :(
+
+    # TODO: Consider using pyxdg to find this path.
+    channel.git_cachedir = os.path.expanduser(
+        '~/.cache/nix-pin-channel/git/%s' %
+        digest_string(
+            channel.url.encode()))
+    if not os.path.exists(channel.git_cachedir):
+        v.status("Initializing git repo")
+        process = subprocess.run(
+            ['git', 'init', '--bare', channel.git_cachedir])
+        v.result(process.returncode == 0)
+
+    v.status('Checking if we already have this rev:')
+    process = subprocess.run(
+        ['git', '-C', channel.git_cachedir, 'cat-file', '-e', channel.git_commit])
+    if process.returncode == 0:
+        v.status('yes')
+    if process.returncode == 1:
+        v.status('no')
+    v.result(process.returncode == 0 or process.returncode == 1)
+    if process.returncode == 1:
+        v.status('Fetching ref "%s"' % channel.git_ref)
+        # We don't use --force here because we want to abort and freak out if forced
+        # updates are happening.
+        process = subprocess.run(['git',
+                                  '-C',
+                                  channel.git_cachedir,
+                                  'fetch',
+                                  channel.git_repo,
+                                  '%s:%s' % (channel.git_ref,
+                                             channel.git_ref)])
+        v.result(process.returncode == 0)
+        v.status('Verifying that fetch retrieved this rev')
+        process = subprocess.run(
+            ['git', '-C', channel.git_cachedir, 'cat-file', '-e', channel.git_commit])
+        v.result(process.returncode == 0)
+
+    v.status('Verifying rev is an ancestor of ref')
+    process = subprocess.run(['git',
+                              '-C',
+                              channel.git_cachedir,
+                              'merge-base',
+                              '--is-ancestor',
+                              channel.git_commit,
+                              channel.git_ref])
+    v.result(process.returncode == 0)
+
+
 def check_channel_contents(v: Verification, channel: Channel) -> None:
-    with tempfile.TemporaryDirectory() as d:
-        v.status('Extracting %s' % channel.table['nixexprs.tar.xz'].file)
-        shutil.unpack_archive(channel.table['nixexprs.tar.xz'].file, d)
+    with tempfile.TemporaryDirectory() as channel_contents, \
+            tempfile.TemporaryDirectory() as git_contents:
+        v.status('Extracting tarball %s' %
+                 channel.table['nixexprs.tar.xz'].file)
+        shutil.unpack_archive(
+            channel.table['nixexprs.tar.xz'].file,
+            channel_contents)
+        v.ok()
+        v.status('Checking out corresponding git revision')
+        git = subprocess.Popen(['git',
+                                '-C',
+                                channel.git_cachedir,
+                                'archive',
+                                channel.git_commit],
+                               stdout=subprocess.PIPE)
+        tar = subprocess.Popen(
+            ['tar', 'x', '-C', git_contents, '-f', '-'], stdin=git.stdout)
+        git.stdout.close()
+        tar.wait()
+        git.wait()
+        v.result(git.returncode == 0 and tar.returncode == 0)
+        v.status('Comparing channel tarball with git checkout')
+        match, mismatch, errors = compare(os.path.join(
+            channel_contents, channel.release_name), git_contents)
         v.ok()
-        v.status('Removing temporary directory')
+        v.check('%d files match' % len(match), len(match) > 0)
+        v.check('%d files differ' % len(mismatch), len(mismatch) == 0)
+        expected_errors = [
+            '.git-revision',
+            '.version-suffix',
+            'nixpkgs',
+            'programs.sqlite',
+            'svn-revision']
+        benign_errors = []
+        for ee in expected_errors:
+            if ee in errors:
+                errors.remove(ee)
+                benign_errors.append(ee)
+        v.check(
+            '%d unexpected incomparable files' %
+            len(errors),
+            len(errors) == 0)
+        v.check(
+            '(%d of %d expected incomparable files)' %
+            (len(benign_errors),
+             len(expected_errors)),
+            len(benign_errors) == len(expected_errors))
+        v.status('Removing temporary directories')
     v.ok()
 
 
 def main() -> None:
     v = Verification()
-    channel = Channel(url='https://channels.nixos.org/nixos-20.03')
+    channel = Channel(url='https://channels.nixos.org/nixos-20.03',
+                      git_repo='https://github.com/NixOS/nixpkgs.git',
+                      git_ref='nixos-20.03')
     fetch(v, channel)
     parse_channel(v, channel)
     fetch_resources(v, channel)
+    git_fetch(v, channel)
     check_channel_contents(v, channel)
     print(channel)
 

diff --git a/test.sh b/test.sh
index 779a65b7c91e67940221a58a2a93a60dc8c4cecf..edab8d8316467eb0397a9d5f5e7b6d67b9342457 100755 (executable)
--- a/test.sh
+++ b/test.sh
@@ -8,7 +8,7 @@ find . -name '*.py' -print0 | xargs -0 mypy --strict --ignore-missing-imports
 
 find . -name '*_test.py' -print0 | xargs -0 -r -n1 python3
 
-find . -name '*.py' -print0 | xargs -0 pylint --reports=n --persistent=n --ignore-imports=y -d invalid-name,missing-docstring,too-few-public-methods
+find . -name '*.py' -print0 | xargs -0 pylint --reports=n --persistent=n --ignore-imports=y -d fixme,invalid-name,missing-docstring,too-few-public-methods
 
 formatting_needs_fixing=$(
   find . -name '*.py' -print0 |