From f36d5c6f216b5b8cc92e04b2a6f3148e95e50585 Mon Sep 17 00:00:00 2001 From: Scott Worley Date: Fri, 17 Jul 2020 17:29:18 -0700 Subject: [PATCH 1/1] Retry fetch with backoff --- Changelog | 2 ++ default.nix | 29 ++++++++++++++++++++++++++--- git_cache.py | 18 ++++++++++++++---- test.sh | 2 +- test_git_cache.py | 2 ++ 5 files changed, 45 insertions(+), 8 deletions(-) diff --git a/Changelog b/Changelog index 44b04c1..b5ae99c 100644 --- a/Changelog +++ b/Changelog @@ -1,4 +1,6 @@ ## [Unreleased] +### Added +- Retry fetch with backoff ## [1.1.0] - 2020-07-16 diff --git a/default.nix b/default.nix index 6c57b0d..6ac06fa 100644 --- a/default.nix +++ b/default.nix @@ -1,11 +1,34 @@ { pkgs ? import { }, lint ? false }: -pkgs.python3Packages.callPackage -({ lib, buildPythonPackage, nix, git, autopep8, mypy, pylint, }: + +let + # Remove this package definition after https://github.com/NixOS/nixpkgs/pull/93377 reaches stable + backoff-fallback = { lib, buildPythonPackage, fetchPypi }: + buildPythonPackage rec { + pname = "backoff"; + version = "1.10.0"; + src = fetchPypi { + inherit pname version; + sha256 = "190rdpfhpjvb6bjh99fhdkgfsfkjwky7nz7b0nn5ah67z8hs1yxq"; + }; + meta = with lib; { + description = "Function decoration for backoff and retry"; + homepage = "https://github.com/litl/backoff"; + license = licenses.mit; + maintainers = with maintainers; [ chkno ]; + }; + }; + +in pkgs.python3Packages.callPackage +({ lib, buildPythonPackage, nix, git, backoff, autopep8, mypy, pylint, }: buildPythonPackage rec { pname = "git-cache"; version = "1.2.0-pre"; src = lib.cleanSource ./.; + propagatedBuildInputs = [ backoff ]; checkInputs = [ nix git mypy ] ++ lib.optionals lint [ autopep8 pylint ]; doCheck = true; checkPhase = "./test.sh"; - }) { } + }) { + backoff = pkgs.python3Packages.backoff or (pkgs.python3Packages.callPackage + backoff-fallback { }); + } diff --git a/git_cache.py b/git_cache.py index 594fb8a..c848161 100644 --- a/git_cache.py +++ b/git_cache.py @@ -12,6 +12,8 @@ import sys from typing import Tuple +import backoff + Path = str # eg: "/home/user/.cache/git-cache/v1" Repo = str # eg: "https://github.com/NixOS/nixpkgs.git" Ref = str # eg: "master" or "v1.0.0" @@ -36,6 +38,17 @@ def verify_ancestry(repo: Repo, ref: Ref, rev: Rev) -> None: rev, ref], check=True) +@backoff.on_exception( + backoff.expo, + subprocess.CalledProcessError, + max_time=lambda: int(os.environ.get('BACKOFF_MAX_TIME', '30'))) +def _git_fetch(cachedir: Path, repo: Repo, ref: Ref) -> None: + # We don't use --force here because we want to abort and freak out if forced + # updates are happening. + subprocess.run(['git', '-C', cachedir, 'fetch', repo, + '%s:%s' % (ref, ref)], check=True) + + def fetch(repo: Repo, ref: Ref) -> Tuple[Path, Rev]: cachedir = git_cachedir(repo) if not os.path.exists(cachedir): @@ -43,10 +56,7 @@ def fetch(repo: Repo, ref: Ref) -> Tuple[Path, Rev]: subprocess.run(['git', 'init', '--bare', cachedir], check=True) logging.debug('Fetching ref "%s" from %s', ref, repo) - # We don't use --force here because we want to abort and freak out if forced - # updates are happening. - subprocess.run(['git', '-C', cachedir, 'fetch', repo, - '%s:%s' % (ref, ref)], check=True) + _git_fetch(cachedir, repo, ref) with open(os.path.join(cachedir, 'refs', 'heads', ref)) as rev_file: rev = Rev(rev_file.read(999).strip()) diff --git a/test.sh b/test.sh index a0e3f5b..dbdf643 100755 --- a/test.sh +++ b/test.sh @@ -5,7 +5,7 @@ set -e PARALLELISM=4 find . -name build -prune -o -name dist -prune -o -name '*.py' -print0 | - xargs -0 mypy --strict --ignore-missing-imports --no-warn-unused-ignores + xargs -0 mypy --strict --ignore-missing-imports --no-warn-unused-ignores --allow-untyped-decorators python3 -m unittest diff --git a/test_git_cache.py b/test_git_cache.py index 7408cad..ecb11e8 100644 --- a/test_git_cache.py +++ b/test_git_cache.py @@ -37,6 +37,8 @@ class TestGitCache(unittest.TestCase): os.environ['GIT_AUTHOR_EMAIL'] = 'test_git_cache@example.com' os.environ['GIT_COMMITTER_EMAIL'] = 'test_git_cache@example.com' + os.environ['BACKOFF_MAX_TIME'] = '0' + self.tempdir = tempfile.TemporaryDirectory(prefix='git_cache_test-') self.upstream = os.path.join(self.tempdir.name, 'upstream') subprocess.run(['git', 'init', self.upstream], check=True) -- 2.44.1