## [Unreleased]
+### Added
+- Retry fetch with backoff
## [1.1.0] - 2020-07-16
{ pkgs ? import <nixpkgs> { }, lint ? false }:
-pkgs.python3Packages.callPackage
-({ lib, buildPythonPackage, nix, git, autopep8, mypy, pylint, }:
+
+let
+ # Remove this package definition after https://github.com/NixOS/nixpkgs/pull/93377 reaches stable
+ backoff-fallback = { lib, buildPythonPackage, fetchPypi }:
+ buildPythonPackage rec {
+ pname = "backoff";
+ version = "1.10.0";
+ src = fetchPypi {
+ inherit pname version;
+ sha256 = "190rdpfhpjvb6bjh99fhdkgfsfkjwky7nz7b0nn5ah67z8hs1yxq";
+ };
+ meta = with lib; {
+ description = "Function decoration for backoff and retry";
+ homepage = "https://github.com/litl/backoff";
+ license = licenses.mit;
+ maintainers = with maintainers; [ chkno ];
+ };
+ };
+
+in pkgs.python3Packages.callPackage
+({ lib, buildPythonPackage, nix, git, backoff, autopep8, mypy, pylint, }:
buildPythonPackage rec {
pname = "git-cache";
version = "1.2.0-pre";
src = lib.cleanSource ./.;
+ propagatedBuildInputs = [ backoff ];
checkInputs = [ nix git mypy ] ++ lib.optionals lint [ autopep8 pylint ];
doCheck = true;
checkPhase = "./test.sh";
- }) { }
+ }) {
+ backoff = pkgs.python3Packages.backoff or (pkgs.python3Packages.callPackage
+ backoff-fallback { });
+ }
from typing import Tuple
+import backoff
+
Path = str # eg: "/home/user/.cache/git-cache/v1"
Repo = str # eg: "https://github.com/NixOS/nixpkgs.git"
Ref = str # eg: "master" or "v1.0.0"
rev, ref], check=True)
+@backoff.on_exception(
+ backoff.expo,
+ subprocess.CalledProcessError,
+ max_time=lambda: int(os.environ.get('BACKOFF_MAX_TIME', '30')))
+def _git_fetch(cachedir: Path, repo: Repo, ref: Ref) -> None:
+ # We don't use --force here because we want to abort and freak out if forced
+ # updates are happening.
+ subprocess.run(['git', '-C', cachedir, 'fetch', repo,
+ '%s:%s' % (ref, ref)], check=True)
+
+
def fetch(repo: Repo, ref: Ref) -> Tuple[Path, Rev]:
cachedir = git_cachedir(repo)
if not os.path.exists(cachedir):
subprocess.run(['git', 'init', '--bare', cachedir], check=True)
logging.debug('Fetching ref "%s" from %s', ref, repo)
- # We don't use --force here because we want to abort and freak out if forced
- # updates are happening.
- subprocess.run(['git', '-C', cachedir, 'fetch', repo,
- '%s:%s' % (ref, ref)], check=True)
+ _git_fetch(cachedir, repo, ref)
with open(os.path.join(cachedir, 'refs', 'heads', ref)) as rev_file:
rev = Rev(rev_file.read(999).strip())
PARALLELISM=4
find . -name build -prune -o -name dist -prune -o -name '*.py' -print0 |
- xargs -0 mypy --strict --ignore-missing-imports --no-warn-unused-ignores
+ xargs -0 mypy --strict --ignore-missing-imports --no-warn-unused-ignores --allow-untyped-decorators
python3 -m unittest
os.environ['GIT_AUTHOR_EMAIL'] = 'test_git_cache@example.com'
os.environ['GIT_COMMITTER_EMAIL'] = 'test_git_cache@example.com'
+ os.environ['BACKOFF_MAX_TIME'] = '0'
+
self.tempdir = tempfile.TemporaryDirectory(prefix='git_cache_test-')
self.upstream = os.path.join(self.tempdir.name, 'upstream')
subprocess.run(['git', 'init', self.upstream], check=True)