[git-cache] / git_cache.py

# git-cache: Cache git content locally
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation, version 3.


# It would be nice if we could share the nix git cache, but as of the
# time of writing it is transitioning from gitv2 (deprecated) to gitv3
# (not ready yet), and trying to straddle them both is too far into nix
# implementation details for my comfort.  So we re-implement here half of
# nix's builtins.fetchGit.  :(

import argparse
import functools
import hashlib
import logging
import os
import subprocess
import sys
import time

from typing import Iterator, NamedTuple, Optional, TypeVar, Tuple, Union

import backoff

Path = str   # eg: "/home/user/.cache/git-cache/v1"
Repo = str   # eg: "https://github.com/NixOS/nixpkgs.git"
Ref = str    # eg: "master" or "v1.0.0"
Rev = str    # eg: "53a27350551844e1ed1a9257690294767389ef0d"
RefOrRev = Union[Ref, Rev]


class _LogEntry(NamedTuple):
    ref: Ref
    rev: Rev


T = TypeVar('T')


def _repo_hashname(repo: Repo) -> str:
    return hashlib.sha256(repo.encode()).hexdigest()


def git_cachedir(repo: Repo) -> Path:
    # Use xdg module when it's less painful to have as a dependency
    XDG_CACHE_HOME = Path(
        os.environ.get('XDG_CACHE_HOME', os.path.expanduser('~/.cache')))

    return Path(os.path.join(
        XDG_CACHE_HOME,
        'git-cache/v1',
        _repo_hashname(repo)))


def _log_filename(repo: Repo) -> Path:
    # Use xdg module when it's less painful to have as a dependency
    XDG_DATA_HOME = Path(
        os.environ.get('XDG_DATA_HOME', os.path.expanduser('~/.local/share')))

    return Path(os.path.join(
        XDG_DATA_HOME,
        'git-cache/v1',
        _repo_hashname(repo)))


def is_ancestor(repo: Repo, descendant: RefOrRev, ancestor: RefOrRev) -> bool:
    cachedir = git_cachedir(repo)
    logging.debug('Checking if %s is an ancestor of %s', ancestor, descendant)
    process = subprocess.run(['git',
                              '-C',
                              cachedir,
                              'merge-base',
                              '--is-ancestor',
                              ancestor,
                              descendant],
                             check=False)
    return process.returncode == 0


def verify_ancestry(
        repo: Repo,
        descendant: RefOrRev,
        ancestor: RefOrRev,
        force: bool = False) -> None:
    if not force and not is_ancestor(repo, descendant, ancestor):
        raise Exception(f'{ancestor} is not an ancestor of {descendant}')


def _read_fetch_log(repo: Repo) -> Iterator[_LogEntry]:
    filename = _log_filename(repo)
    if not os.path.exists(filename):
        return
    with open(filename, 'r', encoding='utf-8') as f:
        for line in f:
            _, _, rev, ref = line.strip().split(maxsplit=3)
            yield _LogEntry(ref, rev)


def _last(it: Iterator[T]) -> Optional[T]:
    return functools.reduce(lambda a, b: b, it, None)


def _previous_fetched_rev(repo: Repo, ref: Ref) -> Optional[Rev]:
    return _last(entry.rev for entry in _read_fetch_log(
        repo) if entry.ref == ref)


def _log_fetch(repo: Repo, ref: Ref, rev: Rev, force: bool = False) -> None:
    if not force:
        prev_rev = _previous_fetched_rev(repo, ref)
        if prev_rev is not None:
            verify_ancestry(repo, rev, prev_rev)
    filename = _log_filename(repo)
    os.makedirs(os.path.dirname(filename), exist_ok=True)
    with open(filename, 'a', encoding='utf-8') as f:
        f.write(
            f'{time.strftime("%Y-%m%d-%H:%M:%S%z")} '
            f'{"FORCEDFETCH" if force else "fetch"} {rev} {ref}\n'
        )


def _show_force_warning() -> None:
    print('''
**************************************************************************
*               WARNING: git-cache INVOKED WITH --force!                 *
*                                                                        *
* This mode allows previously-fetched refs to be overwritten to point to *
* non-descendants -- commits that don't have the previous version of the *
* the ref in their history!                                              *
*                                                                        *
* This should only be invoked by a human operator who knows what they're *
* doing to correct a specific, known, problem.  Care should be taken to  *
* prevent recurrence.                                                    *
*                                                                        *
* Press ^C to abort.                                                     *
*                                                                        *
''', end='', file=sys.stderr)
    warn_time_override = os.environ.get('FORCE_WARNING_TIME', None)
    warn_time: int
    if warn_time_override is None:
        warn_time = 15
    else:
        warn_time = int(warn_time_override)
        print(
            '''*   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!   *
*   !!              WARNING DISPLAY TIME OVERRIDDEN                 !!   *
*   !!                                                              !!   *
*   !! This message is intended to be displayed long enough for a   !!   *
*   !! human operator to read it and have a chance to abort.  An    !!   *
*   !! override for the delay time is provided FOR THE UNIT TESTS   !!   *
*   !! to avoid delaying software builds unnecessarily.  This is    !!   *
*   !! INTENDED FOR USE IN UNIT TESTS ONLY; THIS MESSAGE SHOULD     !!   *
*   !! NEVER BE SEEN OUTSIDE BUILD LOGS!                            !!   *
*   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!   *
*                                                                        *
''', end='', file=sys.stderr)

    for i in range(warn_time, 0, -1):
        msg = f'* {f"Continuing in {i} seconds...":-70s} *'
        print(msg, file=sys.stderr)
        time.sleep(1)
    print('*' * 74, file=sys.stderr)


@backoff.on_exception(
    backoff.expo,
    subprocess.CalledProcessError,
    max_time=lambda: int(os.environ.get('BACKOFF_MAX_TIME', '30')))
def _git_fetch(
        cachedir: Path,
        repo: Repo,
        ref: Ref,
        force: bool = False) -> None:
    subprocess.run(['git', '-C', cachedir, 'fetch'] +
                   (['--force'] if force else []) +
                   [repo, f'{ref}:{ref}'], check=True)


def fetch(repo: Repo, ref: Ref, force: bool = False) -> Tuple[Path, Rev]:
    if force:
        _show_force_warning()
    cachedir = git_cachedir(repo)
    if not os.path.exists(cachedir):
        logging.debug("Initializing git repo")
        subprocess.run(['git',
                        '-c',
                        'init.defaultBranch=git-cache--no-default-branch',
                        'init',
                        '--bare',
                        cachedir],
                       check=True,
                       stdout=sys.stderr)

    logging.debug('Fetching ref "%s" from %s', ref, repo)
    _git_fetch(cachedir, repo, ref, force=force)

    with open(os.path.join(cachedir, 'refs', 'heads', ref), encoding='utf-8') as rev_file:
        rev = Rev(rev_file.read(999).strip())
    verify_ancestry(repo, ref, rev, force=force)
    _log_fetch(repo, ref, rev, force=force)

    return cachedir, rev


def ensure_rev_available(
        repo: Repo,
        ref: Ref,
        rev: Rev,
        force: bool = False) -> Path:
    cachedir = git_cachedir(repo)
    if os.path.exists(cachedir) and is_ancestor(repo, ref, rev):
        return cachedir

    logging.debug(
        'We do not have rev %s.  We will fetch ref "%s" and hope it appears.',
        rev, ref)
    fetch(repo, ref, force=force)
    logging.debug('Verifying that fetch retrieved rev %s', rev)
    subprocess.run(['git', '-C', cachedir, 'cat-file', '-e', rev], check=True)
    verify_ancestry(repo, ref, rev, force=force)

    return cachedir


def _main() -> None:
    parser = argparse.ArgumentParser(
        description='Cache remote git repositories locally.',
        epilog='example usage: git-cache https://github.com/NixOS/nixpkgs.git master')
    parser.add_argument(
        '--force',
        action='store_true',
        help='Recover from a force-push in the remote repo')
    parser.add_argument(
        'repo',
        metavar='Repo',
        type=Repo,
        help='Git repository URL')
    parser.add_argument(
        'ref',
        metavar='Ref',
        type=Ref,
        help='Ref (branch or tag) in the git repo')
    parser.add_argument(
        'rev',
        metavar='Rev',
        type=Rev,
        nargs='?',
        help='Ensure that this revision is present.  ' +
        'If this revision is already present locally, no network operations are performed.')
    args = parser.parse_args()

    if args.rev is None:
        cachedir, rev = fetch(args.repo, args.ref, force=args.force)
        print(f'{rev} {cachedir}')
    else:
        print(
            ensure_rev_available(
                args.repo,
                args.ref,
                args.rev,
                force=args.force))
Commit	Line	Data
50685beb SW	1	# git-cache: Cache git content locally
	2	#
	3	# This program is free software: you can redistribute it and/or modify it
	4	# under the terms of the GNU General Public License as published by the
	5	# Free Software Foundation, version 3.
	6
	7
bef7ce53 SW	8	# It would be nice if we could share the nix git cache, but as of the
	9	# time of writing it is transitioning from gitv2 (deprecated) to gitv3
	10	# (not ready yet), and trying to straddle them both is too far into nix
	11	# implementation details for my comfort. So we re-implement here half of
	12	# nix's builtins.fetchGit. :(
	13
d1ab0853	14	import argparse
35000f72	15	import functools
bef7ce53 SW	16	import hashlib
	17	import logging
	18	import os
	19	import subprocess
347be7cf	20	import sys
35000f72	21	import time
bef7ce53	22
35000f72	23	from typing import Iterator, NamedTuple, Optional, TypeVar, Tuple, Union
bef7ce53	24
f36d5c6f SW	25	import backoff
f36d5c6f SW	26
35000f72 SW	27	Path = str # eg: "/home/user/.cache/git-cache/v1"
	28	Repo = str # eg: "https://github.com/NixOS/nixpkgs.git"
	29	Ref = str # eg: "master" or "v1.0.0"
	30	Rev = str # eg: "53a27350551844e1ed1a9257690294767389ef0d"
a5d42d8d	31	RefOrRev = Union[Ref, Rev]
bef7ce53 SW	32
bef7ce53 SW	33
35000f72 SW	34	class _LogEntry(NamedTuple):
	35	ref: Ref
	36	rev: Rev
	37
	38
	39	T = TypeVar('T')
	40
	41
	42	def _repo_hashname(repo: Repo) -> str:
	43	return hashlib.sha256(repo.encode()).hexdigest()
	44
	45
bef7ce53 SW	46	def git_cachedir(repo: Repo) -> Path:
	47	# Use xdg module when it's less painful to have as a dependency
	48	XDG_CACHE_HOME = Path(
	49	os.environ.get('XDG_CACHE_HOME', os.path.expanduser('~/.cache')))
	50
	51	return Path(os.path.join(
	52	XDG_CACHE_HOME,
	53	'git-cache/v1',
35000f72 SW	54	_repo_hashname(repo)))
	55
	56
	57	def _log_filename(repo: Repo) -> Path:
	58	# Use xdg module when it's less painful to have as a dependency
	59	XDG_DATA_HOME = Path(
	60	os.environ.get('XDG_DATA_HOME', os.path.expanduser('~/.local/share')))
	61
	62	return Path(os.path.join(
	63	XDG_DATA_HOME,
	64	'git-cache/v1',
	65	_repo_hashname(repo)))
bef7ce53 SW	66
bef7ce53 SW	67
a5d42d8d	68	def is_ancestor(repo: Repo, descendant: RefOrRev, ancestor: RefOrRev) -> bool:
bef7ce53	69	cachedir = git_cachedir(repo)
a5d42d8d SW	70	logging.debug('Checking if %s is an ancestor of %s', ancestor, descendant)
	71	process = subprocess.run(['git',
	72	'-C',
	73	cachedir,
	74	'merge-base',
	75	'--is-ancestor',
	76	ancestor,
	77	descendant],
	78	check=False)
eb638847 SW	79	return process.returncode == 0
	80
	81
a5d42d8d SW	82	def verify_ancestry(
	83	repo: Repo,
	84	descendant: RefOrRev,
083b90e7 SW	85	ancestor: RefOrRev,
	86	force: bool = False) -> None:
	87	if not force and not is_ancestor(repo, descendant, ancestor):
f580771a	88	raise Exception(f'{ancestor} is not an ancestor of {descendant}')
bef7ce53 SW	89
bef7ce53 SW	90
35000f72 SW	91	def _read_fetch_log(repo: Repo) -> Iterator[_LogEntry]:
	92	filename = _log_filename(repo)
	93	if not os.path.exists(filename):
	94	return
a28f4bb7	95	with open(filename, 'r', encoding='utf-8') as f:
35000f72 SW	96	for line in f:
	97	_, _, rev, ref = line.strip().split(maxsplit=3)
	98	yield _LogEntry(ref, rev)
	99
	100
	101	def _last(it: Iterator[T]) -> Optional[T]:
	102	return functools.reduce(lambda a, b: b, it, None)
	103
	104
	105	def _previous_fetched_rev(repo: Repo, ref: Ref) -> Optional[Rev]:
	106	return _last(entry.rev for entry in _read_fetch_log(
	107	repo) if entry.ref == ref)
	108
	109
083b90e7 SW	110	def _log_fetch(repo: Repo, ref: Ref, rev: Rev, force: bool = False) -> None:
	111	if not force:
	112	prev_rev = _previous_fetched_rev(repo, ref)
	113	if prev_rev is not None:
	114	verify_ancestry(repo, rev, prev_rev)
35000f72 SW	115	filename = _log_filename(repo)
35000f72 SW	116	os.makedirs(os.path.dirname(filename), exist_ok=True)
a28f4bb7	117	with open(filename, 'a', encoding='utf-8') as f:
f580771a SW	118	f.write(
	119	f'{time.strftime("%Y-%m%d-%H:%M:%S%z")} '
	120	f'{"FORCEDFETCH" if force else "fetch"} {rev} {ref}\n'
	121	)
083b90e7 SW	122
	123
	124	def _show_force_warning() -> None:
	125	print('''
	126	**************************************************************************
	127	* WARNING: git-cache INVOKED WITH --force! *
	128	* *
	129	* This mode allows previously-fetched refs to be overwritten to point to *
	130	* non-descendants -- commits that don't have the previous version of the *
	131	* the ref in their history! *
	132	* *
	133	* This should only be invoked by a human operator who knows what they're *
	134	* doing to correct a specific, known, problem. Care should be taken to *
	135	* prevent recurrence. *
	136	* *
	137	* Press ^C to abort. *
	138	* *
	139	''', end='', file=sys.stderr)
	140	warn_time_override = os.environ.get('FORCE_WARNING_TIME', None)
	141	warn_time: int
	142	if warn_time_override is None:
	143	warn_time = 15
	144	else:
	145	warn_time = int(warn_time_override)
	146	print(
	147	'''* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! *
	148	* !! WARNING DISPLAY TIME OVERRIDDEN !! *
	149	* !! !! *
	150	* !! This message is intended to be displayed long enough for a !! *
	151	* !! human operator to read it and have a chance to abort. An !! *
	152	* !! override for the delay time is provided FOR THE UNIT TESTS !! *
	153	* !! to avoid delaying software builds unnecessarily. This is !! *
	154	* !! INTENDED FOR USE IN UNIT TESTS ONLY; THIS MESSAGE SHOULD !! *
	155	* !! NEVER BE SEEN OUTSIDE BUILD LOGS! !! *
	156	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! *
	157	* *
	158	''', end='', file=sys.stderr)
	159
	160	for i in range(warn_time, 0, -1):
f580771a	161	msg = f'* {f"Continuing in {i} seconds...":-70s} *'
083b90e7 SW	162	print(msg, file=sys.stderr)
	163	time.sleep(1)
	164	print('' 74, file=sys.stderr)
	165
35000f72	166
f36d5c6f SW	167	@backoff.on_exception(
	168	backoff.expo,
	169	subprocess.CalledProcessError,
	170	max_time=lambda: int(os.environ.get('BACKOFF_MAX_TIME', '30')))
083b90e7 SW	171	def _git_fetch(
	172	cachedir: Path,
	173	repo: Repo,
	174	ref: Ref,
	175	force: bool = False) -> None:
	176	subprocess.run(['git', '-C', cachedir, 'fetch'] +
	177	(['--force'] if force else []) +
f580771a	178	[repo, f'{ref}:{ref}'], check=True)
f36d5c6f SW	179
f36d5c6f SW	180
083b90e7 SW	181	def fetch(repo: Repo, ref: Ref, force: bool = False) -> Tuple[Path, Rev]:
	182	if force:
	183	_show_force_warning()
bef7ce53 SW	184	cachedir = git_cachedir(repo)
	185	if not os.path.exists(cachedir):
	186	logging.debug("Initializing git repo")
21971f7f SW	187	subprocess.run(['git',
	188	'-c',
	189	'init.defaultBranch=git-cache--no-default-branch',
	190	'init',
	191	'--bare',
	192	cachedir],
	193	check=True,
	194	stdout=sys.stderr)
bef7ce53 SW	195
bef7ce53 SW	196	logging.debug('Fetching ref "%s" from %s', ref, repo)
083b90e7	197	_git_fetch(cachedir, repo, ref, force=force)
bef7ce53	198
a28f4bb7	199	with open(os.path.join(cachedir, 'refs', 'heads', ref), encoding='utf-8') as rev_file:
bef7ce53	200	rev = Rev(rev_file.read(999).strip())
083b90e7 SW	201	verify_ancestry(repo, ref, rev, force=force)
083b90e7 SW	202	_log_fetch(repo, ref, rev, force=force)
bef7ce53 SW	203
	204	return cachedir, rev
	205
	206
083b90e7 SW	207	def ensure_rev_available(
	208	repo: Repo,
	209	ref: Ref,
	210	rev: Rev,
	211	force: bool = False) -> Path:
bef7ce53	212	cachedir = git_cachedir(repo)
eb638847 SW	213	if os.path.exists(cachedir) and is_ancestor(repo, ref, rev):
eb638847 SW	214	return cachedir
bef7ce53 SW	215
	216	logging.debug(
	217	'We do not have rev %s. We will fetch ref "%s" and hope it appears.',
	218	rev, ref)
083b90e7	219	fetch(repo, ref, force=force)
bef7ce53 SW	220	logging.debug('Verifying that fetch retrieved rev %s', rev)
bef7ce53 SW	221	subprocess.run(['git', '-C', cachedir, 'cat-file', '-e', rev], check=True)
083b90e7	222	verify_ancestry(repo, ref, rev, force=force)
bef7ce53 SW	223
bef7ce53 SW	224	return cachedir
347be7cf SW	225
	226
	227	def _main() -> None:
d1ab0853 SW	228	parser = argparse.ArgumentParser(
	229	description='Cache remote git repositories locally.',
	230	epilog='example usage: git-cache https://github.com/NixOS/nixpkgs.git master')
083b90e7 SW	231	parser.add_argument(
	232	'--force',
	233	action='store_true',
	234	help='Recover from a force-push in the remote repo')
d1ab0853 SW	235	parser.add_argument(
	236	'repo',
	237	metavar='Repo',
	238	type=Repo,
	239	help='Git repository URL')
	240	parser.add_argument(
	241	'ref',
	242	metavar='Ref',
	243	type=Ref,
	244	help='Ref (branch or tag) in the git repo')
	245	parser.add_argument(
	246	'rev',
	247	metavar='Rev',
	248	type=Rev,
	249	nargs='?',
	250	help='Ensure that this revision is present. ' +
	251	'If this revision is already present locally, no network operations are performed.')
	252	args = parser.parse_args()
	253
	254	if args.rev is None:
f580771a SW	255	cachedir, rev = fetch(args.repo, args.ref, force=args.force)
f580771a SW	256	print(f'{rev} {cachedir}')
347be7cf	257	else:
083b90e7 SW	258	print(
	259	ensure_rev_available(
	260	args.repo,
	261	args.ref,
	262	args.rev,
	263	force=args.force))