[git-cache] / git_cache.py

# It would be nice if we could share the nix git cache, but as of the
# time of writing it is transitioning from gitv2 (deprecated) to gitv3
# (not ready yet), and trying to straddle them both is too far into nix
# implementation details for my comfort.  So we re-implement here half of
# nix's builtins.fetchGit.  :(

import argparse
import functools
import hashlib
import logging
import os
import subprocess
import sys
import time

from typing import Iterator, NamedTuple, Optional, TypeVar, Tuple, Union

import backoff

Path = str   # eg: "/home/user/.cache/git-cache/v1"
Repo = str   # eg: "https://github.com/NixOS/nixpkgs.git"
Ref = str    # eg: "master" or "v1.0.0"
Rev = str    # eg: "53a27350551844e1ed1a9257690294767389ef0d"
RefOrRev = Union[Ref, Rev]


class _LogEntry(NamedTuple):
    ref: Ref
    rev: Rev


T = TypeVar('T')


def _repo_hashname(repo: Repo) -> str:
    return hashlib.sha256(repo.encode()).hexdigest()


def git_cachedir(repo: Repo) -> Path:
    # Use xdg module when it's less painful to have as a dependency
    XDG_CACHE_HOME = Path(
        os.environ.get('XDG_CACHE_HOME', os.path.expanduser('~/.cache')))

    return Path(os.path.join(
        XDG_CACHE_HOME,
        'git-cache/v1',
        _repo_hashname(repo)))


def _log_filename(repo: Repo) -> Path:
    # Use xdg module when it's less painful to have as a dependency
    XDG_DATA_HOME = Path(
        os.environ.get('XDG_DATA_HOME', os.path.expanduser('~/.local/share')))

    return Path(os.path.join(
        XDG_DATA_HOME,
        'git-cache/v1',
        _repo_hashname(repo)))


def is_ancestor(repo: Repo, descendant: RefOrRev, ancestor: RefOrRev) -> bool:
    cachedir = git_cachedir(repo)
    logging.debug('Checking if %s is an ancestor of %s', ancestor, descendant)
    process = subprocess.run(['git',
                              '-C',
                              cachedir,
                              'merge-base',
                              '--is-ancestor',
                              ancestor,
                              descendant],
                             check=False)
    return process.returncode == 0


def verify_ancestry(
        repo: Repo,
        descendant: RefOrRev,
        ancestor: RefOrRev,
        force: bool = False) -> None:
    if not force and not is_ancestor(repo, descendant, ancestor):
        raise Exception(f'{ancestor} is not an ancestor of {descendant}')


def _read_fetch_log(repo: Repo) -> Iterator[_LogEntry]:
    filename = _log_filename(repo)
    if not os.path.exists(filename):
        return
    with open(filename, 'r') as f:
        for line in f:
            _, _, rev, ref = line.strip().split(maxsplit=3)
            yield _LogEntry(ref, rev)


def _last(it: Iterator[T]) -> Optional[T]:
    return functools.reduce(lambda a, b: b, it, None)


def _previous_fetched_rev(repo: Repo, ref: Ref) -> Optional[Rev]:
    return _last(entry.rev for entry in _read_fetch_log(
        repo) if entry.ref == ref)


def _log_fetch(repo: Repo, ref: Ref, rev: Rev, force: bool = False) -> None:
    if not force:
        prev_rev = _previous_fetched_rev(repo, ref)
        if prev_rev is not None:
            verify_ancestry(repo, rev, prev_rev)
    filename = _log_filename(repo)
    os.makedirs(os.path.dirname(filename), exist_ok=True)
    with open(filename, 'a') as f:
        f.write(
            f'{time.strftime("%Y-%m%d-%H:%M:%S%z")} '
            f'{"FORCEDFETCH" if force else "fetch"} {rev} {ref}\n'
        )


def _show_force_warning() -> None:
    print('''
**************************************************************************
*               WARNING: git-cache INVOKED WITH --force!                 *
*                                                                        *
* This mode allows previously-fetched refs to be overwritten to point to *
* non-descendants -- commits that don't have the previous version of the *
* the ref in their history!                                              *
*                                                                        *
* This should only be invoked by a human operator who knows what they're *
* doing to correct a specific, known, problem.  Care should be taken to  *
* prevent recurrence.                                                    *
*                                                                        *
* Press ^C to abort.                                                     *
*                                                                        *
''', end='', file=sys.stderr)
    warn_time_override = os.environ.get('FORCE_WARNING_TIME', None)
    warn_time: int
    if warn_time_override is None:
        warn_time = 15
    else:
        warn_time = int(warn_time_override)
        print(
            '''*   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!   *
*   !!              WARNING DISPLAY TIME OVERRIDDEN                 !!   *
*   !!                                                              !!   *
*   !! This message is intended to be displayed long enough for a   !!   *
*   !! human operator to read it and have a chance to abort.  An    !!   *
*   !! override for the delay time is provided FOR THE UNIT TESTS   !!   *
*   !! to avoid delaying software builds unnecessarily.  This is    !!   *
*   !! INTENDED FOR USE IN UNIT TESTS ONLY; THIS MESSAGE SHOULD     !!   *
*   !! NEVER BE SEEN OUTSIDE BUILD LOGS!                            !!   *
*   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!   *
*                                                                        *
''', end='', file=sys.stderr)

    for i in range(warn_time, 0, -1):
        msg = f'* {f"Continuing in {i} seconds...":-70s} *'
        print(msg, file=sys.stderr)
        time.sleep(1)
    print('*' * 74, file=sys.stderr)


@backoff.on_exception(
    backoff.expo,
    subprocess.CalledProcessError,
    max_time=lambda: int(os.environ.get('BACKOFF_MAX_TIME', '30')))
def _git_fetch(
        cachedir: Path,
        repo: Repo,
        ref: Ref,
        force: bool = False) -> None:
    subprocess.run(['git', '-C', cachedir, 'fetch'] +
                   (['--force'] if force else []) +
                   [repo, f'{ref}:{ref}'], check=True)


def fetch(repo: Repo, ref: Ref, force: bool = False) -> Tuple[Path, Rev]:
    if force:
        _show_force_warning()
    cachedir = git_cachedir(repo)
    if not os.path.exists(cachedir):
        logging.debug("Initializing git repo")
        subprocess.run(['git',
                        '-c',
                        'init.defaultBranch=git-cache--no-default-branch',
                        'init',
                        '--bare',
                        cachedir],
                       check=True,
                       stdout=sys.stderr)

    logging.debug('Fetching ref "%s" from %s', ref, repo)
    _git_fetch(cachedir, repo, ref, force=force)

    with open(os.path.join(cachedir, 'refs', 'heads', ref)) as rev_file:
        rev = Rev(rev_file.read(999).strip())
    verify_ancestry(repo, ref, rev, force=force)
    _log_fetch(repo, ref, rev, force=force)

    return cachedir, rev


def ensure_rev_available(
        repo: Repo,
        ref: Ref,
        rev: Rev,
        force: bool = False) -> Path:
    cachedir = git_cachedir(repo)
    if os.path.exists(cachedir) and is_ancestor(repo, ref, rev):
        return cachedir

    logging.debug(
        'We do not have rev %s.  We will fetch ref "%s" and hope it appears.',
        rev, ref)
    fetch(repo, ref, force=force)
    logging.debug('Verifying that fetch retrieved rev %s', rev)
    subprocess.run(['git', '-C', cachedir, 'cat-file', '-e', rev], check=True)
    verify_ancestry(repo, ref, rev, force=force)

    return cachedir


def _main() -> None:
    parser = argparse.ArgumentParser(
        description='Cache remote git repositories locally.',
        epilog='example usage: git-cache https://github.com/NixOS/nixpkgs.git master')
    parser.add_argument(
        '--force',
        action='store_true',
        help='Recover from a force-push in the remote repo')
    parser.add_argument(
        'repo',
        metavar='Repo',
        type=Repo,
        help='Git repository URL')
    parser.add_argument(
        'ref',
        metavar='Ref',
        type=Ref,
        help='Ref (branch or tag) in the git repo')
    parser.add_argument(
        'rev',
        metavar='Rev',
        type=Rev,
        nargs='?',
        help='Ensure that this revision is present.  ' +
        'If this revision is already present locally, no network operations are performed.')
    args = parser.parse_args()

    if args.rev is None:
        cachedir, rev = fetch(args.repo, args.ref, force=args.force)
        print(f'{rev} {cachedir}')
    else:
        print(
            ensure_rev_available(
                args.repo,
                args.ref,
                args.rev,
                force=args.force))
Commit	Line	Data
bef7ce53 SW	1	# It would be nice if we could share the nix git cache, but as of the
	2	# time of writing it is transitioning from gitv2 (deprecated) to gitv3
	3	# (not ready yet), and trying to straddle them both is too far into nix
	4	# implementation details for my comfort. So we re-implement here half of
	5	# nix's builtins.fetchGit. :(
	6
d1ab0853	7	import argparse
35000f72	8	import functools
bef7ce53 SW	9	import hashlib
	10	import logging
	11	import os
	12	import subprocess
347be7cf	13	import sys
35000f72	14	import time
bef7ce53	15
35000f72	16	from typing import Iterator, NamedTuple, Optional, TypeVar, Tuple, Union
bef7ce53	17
f36d5c6f SW	18	import backoff
f36d5c6f SW	19
35000f72 SW	20	Path = str # eg: "/home/user/.cache/git-cache/v1"
	21	Repo = str # eg: "https://github.com/NixOS/nixpkgs.git"
	22	Ref = str # eg: "master" or "v1.0.0"
	23	Rev = str # eg: "53a27350551844e1ed1a9257690294767389ef0d"
a5d42d8d	24	RefOrRev = Union[Ref, Rev]
bef7ce53 SW	25
bef7ce53 SW	26
35000f72 SW	27	class _LogEntry(NamedTuple):
	28	ref: Ref
	29	rev: Rev
	30
	31
	32	T = TypeVar('T')
	33
	34
	35	def _repo_hashname(repo: Repo) -> str:
	36	return hashlib.sha256(repo.encode()).hexdigest()
	37
	38
bef7ce53 SW	39	def git_cachedir(repo: Repo) -> Path:
	40	# Use xdg module when it's less painful to have as a dependency
	41	XDG_CACHE_HOME = Path(
	42	os.environ.get('XDG_CACHE_HOME', os.path.expanduser('~/.cache')))
	43
	44	return Path(os.path.join(
	45	XDG_CACHE_HOME,
	46	'git-cache/v1',
35000f72 SW	47	_repo_hashname(repo)))
	48
	49
	50	def _log_filename(repo: Repo) -> Path:
	51	# Use xdg module when it's less painful to have as a dependency
	52	XDG_DATA_HOME = Path(
	53	os.environ.get('XDG_DATA_HOME', os.path.expanduser('~/.local/share')))
	54
	55	return Path(os.path.join(
	56	XDG_DATA_HOME,
	57	'git-cache/v1',
	58	_repo_hashname(repo)))
bef7ce53 SW	59
bef7ce53 SW	60
a5d42d8d	61	def is_ancestor(repo: Repo, descendant: RefOrRev, ancestor: RefOrRev) -> bool:
bef7ce53	62	cachedir = git_cachedir(repo)
a5d42d8d SW	63	logging.debug('Checking if %s is an ancestor of %s', ancestor, descendant)
	64	process = subprocess.run(['git',
	65	'-C',
	66	cachedir,
	67	'merge-base',
	68	'--is-ancestor',
	69	ancestor,
	70	descendant],
	71	check=False)
eb638847 SW	72	return process.returncode == 0
	73
	74
a5d42d8d SW	75	def verify_ancestry(
	76	repo: Repo,
	77	descendant: RefOrRev,
083b90e7 SW	78	ancestor: RefOrRev,
	79	force: bool = False) -> None:
	80	if not force and not is_ancestor(repo, descendant, ancestor):
f580771a	81	raise Exception(f'{ancestor} is not an ancestor of {descendant}')
bef7ce53 SW	82
bef7ce53 SW	83
35000f72 SW	84	def _read_fetch_log(repo: Repo) -> Iterator[_LogEntry]:
	85	filename = _log_filename(repo)
	86	if not os.path.exists(filename):
	87	return
	88	with open(filename, 'r') as f:
	89	for line in f:
	90	_, _, rev, ref = line.strip().split(maxsplit=3)
	91	yield _LogEntry(ref, rev)
	92
	93
	94	def _last(it: Iterator[T]) -> Optional[T]:
	95	return functools.reduce(lambda a, b: b, it, None)
	96
	97
	98	def _previous_fetched_rev(repo: Repo, ref: Ref) -> Optional[Rev]:
	99	return _last(entry.rev for entry in _read_fetch_log(
	100	repo) if entry.ref == ref)
	101
	102
083b90e7 SW	103	def _log_fetch(repo: Repo, ref: Ref, rev: Rev, force: bool = False) -> None:
	104	if not force:
	105	prev_rev = _previous_fetched_rev(repo, ref)
	106	if prev_rev is not None:
	107	verify_ancestry(repo, rev, prev_rev)
35000f72 SW	108	filename = _log_filename(repo)
	109	os.makedirs(os.path.dirname(filename), exist_ok=True)
	110	with open(filename, 'a') as f:
f580771a SW	111	f.write(
	112	f'{time.strftime("%Y-%m%d-%H:%M:%S%z")} '
	113	f'{"FORCEDFETCH" if force else "fetch"} {rev} {ref}\n'
	114	)
083b90e7 SW	115
	116
	117	def _show_force_warning() -> None:
	118	print('''
	119	**************************************************************************
	120	* WARNING: git-cache INVOKED WITH --force! *
	121	* *
	122	* This mode allows previously-fetched refs to be overwritten to point to *
	123	* non-descendants -- commits that don't have the previous version of the *
	124	* the ref in their history! *
	125	* *
	126	* This should only be invoked by a human operator who knows what they're *
	127	* doing to correct a specific, known, problem. Care should be taken to *
	128	* prevent recurrence. *
	129	* *
	130	* Press ^C to abort. *
	131	* *
	132	''', end='', file=sys.stderr)
	133	warn_time_override = os.environ.get('FORCE_WARNING_TIME', None)
	134	warn_time: int
	135	if warn_time_override is None:
	136	warn_time = 15
	137	else:
	138	warn_time = int(warn_time_override)
	139	print(
	140	'''* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! *
	141	* !! WARNING DISPLAY TIME OVERRIDDEN !! *
	142	* !! !! *
	143	* !! This message is intended to be displayed long enough for a !! *
	144	* !! human operator to read it and have a chance to abort. An !! *
	145	* !! override for the delay time is provided FOR THE UNIT TESTS !! *
	146	* !! to avoid delaying software builds unnecessarily. This is !! *
	147	* !! INTENDED FOR USE IN UNIT TESTS ONLY; THIS MESSAGE SHOULD !! *
	148	* !! NEVER BE SEEN OUTSIDE BUILD LOGS! !! *
	149	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! *
	150	* *
	151	''', end='', file=sys.stderr)
	152
	153	for i in range(warn_time, 0, -1):
f580771a	154	msg = f'* {f"Continuing in {i} seconds...":-70s} *'
083b90e7 SW	155	print(msg, file=sys.stderr)
	156	time.sleep(1)
	157	print('' 74, file=sys.stderr)
	158
35000f72	159
f36d5c6f SW	160	@backoff.on_exception(
	161	backoff.expo,
	162	subprocess.CalledProcessError,
	163	max_time=lambda: int(os.environ.get('BACKOFF_MAX_TIME', '30')))
083b90e7 SW	164	def _git_fetch(
	165	cachedir: Path,
	166	repo: Repo,
	167	ref: Ref,
	168	force: bool = False) -> None:
	169	subprocess.run(['git', '-C', cachedir, 'fetch'] +
	170	(['--force'] if force else []) +
f580771a	171	[repo, f'{ref}:{ref}'], check=True)
f36d5c6f SW	172
f36d5c6f SW	173
083b90e7 SW	174	def fetch(repo: Repo, ref: Ref, force: bool = False) -> Tuple[Path, Rev]:
	175	if force:
	176	_show_force_warning()
bef7ce53 SW	177	cachedir = git_cachedir(repo)
	178	if not os.path.exists(cachedir):
	179	logging.debug("Initializing git repo")
21971f7f SW	180	subprocess.run(['git',
	181	'-c',
	182	'init.defaultBranch=git-cache--no-default-branch',
	183	'init',
	184	'--bare',
	185	cachedir],
	186	check=True,
	187	stdout=sys.stderr)
bef7ce53 SW	188
bef7ce53 SW	189	logging.debug('Fetching ref "%s" from %s', ref, repo)
083b90e7	190	_git_fetch(cachedir, repo, ref, force=force)
bef7ce53 SW	191
	192	with open(os.path.join(cachedir, 'refs', 'heads', ref)) as rev_file:
	193	rev = Rev(rev_file.read(999).strip())
083b90e7 SW	194	verify_ancestry(repo, ref, rev, force=force)
083b90e7 SW	195	_log_fetch(repo, ref, rev, force=force)
bef7ce53 SW	196
	197	return cachedir, rev
	198
	199
083b90e7 SW	200	def ensure_rev_available(
	201	repo: Repo,
	202	ref: Ref,
	203	rev: Rev,
	204	force: bool = False) -> Path:
bef7ce53	205	cachedir = git_cachedir(repo)
eb638847 SW	206	if os.path.exists(cachedir) and is_ancestor(repo, ref, rev):
eb638847 SW	207	return cachedir
bef7ce53 SW	208
	209	logging.debug(
	210	'We do not have rev %s. We will fetch ref "%s" and hope it appears.',
	211	rev, ref)
083b90e7	212	fetch(repo, ref, force=force)
bef7ce53 SW	213	logging.debug('Verifying that fetch retrieved rev %s', rev)
bef7ce53 SW	214	subprocess.run(['git', '-C', cachedir, 'cat-file', '-e', rev], check=True)
083b90e7	215	verify_ancestry(repo, ref, rev, force=force)
bef7ce53 SW	216
bef7ce53 SW	217	return cachedir
347be7cf SW	218
	219
	220	def _main() -> None:
d1ab0853 SW	221	parser = argparse.ArgumentParser(
	222	description='Cache remote git repositories locally.',
	223	epilog='example usage: git-cache https://github.com/NixOS/nixpkgs.git master')
083b90e7 SW	224	parser.add_argument(
	225	'--force',
	226	action='store_true',
	227	help='Recover from a force-push in the remote repo')
d1ab0853 SW	228	parser.add_argument(
	229	'repo',
	230	metavar='Repo',
	231	type=Repo,
	232	help='Git repository URL')
	233	parser.add_argument(
	234	'ref',
	235	metavar='Ref',
	236	type=Ref,
	237	help='Ref (branch or tag) in the git repo')
	238	parser.add_argument(
	239	'rev',
	240	metavar='Rev',
	241	type=Rev,
	242	nargs='?',
	243	help='Ensure that this revision is present. ' +
	244	'If this revision is already present locally, no network operations are performed.')
	245	args = parser.parse_args()
	246
	247	if args.rev is None:
f580771a SW	248	cachedir, rev = fetch(args.repo, args.ref, force=args.force)
f580771a SW	249	print(f'{rev} {cachedir}')
347be7cf	250	else:
083b90e7 SW	251	print(
	252	ensure_rev_available(
	253	args.repo,
	254	args.ref,
	255	args.rev,
	256	force=args.force))