[git-cache] / git_cache.py

# It would be nice if we could share the nix git cache, but as of the
# time of writing it is transitioning from gitv2 (deprecated) to gitv3
# (not ready yet), and trying to straddle them both is too far into nix
# implementation details for my comfort.  So we re-implement here half of
# nix's builtins.fetchGit.  :(

import argparse
import functools
import hashlib
import logging
import os
import subprocess
import sys
import time

from typing import Iterator, NamedTuple, Optional, TypeVar, Tuple, Union

import backoff

Path = str   # eg: "/home/user/.cache/git-cache/v1"
Repo = str   # eg: "https://github.com/NixOS/nixpkgs.git"
Ref = str    # eg: "master" or "v1.0.0"
Rev = str    # eg: "53a27350551844e1ed1a9257690294767389ef0d"
RefOrRev = Union[Ref, Rev]


class _LogEntry(NamedTuple):
    ref: Ref
    rev: Rev


T = TypeVar('T')


def _repo_hashname(repo: Repo) -> str:
    return hashlib.sha256(repo.encode()).hexdigest()


def git_cachedir(repo: Repo) -> Path:
    # Use xdg module when it's less painful to have as a dependency
    XDG_CACHE_HOME = Path(
        os.environ.get('XDG_CACHE_HOME', os.path.expanduser('~/.cache')))

    return Path(os.path.join(
        XDG_CACHE_HOME,
        'git-cache/v1',
        _repo_hashname(repo)))


def _log_filename(repo: Repo) -> Path:
    # Use xdg module when it's less painful to have as a dependency
    XDG_DATA_HOME = Path(
        os.environ.get('XDG_DATA_HOME', os.path.expanduser('~/.local/share')))

    return Path(os.path.join(
        XDG_DATA_HOME,
        'git-cache/v1',
        _repo_hashname(repo)))


def is_ancestor(repo: Repo, descendant: RefOrRev, ancestor: RefOrRev) -> bool:
    cachedir = git_cachedir(repo)
    logging.debug('Checking if %s is an ancestor of %s', ancestor, descendant)
    process = subprocess.run(['git',
                              '-C',
                              cachedir,
                              'merge-base',
                              '--is-ancestor',
                              ancestor,
                              descendant],
                             check=False)
    return process.returncode == 0


def verify_ancestry(
        repo: Repo,
        descendant: RefOrRev,
        ancestor: RefOrRev,
        force: bool = False) -> None:
    if not force and not is_ancestor(repo, descendant, ancestor):
        raise Exception('%s is not an ancestor of %s' % (ancestor, descendant))


def _read_fetch_log(repo: Repo) -> Iterator[_LogEntry]:
    filename = _log_filename(repo)
    if not os.path.exists(filename):
        return
    with open(filename, 'r') as f:
        for line in f:
            _, _, rev, ref = line.strip().split(maxsplit=3)
            yield _LogEntry(ref, rev)


def _last(it: Iterator[T]) -> Optional[T]:
    return functools.reduce(lambda a, b: b, it, None)


def _previous_fetched_rev(repo: Repo, ref: Ref) -> Optional[Rev]:
    return _last(entry.rev for entry in _read_fetch_log(
        repo) if entry.ref == ref)


def _log_fetch(repo: Repo, ref: Ref, rev: Rev, force: bool = False) -> None:
    if not force:
        prev_rev = _previous_fetched_rev(repo, ref)
        if prev_rev is not None:
            verify_ancestry(repo, rev, prev_rev)
    filename = _log_filename(repo)
    os.makedirs(os.path.dirname(filename), exist_ok=True)
    with open(filename, 'a') as f:
        f.write('%s %s %s %s\n' %
                (time.strftime('%Y-%m%d-%H:%M:%S%z'),
                 ('FORCEDFETCH' if force else 'fetch'), rev, ref))


def _show_force_warning() -> None:
    print('''
**************************************************************************
*               WARNING: git-cache INVOKED WITH --force!                 *
*                                                                        *
* This mode allows previously-fetched refs to be overwritten to point to *
* non-descendants -- commits that don't have the previous version of the *
* the ref in their history!                                              *
*                                                                        *
* This should only be invoked by a human operator who knows what they're *
* doing to correct a specific, known, problem.  Care should be taken to  *
* prevent recurrence.                                                    *
*                                                                        *
* Press ^C to abort.                                                     *
*                                                                        *
''', end='', file=sys.stderr)
    warn_time_override = os.environ.get('FORCE_WARNING_TIME', None)
    warn_time: int
    if warn_time_override is None:
        warn_time = 15
    else:
        warn_time = int(warn_time_override)
        print(
            '''*   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!   *
*   !!              WARNING DISPLAY TIME OVERRIDDEN                 !!   *
*   !!                                                              !!   *
*   !! This message is intended to be displayed long enough for a   !!   *
*   !! human operator to read it and have a chance to abort.  An    !!   *
*   !! override for the delay time is provided FOR THE UNIT TESTS   !!   *
*   !! to avoid delaying software builds unnecessarily.  This is    !!   *
*   !! INTENDED FOR USE IN UNIT TESTS ONLY; THIS MESSAGE SHOULD     !!   *
*   !! NEVER BE SEEN OUTSIDE BUILD LOGS!                            !!   *
*   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!   *
*                                                                        *
''', end='', file=sys.stderr)

    for i in range(warn_time, 0, -1):
        msg = '* %-70s *' % ("Continuing in %d seconds..." % i)
        print(msg, file=sys.stderr)
        time.sleep(1)
    print('*' * 74, file=sys.stderr)


@backoff.on_exception(
    backoff.expo,
    subprocess.CalledProcessError,
    max_time=lambda: int(os.environ.get('BACKOFF_MAX_TIME', '30')))
def _git_fetch(
        cachedir: Path,
        repo: Repo,
        ref: Ref,
        force: bool = False) -> None:
    subprocess.run(['git', '-C', cachedir, 'fetch'] +
                   (['--force'] if force else []) +
                   [repo, '%s:%s' % (ref, ref)], check=True)


def fetch(repo: Repo, ref: Ref, force: bool = False) -> Tuple[Path, Rev]:
    if force:
        _show_force_warning()
    cachedir = git_cachedir(repo)
    if not os.path.exists(cachedir):
        logging.debug("Initializing git repo")
        subprocess.run(['git', 'init', '--bare', cachedir],
                       check=True, stdout=sys.stderr)

    logging.debug('Fetching ref "%s" from %s', ref, repo)
    _git_fetch(cachedir, repo, ref, force=force)

    with open(os.path.join(cachedir, 'refs', 'heads', ref)) as rev_file:
        rev = Rev(rev_file.read(999).strip())
    verify_ancestry(repo, ref, rev, force=force)
    _log_fetch(repo, ref, rev, force=force)

    return cachedir, rev


def ensure_rev_available(
        repo: Repo,
        ref: Ref,
        rev: Rev,
        force: bool = False) -> Path:
    cachedir = git_cachedir(repo)
    if os.path.exists(cachedir) and is_ancestor(repo, ref, rev):
        return cachedir

    logging.debug(
        'We do not have rev %s.  We will fetch ref "%s" and hope it appears.',
        rev, ref)
    fetch(repo, ref, force=force)
    logging.debug('Verifying that fetch retrieved rev %s', rev)
    subprocess.run(['git', '-C', cachedir, 'cat-file', '-e', rev], check=True)
    verify_ancestry(repo, ref, rev, force=force)

    return cachedir


def _main() -> None:
    parser = argparse.ArgumentParser(
        description='Cache remote git repositories locally.',
        epilog='example usage: git-cache https://github.com/NixOS/nixpkgs.git master')
    parser.add_argument(
        '--force',
        action='store_true',
        help='Recover from a force-push in the remote repo')
    parser.add_argument(
        'repo',
        metavar='Repo',
        type=Repo,
        help='Git repository URL')
    parser.add_argument(
        'ref',
        metavar='Ref',
        type=Ref,
        help='Ref (branch or tag) in the git repo')
    parser.add_argument(
        'rev',
        metavar='Rev',
        type=Rev,
        nargs='?',
        help='Ensure that this revision is present.  ' +
        'If this revision is already present locally, no network operations are performed.')
    args = parser.parse_args()

    if args.rev is None:
        print('{1} {0}'.format(*fetch(args.repo, args.ref, force=args.force)))
    else:
        print(
            ensure_rev_available(
                args.repo,
                args.ref,
                args.rev,
                force=args.force))
Commit	Line	Data
bef7ce53 SW	1	# It would be nice if we could share the nix git cache, but as of the
	2	# time of writing it is transitioning from gitv2 (deprecated) to gitv3
	3	# (not ready yet), and trying to straddle them both is too far into nix
	4	# implementation details for my comfort. So we re-implement here half of
	5	# nix's builtins.fetchGit. :(
	6
d1ab0853	7	import argparse
35000f72	8	import functools
bef7ce53 SW	9	import hashlib
	10	import logging
	11	import os
	12	import subprocess
347be7cf	13	import sys
35000f72	14	import time
bef7ce53	15
35000f72	16	from typing import Iterator, NamedTuple, Optional, TypeVar, Tuple, Union
bef7ce53	17
f36d5c6f SW	18	import backoff
f36d5c6f SW	19
35000f72 SW	20	Path = str # eg: "/home/user/.cache/git-cache/v1"
	21	Repo = str # eg: "https://github.com/NixOS/nixpkgs.git"
	22	Ref = str # eg: "master" or "v1.0.0"
	23	Rev = str # eg: "53a27350551844e1ed1a9257690294767389ef0d"
a5d42d8d	24	RefOrRev = Union[Ref, Rev]
bef7ce53 SW	25
bef7ce53 SW	26
35000f72 SW	27	class _LogEntry(NamedTuple):
	28	ref: Ref
	29	rev: Rev
	30
	31
	32	T = TypeVar('T')
	33
	34
	35	def _repo_hashname(repo: Repo) -> str:
	36	return hashlib.sha256(repo.encode()).hexdigest()
	37
	38
bef7ce53 SW	39	def git_cachedir(repo: Repo) -> Path:
	40	# Use xdg module when it's less painful to have as a dependency
	41	XDG_CACHE_HOME = Path(
	42	os.environ.get('XDG_CACHE_HOME', os.path.expanduser('~/.cache')))
	43
	44	return Path(os.path.join(
	45	XDG_CACHE_HOME,
	46	'git-cache/v1',
35000f72 SW	47	_repo_hashname(repo)))
	48
	49
	50	def _log_filename(repo: Repo) -> Path:
	51	# Use xdg module when it's less painful to have as a dependency
	52	XDG_DATA_HOME = Path(
	53	os.environ.get('XDG_DATA_HOME', os.path.expanduser('~/.local/share')))
	54
	55	return Path(os.path.join(
	56	XDG_DATA_HOME,
	57	'git-cache/v1',
	58	_repo_hashname(repo)))
bef7ce53 SW	59
bef7ce53 SW	60
a5d42d8d	61	def is_ancestor(repo: Repo, descendant: RefOrRev, ancestor: RefOrRev) -> bool:
bef7ce53	62	cachedir = git_cachedir(repo)
a5d42d8d SW	63	logging.debug('Checking if %s is an ancestor of %s', ancestor, descendant)
	64	process = subprocess.run(['git',
	65	'-C',
	66	cachedir,
	67	'merge-base',
	68	'--is-ancestor',
	69	ancestor,
	70	descendant],
	71	check=False)
eb638847 SW	72	return process.returncode == 0
	73
	74
a5d42d8d SW	75	def verify_ancestry(
	76	repo: Repo,
	77	descendant: RefOrRev,
083b90e7 SW	78	ancestor: RefOrRev,
	79	force: bool = False) -> None:
	80	if not force and not is_ancestor(repo, descendant, ancestor):
a5d42d8d	81	raise Exception('%s is not an ancestor of %s' % (ancestor, descendant))
bef7ce53 SW	82
bef7ce53 SW	83
35000f72 SW	84	def _read_fetch_log(repo: Repo) -> Iterator[_LogEntry]:
	85	filename = _log_filename(repo)
	86	if not os.path.exists(filename):
	87	return
	88	with open(filename, 'r') as f:
	89	for line in f:
	90	_, _, rev, ref = line.strip().split(maxsplit=3)
	91	yield _LogEntry(ref, rev)
	92
	93
	94	def _last(it: Iterator[T]) -> Optional[T]:
	95	return functools.reduce(lambda a, b: b, it, None)
	96
	97
	98	def _previous_fetched_rev(repo: Repo, ref: Ref) -> Optional[Rev]:
	99	return _last(entry.rev for entry in _read_fetch_log(
	100	repo) if entry.ref == ref)
	101
	102
083b90e7 SW	103	def _log_fetch(repo: Repo, ref: Ref, rev: Rev, force: bool = False) -> None:
	104	if not force:
	105	prev_rev = _previous_fetched_rev(repo, ref)
	106	if prev_rev is not None:
	107	verify_ancestry(repo, rev, prev_rev)
35000f72 SW	108	filename = _log_filename(repo)
	109	os.makedirs(os.path.dirname(filename), exist_ok=True)
	110	with open(filename, 'a') as f:
083b90e7 SW	111	f.write('%s %s %s %s\n' %
	112	(time.strftime('%Y-%m%d-%H:%M:%S%z'),
	113	('FORCEDFETCH' if force else 'fetch'), rev, ref))
	114
	115
	116	def _show_force_warning() -> None:
	117	print('''
	118	**************************************************************************
	119	* WARNING: git-cache INVOKED WITH --force! *
	120	* *
	121	* This mode allows previously-fetched refs to be overwritten to point to *
	122	* non-descendants -- commits that don't have the previous version of the *
	123	* the ref in their history! *
	124	* *
	125	* This should only be invoked by a human operator who knows what they're *
	126	* doing to correct a specific, known, problem. Care should be taken to *
	127	* prevent recurrence. *
	128	* *
	129	* Press ^C to abort. *
	130	* *
	131	''', end='', file=sys.stderr)
	132	warn_time_override = os.environ.get('FORCE_WARNING_TIME', None)
	133	warn_time: int
	134	if warn_time_override is None:
	135	warn_time = 15
	136	else:
	137	warn_time = int(warn_time_override)
	138	print(
	139	'''* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! *
	140	* !! WARNING DISPLAY TIME OVERRIDDEN !! *
	141	* !! !! *
	142	* !! This message is intended to be displayed long enough for a !! *
	143	* !! human operator to read it and have a chance to abort. An !! *
	144	* !! override for the delay time is provided FOR THE UNIT TESTS !! *
	145	* !! to avoid delaying software builds unnecessarily. This is !! *
	146	* !! INTENDED FOR USE IN UNIT TESTS ONLY; THIS MESSAGE SHOULD !! *
	147	* !! NEVER BE SEEN OUTSIDE BUILD LOGS! !! *
	148	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! *
	149	* *
	150	''', end='', file=sys.stderr)
	151
	152	for i in range(warn_time, 0, -1):
	153	msg = '* %-70s *' % ("Continuing in %d seconds..." % i)
	154	print(msg, file=sys.stderr)
	155	time.sleep(1)
	156	print('' 74, file=sys.stderr)
	157
35000f72	158
f36d5c6f SW	159	@backoff.on_exception(
	160	backoff.expo,
	161	subprocess.CalledProcessError,
	162	max_time=lambda: int(os.environ.get('BACKOFF_MAX_TIME', '30')))
083b90e7 SW	163	def _git_fetch(
	164	cachedir: Path,
	165	repo: Repo,
	166	ref: Ref,
	167	force: bool = False) -> None:
	168	subprocess.run(['git', '-C', cachedir, 'fetch'] +
	169	(['--force'] if force else []) +
	170	[repo, '%s:%s' % (ref, ref)], check=True)
f36d5c6f SW	171
f36d5c6f SW	172
083b90e7 SW	173	def fetch(repo: Repo, ref: Ref, force: bool = False) -> Tuple[Path, Rev]:
	174	if force:
	175	_show_force_warning()
bef7ce53 SW	176	cachedir = git_cachedir(repo)
	177	if not os.path.exists(cachedir):
	178	logging.debug("Initializing git repo")
513b354c SW	179	subprocess.run(['git', 'init', '--bare', cachedir],
513b354c SW	180	check=True, stdout=sys.stderr)
bef7ce53 SW	181
bef7ce53 SW	182	logging.debug('Fetching ref "%s" from %s', ref, repo)
083b90e7	183	_git_fetch(cachedir, repo, ref, force=force)
bef7ce53 SW	184
	185	with open(os.path.join(cachedir, 'refs', 'heads', ref)) as rev_file:
	186	rev = Rev(rev_file.read(999).strip())
083b90e7 SW	187	verify_ancestry(repo, ref, rev, force=force)
083b90e7 SW	188	_log_fetch(repo, ref, rev, force=force)
bef7ce53 SW	189
	190	return cachedir, rev
	191
	192
083b90e7 SW	193	def ensure_rev_available(
	194	repo: Repo,
	195	ref: Ref,
	196	rev: Rev,
	197	force: bool = False) -> Path:
bef7ce53	198	cachedir = git_cachedir(repo)
eb638847 SW	199	if os.path.exists(cachedir) and is_ancestor(repo, ref, rev):
eb638847 SW	200	return cachedir
bef7ce53 SW	201
	202	logging.debug(
	203	'We do not have rev %s. We will fetch ref "%s" and hope it appears.',
	204	rev, ref)
083b90e7	205	fetch(repo, ref, force=force)
bef7ce53 SW	206	logging.debug('Verifying that fetch retrieved rev %s', rev)
bef7ce53 SW	207	subprocess.run(['git', '-C', cachedir, 'cat-file', '-e', rev], check=True)
083b90e7	208	verify_ancestry(repo, ref, rev, force=force)
bef7ce53 SW	209
bef7ce53 SW	210	return cachedir
347be7cf SW	211
	212
	213	def _main() -> None:
d1ab0853 SW	214	parser = argparse.ArgumentParser(
	215	description='Cache remote git repositories locally.',
	216	epilog='example usage: git-cache https://github.com/NixOS/nixpkgs.git master')
083b90e7 SW	217	parser.add_argument(
	218	'--force',
	219	action='store_true',
	220	help='Recover from a force-push in the remote repo')
d1ab0853 SW	221	parser.add_argument(
	222	'repo',
	223	metavar='Repo',
	224	type=Repo,
	225	help='Git repository URL')
	226	parser.add_argument(
	227	'ref',
	228	metavar='Ref',
	229	type=Ref,
	230	help='Ref (branch or tag) in the git repo')
	231	parser.add_argument(
	232	'rev',
	233	metavar='Rev',
	234	type=Rev,
	235	nargs='?',
	236	help='Ensure that this revision is present. ' +
	237	'If this revision is already present locally, no network operations are performed.')
	238	args = parser.parse_args()
	239
	240	if args.rev is None:
083b90e7	241	print('{1} {0}'.format(*fetch(args.repo, args.ref, force=args.force)))
347be7cf	242	else:
083b90e7 SW	243	print(
	244	ensure_rev_available(
	245	args.repo,
	246	args.ref,
	247	args.rev,
	248	force=args.force))