#!/usr/bin/env bash set -euo pipefail die() { echo "$*" >&2; exit 1; } vol_name() { echo "${1% *}"; } vol_dir() { echo "${1#* }"; } make_test_vols() { vols=() deleted_vols=() for (( i=0; i<$1; i++ ));do vol=$(mktemp -d) name="r$i" if (( i == 0 ));then git -C "$vol" init git -C "$vol" commit --allow-empty -m "Begin" else git clone "$(vol_dir "${vols[0]}")" "$vol" git -C "$vol" remote remove origin fi git -C "$vol" annex init "$name" vols+=( "$name $vol" ) done for vol in "${vols[@]}";do for r in "${vols[@]}";do if [[ "$vol" != "$r" ]];then git -C "$(vol_dir "$vol")" remote add "$(vol_name "$r")" "$(vol_dir "$r")" fi done done sync_everything } sync_everything() { for vol in "${vols[@]}";do for already_deleted in "${deleted_vols[@]}";do if [[ "$vol" == "$already_deleted" ]];then continue 2; fi done git -C "$(vol_dir "$vol")" annex sync done } fsck_everything() { for vol in "${vols[@]}";do for already_deleted in "${deleted_vols[@]}";do if [[ "$vol" == "$already_deleted" ]];then continue 2; fi done git -C "$(vol_dir "$vol")" annex fsck done } delete_test_vol() { for already_deleted in "${deleted_vols[@]}";do if [[ "$1" == "$already_deleted" ]];then return; fi done d="$(vol_dir "$1")" if [[ -d "$d/.git/annex/objects" ]];then chmod -R +w "$d/.git/annex/objects" fi rm -rf "$d" deleted_vols+=( "$1" ) # Find a not-yet-deleted volume (if there is one) and report the deleted volume as dead for vol in "${vols[@]}";do for already_deleted in "${deleted_vols[@]}";do if [[ "$vol" == "$already_deleted" ]];then continue 2; fi done git -C "$(vol_dir "$vol")" annex dead "$(vol_name "$1")" break done } delete_some_test_vols() { while read -r vol;do delete_test_vol "$vol" done < <(for vol in "${vols[@]}";do echo "$vol" done | shuf | head -n "$1") } delete_all_test_vols() { for vol in "${vols[@]}";do delete_test_vol "$vol" done vols=() deleted_vols=() } make_test_file() { name=$(tr -cd 0-9a-f < /dev/urandom | head -c 32) size=$((RANDOM + RANDOM)) f="$name-$size" set +o pipefail openssl aes-128-cbc -nosalt -iv 0 -K "$name" < /dev/zero | head -c "$size" > "$1/$f" set -o pipefail git -C "$1" annex add "$f" >&2 echo "$f" } choose_volumes() { x=$(for vol in "${vols[@]}";do vol_name "$vol" done | shuf | head -n "$1" | tr \\n ,) echo "${x%,}" } MIN_REDUNDANCY=1 MIN_FILES=2 # If you only have one file in a group, you'd just make copies of it, no need for annex-ec MIN_VOLUMES=$((MIN_REDUNDANCY + MIN_FILES)) for (( num_vols=MIN_VOLUMES; num_vols <= 10; num_vols++ ));do for (( redundancy=1; redundancy < num_vols-2; redundancy++ ));do max_files=$(( num_vols - redundancy )) for (( num_files=MIN_FILES; num_files <= max_files; num_files++ ));do make_test_vols "$num_vols" files=() for (( i=0; i < num_files; i++ )); do files[i]=$(make_test_file "$(vol_dir "${vols[i]}")") done sync_everything sync_everything pushd "$(vol_dir "${vols[$RANDOM % $num_vols]}")" cmd=(annex-ec -r "$redundancy" -v "$(choose_volumes $((num_files+redundancy)))" "${files[@]}") echo "In $PWD , running ${cmd[*]}" >&2 "${cmd[@]}" popd sync_everything fsck_everything delete_some_test_vols "$redundancy" # TODO: Recover sync_everything # fsck_everything # Skip this check until recovery is implemented delete_all_test_vols done done done