git.scottworley.com Git - annex-ec/blob - annex-ec-test

   1 #!/usr/bin/env bash
   2
   3 set -euo pipefail
   4
   5 die() { echo "$*" >&2; exit 1; }
   6
   7 vol_name() { echo "${1% *}"; }
   8 vol_dir() { echo "${1#* }"; }
   9
  10 make_test_vols() {
  11   vols=()
  12   deleted_vols=()
  13   for (( i=0; i<$1; i++ ));do
  14     vol=$(mktemp -d)
  15     name="r$i"
  16     if (( i == 0 ));then
  17       git -C "$vol" init
  18       git -C "$vol" commit --allow-empty -m "Begin"
  19     else
  20       git clone "$(vol_dir "${vols[0]}")" "$vol"
  21       git -C "$vol" remote remove origin
  22     fi
  23     git -C "$vol" annex init "$name"
  24     vols+=( "$name $vol" )
  25   done
  26   for vol in "${vols[@]}";do
  27     for r in "${vols[@]}";do
  28       if [[ "$vol" != "$r" ]];then
  29         git -C "$(vol_dir "$vol")" remote add "$(vol_name "$r")" "$(vol_dir "$r")"
  30       fi
  31     done
  32   done
  33   sync_everything
  34 }
  35
  36 sync_everything() {
  37   for vol in "${vols[@]}";do
  38     for already_deleted in "${deleted_vols[@]}";do
  39       if [[ "$vol" == "$already_deleted" ]];then continue 2; fi
  40     done
  41     git -C "$(vol_dir "$vol")" annex sync
  42   done
  43 }
  44
  45 fsck_everything() {
  46   for vol in "${vols[@]}";do
  47     for already_deleted in "${deleted_vols[@]}";do
  48       if [[ "$vol" == "$already_deleted" ]];then continue 2; fi
  49     done
  50     git -C "$(vol_dir "$vol")" annex fsck
  51   done
  52 }
  53
  54 delete_test_vol() {
  55   for already_deleted in "${deleted_vols[@]}";do
  56     if [[ "$1" == "$already_deleted" ]];then return; fi
  57   done
  58   d="$(vol_dir "$1")"
  59   if [[ -d "$d/.git/annex/objects" ]];then
  60     chmod -R +w "$d/.git/annex/objects"
  61   fi
  62   rm -rf "$d"
  63   deleted_vols+=( "$1" )
  64
  65   # Find a not-yet-deleted volume (if there is one) and report the deleted volume as dead
  66   for vol in "${vols[@]}";do
  67     for already_deleted in "${deleted_vols[@]}";do
  68       if [[ "$vol" == "$already_deleted" ]];then continue 2; fi
  69     done
  70     git -C "$(vol_dir "$vol")" annex dead "$(vol_name "$1")"
  71     break
  72   done
  73 }
  74
  75 delete_some_test_vols() {
  76   while read -r vol;do
  77     delete_test_vol "$vol"
  78   done < <(for vol in "${vols[@]}";do
  79              echo "$vol"
  80            done | shuf | head -n "$1")
  81 }
  82
  83 delete_all_test_vols() {
  84   for vol in "${vols[@]}";do
  85     delete_test_vol "$vol"
  86   done
  87   vols=()
  88   deleted_vols=()
  89 }
  90
  91 make_test_file() {
  92   name=$(tr -cd 0-9a-f < /dev/urandom | head -c 32)
  93   size=$((RANDOM + RANDOM))
  94   f="$name-$size"
  95   set +o pipefail
  96   openssl aes-128-cbc -nosalt -iv 0 -K "$name" < /dev/zero | head -c "$size" > "$1/$f"
  97   set -o pipefail
  98   git -C "$1" annex add "$f" >&2
  99   echo "$f"
 100 }
 101
 102 choose_volumes() {
 103   x=$(for vol in "${vols[@]}";do
 104         vol_name "$vol"
 105       done | shuf | head -n "$1" | tr \\n ,)
 106   echo "${x%,}"
 107 }
 108
 109 MIN_REDUNDANCY=1
 110 MIN_FILES=2 # If you only have one file in a group, you'd just make copies of it, no need for annex-ec
 111 MIN_VOLUMES=$((MIN_REDUNDANCY + MIN_FILES))
 112
 113 for (( num_vols=MIN_VOLUMES; num_vols <= 10; num_vols++ ));do
 114   for (( redundancy=1; redundancy < num_vols-2; redundancy++ ));do
 115     max_files=$(( num_vols - redundancy ))
 116     for (( num_files=MIN_FILES; num_files <= max_files; num_files++ ));do
 117       make_test_vols "$num_vols"
 118       files=()
 119       for (( i=0; i < num_files; i++ )); do
 120         files[i]=$(make_test_file "$(vol_dir "${vols[i]}")")
 121       done
 122       sync_everything
 123       sync_everything
 124       pushd "$(vol_dir "${vols[$RANDOM % $num_vols]}")"
 125         cmd=(annex-ec -r "$redundancy" -v "$(choose_volumes $((num_files+redundancy)))" "${files[@]}")
 126         echo "In $PWD , running ${cmd[*]}" >&2
 127         "${cmd[@]}"
 128       popd
 129       sync_everything
 130       fsck_everything
 131       delete_some_test_vols "$redundancy"
 132       # TODO: Recover
 133       sync_everything
 134       # fsck_everything # Skip this check until recovery is implemented
 135       delete_all_test_vols
 136     done
 137   done
 138 done
 139