git.scottworley.com Git - annex-ec/blob - annex-ec-test

   1 #!/usr/bin/env bash
   2
   3 # annex-ec: Use erasure codes for more efficient storage use in git-annex
   4 # Copyright (C) 2026  Scott Worley
   5
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU Affero General Public License as
   8 # published by the Free Software Foundation, either version 3 of the
   9 # License, or (at your option) any later version.
  10
  11 set -euo pipefail
  12
  13 die() { echo "$*" >&2; exit 1; }
  14
  15 vol_name() { echo "${1% *}"; }
  16 vol_dir() { echo "${1#* }"; }
  17
  18 make_test_vols() {
  19   vols=()
  20   deleted_vols=()
  21   for (( i=0; i<$1; i++ ));do
  22     vol=$(mktemp -d)
  23     name="r$i"
  24     if (( i == 0 ));then
  25       git -C "$vol" init
  26       git -C "$vol" commit --allow-empty -m "Begin"
  27     else
  28       git clone "$(vol_dir "${vols[0]}")" "$vol"
  29       git -C "$vol" remote remove origin
  30     fi
  31     git -C "$vol" annex init "$name"
  32     vols+=( "$name $vol" )
  33   done
  34   for vol in "${vols[@]}";do
  35     for r in "${vols[@]}";do
  36       if [[ "$vol" != "$r" ]];then
  37         git -C "$(vol_dir "$vol")" remote add "$(vol_name "$r")" "$(vol_dir "$r")"
  38       fi
  39     done
  40   done
  41   sync_everything
  42 }
  43
  44 sync_everything() {
  45   for vol in "${vols[@]}";do
  46     for already_deleted in "${deleted_vols[@]}";do
  47       if [[ "$vol" == "$already_deleted" ]];then continue 2; fi
  48     done
  49     git -C "$(vol_dir "$vol")" annex sync
  50   done
  51 }
  52
  53 fsck_everything() {
  54   for vol in "${vols[@]}";do
  55     for already_deleted in "${deleted_vols[@]}";do
  56       if [[ "$vol" == "$already_deleted" ]];then continue 2; fi
  57     done
  58     pushd "$(vol_dir "$vol")"
  59       git annex fsck -- *-*
  60     popd
  61   done
  62 }
  63
  64 delete_test_vol() {
  65   for already_deleted in "${deleted_vols[@]}";do
  66     if [[ "$1" == "$already_deleted" ]];then return; fi
  67   done
  68   d="$(vol_dir "$1")"
  69   if [[ -d "$d/.git/annex/objects" ]];then
  70     chmod -R +w "$d/.git/annex/objects"
  71   fi
  72   rm -rf "$d"
  73   deleted_vols+=( "$1" )
  74
  75   # Find a not-yet-deleted volume (if there is one) and report the deleted volume as dead
  76   for vol in "${vols[@]}";do
  77     for already_deleted in "${deleted_vols[@]}";do
  78       if [[ "$vol" == "$already_deleted" ]];then continue 2; fi
  79     done
  80     git -C "$(vol_dir "$vol")" annex dead "$(vol_name "$1")"
  81     break
  82   done
  83 }
  84
  85 delete_some_test_vols() {
  86   while read -r vol;do
  87     delete_test_vol "$vol"
  88   done < <(for vol in "${vols[@]}";do
  89              echo "$vol"
  90            done | shuf | head -n "$1")
  91 }
  92
  93 delete_all_test_vols() {
  94   for vol in "${vols[@]}";do
  95     delete_test_vol "$vol"
  96   done
  97   vols=()
  98   deleted_vols=()
  99 }
 100
 101 make_test_file() {
 102   name=$(tr -cd 0-9a-f < /dev/urandom | head -c 32)
 103   size=$((RANDOM + RANDOM))
 104   f="$name-$size"
 105   set +o pipefail
 106   openssl aes-128-cbc -nosalt -iv 0 -K "$name" < /dev/zero | head -c "$size" > "$1/$f"
 107   set -o pipefail
 108   git -C "$1" annex add "$f" >&2
 109   echo "$f"
 110 }
 111
 112 some_random_volume_names() {
 113   x=$(for vol in "${vols[@]}";do
 114         vol_name "$vol"
 115       done | shuf | head -n "$1" | tr \\n ,)
 116   echo "${x%,}"
 117 }
 118
 119 a_random_volume_dir() {
 120   while true;do
 121     vol="${vols[$RANDOM % $num_vols]}"
 122     for already_deleted in "${deleted_vols[@]}";do
 123       if [[ "$vol" == "$already_deleted" ]];then continue 2; fi
 124     done
 125     vol_dir "$vol"
 126     break
 127   done
 128 }
 129
 130 MIN_REDUNDANCY=1
 131 MIN_FILES=2 # If you only have one file in a group, you'd just make copies of it, no need for annex-ec
 132 MIN_VOLUMES=$((MIN_REDUNDANCY + MIN_FILES))
 133
 134 for (( num_vols=MIN_VOLUMES; num_vols <= 10; num_vols++ ));do
 135   for (( redundancy=1; redundancy < num_vols-2; redundancy++ ));do
 136     max_files=$(( num_vols - redundancy ))
 137     for (( num_files=MIN_FILES; num_files <= max_files; num_files++ ));do
 138       make_test_vols "$num_vols"
 139       files=()
 140       for (( i=0; i < num_files; i++ )); do
 141         files[i]=$(make_test_file "$(vol_dir "${vols[i]}")")
 142       done
 143       sync_everything
 144       sync_everything
 145       pushd "$(a_random_volume_dir)"
 146         cmd=(annex-ec -r "$redundancy" -v "$(some_random_volume_names $((num_files+redundancy)))" "${files[@]}")
 147         echo "In $PWD , running ${cmd[*]}" >&2
 148         "${cmd[@]}"
 149       popd
 150       sync_everything
 151       sync_everything
 152       fsck_everything
 153
 154       delete_some_test_vols "$redundancy"
 155       pushd "$(a_random_volume_dir)"
 156         annex-ec-recover -- *-*
 157       popd
 158
 159       sync_everything
 160       sync_everything
 161       fsck_everything
 162       delete_all_test_vols
 163     done
 164   done
 165 done
 166