git.scottworley.com Git - annex-ec/blob - annex-ec-test

   1 #!/usr/bin/env bash
   2
   3 # annex-ec: Use erasure codes for more efficient storage use in git-annex
   4 # Copyright (C) 2026  Scott Worley
   5
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU Affero General Public License as
   8 # published by the Free Software Foundation, either version 3 of the
   9 # License, or (at your option) any later version.
  10
  11 set -euo pipefail
  12
  13 die() { echo "$*" >&2; exit 1; }
  14
  15 vol_name() { echo "${1% *}"; }
  16 vol_dir() { echo "${1#* }"; }
  17
  18 make_test_vols() {
  19   vols=()
  20   deleted_vols=()
  21   for (( i=0; i<$1; i++ ));do
  22     vol=$(mktemp -d)
  23     name="r$i"
  24     if (( i == 0 ));then
  25       git -C "$vol" init
  26       git -C "$vol" commit --allow-empty -m "Begin"
  27     else
  28       git clone "$(vol_dir "${vols[0]}")" "$vol"
  29       git -C "$vol" remote remove origin
  30     fi
  31     git -C "$vol" annex init "$name"
  32     vols+=( "$name $vol" )
  33   done
  34   for vol in "${vols[@]}";do
  35     for r in "${vols[@]}";do
  36       if [[ "$vol" != "$r" ]];then
  37         git -C "$(vol_dir "$vol")" remote add "$(vol_name "$r")" "$(vol_dir "$r")"
  38       fi
  39     done
  40   done
  41   sync_everything
  42 }
  43
  44 has_been_deleted() {
  45   for already_deleted in "${deleted_vols[@]}";do
  46     if [[ "$1" == "$already_deleted" ]];then return 0; fi
  47   done
  48   return 1
  49 }
  50
  51 sync_everything() {
  52   for vol in "${vols[@]}";do
  53     if has_been_deleted "$vol";then continue; fi
  54     git -C "$(vol_dir "$vol")" annex sync
  55   done
  56 }
  57
  58 fsck_everything() {
  59   for vol in "${vols[@]}";do
  60     if has_been_deleted "$vol";then continue; fi
  61     pushd "$(vol_dir "$vol")"
  62       git annex fsck -- *-*
  63     popd
  64   done
  65 }
  66
  67 delete_test_vol() {
  68   if has_been_deleted "$vol";then return; fi
  69   d="$(vol_dir "$1")"
  70   if [[ -d "$d/.git/annex/objects" ]];then
  71     chmod -R +w "$d/.git/annex/objects"
  72   fi
  73   rm -rf "$d"
  74   deleted_vols+=( "$1" )
  75
  76   # Find a not-yet-deleted volume (if there is one) and report the deleted volume as dead
  77   for vol in "${vols[@]}";do
  78     if has_been_deleted "$vol";then continue; fi
  79     git -C "$(vol_dir "$vol")" annex dead "$(vol_name "$1")"
  80     break
  81   done
  82 }
  83
  84 delete_some_test_vols() {
  85   while read -r vol;do
  86     delete_test_vol "$vol"
  87   done < <(for vol in "${vols[@]}";do
  88              echo "$vol"
  89            done | shuf | head -n "$1")
  90 }
  91
  92 delete_all_test_vols() {
  93   for vol in "${vols[@]}";do
  94     delete_test_vol "$vol"
  95   done
  96   vols=()
  97   deleted_vols=()
  98 }
  99
 100 make_test_file() {
 101   name=$(tr -cd 0-9a-f < /dev/urandom | head -c 32)
 102   size=$((RANDOM + RANDOM))
 103   f="$name-$size"
 104   set +o pipefail
 105   openssl aes-128-cbc -nosalt -iv 0 -K "$name" < /dev/zero | head -c "$size" > "$1/$f"
 106   set -o pipefail
 107   git -C "$1" annex add "$f" >&2
 108   echo "$f"
 109 }
 110
 111 some_random_volume_names() {
 112   x=$(for vol in "${vols[@]}";do
 113         vol_name "$vol"
 114       done | shuf | head -n "$1" | tr \\n ,)
 115   echo "${x%,}"
 116 }
 117
 118 a_random_volume_dir() {
 119   while true;do
 120     vol="${vols[$RANDOM % $num_vols]}"
 121     if has_been_deleted "$vol";then continue; fi
 122     vol_dir "$vol"
 123     break
 124   done
 125 }
 126
 127 MIN_REDUNDANCY=1
 128 MIN_FILES=2 # If you only have one file in a group, you'd just make copies of it, no need for annex-ec
 129 MIN_VOLUMES=$((MIN_REDUNDANCY + MIN_FILES))
 130
 131 for (( num_vols=MIN_VOLUMES; num_vols <= 10; num_vols++ ));do
 132   for (( redundancy=1; redundancy <= num_vols-2; redundancy++ ));do
 133     max_files=$(( num_vols - redundancy ))
 134     for (( num_files=MIN_FILES; num_files <= max_files; num_files++ ));do
 135       make_test_vols "$num_vols"
 136       files=()
 137       for (( i=0; i < num_files; i++ )); do
 138         files[i]=$(make_test_file "$(vol_dir "${vols[i]}")")
 139       done
 140       sync_everything
 141       sync_everything
 142       pushd "$(a_random_volume_dir)"
 143         cmd=(annex-ec -r "$redundancy" -v "$(some_random_volume_names $((num_files+redundancy)))" "${files[@]}")
 144         echo "In $PWD , running ${cmd[*]}" >&2
 145         "${cmd[@]}"
 146       popd
 147       sync_everything
 148       sync_everything
 149       fsck_everything
 150
 151       delete_some_test_vols "$redundancy"
 152       pushd "$(a_random_volume_dir)"
 153         annex-ec-recover -- *-*
 154       popd
 155
 156       sync_everything
 157       sync_everything
 158       fsck_everything
 159       delete_all_test_vols
 160     done
 161   done
 162 done
 163