git.scottworley.com Git - annex-ec/blob - annex-ec-test

   1 #!/usr/bin/env bash
   2
   3 # annex-ec: Use erasure codes for more efficient storage use in git-annex
   4 # Copyright (C) 2026  Scott Worley
   5
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU Affero General Public License as
   8 # published by the Free Software Foundation, either version 3 of the
   9 # License, or (at your option) any later version.
  10
  11 set -euo pipefail
  12
  13 die() { echo "$*" >&2; exit 1; }
  14
  15 vol_name() { echo "${1% *}"; }
  16 vol_dir() { echo "${1#* }"; }
  17
  18 make_test_vols() {
  19   vols=()
  20   deleted_vols=()
  21   for (( i=0; i<$1; i++ ));do
  22     vol=$(mktemp -d)
  23     name="r$i"
  24     if (( i == 0 ));then
  25       git -C "$vol" init
  26       git -C "$vol" commit --allow-empty -m "Begin"
  27     else
  28       git clone "$(vol_dir "${vols[0]}")" "$vol"
  29       git -C "$vol" remote remove origin
  30     fi
  31     git -C "$vol" annex init "$name"
  32     vols+=( "$name $vol" )
  33   done
  34   for vol in "${vols[@]}";do
  35     for r in "${vols[@]}";do
  36       if [[ "$vol" != "$r" ]];then
  37         git -C "$(vol_dir "$vol")" remote add "$(vol_name "$r")" "$(vol_dir "$r")"
  38       fi
  39     done
  40   done
  41   sync_everything
  42 }
  43
  44 sync_everything() {
  45   for vol in "${vols[@]}";do
  46     for already_deleted in "${deleted_vols[@]}";do
  47       if [[ "$vol" == "$already_deleted" ]];then continue 2; fi
  48     done
  49     git -C "$(vol_dir "$vol")" annex sync
  50   done
  51 }
  52
  53 fsck_everything() {
  54   for vol in "${vols[@]}";do
  55     for already_deleted in "${deleted_vols[@]}";do
  56       if [[ "$vol" == "$already_deleted" ]];then continue 2; fi
  57     done
  58     git -C "$(vol_dir "$vol")" annex fsck
  59   done
  60 }
  61
  62 delete_test_vol() {
  63   for already_deleted in "${deleted_vols[@]}";do
  64     if [[ "$1" == "$already_deleted" ]];then return; fi
  65   done
  66   d="$(vol_dir "$1")"
  67   if [[ -d "$d/.git/annex/objects" ]];then
  68     chmod -R +w "$d/.git/annex/objects"
  69   fi
  70   rm -rf "$d"
  71   deleted_vols+=( "$1" )
  72
  73   # Find a not-yet-deleted volume (if there is one) and report the deleted volume as dead
  74   for vol in "${vols[@]}";do
  75     for already_deleted in "${deleted_vols[@]}";do
  76       if [[ "$vol" == "$already_deleted" ]];then continue 2; fi
  77     done
  78     git -C "$(vol_dir "$vol")" annex dead "$(vol_name "$1")"
  79     break
  80   done
  81 }
  82
  83 delete_some_test_vols() {
  84   while read -r vol;do
  85     delete_test_vol "$vol"
  86   done < <(for vol in "${vols[@]}";do
  87              echo "$vol"
  88            done | shuf | head -n "$1")
  89 }
  90
  91 delete_all_test_vols() {
  92   for vol in "${vols[@]}";do
  93     delete_test_vol "$vol"
  94   done
  95   vols=()
  96   deleted_vols=()
  97 }
  98
  99 make_test_file() {
 100   name=$(tr -cd 0-9a-f < /dev/urandom | head -c 32)
 101   size=$((RANDOM + RANDOM))
 102   f="$name-$size"
 103   set +o pipefail
 104   openssl aes-128-cbc -nosalt -iv 0 -K "$name" < /dev/zero | head -c "$size" > "$1/$f"
 105   set -o pipefail
 106   git -C "$1" annex add "$f" >&2
 107   echo "$f"
 108 }
 109
 110 choose_volumes() {
 111   x=$(for vol in "${vols[@]}";do
 112         vol_name "$vol"
 113       done | shuf | head -n "$1" | tr \\n ,)
 114   echo "${x%,}"
 115 }
 116
 117 MIN_REDUNDANCY=1
 118 MIN_FILES=2 # If you only have one file in a group, you'd just make copies of it, no need for annex-ec
 119 MIN_VOLUMES=$((MIN_REDUNDANCY + MIN_FILES))
 120
 121 for (( num_vols=MIN_VOLUMES; num_vols <= 10; num_vols++ ));do
 122   for (( redundancy=1; redundancy < num_vols-2; redundancy++ ));do
 123     max_files=$(( num_vols - redundancy ))
 124     for (( num_files=MIN_FILES; num_files <= max_files; num_files++ ));do
 125       make_test_vols "$num_vols"
 126       files=()
 127       for (( i=0; i < num_files; i++ )); do
 128         files[i]=$(make_test_file "$(vol_dir "${vols[i]}")")
 129       done
 130       sync_everything
 131       sync_everything
 132       pushd "$(vol_dir "${vols[$RANDOM % $num_vols]}")"
 133         cmd=(annex-ec -r "$redundancy" -v "$(choose_volumes $((num_files+redundancy)))" "${files[@]}")
 134         echo "In $PWD , running ${cmd[*]}" >&2
 135         "${cmd[@]}"
 136       popd
 137       sync_everything
 138       fsck_everything
 139       delete_some_test_vols "$redundancy"
 140       # TODO: Recover
 141       sync_everything
 142       # fsck_everything # Skip this check until recovery is implemented
 143       delete_all_test_vols
 144     done
 145   done
 146 done
 147