git.scottworley.com Git - annex-ec/blob - annex-ec

   1 #!/usr/bin/env bash
   2
   3 set -euo pipefail
   4
   5 die() {
   6   echo "$*" >&2
   7   exit 1
   8 }
   9
  10
  11 parse_volume_list() {
  12   this_volume_name=$(join -j1 <(git config get annex.uuid) <(git cat-file -p git-annex:uuid.log|sort)  | cut -d' ' -f2)
  13   tmp=()
  14   mapfile -d , tmp <<< "$1"
  15   volumes=()
  16   for x in "${tmp[@]}";do
  17     x=${x%,}
  18     x=${x%$'\n'}
  19     if [[ "$x" ]];then
  20       if [[ "$x" == "$this_volume_name" ]];then
  21         volumes+=( "here" )
  22       else
  23         volumes+=( "$x" )
  24       fi
  25     fi
  26   done
  27 }
  28
  29 make_name() {
  30   FILENAME_MAX=255
  31   EXAMPLE_SUFFIX='.vol0000+9999.par2'
  32   EXAMPLE_ANNEX_INTERNAL_PREFIX='ingest-'
  33   EXAMPLE_ANNEX_INTERNAL_SUFFIX='-1-1173fd7'
  34   num_separating_dashes=$((N - 1))
  35   overhead=$(( ${#EXAMPLE_SUFFIX} + ${#EXAMPLE_ANNEX_INTERNAL_PREFIX} + ${#EXAMPLE_ANNEX_INTERNAL_SUFFIX} + num_separating_dashes ))
  36   available=$((FILENAME_MAX - overhead))
  37   len=$((available / N))
  38   name=$(find "$@" -printf '%l\n' | sed -r 's/.*SHA256E-s[0-9]+--//;s/\..*//' | cut -c-$len | tr \\n -)
  39   name=${name%-}
  40 }
  41
  42 volumes=()
  43 redundancy=1
  44 block_size_is_a_multiple_of=4  # par2 requires that this be at least 4
  45 blocks_per_file=10
  46 while getopts b:m:r:v: opt;do
  47  case $opt in
  48   b)    blocks_per_file=$OPTARG;;
  49   m)    block_size_is_a_multiple_of=$OPTARG;;
  50   r)    redundancy=$OPTARG;;
  51   v)    parse_volume_list "$OPTARG";;
  52   *)    echo 'usage: annex-ec [-v remote1,remote2,...] [-r N] file file...' >&2; exit 1;;
  53  esac
  54 done
  55 shift $((OPTIND - 1))
  56
  57 if (( ${#volumes[@]} == 0 ));then
  58   parse_volume_list "here,$(git remote | tr \\n ,)"
  59 fi
  60
  61 N=$((${#volumes[@]} - redundancy))
  62
  63 (( $# == N )) || die "Expected $N files in this group ($N + $redundancy = ${#volumes[@]}), but got $#"
  64
  65
  66 git annex get -- "$@"
  67
  68 max_size=$(find -L "$@" -printf '%s\n' | sort -nr | head -n1)
  69 block_size=$(( ((max_size/(block_size_is_a_multiple_of*blocks_per_file))+1) * block_size_is_a_multiple_of))
  70
  71 make_name "$@"
  72
  73 if [[ ! -d ec ]];then
  74   mkdir ec
  75   # TODO: Make this robust against being interrupted here
  76   echo '* annex.numcopies=1' >> ec/.gitattributes
  77   git add ec/.gitattributes
  78 fi
  79
  80 par2 c -n"$redundancy" -c"$((blocks_per_file * redundancy))" -s"$block_size" "$name.par2" "$@"
  81 rm "$name.par2"
  82 mv "$name.vol"* ec/
  83
  84 i=0
  85 for f;do
  86   target_volume="${volumes[i]}"
  87   for volume in "${volumes[@]}";do
  88     if [[ "$volume" != here ]];then
  89       if [[ "$volume" == "$target_volume" ]]; then
  90         git annex copy --to "$volume" "$f"
  91       fi
  92     fi
  93   done
  94   i=$((i+1))
  95 done
  96
  97 for f in ec/"$name.vol"*;do
  98   target_volume="${volumes[i]}"
  99   git annex add "$f"
 100   if [[ "$target_volume" != here ]];then
 101     git annex move --to "$target_volume" "$f"
 102   fi
 103   i=$((i+1))
 104 done
 105
 106 i=0
 107 for f;do
 108   target_volume="${volumes[i]}"
 109   echo "${f// /[[:space:]]} annex.numcopies=1" >> .gitattributes
 110   for volume in here "${volumes[@]}";do
 111     if [[ "$volume" != "$target_volume" ]]; then
 112       if [[ "$volume" == here ]];then
 113         git annex drop "$f"
 114       else
 115         git annex drop --from "$volume" "$f"
 116       fi
 117     fi
 118   done
 119   i=$((i+1))
 120 done
 121 git add .gitattributes
 122
 123 (
 124   flock 1
 125   echo "$name"
 126   for f;do
 127     echo " $f"
 128   done
 129 ) >> ec/.meta
 130 git add ec/.meta