iabak-helper

#!/bin/bash
set -e

NUMCOPIES=4
ANNEXGETOPTS=

LANG=C
export LANG

echo "Welcome to iabak version 0.1"
echo "ALPHA TEST, use with caution"
echo

case "$(uname)" in
	Darwin)
		PATH=$(pwd)/git-annex.osx:$PATH
		;;
	*)
		PATH=$(pwd)/git-annex.linux:$PATH
		;;
esac
export PATH

if [ -e ANNEXGETOPTS ]; then
	ANNEXGETOPTS=$(cat ANNEXGETOPTS)
	export ANNEXGETOPTS
fi

. ./locks.sh

pow () {
	local x=${1:-1} y=${2:-0} result=${3:-1}
	if [ ${y} -gt 0 ]; then
		echo "$(pow "${x}" "$((y-1))" "$((result*x))")"
	else
		echo "${result}"
	fi
}

# I can wikipedia
# sh is annoying, so you must call rand_update (and not from a
# subshell) to update the state variable, then you can call rand to
# read it
RANDOM_SEED=$$
rand_update () {
	RANDOM_SEED="$((1664525 * RANDOM_SEED + 1013904223))"
}
rand () {
	local max=${1}
	if [ -z "${max}" ]; then
		echo "${RANDOM_SEED}"
	else
		local r="$((RANDOM_SEED % max))"
		if [ ${r} -lt 0 ]; then
			echo "$((r * -1))"
		else
			echo "${r}"
		fi
	fi
}

bytesFromSize () {
	if [[ -z "$1" ]]; then
		echo "0"
	fi
	if which numfmt >/dev/null 2>&1; then
		echo "$1" | numfmt --from=si
	else
		declare -a size
		size=($(echo ${1} | tr 'a-z' 'A-Z' | sed -e 's/\([.0-9]\)\([A-Z]\)/\1 \2/'))
		count="$(echo ${size[0]} | cut -f 1 -d '.')"
		unit="${size[1]}"
		exponent=1
		case ${unit} in
			B) exponent=0;;
			K) exponent=3;;
			M) exponent=6;;
			G) exponent=9;;
			T) exponent=12;;
			*) exponent=15;;
		esac
		echo $((${count} * $(pow 10 ${exponent})))
	fi
}

# syncs a shard with random exponential backoff in case of failure, and a maximum of 4 retries.
syncshard () {
	# assumes that pwd is a shard
	local retries=0
	while [ ${retries} -lt 5 ]; do
		if ${GITANNEX} sync; then
			return 0
		fi
		retries=$((retries+1))
		rand_update
		sleep "$(rand "$(pow "2" "${retries}")")"
	done
	return 1
}

download () {
	syncshard || echo "Warning: Failed to sync this shard; working off of data which is potentially out of date."
	periodicsync &

	echo "Here goes! Downloading from Internet Archive."
	echo "(This can be safely interrupted at any time with ctrl-C)"
	echo "(Not using enough bandwith? Enable concurrent downloads with: echo -J5 > ANNEXGETOPTS)"
	echo ""
	if rundownloaddirs; then
		echo
		echo "Wow! I'm done downloading this shard of the IA!"
	else
		echo 
		echo "Download finished, but the backup of this shard is not fully complete."
		echo "Some files may have failed to download, or all allocated disk space is in use."
	fi
}

find_insufficient_copies () {
	${GITANNEX} find --format='${bytesize} ${file}\000' --not --copies "$NUMCOPIES" --not --in=here
}

sumofbytes () {
	awk 'BEGIN {RS=ORS="\000";FS=OFS=SUBSEP=" "} {arr[substr($0, index($0, " ")+1)]+=$1} END {for (i in arr) print arr[i],i}'
}

shuffle () {
	if [ ! -e ../NOSHUF ] && [ -x /usr/bin/shuf ]; then
		#echo "(Oh good, you have shuf(1)! Randomizing order.. Will take a couple minutes..)"
		shuf -z
	else
		cat
	fi
}

dirname_pipe () {
	if dirname -z foo >/dev/null 2>&1; then
		xargs --no-run-if-empty -0 dirname -z --
	else
		cat
	fi
}

rundownloaddirs () {
	echo "Continuing any downloads that were previously interrupted..."
	${GITANNEX} get ${ANNEXGETOPTS} --incomplete
	echo "Checking for any items that still need to be downloaded..."
	find_insufficient_copies | dirname_pipe | sumofbytes | shuffle | rundownloads
}


rundownloadfiles () {
	echo "Checking for any files that still need to be downloaded..."
	# don't use dirname here, because in maint mode,
	# only 1 file from a dir might go missing and we
	# don't want to download the whole dir then
	find_insufficient_copies | shuffle | rundownloads
}

rundownloads () {
	spacelimit=${1}

	if [[ -z "${spacelimit}" ]]; then
		available=$(($(bytesFromSize "$(diskfree)") - $(bytesFromSize "$(annexreserved ".")")))
		if [[ ${available:-0} -gt 0 ]]; then
			spacelimit="${available}"
		else
			echo "oops, out of disk space"
			return 1
		fi
	fi

	while :; do
		spaceneeded=0
		files=()
		# queue files/items that fit within the spacelimit, without consuming the whole stream. note
		# the ${numfiles} -eq 0 clause which makes us always accept the first file in the list, even
		# if it's larger than the spacelimit. This ensures that we always download something, and
		# don't completely neglect gigantic files.
		while read -d $'\0' bytes filename && [[ ${numfiles} -eq 0 || $((${spaceneeded} + ${bytes})) -lt ${spacelimit} ]]; do
			spaceneeded=$((${spaceneeded} + ${bytes}))
			files+=(${filename})
		done
		numfiles=$((${#files[@]}))
		if [[ ! ${numfiles} -gt 0 ]]; then
			return ${rv}
		fi
		chunksize=$((1 + ${numfiles} / 10))
		offset=0
		rv=0
		while [[ ${offset} -lt ${numfiles} ]]; do
			${GIT} -c annex.alwayscommit=false annex get ${ANNEXGETOPTS} -- "${files[@]:${offset}:${chunksize}}"
			rv=$((${rv} && ${?}))
			offset=$((${offset} + ${chunksize}))
		done
		spacelimit=$((${spacelimit} - ${spaceneeded}))
	done
	if ! stillhavespace; then
		return ${rv}
	fi
}

fsckme () {
	set +e
	if [ -e ../FSCKTIMELIMIT ]; then
		FSCKTIMELIMIT="$(cat ../FSCKTIMELIMIT)"
	fi
	if [ -z "$FSCKTIMELIMIT" ]; then
		FSCKTIMELIMIT=5h
	fi
	${FLOCK_EX} .git/annex/fullfsck.lock ${GITANNEX} fsck --incremental-schedule 30d --time-limit "$FSCKTIMELIMIT"
	fsckresult=$?
	set -e
	if [ "$fsckresult" -eq 101 ]; then
		echo "Incremental full fsck ran out of time, will continue it next run."
	fi
}

fastfsck () {
	# The point of this is just to report that the repository still
	# exists, to prevent it from expiring.
	# Even a fsck --fast --in here is a litte too slow to run
	# frequently, so fsck an empty directory.
	if [ ! -d .empty ]; then
		mkdir .empty
	fi
	# Only run the fast fsck once per week, to avoid unncessary churn
	# in the git-annex branch. Look at the timestamp of the .empty
	# directory.
	if [ -n "$(find .empty -mtime +6)" ]; then
		${GITANNEX} fsck --fast .empty --quiet >/dev/null 2>/dev/null || true
		if syncshard; then
			touch .empty
		else
			echo "Warning: failed to sync this shard; fsck results not recorded on server"
			# .empty not touched, so will try to sync next time
		fi
	fi
}

maint () {
	echo "This shard is in maintenance mode; checking it."
	fsckme

	if [ ! "$CRONJOB" ]; then
		syncshard || (echo "Error: failed to sync shard in maintenance mode"; exit 1)
		periodicsync &
		if rundownloadfiles; then
			echo ""
			echo "This shard remains in good shape!"
		else
			echo 
			echo "Could not fully maintain this shard."
			echo "Some files may have failed to download, or all allocated disk space is in use."
		fi
	fi
}

periodicsync () {
	(cd .. && ${FLOCK_EX} .git/annex/iasyncer.lock ./iabak-hourlysync) || true
}

annexreserved () {
	prevshard="${1}"
	if [[ -z "${prevshard}" ]]; then
		prevshard="$(echo shard* 2>/dev/null | cut -d ' ' -f 1)"
		if [[ -n "${prevshard}" ]]; then
			(cd "${prevshard}" && ${GIT} config annex.diskreserve) || echo "0"
		fi
	else
		${GIT} config annex.diskreserve || echo "0"
	fi
}

diskfree () {
	df -Ph . | tail -1 | awk '{print $4}'
}

outofspace () {
	reserve="$1"
	[[ -n "${reserve}" && $(bytesFromSize $(diskfree)) -lt $(bytesFromSize "${reserve}") ]]
}

stillhavespace () {
	! outofspace "$(annexreserved)"
}

setup () {
	echo ""
	df -h .
	echo "How much of this disk space do you want to keep free, and not use for IA backup?"
	echo "(You can free up space at any time by simply deleting files from the archives.)"
	echo "To use entire disk, just hit Enter, or type in something like 100M, 200G, or 1T."
	echo "You must input a value of at least 100 megabytes."
        askreserve
}

askreserve () {
	read -e -p "keep free: " -i "100M" reserve
	if [ -n "$reserve" ]; then
            if [ $(bytesFromSize ${reserve}) -lt $((100*1024*1024)) ]; then
                echo "*** you must reserve at least 100M ***"
                askreserve
            else
		if outofspace "$reserve"; then
			echo "*** not enough diskspace to reserve ***"
			askreserve
		else
			${GIT} config annex.diskreserve "$reserve"
		fi
            fi
        else
            echo "*** sorry, you may not leave this blank ***"
            askreserve
	fi
}

installgitannex () {
	${FLOCK_EX} .iabak-install-git-annex ./install-git-annex || true
	${GITANNEX} version
}

installfsckservice () {
	if [ ! "$CRONJOB" ]; then
		${FLOCK_EX} ./iabak-install-fsck-service ./install-fsck-service || true
	fi
}

handleshard () {
	shard="$1"

	echo
	echo "========= $shard ========="
	echo
	
	l="$(grep "^$shard " repolist | head -n 1)"
	set -- $l
	state="$3"

	cd "$shard"
	case "$state" in
		active)
			if [ ! "$CRONJOB" ]; then
				download
			fi
		;;
		reserve)
			if [ ! "$CRONJOB" ]; then
				download
			fi
		;;
		maint)
			maint
		;;
		restore)
			echo "TODO: upload" >&2
		;;
		*)
			echo "unknown state $state for shard $shard" >&2
		;;
	esac

	syncshard || echo "Warning: failed to sync shard; results of download not known to be recoded on the server."
}

# Order with newest checked out shards first.
sharddirs () {
	perl -le 'foreach (@ARGV) { $ctime=((stat($_))[10]); print "$ctime $_" if -d $_ }' shard* | sort -rn | cut -d ' ' -f 2
}

randomshard () {
	wantstate="$1"
	${GIT} pull >&2 || true
	grep " $wantstate$" repolist | cut -d ' ' -f 1 | perl -e 'while (<>) { chomp; if (! -d $_) { push @l, $_ } };  print $l[rand @l]'
}

randomnew () {
	active="$(randomshard active)"
	if [ -z "$active" ]; then
		randomshard reserve
	else
		echo "$active"
	fi
}

cleanup () {
	echo "Cleaning up..."
	for d in $(sharddirs); do
		if [ -e "$d/.git/iabak-hourlysync.pid" ]; then
			pid="$(cat "$d/.git/iabak-hourlysync.pid")"
			# guard against stale pid file
			if "$(ps "$pid")" | grep -q iabak-hourlysync; then
				kill "$pid" || true
			fi
			rm -f "$d/.git/iabak-hourlysync.pid" || true
		fi
	done
}

if [ -n "$(sharddirs)" ]; then
	trap cleanup EXIT INT
	installgitannex
	installfsckservice

	# First do a fast fsck of all shards, in order to detect
	# obvious problems, and report in that we still have the data
	# we said we have.
	for d in $(sharddirs); do
		(cd "$d" && fastfsck)
	done

	# Now the expensive processing of each shard.
	for d in $(sharddirs); do
		( handleshard "$d" )
	done
	
	# Move clients on to next shards.
	if ! [ -e NOMORE ] && [ ! "$CRONJOB" ]; then
		while stillhavespace; do
			new="$(randomnew)"
			if ! [ -z "$new" ]; then
				echo ""
				echo "Looks like you still have free disk space, so I will continue to fill it up!"
				echo "(ctrl-c and touch NOMORE to prevent this behavior..)"
				echo ""
				./checkoutshard "$new"
				( handleshard "$new" )
			else
				echo ""
				echo "Looks like we ran out of shards for you to download before you ran out of disk space. Please try again later!" >&2
				exit 0
			fi
		done
		echo ""
		echo "Filled up available disk space, so stopping here!"
	fi
else
	if [ "$CRONJOB" ]; then
		echo "running as cron job, but have no shards, so doing nothing" >&2
		exit 1
	fi

	localdir="$(randomnew)"
	if [ -z "$localdir" ]; then
		echo "No new shards are currently available. Please try again later!" >&2
		exit 1
	fi

	echo "Looks like this is a new IA backup. I'm going to put it in"
	echo "$(pwd)/$localdir"
	echo "This will use some disk space. :)"
	printf "Press Enter to confirm, or ctrl-C to cancel."
	read confirm

	trap cleanup EXIT INT
	installgitannex
	echo "Now we need to set up a service to regularly double-check the content of"
	echo "your backup. This protects the backup against bit-rot."
	installfsckservice
	./checkoutshard "$localdir"
	(
		cd "$localdir"
		setup
		periodicsync &
		download
	)
fi