offload-one

#!/bin/bash
# Offloads megawarcs from the upload queue.
# (Needs a config.sh in the working directory.)
#
#   ./offload-one
#
# 1. Grabs an item from UPLOAD_QUEUE_DIR
# 2. Reserves the item by moving the directory to the
#    UPLOADER_WORKING_DIR
# 3. Offloads the item to the target defined in OFFLOAD_TARGET
#    or in the offload_targets file
# 4. Removes the source files from the working directory
#    If COMPLETED_DIR is set, offloaded files are moved there.
#
# The program exits with 1 on any nontransient error.
#

SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

source ./config.sh || exit 1

mkdir -p "${UPLOAD_QUEUE_DIR}" || exit 1
mkdir -p "${UPLOADER_WORKING_DIR}" || exit 1

if [ ! -z "${COMPLETED_DIR}" ]
then
  mkdir -p "${COMPLETED_DIR}" || exit 1
fi

function mayicontinue {
  echo
# echo "May I continue?"
# read
# echo
}

mayicontinue

if test -z "${OFFLOAD_TARGET}" && ! cat ./offload_targets 2> /dev/null | grep -qE '^rsync://[^/]+/[^/]+'; then
  echo "No valid offload target specified in OFFLOAD_TARGET environment variable or ./offload_targets file, aborting offload"
  sleep 30
  exit 1
fi

# try to grab an item from UPLOAD_QUEUE_DIR
ITEM=none
while [[ "${ITEM}" = none ]]
do
  possible_item=$( ls -1 "${UPLOAD_QUEUE_DIR}" | grep -E '[0-9]{14}_[a-f0-9]{8}$' | sort | head -n 1 )
  if test -n "${possible_item}"
  then
    echo "Trying to grab ${possible_item}"
    if mv "${UPLOAD_QUEUE_DIR}/${possible_item}" "${UPLOADER_WORKING_DIR}/"
    then
      ITEM="${possible_item}"
    else
      echo "Failed to move ${possible_item}"
      sleep 5
    fi
  else
    date
    echo "No current item found!"
    sleep 30
    exit 0
  fi
done


echo "$( date ): Start offloading for item ${ITEM}" >> uploader.log

result=1
while [[ "${result}" -ne 0 ]]
do
  _OFFLOAD_TARGET="${OFFLOAD_TARGET}"
  if test -z "${_OFFLOAD_TARGET}"; then
    _OFFLOAD_TARGET=$(cat "./offload_targets" 2> /dev/null | grep -E '^rsync://[^/]+/[^/]+' | shuf -n 1)
  fi
  if test -z "${_OFFLOAD_TARGET}"; then
    echo "No valid offload target specified in OFFLOAD_TARGET environment variable or ./offload_targets file"
    echo "Will retry in 30 seconds"
    sleep 30
    continue
  fi
  echo "Offloading to ${_OFFLOAD_TARGET}/${ITEM}/"
  rsync -rltv --timeout=900 --contimeout=60 --sockopts=SO_SNDBUF=8388608,SO_RCVBUF=8388608 --progress --stats --no-owner --no-group --partial --partial-dir .rsync-tmp --delay-updates --no-compress --compress-level 0 "${UPLOADER_WORKING_DIR}/${ITEM}/" "${_OFFLOAD_TARGET}/${ITEM}/"
  result="${?}"
  if [[ "${result}" -ne 0 ]]
  then
    date
    echo "Error while offloading ${ITEM}, rsync said ${result}"
    echo "Will retry in 30 seconds"
    sleep 30
  fi
done

echo "Offloaded ${ITEM}"

echo "$( date ): Completed offloading for item ${ITEM}" >> uploader.log


mayicontinue


# move or remove megawarc
if [ -z "${COMPLETED_DIR}" ]
then
  # remove
  rm -rf "${UPLOADER_WORKING_DIR}/${ITEM}"
  result="${?}"

  if [[ "${result}" -ne 0 ]]
  then
    date
    echo "rm -rf megawarc exited with ${result} for ${ITEM}"
    exit 1
  fi
else
  # move
  mv "${UPLOADER_WORKING_DIR}/${ITEM}" "${COMPLETED_DIR}/"
  result="${?}"

  if [[ "${result}" -ne 0 ]]
  then
    date
    echo "rm -rf megawarc exited with ${result} for ${ITEM}"
    exit 1
  fi
fi

exit 0