cache_get

#!/bin/bash
#
# Retrieve a url and cache it for an arbitrary period of time,
# avoiding duplicate requests if unexpired cached output exists.
# Usage: cache_get <lifetime in seconds> <url>
#

set -o nounset
set -o errexit
cachedir=$HOME/.generic_cache

TIMEOUT=2
RETRIES=10

if [ ! -d $cachedir ]
then
	mkdir $cachedir
fi

cache_life=$1
url=$2

if [ ! "$url" ]
then
	echo "usage: $0 <data_lifetime_seconds> <url>"
	exit 2
fi

if which md5 &>/dev/null
then
  md5=md5
elif which md5sum &> /dev/null
then
  md5=md5sum
else
  # don't use checksum for cache file
  #
  md5=cat
fi

cachefile=$cachedir/`echo "$url" | sed 's/[^0-9a-zA-Z_-]/_/g' | $md5 | tr -d \ -`

if [ -e "$cachefile" ]
then
  if [ "$cache_life" == "-1" ]
  then
    cat $cachefile
		exit
  fi
	# check if content has expired
	#
	file_mtime=`perl -MFile::stat -e 'print stat("'$cachefile'")->mtime'`

	current_time=`date +%s`
	if [ `expr $current_time - $file_mtime` -lt $cache_life ]
	then
    cat $cachefile
		exit
	fi
fi

tempfile=`mktemp`
tries=0
while [ ! -s "$tempfile" -a "$tries" -lt "$RETRIES" ]
do
  # pass --cookie /dev/null to activate cookie engine without specifying any cookies
  #
  curl --cookie /dev/null --max-time $TIMEOUT --location --silent --connect-timeout $TIMEOUT --header 'User-Agent: Mozilla/5.0' $url | tee $tempfile
  let tries=tries+1
done
if [ -s "$tempfile" ]
then
  mv "$tempfile" "$cachefile"
elif [ -s "$cachefile" ]
then
  cat $cachefile
fi