#
#---- Function:  cg_geturl - Download an image if it hasn't changed (According to Content-Length)
#
#  Copyright: John B. Battles 2006
#  License:   GPL 2
#mk

# Currently using md5sum

# TODO LIST:

# 1. Clean up cache create
# 2. Set max cache  maybe have something compress the cache periodically  then uncompress if it is compressed
# 3. Use a Cache hierarchy maybe by website this should speed up access in a reiserfs
# 4. Process 404 and other failed errors
# 5.  ETag is not good if site is mirrored  Use length may need to use Last Modified must be newer
# 6. Create a cg_geturl.conf for font etc


checksum=md5

. cg_filecomp

cg_geturl () {

GETBASEURL=$1
GETOBJECT=$2
DESTDIR=$3
DESTFILE=$4
QUALITY=$5
ARCHIVE_FLAG=$6
TIMESTAMP_FLAG=$7

#echo " GETBASEURL <$GETBASEURL>  GETOBJECT <$GETOBJECT> DESTDIR <$DESTDIR> DESTFILE <$DESTFILE> QUALITY <$QUALITY> ARCHIVE_FLAG <$ARCHIVE_FLAG>"

if [ -z $CG_GETURL_CACHE ]
then 
  CG_GETURL_CACHE="/capture/tmp/`whoami`.cg_geturl.cache"
fi # if [ -z $CG_GETURL_CACHE ]

if [ ! -d $CG_GETURL_CACHE ]
then
  echo "No cache for \$CG_GETURL_CACHE found creating $CG_GETURL_CACHE"
  echo "You might want to create one somewhere pointed to by the environment variable CG_GETURL_CACHE"
  echo "Example export CG_GETURL_CACHE=/var/local/cache/cg_geturl"
  mkdir -v $CG_GETURL_CACHE
  mkdir -v $CG_GETURL_CACHE/last
fi # [ ! -d $CG_GETURL_CACHE ]


squid_running="`ps -C squid|grep squid|wc -l`"

if [ $squid_running == "0" ]
then
  unset http_proxy
  unset ftp_proxy
else
  export http_proxy=http://127.0.0.1:3128
  export ftp_proxy=$http_proxy 
fi

cd $CG_GETURL_CACHE  >/dev/null 2>&1  # Need to define this otherwise it will dump all to /tmp  Use something like /var/local/cache/cg_geturl with a+rwx permits

if [ ! -f ./last/$DESTFILE.Content_Length.last ]
then
  echo "Getting $GETBASEURL$GETOBJECT for the first time"
  echo "Content_Length: \"0\"" > ./last/$DESTFILE.Content_Length.last        #create a dummy last Content_Length file
fi

orgContent_Length=" "
newContent_Length=" "

orgContent_Length="`cat ./last/$DESTFILE.Content_Length.last | awk '{print $2}'`"

  starttime=`getnanotime`
  proc_getdatetime=`getdatetime`

curl -k --connect-timeout 30 --retry-delay 11 --max-time 60 --retry-max-time 60 --retry 2 --silent --show-error  --head $GETBASEURL$GETOBJECT|grep ^Content|grep Length >$DESTFILE.Content_Length 2>/dev/null
#curl -k --connect-timeout 10 --retry-delay 5 --max-time 10 --retry-max-time 5 --retry 2  --show-error  --head $GETBASEURL$GETOBJECT|grep ^Content|grep Length >$DESTFILE.Content_Length 2>/dev/null
newContent_Length="`cat $DESTFILE.Content_Length | awk '{print $2}'`"

#echo "orgContent_Length <$orgContent_Length>"
#echo "newContent_Length <$newContent_Length>"

if [ $orgContent_Length == $newContent_Length ]
then
  echo "Unchanged $DESTFILE `date`"
  if [ -f $DESTDIR/$DESTFILE ]
  then
    cg_filecomp $DESTFILE $DESTDIR/$DESTFILE
    if [ $RC != "1" ]
    then
      rm -f $DESTDIR/$DESTFILE 
      cp -p $DESTFILE $DESTDIR/$DESTFILE 
    fi
  else
    cp -p $DESTFILE $DESTDIR/$DESTFILE 
  fi # if [ -f $DESTDIR/$DESTFILE ]
else
  if [ -f $DESTFILE.Content_Length ]
  then
    mv -f $DESTFILE.Content_Length ./last/$DESTFILE.Content_Length.last
  fi

  if [ -f $DESTFILE ]
  then
    mv -f $DESTFILE $DESTFILE.save
  fi

#  rm -f $GETOBJECT
# This might be timing out on large images Modified --max-time from 10 to 20 6/24/2006
  curl -k --connect-timeout 30 --retry-delay 10 --max-time 240 --retry-max-time 120 --retry 2 --silent --show-error  $GETBASEURL$GETOBJECT > $DESTFILE # Use -S flag and rewget if an error

#  if [ $GETOBJECT != $DESTFILE ]
# then
#    mv -f $GETOBJECT $DESTFILE
#  fi

  cg_filecomp $DESTFILE $DESTFILE.save

  if [ ! $RC == "1" ]
  then # New file
    echo "Updating $GETBASEURL$DESTFILE `date`..."
    mv -f $DESTFILE.save  ./last/$DESTFILE
    rm -f $DESTDIR/last/$DESTFILE
    cp -pf ./last/$DESTFILE  $DESTDIR/last/$DESTFILE
    rm -f $DESTDIR/$DESTFILE
    cp -pf $DESTFILE $DESTDIR/$DESTFILE 

    if [ $ARCHIVE_FLAG == "Y" ]
    then
      BASEFILE="`basename $DESTFILE |\
               sed 's/\.jpg//'|\
               sed 's/\.JPG//'|\
               sed 's/\.GIF//'|\
               sed 's/\.gif//'|\
               sed 's/\.png//'`"

      if [ ! -d $DESTDIR/archive ]
      then
        mkdir -v $DESTDIR/archive 
      fi # if [ ! -d $DESTDIR/archive ]

      if [ ! -d $DESTDIR/archive/$BASEFILE ]
      then
        mkdir -v $DESTDIR/archive/$BASEFILE
      fi # if [ ! -d $DESTDIR/archive/$BASEFILE ]


   #echo "ARCFILE <$ARCFILE>"
      nowdate=`date +%Y%m%d%H%M%S`
      rm -f $BASEFILE.$nowdate.jpg
#
#---- Do a final check that the image isn't already archived based on the md5sum
#
#echo "convert $QUALITY $DESTDIR/$DESTFILE $BASEFILE.$nowdate.jpg"
      convert -format jpg -quality $QUALITY $DESTDIR/$DESTFILE $BASEFILE.$nowdate.jpg
      rm $DESTDIR/$DESTFILE
      cp -p $BASEFILE.$nowdate.jpg $DESTDIR/$DESTFILE   #Fix up if orig file was not jpg
      md5sum $BASEFILE.$nowdate.jpg > $BASEFILE.$nowdate.md5sum
      md5val=`cat $BASEFILE.$nowdate.md5sum|awk '{print $1}'`
      destbase=`basename $DESTDIR/archive/$BASEFILE`

      if [ -f $DESTDIR/archive/$BASEFILE/$destbase.md5list.txt ]
      then
        nfound=`grep -c --mmap $md5val $DESTDIR/archive/$BASEFILE/$destbase.md5list.txt`
        if [ $nfound -eq 0 ]
        then
          cat $BASEFILE.$nowdate.md5sum >> $DESTDIR/archive/$BASEFILE/$destbase.md5list.txt 
#          convert -draw "text 5,14 '`date`'" -font helvetica -pointsize 16 -fill yellow $BASEFILE.$nowdate.jpg $DESTDIR/archive/$BASEFILE
#          rm -f $BASEFILE.$nowdate.jpg

          if [ $TIMESTAMP_FLAG == "Y" ]
          then
            convert -draw "text 2,24 '`date`'" -font helvetica -pointsize 24 -fill red $BASEFILE.$nowdate.jpg $DESTDIR/archive/$BASEFILE/$BASEFILE.$nowdate.jpg
            rm -f $BASEFILE.$nowdate.jpg
            cp -pv $DESTDIR/archive/$BASEFILE/$BASEFILE.$nowdate.jpg $DESTDIR/$DESTFILE
 					else
            mv -f $BASEFILE.$nowdate.jpg $DESTDIR/archive/$BASEFILE  
          fi

          echo "Archived $BASEFILE.$nowdate.jpg"
      
#
#---- Time statistics
#
          endtime=`getnanotime`
          proctime=`echo 8 $endtime  $starttime|bc /usr/local/bc/subfloat.bc`

          echo "STATS 1"

          if [ -f $DESTDIR/archive/$BASEFILE/$BASEFILE.$nowdate.jpg ]
          then
            filesize=`ls -l $DESTDIR/archive/$BASEFILE/$BASEFILE.$nowdate.jpg|awk '{print $5}'`
            echo "$BASEFILE,$proc_getdatetime,$proctime,$filesize" >> /capture/log/stats/cg_geturl.log
            echo "$BASEFILE,$proc_getdatetime,$proctime,$filesize" >> /capture/log/stats/cg_geturl.$BASEFILE.log
            echo "$BASEFILE,$proc_getdatetime,$proctime,$filesize"
          else
            echo "Can't archive $BASEFILE.$nowdate.jpg  - NOT FOUND"
            ls -l $DESTDIR/archive/$BASEFILE/$BASEFILE.$nowdate.jpg
          fi

        else
          echo "$BASEFILE.$nowdate.jpg was already archived"
        fi # if [ $nfound -ne 0 ]
      else
          cat $BASEFILE.$nowdate.md5sum >> $DESTDIR/archive/$BASEFILE/$destbase.md5list.txt 

          if [ $TIMESTAMP_FLAG == "Y" ]
          then
            convert -draw "text 2,24 '`date`'" -font helvetica -pointsize 24 -fill red $BASEFILE.$nowdate.jpg $DESTDIR/archive/$BASEFILE/$BASEFILE.$nowdate.jpg
            rm -f $BASEFILE.$nowdate.jpg
            cp -pv $DESTDIR/archive/$BASEFILE/$BASEFILE.$nowdate.jpg $DESTDIR/$DESTFILE
 					else
            mv -f $BASEFILE.$nowdate.jpg $DESTDIR/archive/$BASEFILE  
          fi

#          convert -draw "text 5,14 '`date`'" -font helvetica -pointsize 16 -fill yellow $BASEFILE.$nowdate.jpg $DESTDIR/archive/$BASEFILE
#          rm -f $BASEFILE.$nowdate.jpg
          mv -f $BASEFILE.$nowdate.jpg $DESTDIR/archive/$BASEFILE
          echo "Archived $BASEFILE.$nowdate.jpg"
#
#---- Time statistics
#
          endtime=`getnanotime`
          proctime=`echo 8 $endtime  $starttime|bc /usr/local/bc/subfloat.bc`

          echo "STATS 2"

          if [ -f $DESTDIR/archive/$BASEFILE ]
          then
            filesize=`ls -l $DESTDIR/archive/$BASEFILE|awk '{print $5}'`
            echo "$BASEFILE,$proc_getdatetime,$proctime,$filesize" >> /capture/log/stats/cg_geturl.log
            echo "$BASEFILE,$proc_getdatetime,$proctime,$filesize" >> /capture/log/stats/cg_geturl.$BASEFILE.log
            echo "$BASEFILE,$proc_getdatetime,$proctime,$filesize"
          else
            echo "Can't archive $BASEFILE.jpg  - NOT FOUND"
            ls -l $DESTDIR/archive/$BASEFILE/$BASEFILE
          fi


      fi # if [ -f $DESTDIR/archive/$BASEFILE/destbase.md5list.txt ]
    else # Not archived Clean up but need better fix
      rm -f $BASEFILE.$nowdate.jpg
    fi # if [ $ARCHIVE_FLAG == "Y" ]
  else # File did not really chage 
    echo "Unchanged (False change) $DESTFILE `date`"
  fi # if [ ! $RC == "1" ]
fi # if [ $orgContent_Length == $newContent_Length ] else

rm -f $BASEFILE.$nowdate.md5sum

cd - >/dev/null 2>&1

}