. cg_filecomp

cg_geturl () {

GETBASEURL=$1
GETOBJECT=$2
DESTDIR=$3
DESTFILE=$4
QUALITY=$5
ARCHIVE_FLAG=$6
TIMESTAMP_FLAG=$7


if [ -z $CG_GETURL_CACHE ]
then 
  CG_GETURL_CACHE="/capture/tmp/`whoami`.cg_geturl.cache"
fi # if [ -z $CG_GETURL_CACHE ]

if [ ! -d $CG_GETURL_CACHE ]
then
  echo "No cache for \$CG_GETURL_CACHE found creating $CG_GETURL_CACHE"
  echo "You might want to create one somewhere pointed to by the environment variable CG_GETURL_CACHE"
  echo "Example export CG_GETURL_CACHE=/var/local/cache/cg_geturl"
  mkdir -v $CG_GETURL_CACHE
  mkdir -v $CG_GETURL_CACHE/last
fi # [ ! -d $CG_GETURL_CACHE ]

#
#---- Use squid if available
#
squid_running="`ps -C squid|grep squid|wc -l`"

if [ $squid_running == "0" ]
then
  unset http_proxy
  unset ftp_proxy
else
  export http_proxy=http://127.0.0.1:3128
  export ftp_proxy=$http_proxy 
fi

cd $CG_GETURL_CACHE  >/dev/null 2>&1  # Need to define this otherwise it will dump all to /tmp  Use something like /var/local/cache/cg_geturl with a+rwx permits

if [ ! -f ./last/$DESTFILE.Content_Length.last ]
then
  echo "Getting $GETBASEURL$GETOBJECT for the first time"
  echo "Content_Length: \"0\"" > ./last/$DESTFILE.Content_Length.last        #create a dummy last Content_Length file
fi

orgContent_Length=" "
newContent_Length=" "

orgContent_Length="`cat ./last/$DESTFILE.Content_Length.last | awk '{print $2}'`"

  starttime=`getnanotime`
  proc_getdatetime=`getdatetime`

curl -k --connect-timeout 30 --retry-delay 11 --max-time 60 --retry-max-time 60 --retry 2 --silent --show-error  --head $GETBASEURL$GETOBJECT|grep ^Content|grep Length >$DESTFILE.Content_Length 2>/dev/null
newContent_Length="`cat $DESTFILE.Content_Length | awk '{print $2}'`"


if [ $orgContent_Length == $newContent_Length ]
then
  echo "Unchanged $DESTFILE `date`"
  if [ -f $DESTDIR/$DESTFILE ]
  then
    cg_filecomp $DESTFILE $DESTDIR/$DESTFILE
    if [ $RC != "1" ]
    then
      rm -f $DESTDIR/$DESTFILE 
      cp -p $DESTFILE $DESTDIR/$DESTFILE 
    fi
  else
    cp -p $DESTFILE $DESTDIR/$DESTFILE 
  fi # if [ -f $DESTDIR/$DESTFILE ]
else
  if [ -f $DESTFILE.Content_Length ]
  then
    mv -f $DESTFILE.Content_Length ./last/$DESTFILE.Content_Length.last
  fi

  if [ -f $DESTFILE ]
  then
    mv -f $DESTFILE $DESTFILE.save
  fi

# This might be timing out on large images Modified --max-time from 10 to 20 6/24/2006

  curl -k --connect-timeout 30 --retry-delay 10 --max-time 240 --retry-max-time 120 --retry 2 --silent --show-error  $GETBASEURL$GETOBJECT > $DESTFILE # Use -S flag and rewget if an error

  cg_filecomp $DESTFILE $DESTFILE.save

  if [ ! $RC == "1" ]
  then # New file
    echo "Updating $GETBASEURL$DESTFILE `date`..."
    mv -f $DESTFILE.save  ./last/$DESTFILE
    rm -f $DESTDIR/last/$DESTFILE
    cp -pf ./last/$DESTFILE  $DESTDIR/last/$DESTFILE
    rm -f $DESTDIR/$DESTFILE
    cp -pf $DESTFILE $DESTDIR/$DESTFILE 

    if [ $ARCHIVE_FLAG == "Y" ]
    then
      BASEFILE="`basename $DESTFILE |\
               sed 's/\.jpg//'|\
               sed 's/\.JPG//'|\
               sed 's/\.GIF//'|\
               sed 's/\.gif//'|\
               sed 's/\.png//'`"

      if [ ! -d $DESTDIR/archive ]
      then
        mkdir -v $DESTDIR/archive 
      fi # if [ ! -d $DESTDIR/archive ]

      if [ ! -d $DESTDIR/archive/$BASEFILE ]
      then
        mkdir -v $DESTDIR/archive/$BASEFILE
      fi # if [ ! -d $DESTDIR/archive/$BASEFILE ]

      nowdate=`date +%Y%m%d%H%M%S`
      rm -f $BASEFILE.$nowdate.jpg

#---- Force image to jpeg  Need to add test to see if it is already jpg maybe use file $DESTDIR/$DESTFILE

      convert -format jpg -quality $QUALITY $DESTDIR/$DESTFILE $BASEFILE.$nowdate.jpg
      rm $DESTDIR/$DESTFILE
      cp -p $BASEFILE.$nowdate.jpg $DESTDIR/$DESTFILE 

#
#---- Do a final check that the image isn't already archived based on the sha1sum. md5sum is probably good enough, but heck, we got cpu to burn.
#

      sha1sum $BASEFILE.$nowdate.jpg > $BASEFILE.$nowdate.sha1sum
      sha1val=`cat $BASEFILE.$nowdate.sha1sum|awk '{print $1}'`
      destbase=`basename $DESTDIR/archive/$BASEFILE`

      if [ -f $DESTDIR/archive/$BASEFILE/$destbase.sha1list.txt ]
      then
        nfound=`grep -c --mmap $sha1val $DESTDIR/archive/$BASEFILE/$destbase.sha1list.txt`

        if [ $nfound -eq 0 ]
        then
          cat $BASEFILE.$nowdate.sha1sum >> $DESTDIR/archive/$BASEFILE/$destbase.sha1list.txt 

#---- Add timestamp to upper left if requested

          if [ $TIMESTAMP_FLAG == "Y" ]
          then
            convert -draw "text 2,24 '`date`'" -font helvetica -pointsize 24 -fill red $BASEFILE.$nowdate.jpg $DESTDIR/archive/$BASEFILE/$BASEFILE.$nowdate.jpg
            rm -f $BASEFILE.$nowdate.jpg
            cp -pv $DESTDIR/archive/$BASEFILE/$BASEFILE.$nowdate.jpg $DESTDIR/$DESTFILE
 					else
            mv -f $BASEFILE.$nowdate.jpg $DESTDIR/archive/$BASEFILE  
          fi

          echo "Archived $BASEFILE.$nowdate.jpg"
      
#
#---- Time statistics
#
          endtime=`getnanotime`
          proctime=`echo 8 $endtime  $starttime|bc /usr/local/bc/subfloat.bc`

          echo "STATS 1"

          if [ -f $DESTDIR/archive/$BASEFILE/$BASEFILE.$nowdate.jpg ]
          then
            filesize=`ls -l $DESTDIR/archive/$BASEFILE/$BASEFILE.$nowdate.jpg|awk '{print $5}'`
            echo "$BASEFILE,$proc_getdatetime,$proctime,$filesize" >> /capture/log/stats/cg_geturl.log
            echo "$BASEFILE,$proc_getdatetime,$proctime,$filesize" >> /capture/log/stats/cg_geturl.$BASEFILE.log
            echo "$BASEFILE,$proc_getdatetime,$proctime,$filesize"
          else
            echo "Can't archive $BASEFILE.$nowdate.jpg  - NOT FOUND"
            ls -l $DESTDIR/archive/$BASEFILE/$BASEFILE.$nowdate.jpg
          fi

        else
          echo "$BASEFILE.$nowdate.jpg was already archived"
        fi # if [ $nfound -ne 0 ]
      else
          cat $BASEFILE.$nowdate.sha1sum >> $DESTDIR/archive/$BASEFILE/$destbase.sha1list.txt 

          if [ $TIMESTAMP_FLAG == "Y" ]
          then
            convert -draw "text 2,24 '`date`'" -font helvetica -pointsize 24 -fill red $BASEFILE.$nowdate.jpg $DESTDIR/archive/$BASEFILE/$BASEFILE.$nowdate.jpg
            rm -f $BASEFILE.$nowdate.jpg
            cp -pv $DESTDIR/archive/$BASEFILE/$BASEFILE.$nowdate.jpg $DESTDIR/$DESTFILE
 					else
            mv -f $BASEFILE.$nowdate.jpg $DESTDIR/archive/$BASEFILE  
          fi

          mv -f $BASEFILE.$nowdate.jpg $DESTDIR/archive/$BASEFILE
          echo "Archived $BASEFILE.$nowdate.jpg"
#
#---- Time statistics
#
          endtime=`getnanotime`
          proctime=`echo 8 $endtime  $starttime|bc /usr/local/bc/subfloat.bc`

          echo "STATS 2"

          if [ -f $DESTDIR/archive/$BASEFILE ]
          then
            filesize=`ls -l $DESTDIR/archive/$BASEFILE|awk '{print $5}'`
            echo "$BASEFILE,$proc_getdatetime,$proctime,$filesize" >> /capture/log/stats/cg_geturl.log
            echo "$BASEFILE,$proc_getdatetime,$proctime,$filesize" >> /capture/log/stats/cg_geturl.$BASEFILE.log
            echo "$BASEFILE,$proc_getdatetime,$proctime,$filesize"
          else
            echo "Can't archive $BASEFILE.jpg  - NOT FOUND"
            ls -l $DESTDIR/archive/$BASEFILE/$BASEFILE
          fi


      fi # if [ -f $DESTDIR/archive/$BASEFILE/destbase.sha1list.txt ]
    else # Not archived Clean up but need better fix
      rm -f $BASEFILE.$nowdate.jpg
    fi # if [ $ARCHIVE_FLAG == "Y" ]
  else # File did not really change 
    echo "Unchanged (False change) $DESTFILE `date`"
  fi # if [ ! $RC == "1" ]
fi # if [ $orgContent_Length == $newContent_Length ] else

rm -f $BASEFILE.$nowdate.sha1sum

cd - >/dev/null 2>&1

}