#!/bin/sh
#
# grabdata.sh - Version 3.2
#
# 20/02/2008 - Fixed problem with identifying version# of app used for crunching
# 25/02/2008 - make sure an initial data capture is sorted, not just update ones
#
# Script to retrieve successful task data from the Einstein@Home online database for
# one or many hostIDs (either entered singly or read from a file) and to parse it to
# produce a tabulated set of information (frequency, seq#, cpu_time, app_version) as
# a CSV list for each host of interest.
#
# Hosts entered manually may be added automatically to the HostIDs file for future
# reuse.  Extracted results data are added to individual host results files to provide
# an ongoing, updating log of host results statistics.
#
# This script relies on an awk script written by Bikeman.
#
echo
echo "Program to grab stats for successful EAH tasks and write them to a results file."
echo "Data is collected from the EAH website for hosts of interest, using the HostID."
echo "The IDs can be entered manually or read from an existing file named hostids."
echo "HostIDs manually entered will be added to the hostids file if not already there."
echo "The lines in this file contain the HostID and a HostName separated by a space."
hidfile=hostids
mindays=+2
echo
echo "Integer days of enforced wait from previous update before a new one is allowed."
echo -n "(Signed ints like +0 +1 +2 - see -mtime flag on 'find' manpage) (default=+2) : "
read ans
echo
if [ "X$ans" != "X" ]
then
	mindays=$ans
fi
#
# Find all results files older than $mindays & store in $hids - ignore newer files
#
hids=`find results.* -mtime $mindays | sed 's/^results\.//'`
#
# If there is no file of HostIDs (ie 1st time run) create a new one
#
if [ ! -f  $hidfile ]
then
	touch $hidfile
fi
#
# See if we are doing an auto update of all out-of-date host or perhaps a manual run
#
echo
echo -n "Using hosts from hostids file? (y or n - use <cr> if entering manually) : "
read ans
echo
if [ "$ans" == "y" ]
then
	if [ "`wc -l $hidfile`" == "0" ]
#
# Can't autorun if there are no hosts filed.  Suggest manual host entry
#
	then
		echo "HostIDs file $hidfile does not contain hosts - use manual entry ..."
		echo
		exit
	else
		echo "Any outdated results for hosts in $hidfile file will be retrieved."
		echo
	fi
else
#
# Manual run.  Collect HostIDs.  Check if HostID is on the list of out-of-date results
# Only accept it if it is.  If it's not, check if we have a results file for it.
# Offer to update HostIDs file if we don't already have this host filed.
#
	mhids=""
	allhids=`cat $hidfile | sed 's/\ \ .*//'`
	while true
	do
		echo -n "Next hostID to use for results ( <cr> only when finished ) : "
		read hid
		if [ "X$hid" != "X" ]
		then
			inhids=n
			for i in $hids
			do
				if [ "$hid" == "$i" ]
				then
					inhids=y
					mhids="$mhids$hid "
					break
				fi
			done
			if [ "$inhids" == "n" ]
			then
				if [ -f results.$hid ]
				then
					inhids=r
					echo "HostID $hid was recently updated -- ignoring this HostID ..."
				else
					mhids="$mhids$hid "
				fi
				inallhids=n
				for j in $allhids
				do
					if [ "$j" == "$hid" ]
					then
					  inallhids=y
					  break
					fi
				done
				if [ "$inallhids" == "n" ]
				then
					echo -n "Hostname for updating HostIDs file $hidfile (<cr> = Dont update) : "
					read host
					if [ "X$host" != "X" ]
					then
						echo "$hid $host" >> $hidfile
					fi
				else
					echo "HostID $hid already in $hidfile - no need to update that file ..."
				fi
			else
				echo "HostID $hid already in $hidfile - no need to update that file ..."
			fi
		else
			break
		fi
	done
	echo "Manually entered HostIDs requiring an update are :-"
	echo "$mhids"
	echo
	echo -n "If these are OK hit <cr> to continue or q to quit : "
	read tmp
	if [ "X$tmp" != "X" ]
	then
		exit
	else
		hids=$mhids
	fi
fi
#
# Variable $hids now contains just those hosts (either auto-determined or manually
# entered) that require updating - if there are any.
#
if [ "X$hids" == "X" ]
then
	echo "There are no out-of-date results files that need updating -- exiting ... "
	echo
	exit
fi
#
# Grab the data from the website for each valid host
#
for hid in $hids
do
#
# Get the first page of (up to 20) results   Test if there is a "Next" page
#
	curl -s "http://einstein.phys.uwm.edu/results.php?hostid=$hid" > results.curl
	while true
	do
		tmp=`grep Next results.curl | tail -1 | sed -e 's/^.*offset.//' -e 's/.Next.*//'`
		if [ "X$tmp" != "X" ]
		then
#
# There is a next page - keep grabbing until no more
#
			while true
			do
				curl -s "http://einstein.phys.uwm.edu/results.php?hostid=$hid&offset=$tmp" > results.ext
				tmp1=`grep Next results.ext | tail -1 | sed -e 's/^.*offset.//' -e 's/.Next.*//'`
				cat results.ext >> results.curl
				tmp=$tmp1
				if [ "X$tmp1" == "X" ]
				then
					break 2
				fi
			done
		else
			break
		fi
	done
#
# All grabbed pages for a host have been concatenated.  Pass through Bikeman's awk script
# Massage to create a CSV list of Freq,Seq#,Runtime and a list of task IDs.  Use the TIDs
# to grab the page with stderr.out so that the app version can be obtained.
#
	awk -f parser.awk results.curl > results.raw
	cut -c-9 results.raw > results.tid
	tids=`cat results.tid`
	cut -c10- results.raw > results.cut
	sed -e s/\ /,/g -e /,[0-9][0-9],/s/,/,0/ -e /,[0-9],/s/,/,00/ -e s/...$// results.cut > results.tmp
	paste -d, results.tid results.tmp | sed s/\ // > results.csv
#
# Progress indicator.  Each dot means another set of raw host results has been parsed
#
	echo -n .
	for tid in $tids
	do
#
# For each TID obtained from the website, check to see if those details are already recorded
# in the results file,  Regard a match of seq# and runtime as sufficient to prove a match.
# Do not grab the page with stderr.out if we already have the data recorded
#
		runtime=`grep $tid results.csv | sed 's/^.*,//'`
		seqno=`grep $tid results.csv | cut -c18- | sed s/,.*//`
		tmp=""
		if [ -f results.$hid ]
		then
			tmp=`grep $runtime results.$hid | grep $seqno`
		fi
		if [ "X$tmp" == "X" ]
		then
#
# No match for seq# and runtime for this TID so we need to grab the page.
#
			freq=`grep $tid results.csv | sed -e 's/,[0-9][0-9][0-9],.*//' -e 's/^.*,//'`
			ver=`curl -s "http://einstein.phys.uwm.edu/result.php?resultid=$tid" | grep einstein_S5R3 | sed -e 's/^.*ein_S5R3_//' -e 's/_[iwp][6io].*//'`
			numv=`echo $ver | wc -w`
#
# Check if more than one app version was used to crunch the data - if so record as 'mixed'
#
			if [ $numv != 1 ]
			then
				flag=1
				ver1=`echo $ver | sed 's/\ .*//'`
				for i in $ver
				do
					if [ $i != $ver1 ]
					then
						flag=2
						break
					fi
				done
				if [ $flag == 1 ]
				then
					ver=$ver1
				else
					ver=mixed
				fi
			fi
#
# Assemble a line of new data and store it in a temporary results file
#
			echo $freq,$seqno,$runtime,$ver >> results.new
		fi
	done
#
# Progress indicator.  Each plus means another host's new results have been assembled.
#
	echo -n +
	if [ ! -f results.new ]
	then
#
# There are actually no new results at this time so create an empty file as a placeholder
#
		touch results.new
	fi
	if [ -f results.$hid ]
	then
#
# If there are existing results for this host add in the new ones.  For no existing results
# any new ones will become the future existing ones.  Clean up any temp files
#
		mv results.$hid results.sav
		cat results.sav results.new | sort > results.$hid
		rm -f results.sav
	else
		sort results.new > results.$hid
	fi
	rm -f results.raw results.c* results.t* results.ext results.new
done
echo