#! /usr/bin/python
# -*- coding: utf-8 -*-

import sys, os, re, urllib, csv

re_successes = 0
re_fails = 0
re_none = 0
no = 0

verbose = False
def uniqify(list):
    uniques = []
    
    for item in list:
        if item not in uniques:
            uniques.append(item)
    return uniques
        

def article_parse(conf_name, articlenr, article, writer):
    global re_successes, re_fails, re_none, no
    global verbose
    print article, articlenr
    sys.exit()

    no += 1

    if no % 10 == 0:
        if not verbose: print '\x08'*80, "Processing #%d." % no, 
#   print conf_name, article
    fullfile = ''.join(open(article).readlines())

#   Extract the title, with some sanity checks
    m = re.search(r"<title>(.*?)</title>", fullfile)
    if not m: return
    title = m.group(1)
    if verbose: print '\t', title
    if verbose: print '\t', article

    subconf_name = ''

    title_parts = title.split(',', 1)
    if not ':' in title or not ',' in title:
        title = title_parts[0]
        conf_name, subconf_name = conf_name.split('_')
    else:
        conf, title = title_parts
        subconf_name = conf.split(':', 1)[1]
        conf_name = conf_name.split('_')[0]
   
    title = title.strip()
    if title[-3:] == '...': title = title[:-3]
    subconf_name = subconf_name.strip()
    conf_name = conf_name.strip()

   # Don't want too long strings
    if len(title) < 20: 
        title = title[0:]
    else:
        title = title[:20]
    
    
    if verbose: print conf_name, subconf_name, title

    # les hele filen inn til en string
    try:
        fullfile = fullfile[fullfile.index('<p class="abstract"><font color="Red">Note: OCR errors may be found in this Reference List extracted from the full text article.  ACM has opted to expose the complete List rather than only correct and linked references.</font></p>'):]
        fullfile = fullfile[:fullfile.index('</table>')]
    except:
        return 
    
    references = fullfile.split('<div class="abstract">')[1:]
    refnr = []
    refyears= []
    
    # Every second <div class="abstract"> is a reference 
    isRef = False
    for ref in references:
        isRef = not isRef
        if isRef:
            nr = ref[:ref.index('</div>')].strip()
            refnr.append(nr)
        else:
            ref_text = ref[:ref.index('</div>')].strip()

            ref_text_old = ref_text

            # if we have a link in the text, try extracting the contents of the tag
            if 'a href' in ref_text:
                ref_start = ref_text.find('>')
                ref_stop = ref_text.find('</a>', ref_start)

                ref_text = ref_text[ref_start+1:ref_stop].strip()
#           writer.write(ref_text + '\n')

#            for x in ["Published", "accessed"]:
#                if x in ref_text:
#                    ref_text = ref_text[:ref_text.find(x)]
#                    break
            else:
            # if we have a failpedia or failttp reference, skip the ref
                if "http" in ref_text: continue
                if "ikipedia" in ref_text: continue

#  regex: (start of line or not (digit or letter) followed by 19 or 20 followed by 2 digits followed by end of line or not a digit.
            pattern = r"([^0-9a-zA-Z]|^)((19|20)\d\d)([^0-9]|$)"
            all_matches = re.findall(pattern, ref_text )

            m = map(lambda x: x[1], all_matches) # scramble array into a usable form
            
            if m:
                # Treat manually
                if len(uniqify(m)) > 1:
                    # Get the refnr that failed, and put it back
                    refn = refnr.pop()
                    refnr.append(refn)
                    refyears.append('-')
                    writer.writerow([title, int(articlenr), int(refn), "CHECK ME", "FILL ME", ref_text])
                    re_fails += 1
#                    writer.write(ref_text  + '\n')
#                    writer.write(ref_text+ '\n')
#                    writer.write(ref_text_old+ '\n')
#                    writer.write(":".join(m)+'\n')
#                    writer.write("*"*78 + '\n')
                    print "Check manually as", m

                # Same year in the ref, probably okay
                elif len(uniqify(m)) == 1:
                    refn = refnr.pop()
                    refnr.append(refn)
                    age = str(2008-int(m[0][0]))
                    writer.writerow([title, int(articlenr), int(refn), int(m[0][0]), int(age)])
                    refyears.append(m[0][0])
                    print "found one", m[0][0]
                    re_successes += 1
                else:
                    print "SHOULD NOT HAPPEN!"
                    sys.exit(0)
            else:
                print "No reference"
                re_none += 1
                refn = refnr.pop()
                refnr.append(refn)
                refyears.append('-')
                writer.writerow([title, articlenr, refn," "," "])
        
        return
            




if __name__ == '__main__':
    #filter = ''
    #if len(sys.argv) == 2:
    #    filter = sys.argv[1]

    for conference in os.listdir("."):
      #  if filter and conference != filter: continue
        if os.path.isdir(conference):
            for sub_conference in os.listdir(conference):
                file_path = "/home/alexao/SE-forskning/results/"+conference+"/"+sub_conference+"/"+'_'.join((conference, sub_conference))+"_result.csv"
                if not os.path.isdir("/home/alexao/SE-forskning/results/"+conference+"/"+sub_conference):
                    os.makedirs("/home/alexao/SE-forskning/results/"+conference+"/"+sub_conference)
                output = csv.writer(open(file_path, "w"))
                for article in os.listdir(os.path.join(conference,sub_conference)):
                    article_parse('_'.join((conference, sub_conference)), article, os.path.join(conference, sub_conference, article), output)

    print '\nsuccesses:\t', re_successes
    print 'fails:\t\t', re_fails
    print 'none:\t\t', re_none
    print '%\t\t', re_fails * 100 / (re_fails + re_successes+re_none)

output.close()