#!/usr/bin/env python
# Copyright, 2007, Brian Mingus, Trevor Pincock, Laura Rassbach
#
# This file is part of get_ratings
#
#   This library is free software; you can redistribute it and/or
#   modify it under the terms of the GNU Lesser General Public
#   License as published by the Free Software Foundation; either
#   version 2.1 of the License, or (at your option) any later version.
#
#   This library is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#   Lesser General Public License for more details. 

"""
 Download and create a list of quality articles and the ranked
 dimensions
"""
debug = True

# Standard Library Imports
# For downloading, searching, file operations, stack access
import datetime
import urllib2
import thread
import sys
import re

# Third Party Imports

# For walking the DOM
from BeautifulSoup import BeautifulSoup

# So we don't parse all of every page
from BeautifulSoup import SoupStrainer

class GetRatings:

    def debug(self, msg, name, error=False):
        """
        error is a boolean value - is this an error or not?
        Pass str(sys._getframe().f_code.co_name) as name
        """
        try:
            if error:
                print "Error: " + str(name) + ": " + str(msg)
            elif debug:
                print "Debug: " + str(name) + ": " + str(msg)
        except:
            print "Even your debug statements are buggy!"

    def URLOpener(self):
        """
        Returns a urllib2 object
        """
        self.opener = urllib2.build_opener()
        self.opener.addheaders = [('User-agent', 'Mozilla/5.0')]    
        return self.opener

    def GetHTML(self, url):
        """
        Returns the HTML of a url passed to it
        """
        website = self.URLOpener()
        website = website.open(url.decode("utf-8"))
        website = website.read()
        return website

    def Beautify(self, html):
        """
        Returns a BeautifulSoup object based on input html
        """    
        return BeautifulSoup(html)

    def GetIndex(self):
        """
        Returns a BeautifulSoup object of the main article quality index at
        http://en.wikipedia.org/wiki/Wikipedia:Version_1.0_Editorial_Team/Index
        """
        index = self.Beautify(self.GetHTML('http://en.wikipedia.org/wiki/Wikipedia:Version_1.0_Editorial_Team/Index'))
        return index

    def isQualityTable(self, page):
        """
        Takes a BeautifulSoup object and determines if it is an article quality table containing the quality metrics of articles
        It's heuristic is simply the *lack* of a digit in the last character of the URL combined with
        the existence of the word 'Comments'.
        """
        if not page.html.find(href=re.compile('/wiki/Wikipedia:Version_1.0_Editorial_Team/.*/[0-9]+')):
            if page.html.find(text='Comments'):
                return True
            else:
                return False

    def isListofQualityTables(self, page):
        """
        Takes a BeautifulSoup object and determines if it is a list of quality metrics of articles (a list of lists)
        It's heuristic is whether there is a URL whose last character is a digit anywhere on the page
        """
        if page.html.find(href=re.compile('/wiki/Wikipedia:Version_1.0_Editorial_Team/.*/[0-9]+')):
            return True
        else:
            return False

    def GetTitle(self, page):
        """
        Takes a BeautifulSoup object and returns the title of the article
        """
        name = str(sys._getframe().f_code.co_name) # Debugging

        title_match = '^.*Team\/(.*) - Wikipedia, the free encyclopedia$'
        title_match = re.compile(title_match)
        try:
            title = title_match.search(page.html.title.string).group(1)
            return title
        except ValueError:
            self.debug("Couldnt get the article title from the <title>", name, True)

    def ParseDumpArticleQualityTable(self, page):
        """
        This function has the unsightly job of parsing the table row by row and writing the output to a CSV
        We don't care about most of the data it sees - just uncomment whatever else you want
        """
        name = str(sys._getframe().f_code.co_name) # Debugging
        Article, Importance, Date, Assessment, Version, Comments = '','','','','',''
        # TODO: str(page) is a major inefficiency in this script
        table = BeautifulSoup(str(page), parseOnlyThese=SoupStrainer('table', border='1'))
        trs = table.tr.nextSibling.nextSibling.findNextSiblings('tr')
        del trs[len(trs) - 1] # Last item is extraneous            
        date = datetime.date.today()
        f = open('ETQR' + str(date) + '.csv', 'a')
        write = True

        for i in range(len(trs)):

            pointer = trs[i].findNext('td')
            try:
                Article = pointer.next.string # Article
            except:
                write = False

            pointer = pointer.findNext('td') #Importance = pointer.next.string
            pointer = pointer.findNext('td') # Date = pointer.next.string
            pointer = pointer.findNext('td') # Assessment

            try:
                Assessment = pointer.next.a.string # Odd man out
            except:
                # This is for FA/GA articles
                try:
                    Assessment = pointer.findNext('b').findNext('a').string
                except:
                    write = False

            pointer = pointer.findNext('td') # Version = pointer.next.string
            pointer = pointer.findNext('td') # Comments = pointer.next.string

            if write:
                # Only log thesex
                if Assessment in ['FA','A', 'GA', 'B', 'Start']:
                    data = str(Article) + '____' + str(Assessment)
                    data = data.encode("utf-8") + '\n'
                    self.debug("Write: " + str(write) + ": " + data, name)                    
                    f.write(data)

        f.close()

    def ParseDumpListOfQualityTables(self, page):
        """
        Knows how to parse a LIST of quality tables, and then call ParseDumpArticleQualityTable on them
        """
        name = str(sys._getframe().f_code.co_name) # Debugging
        links = page.html.findAll('a', href=re.compile('/wiki/Wikipedia:Version_1.0_Editorial_Team/.*/[0-9]+'))
        title = self.GetTitle(page)

        self.debug("len(links): " + str(len(links)), name)
        for i in range(len(links)):
            try:
                url = 'http://en.wikipedia.org' + str(links[i]['href']).decode("utf-8")
            except:
                return 0
            try:
                new_page = self.Beautify(self.GetHTML(url).decode("utf-8"))
            except:
                self.debug("Check page ", name, True)
            try:
                thread.start_new_thread(self.ParseDumpArticleQualityTable, (new_page,))
                #self.ParseDumpArticleQualityTable(new_page)
                #self.WriteToCSVRecursive(new_page)
            except:
                self.debug("Exception in " + str(title), name, True)

    def ParseIndex(self, index):
        """
        Parses the main index and returns a BeautifulSoup list of all the appropriate <a href=""></a> tags
        """
        # This gets n - 1 articles - small bug.
        index_list = index.html.findAllNext('a',href=re.compile('.*articles_by_quality.*'),text=re.compile('.*articles by quality.*'))
        return index_list

    def WriteToCSVRecursive(self, page):
        """
        Not actually recursive, but if it's actually needed in the future it almost works. Uncomment below...
        """
        name = str(sys._getframe().f_code.co_name) # Debugging
        title = self.GetTitle(page)
        self.debug('', name)        

        if self.isQualityTable(page):
            self.debug("Table: " + title, name)
            thread.start_new_thread(self.ParseDumpArticleQualityTable, (page,))
            #self.ParseDumpArticleQualityTable(page)
            # ParseDumpListOfQualityTables calls WriteToCSVRecursive
        elif self.isListofQualityTables(page):
            self.debug("List: " + title, name)
            self.ParseDumpListOfQualityTables(page)
        else:
            self.debug("Unknown: " + title, name)

    def GetList(self):
        """
        1. Download the index
        2. Parse it with BeautifulSoup to get a list of articles
        3. Determine if we have a list of articles or a list of lists of articles
        """
        index = self.GetIndex()                                # 1.
        list_of_article_titles = self.ParseIndex(index)        # 2.
        for i in range(len(list_of_article_titles)):           # 3.
            url = 'http://en.wikipedia.org' + list_of_article_titles[i].parent['href']
            page = self.Beautify(self.GetHTML(url.decode("utf-8")))
            self.WriteToCSVRecursive(page)

    def main(self):
        self.GetList()
        
if __name__ == "__main__":
    articleList = GetRatings()
    sys.exit(articleList.main())