#!/opt/local/bin/python
##!/usr/bin/python
# minidumper v. .01
# Copyright, 1995-2005, Brian Mingus
#
# This file is part of minidumper
#
#   This library is free software; you can redistribute it and/or
#   modify it under the terms of the GNU Lesser General Public
#   License as published by the Free Software Foundation; either
#   version 2.1 of the License, or (at your option) any later version.
#
#   This library is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#   Lesser General Public License for more details. 

# Takes a mediawiki xml dump, a list of the pages that you want from
# the xml dump and extracts them to stdout

# Example usage:
# minidumper wikipedia-xml-dump.xml allowed_titles.list --tags=title,id > minidump.xml

from optparse import OptionParser
import re

class MiniDumper:

    def __init__(self):
        self.titleRE = re.compile('    <title>(.*)</title>\n')
        self.allowListRE = ''
        self.currentPage = ''
        self.xmlPOST = '</mediawiki>'
        self.firstTime = True        # For determining when to pring xmlPRE
        self.counter = 0             # So we know when to quit  (don't parse the whole dump unnecessarily)
        self.limit = 0               # Needs to be set to the length of the list of titles you gave me
        self.finished = False        # We check this to see if counter == limit, and print the end of the dump if so
        self.xmlPRE = '\
<mediawiki xmlns="http://www.mediawiki.org/xml/export-0.3/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.mediawiki.org/xml/export-0.3/ http://www.mediawiki.org/xml/export-0.3.xsd" version="0.3" xml:lang="en">\n\
  <siteinfo>\n\
    <sitename>Wikipedia</sitename>\n\
    <base>http://en.wikipedia.org/wiki/Main_Page</base>\n\
    <generator>MediaWiki 1.10alpha</generator>\n\
    <case>first-letter</case>\n\
      <namespaces>\n\
        <namespace key="0" />\n\
      </namespaces>\n\
  </siteinfo>'

    def PageBuilder(self, thisLine):
        """
        This takes lines of input and uses them to build pages.

        It should be fairly robust to interference, as the correct number of spaces
            are required on the <page> tags.

         When a page is completed (indicated by </page>) it is passed to AnalyzePage

        """
        if thisLine == '  <page>\n':
            self.currentPage = thisLine
        elif thisLine == '  </page>\n':
            self.currentPage += thisLine
            self.AnalyzePage(self.currentPage)
        else:
            self.currentPage += thisLine

    def AllowList(self, listFileName):
        """
        The allow list takes newline separated text file of page titles and
        returns a regular expression object that places boolean OR (|) before each title

        This was designed to work with a list of 150,000 titles, so pass it a big list if you want.
        """
        f = open(listFileName,'r')
        titles = f.readlines()
        titles = [titles[i][:-1] for i in range(len(titles))] # removes trailing newlines
        titleList = [self.Escaper(title) for title in titles]
        self.limit = len(titleList)

        longLine = ''

        for title in range(len(titleList)):
            longLine += titleList[title]
        longLine = longLine[:-1] # remove trailing pipe

        print "Regex: " + longLine # TODO: why is this a list
        f.close()
        self.allowListRE = re.compile(longLine)

    def Escaper(self, string):
        """
        Just puts a pipe inbetween each one and escapes it
        TODO: Remove trailing pipe!
        """
        return str(re.escape(string) + '|')
        
    def AnalyzePage(self, page, allowedTags = []):
        """
         AnalyzePage parses the page for its <title>tags</title> and
            compares it against the allow list

         You can optionally specify that you only want a subset of
            tags and the page will then be parsed by BeautifulSoup.
            
         It's a very cheap parse because we can use the SoupStrainer
            to only get the stuff we want.

         If the regular expression matches this pages title, send it
            to stdout
        """
        # print "AnalyzePage: \n" + page
        titleMatch = self.titleRE.search(page)

        if titleMatch == None:
            return

        title = titleMatch.group(1)
        print "Title: " + title

        # Check title against supplied list
        if self.allowListRE.match(title) == None:
            return

        # It passed all of our checks - have it!
        print page
        self.counter += 1
        if self.counter == self.limit:
            self.finished = True

def main():
    parser = OptionParser()
    parser.add_option("-t", "--tags", dest="tags",
                  help="Comma-separated list of tags which are subordinate to the <page> tag to pay attention to. Current tags in the dump are page, title, id, revision, etc...")
    (options, args) = parser.parse_args()
    xmlFile = args[0]
    listFile = args[1]
    extractor = MiniDumper()
    extractor.AllowList(listFile)
    f = open(xmlFile)
    thisLine = None
    print extractor.xmlPRE
    while not thisLine == '':
        thisLine = f.readline()
        extractor.PageBuilder(thisLine)
        if extractor.finished == True:
            print extractor.xmlPOST
            return

main()