#!/usr/bin/env python
# Copyright 2007, Brian Mingus
#
# This file is part of bagofwords.py
#
#   This library is free software; you can redistribute it and/or
#   modify it under the terms of the GNU Lesser General Public
#   License as published by the Free Software Foundation; either
#   version 2.1 of the License, or (at your option) any later version.
#
#   This library is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#   Lesser General Public License for more details. 


"""
Extracts bags of words from wikipedia xml dumps

Given the path to a wikipedia xml dump and the path to a newline
separated list of titles, writes a file with the title of each
article and all of the unique words that occured in that article

You should first use the mwdumper tool to create a smaller xml dump,
as this script expects that it can load the entire dump into memory
and operate on it.
"""

# Standard Library Imports
import sys
import re

# 3rd Party Imports
from BeautifulSoup import BeautifulStoneSoup

# Incredibly simple CL parsing
from optparse import OptionParser

class BagOfWords:
    def readDump(self, fileName):
        """
        Pass me a filename, and I will run BeautifulStoneSoup on the
        dump and return it
        """
        f = open(fileName)
        xml = f.read()
        f.close()
        the_dump = BeautifulStoneSoup(xml)
        return dump
	
    def main(self, xmldump, titles):
        """
        This is your main function, which is executed when this script
        is called from the command-line. See below for more info.
        """
        the_dump = self.readDump(xmldump)
        print len(the_dump)
        return 1

if __name__ == "__main__":
    """
    The CLI for this program accepts two options:
      --xml-dump=<full path to xml dump>
      --titles=<full path to newline separated list of titles>
    """

    usage = "Usage: %prog file"
    parser = OptionParser(usage)

    # Strings. Also, google for "Ducktyping".
    parser.add_option("-m",
                      "--xml-dump",
                      action="store",
                      type="string",
                      dest="xmldump",
                      default=None,
                      help="The absolute path to the xml dump")

    parser.add_option("-d",
		      "--titles",
		      action="store",
		      type="string",
		      dest="titles",
		      default=None,
		      help="The absolute path to the list of titles ")
		      
    (options, args) = parser.parse_args()

    if options.xmldump:
        xmldump = options.xmldump
    else:
        print "You must specify the --xml-dump option"
        sys.exit(0)

    if options.titles:
        titles = options.titles
    else:
        print "You must specify the --titles option"
        sys.exit(0)

    bagofwords = BagOfWords()
    
    # Python does not have pointers. This syntactic sugar is only
    # valid in funcion calls. Python unpacks the data structure into
    # arguments to the function, and the next function can capture the
    # data.
    
    sys.exit(bagofwords.main(xmldump, titles))